From 924765b0426fec802763e3165989330200e1dc6b Mon Sep 17 00:00:00 2001 From: Sunil Date: Mon, 10 Aug 2026 21:51:27 +0530 Subject: [PATCH 01/40] security: fix XXE vulnerability in RDF/XML parser (CWE-611) --- semantica/explorer/utils/rdf_parser.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/semantica/explorer/utils/rdf_parser.py b/semantica/explorer/utils/rdf_parser.py index 39400156..e685fb1e 100644 --- a/semantica/explorer/utils/rdf_parser.py +++ b/semantica/explorer/utils/rdf_parser.py @@ -22,13 +22,34 @@ def _safe_parse_rdf(g: rdflib.Graph, data: bytes, rdf_format: str) -> None: import defusedxml defusedxml.defuse_stdlib() else: - # Warn once; best-effort protection via rdflib's own parser + # SECURITY: Strip DOCTYPE declarations and entity definitions from + # the raw XML to prevent XXE attacks (file disclosure, SSRF, DoS + # via entity expansion). This is a best-effort defence when + # defusedxml is not installed. + import re import warnings warnings.warn( "defusedxml is not installed. Install it (`pip install defusedxml`) " - "to protect RDF/XML parsing against XXE attacks.", + "for robust XXE protection. Applying basic DOCTYPE stripping as " + "a fallback.", stacklevel=4, ) + text = data.decode("utf-8", errors="replace") + # Remove blocks (including internal subsets) + text = re.sub( + r"\[]*(\[[^\]]*\])?\s*>", + "", + text, + flags=re.IGNORECASE | re.DOTALL, + ) + # Remove any remaining declarations + text = re.sub( + r"]*>", + "", + text, + flags=re.IGNORECASE, + ) + data = text.encode("utf-8") g.parse(data=data, format=rdf_format) def _get_best_label(graph: rdflib.Graph, subject: rdflib.URIRef, predicate: rdflib.URIRef) -> str: From 55f3ee6f84d87d648a4780a81df3c13833ad8baf Mon Sep 17 00:00:00 2001 From: Sunil Date: Mon, 10 Aug 2026 21:51:56 +0530 Subject: [PATCH 02/40] security: fix SPARQL DoS via unbounded graph materialization (CWE-770) --- semantica/explorer/routes/sparql.py | 32 +++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/semantica/explorer/routes/sparql.py b/semantica/explorer/routes/sparql.py index 0d7e2f77..acc825e5 100644 --- a/semantica/explorer/routes/sparql.py +++ b/semantica/explorer/routes/sparql.py @@ -8,7 +8,7 @@ Security contract * Multi-statement injections that start with an allowed keyword (e.g. ``SELECT ... ; DROP ALL``) pass the prefix check and reach rdflib, which rejects non-SELECT/ASK/CONSTRUCT/DESCRIBE update syntax in the parser. -* The in-memory rdflib graph is a read-only projection — the live +* The in-memory rdflib graph is a read-only projection — the live ``GraphSession`` is never mutated by this route. """ @@ -59,8 +59,27 @@ def _build_rdflib_graph(session: GraphSession) -> rdflib.Graph: graph.bind("ent", NS) graph.bind("prop", PROP) - nodes, _ = session.get_nodes(skip=0, limit=999_999) - edges, _ = session.get_edges(skip=0, limit=999_999) + # SECURITY: Cap the number of entities materialized into memory to + # prevent denial-of-service via memory exhaustion. Without this guard + # an attacker can send concurrent SPARQL queries that each load ~1M + # nodes/edges into rdflib Graph objects, consuming gigabytes of RAM. + nodes, total_nodes = session.get_nodes(skip=0, limit=_SPARQL_MAX_GRAPH_NODES + 1) + if len(nodes) > _SPARQL_MAX_GRAPH_NODES: + raise ValueError( + f"Graph has more than {_SPARQL_MAX_GRAPH_NODES:,} nodes. " + f"SPARQL queries are limited to graphs with at most " + f"{_SPARQL_MAX_GRAPH_NODES:,} nodes to prevent excessive " + f"memory usage. Use the REST API for large graph operations." + ) + + edges, _ = session.get_edges(skip=0, limit=_SPARQL_MAX_GRAPH_NODES + 1) + if len(edges) > _SPARQL_MAX_GRAPH_NODES: + raise ValueError( + f"Graph has more than {_SPARQL_MAX_GRAPH_NODES:,} edges. " + f"SPARQL queries are limited to graphs with at most " + f"{_SPARQL_MAX_GRAPH_NODES:,} edges to prevent excessive " + f"memory usage. Use the REST API for large graph operations." + ) for node in nodes: subject = NS[str(node.get("id", ""))] @@ -91,6 +110,7 @@ def _build_rdflib_graph(session: GraphSession) -> rdflib.Graph: _SPARQL_MAX_ROWS = 5_000 # hard cap on returned rows _SPARQL_TIMEOUT_S = 30 # seconds before abandoning the await _SPARQL_MAX_CONCURRENT = 4 # semaphore: max simultaneous executions +_SPARQL_MAX_GRAPH_NODES = 50_000 # cap on graph nodes/edges to prevent OOM # Semaphore caps how many graph.query calls run concurrently so that # timed-out threads (which keep running in the pool) cannot crowd out @@ -157,9 +177,9 @@ async def execute_sparql( # --------------------------------------------------------------------------- # Serialize results into a type-aware tabular representation. - # ASK → single row: {"result": "true"|"false"} - # CONSTRUCT/DESCRIBE → rows of {"subject", "predicate", "object"} triples - # SELECT → rows keyed by projected variable names + # ASK → single row: {"result": "true"|"false"} + # CONSTRUCT/DESCRIBE → rows of {"subject", "predicate", "object"} triples + # SELECT → rows keyed by projected variable names # --------------------------------------------------------------------------- query_type: str = query_results.type # always set by rdflib From c85df419ae654c24894239b5186213ce56d80b25 Mon Sep 17 00:00:00 2001 From: Sunil Date: Mon, 10 Aug 2026 21:52:03 +0530 Subject: [PATCH 03/40] security: add defusedxml to explorer dependencies for XXE protection --- pyproject.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 097b05e1..13b43726 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "semantica" version = "0.6.0" -description = "Accountability and context layer for AI agents. Context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable." +description = "Accountability and context layer for AI agents. Context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable." readme = "README.md" license = { text = "MIT" } @@ -233,14 +233,15 @@ explorer = [ "fastapi>=0.100.0", "uvicorn[standard]>=0.22.0", "websockets>=15.0.1", - "python-multipart>=0.0.6" + "python-multipart>=0.0.6", + "defusedxml>=0.7.1" ] explorer-lite = [ "streamlit>=1.25.0", "streamlit-agraph>=0.0.45" ] -# Everything (cross-platform — gpu excluded; install semantica[gpu] separately on Linux) +# Everything (cross-platform — gpu excluded; install semantica[gpu] separately on Linux) all = [ "semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,explorer]", "semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,agno]" From 30d5fef180d84e465930ceffe1db98c39d475084 Mon Sep 17 00:00:00 2001 From: Sunil Date: Mon, 10 Aug 2026 21:52:47 +0530 Subject: [PATCH 04/40] security: fix SSRF via redirect bypass in ontology URL fetcher (CWE-918) --- semantica/explorer/routes/ontology.py | 45 +++++++++++++++++---------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/semantica/explorer/routes/ontology.py b/semantica/explorer/routes/ontology.py index 4944f2ef..ca9101fc 100644 --- a/semantica/explorer/routes/ontology.py +++ b/semantica/explorer/routes/ontology.py @@ -1005,23 +1005,36 @@ def _validate_fetch_url(url: str) -> None: def _fetch_url_sync(url: str) -> bytes: _validate_fetch_url(url) import requests as _req + _MAX_REDIRECTS = 5 + current_url = url try: - resp = _req.get( - url, - headers={"Accept": "text/turtle, application/rdf+xml, application/ld+json, */*;q=0.1"}, - timeout=30, - stream=True, - allow_redirects=True, - ) - resp.raise_for_status() - chunks: List[bytes] = [] - total = 0 - for chunk in resp.iter_content(65536): - total += len(chunk) - if total > _MAX_FETCH_BYTES: - raise HTTPException(status_code=413, detail="Remote resource exceeds 20 MB limit.") - chunks.append(chunk) - return b"".join(chunks) + for _ in range(_MAX_REDIRECTS + 1): + resp = _req.get( + current_url, + headers={"Accept": "text/turtle, application/rdf+xml, application/ld+json, */*;q=0.1"}, + timeout=30, + stream=True, + allow_redirects=False, # SECURITY: follow redirects manually + ) + if resp.is_redirect or resp.is_permanent_redirect: + redirect_url = resp.headers.get("Location") + if not redirect_url: + raise HTTPException(status_code=502, detail="Redirect without Location header.") + # Re-validate the redirect target to prevent SSRF via + # open-redirect to internal/cloud-metadata endpoints. + _validate_fetch_url(redirect_url) + current_url = redirect_url + continue + resp.raise_for_status() + chunks: List[bytes] = [] + total = 0 + for chunk in resp.iter_content(65536): + total += len(chunk) + if total > _MAX_FETCH_BYTES: + raise HTTPException(status_code=413, detail="Remote resource exceeds 20 MB limit.") + chunks.append(chunk) + return b"".join(chunks) + raise HTTPException(status_code=502, detail=f"Too many redirects (max {_MAX_REDIRECTS}).") except HTTPException: raise except Exception as exc: From 22ea189d0bf4516725e06d5b9ab7dab0ce8b317d Mon Sep 17 00:00:00 2001 From: Sunil Date: Mon, 10 Aug 2026 21:53:11 +0530 Subject: [PATCH 05/40] security: replace unsafe pickle with JSON in vector store (CWE-502) --- semantica/vector_store/vector_store.py | 41 +++++++++++++++++++------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/semantica/vector_store/vector_store.py b/semantica/vector_store/vector_store.py index 2b685fe2..02f21d09 100644 --- a/semantica/vector_store/vector_store.py +++ b/semantica/vector_store/vector_store.py @@ -573,8 +573,8 @@ class VectorStore: Args: path: Directory path to save to """ + import json import os - import pickle os.makedirs(path, exist_ok=True) @@ -586,17 +586,20 @@ class VectorStore: elif self._backend_store is not None and hasattr(self._backend_store, "save_index"): self._backend_store.save_index(os.path.join(path, "index.bin")) - # Save Python-level data + # Save Python-level data using JSON (safe serialization). + # pickle is intentionally avoided to prevent arbitrary code execution + # if a malicious .pkl file is placed in the store directory. data = { - "vectors": getattr(self, "vectors", {}), + "vectors": {k: list(v) if hasattr(v, "tolist") else v + for k, v in getattr(self, "vectors", {}).items()}, "metadata": getattr(self, "metadata", {}), "config": self.config, "backend": self.backend, "dimension": self.dimension } - with open(os.path.join(path, "store_data.pkl"), "wb") as f: - pickle.dump(data, f) + with open(os.path.join(path, "store_data.json"), "w", encoding="utf-8") as f: + json.dump(data, f) self.logger.info(f"Saved vector store to {path}") @@ -606,17 +609,33 @@ class VectorStore: Args: path: Directory path to load from + + Raises: + RuntimeError: If only a legacy pickle file is found (security risk). """ + import json import os - import pickle - data_path = os.path.join(path, "store_data.pkl") - if not os.path.exists(data_path): - self.logger.warning(f"Store data not found: {data_path}") + json_path = os.path.join(path, "store_data.json") + legacy_pkl_path = os.path.join(path, "store_data.pkl") + + if os.path.exists(json_path): + data_path = json_path + elif os.path.exists(legacy_pkl_path): + # Refuse to load pickle files to prevent arbitrary code execution. + # A crafted .pkl file can execute arbitrary Python when deserialized. + raise RuntimeError( + f"Legacy pickle file found at {legacy_pkl_path}. " + "Pickle deserialization is disabled for security (arbitrary code " + "execution risk). Please re-save the vector store to migrate " + "to the safe JSON format: vs.save(path)" + ) + else: + self.logger.warning(f"Store data not found in: {path}") return - with open(data_path, "rb") as f: - data = pickle.load(f) + with open(data_path, "r", encoding="utf-8") as f: + data = json.load(f) self.vectors = data.get("vectors", {}) self.metadata = data.get("metadata", {}) From 0a113b97021a280df25e8415be71d281251634b1 Mon Sep 17 00:00:00 2001 From: Sunil Date: Mon, 10 Aug 2026 22:26:49 +0530 Subject: [PATCH 06/40] fix: make defusedxml required, fail closed if missing (reviewer feedback) --- semantica/explorer/utils/rdf_parser.py | 47 ++++++++------------------ 1 file changed, 14 insertions(+), 33 deletions(-) diff --git a/semantica/explorer/utils/rdf_parser.py b/semantica/explorer/utils/rdf_parser.py index e685fb1e..07206410 100644 --- a/semantica/explorer/utils/rdf_parser.py +++ b/semantica/explorer/utils/rdf_parser.py @@ -14,42 +14,23 @@ _HAS_DEFUSEDXML = importlib.util.find_spec("defusedxml") is not None def _safe_parse_rdf(g: rdflib.Graph, data: bytes, rdf_format: str) -> None: - """Parse RDF bytes into *g*, guarding against XXE for XML-based formats.""" + """Parse RDF bytes into *g*, guarding against XXE for XML-based formats. + + Raises: + ImportError: If ``defusedxml`` is not installed and the format is XML-based. + """ xml_formats = {"xml", "rdf", "rdf/xml", "application/rdf+xml"} if rdf_format.lower() in xml_formats: - if _HAS_DEFUSEDXML: - # defusedxml patches xml.etree so rdflib's XML parser inherits the fix - import defusedxml - defusedxml.defuse_stdlib() - else: - # SECURITY: Strip DOCTYPE declarations and entity definitions from - # the raw XML to prevent XXE attacks (file disclosure, SSRF, DoS - # via entity expansion). This is a best-effort defence when - # defusedxml is not installed. - import re - import warnings - warnings.warn( - "defusedxml is not installed. Install it (`pip install defusedxml`) " - "for robust XXE protection. Applying basic DOCTYPE stripping as " - "a fallback.", - stacklevel=4, + if not _HAS_DEFUSEDXML: + # Fail closed: refuse to parse untrusted XML without XXE protection. + raise ImportError( + "defusedxml is required to safely parse RDF/XML content but is " + "not installed. Install it with: pip install defusedxml " + "(or install semantica with the explorer extra: " + "pip install 'semantica[explorer]')" ) - text = data.decode("utf-8", errors="replace") - # Remove blocks (including internal subsets) - text = re.sub( - r"\[]*(\[[^\]]*\])?\s*>", - "", - text, - flags=re.IGNORECASE | re.DOTALL, - ) - # Remove any remaining declarations - text = re.sub( - r"]*>", - "", - text, - flags=re.IGNORECASE, - ) - data = text.encode("utf-8") + import defusedxml + defusedxml.defuse_stdlib() g.parse(data=data, format=rdf_format) def _get_best_label(graph: rdflib.Graph, subject: rdflib.URIRef, predicate: rdflib.URIRef) -> str: From c35899711d724bac0a452e4969f55cf8ec4dd674 Mon Sep 17 00:00:00 2001 From: Sunil Date: Mon, 10 Aug 2026 22:27:45 +0530 Subject: [PATCH 07/40] fix: re-push sparql.py with correct UTF-8 encoding --- semantica/explorer/routes/sparql.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/semantica/explorer/routes/sparql.py b/semantica/explorer/routes/sparql.py index acc825e5..c110d023 100644 --- a/semantica/explorer/routes/sparql.py +++ b/semantica/explorer/routes/sparql.py @@ -8,7 +8,7 @@ Security contract * Multi-statement injections that start with an allowed keyword (e.g. ``SELECT ... ; DROP ALL``) pass the prefix check and reach rdflib, which rejects non-SELECT/ASK/CONSTRUCT/DESCRIBE update syntax in the parser. -* The in-memory rdflib graph is a read-only projection — the live +* The in-memory rdflib graph is a read-only projection — the live ``GraphSession`` is never mutated by this route. """ @@ -177,9 +177,9 @@ async def execute_sparql( # --------------------------------------------------------------------------- # Serialize results into a type-aware tabular representation. - # ASK → single row: {"result": "true"|"false"} - # CONSTRUCT/DESCRIBE → rows of {"subject", "predicate", "object"} triples - # SELECT → rows keyed by projected variable names + # ASK → single row: {"result": "true"|"false"} + # CONSTRUCT/DESCRIBE → rows of {"subject", "predicate", "object"} triples + # SELECT → rows keyed by projected variable names # --------------------------------------------------------------------------- query_type: str = query_results.type # always set by rdflib From 26f236923cc39e6985eb94b1eea0b42d37352c08 Mon Sep 17 00:00:00 2001 From: Sunil Date: Mon, 10 Aug 2026 22:28:06 +0530 Subject: [PATCH 08/40] fix: re-push pyproject.toml with correct UTF-8 encoding --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 13b43726..d6f9f3bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "semantica" version = "0.6.0" -description = "Accountability and context layer for AI agents. Context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable." +description = "Accountability and context layer for AI agents. Context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable." readme = "README.md" license = { text = "MIT" } @@ -241,7 +241,7 @@ explorer-lite = [ "streamlit-agraph>=0.0.45" ] -# Everything (cross-platform — gpu excluded; install semantica[gpu] separately on Linux) +# Everything (cross-platform — gpu excluded; install semantica[gpu] separately on Linux) all = [ "semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,explorer]", "semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,agno]" From 5573ab7a9f3674a3e0fe12e729abb6bdcb0f71b5 Mon Sep 17 00:00:00 2001 From: Sunil Date: Mon, 10 Aug 2026 22:28:27 +0530 Subject: [PATCH 09/40] fix: re-push ontology.py with correct UTF-8 encoding From 8fa203761912a093845a9691a56e741816a505a1 Mon Sep 17 00:00:00 2001 From: Sunil Date: Mon, 10 Aug 2026 22:28:48 +0530 Subject: [PATCH 10/40] fix: re-push vector_store.py with correct UTF-8 encoding From 51cf97765d93baf7b86a3bdb775652e2094e0d16 Mon Sep 17 00:00:00 2001 From: pravit-amp <43916793+pravit-amp@users.noreply.github.com> Date: Tue, 11 Aug 2026 00:52:15 -0700 Subject: [PATCH 11/40] test(deduplication): cover ClusterBuilder, MergeStrategyManager, and batch paths (#907) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(deduplication): cover ClusterBuilder, MergeStrategyManager, and batch paths Add focused coverage for union-find clustering, property merge rules, merge_duplicates, embedding similarity, and incremental detection (#866). * test(deduplication): assert unrelated clusters without conditional skip Make cluster-separation coverage fail closed by using mocked pairs and unconditional assertions for distinct Apple vs Microsoft cluster IDs. * test(deduplication): tighten update_clusters attachment assertions Require the incremental path to place the new near-duplicate in the same rebuilt cluster instead of accepting a vacuous cluster-count check. * test(deduplication): strengthen incremental detect_duplicates wrapper checks Assert real DuplicateCandidate matches, score threshold, and new×existing routing instead of only checking that the wrapper returns a list. * test(deduplication): verify metadata provenance behavior Assert that preserve_provenance writes metadata.provenance fields and add a disabled-path test so regressions do not pass through merge_entities metadata alone. --------- Co-authored-by: Pravit Ampapathini --- .../test_cluster_merge_coverage.py | 631 ++++++++++++++++++ 1 file changed, 631 insertions(+) create mode 100644 tests/deduplication/test_cluster_merge_coverage.py diff --git a/tests/deduplication/test_cluster_merge_coverage.py b/tests/deduplication/test_cluster_merge_coverage.py new file mode 100644 index 00000000..97783d4c --- /dev/null +++ b/tests/deduplication/test_cluster_merge_coverage.py @@ -0,0 +1,631 @@ +""" +Coverage for ClusterBuilder, MergeStrategyManager, PropertyMergeRule, +EntityMerger.merge_duplicates, embedding similarity, and incremental detection. + +Addresses issue #866. +""" + +import unittest +from unittest.mock import patch + +from semantica.deduplication.cluster_builder import Cluster, ClusterBuilder +from semantica.deduplication.duplicate_detector import ( + DuplicateDetector, + DuplicateGroup, +) +from semantica.deduplication.entity_merger import EntityMerger +from semantica.deduplication.merge_strategy import ( + MergeStrategy, + MergeStrategyManager, + PropertyMergeRule, +) +from semantica.deduplication.methods import calculate_similarity +from semantica.deduplication.similarity_calculator import SimilarityCalculator +from semantica.utils.exceptions import ValidationError + + +class TestClusterBuilderCoverage(unittest.TestCase): + """ClusterBuilder: union-find clustering and size filtering.""" + + def setUp(self): + self.entities = [ + { + "id": "a1", + "name": "Apple Inc.", + "type": "Company", + "properties": {"industry": "Technology"}, + }, + { + "id": "a2", + "name": "Apple", + "type": "Company", + "properties": {"industry": "Tech"}, + }, + { + "id": "m1", + "name": "Microsoft Corporation", + "type": "Company", + "properties": {"industry": "Software"}, + }, + { + "id": "g1", + "name": "Google LLC", + "type": "Company", + "properties": {"industry": "Internet"}, + }, + ] + + def test_known_duplicates_share_cluster_id(self): + """Entities linked by high-similarity pairs land in the same cluster.""" + builder = ClusterBuilder(similarity_threshold=0.4, min_cluster_size=2) + result = builder.build_clusters(self.entities) + + id_to_cluster = {} + for cluster in result.clusters: + for entity in cluster.entities: + id_to_cluster[entity["id"]] = cluster.cluster_id + + self.assertIn("a1", id_to_cluster) + self.assertIn("a2", id_to_cluster) + self.assertEqual(id_to_cluster["a1"], id_to_cluster["a2"]) + + def test_unrelated_entities_get_different_cluster_ids(self): + """Unrelated brands must not share a cluster ID when they form clusters.""" + builder = ClusterBuilder(similarity_threshold=0.8, min_cluster_size=2) + a1 = {"id": "a1", "name": "Apple Inc.", "type": "Company"} + a2 = {"id": "a2", "name": "Apple", "type": "Company"} + m1 = {"id": "m1", "name": "Microsoft Corporation", "type": "Company"} + m2 = {"id": "m2", "name": "Microsoft Corp", "type": "Company"} + # Deterministic pairs: intra-brand duplicates only — no Apple↔Microsoft edge + pairs = [(a1, a2, 0.95), (m1, m2, 0.94)] + + with patch.object( + builder.similarity_calculator, + "batch_calculate_similarity", + return_value=pairs, + ): + clusters = builder._graph_based_clustering( + [a1, a2, m1, m2], threshold=0.8 + ) + + id_to_cluster = { + e["id"]: c.cluster_id for c in clusters for e in c.entities + } + self.assertEqual(set(id_to_cluster), {"a1", "a2", "m1", "m2"}) + self.assertEqual(id_to_cluster["a1"], id_to_cluster["a2"]) + self.assertEqual(id_to_cluster["m1"], id_to_cluster["m2"]) + self.assertNotEqual(id_to_cluster["a1"], id_to_cluster["m1"]) + + def test_singleton_entities_are_unclustered(self): + """Entities with no similar peers remain unclustered (min_cluster_size=2).""" + builder = ClusterBuilder(similarity_threshold=0.9, min_cluster_size=2) + result = builder.build_clusters(self.entities) + unclustered_ids = {e["id"] for e in result.unclustered} + # High threshold: Google and Microsoft should not form a pair cluster + self.assertTrue( + "g1" in unclustered_ids or "m1" in unclustered_ids, + "Dissimilar entities should appear in unclustered", + ) + + def test_graph_clustering_from_similarity_pairs(self): + """Union-find path merges transitively via mocked similarity pairs.""" + builder = ClusterBuilder(similarity_threshold=0.8, min_cluster_size=2) + e1 = {"id": "1", "name": "A"} + e2 = {"id": "2", "name": "B"} + e3 = {"id": "3", "name": "C"} + # 1~2 and 2~3 => all three in one cluster + pairs = [(e1, e2, 0.95), (e2, e3, 0.92)] + + with patch.object( + builder.similarity_calculator, + "batch_calculate_similarity", + return_value=pairs, + ): + clusters = builder._graph_based_clustering([e1, e2, e3], threshold=0.8) + + self.assertEqual(len(clusters), 1) + self.assertEqual({e["id"] for e in clusters[0].entities}, {"1", "2", "3"}) + + def test_unrelated_pairs_form_separate_clusters(self): + """Two disjoint similarity pairs produce two cluster IDs.""" + builder = ClusterBuilder(similarity_threshold=0.8, min_cluster_size=2) + a1, a2 = {"id": "a1", "name": "Apple"}, {"id": "a2", "name": "Apple Inc"} + m1, m2 = {"id": "m1", "name": "MSFT"}, {"id": "m2", "name": "Microsoft"} + pairs = [(a1, a2, 0.95), (m1, m2, 0.94)] + + with patch.object( + builder.similarity_calculator, + "batch_calculate_similarity", + return_value=pairs, + ): + clusters = builder._graph_based_clustering([a1, a2, m1, m2], threshold=0.8) + + self.assertEqual(len(clusters), 2) + cluster_ids = {c.cluster_id for c in clusters} + self.assertEqual(len(cluster_ids), 2) + + def test_quality_metrics_populated(self): + builder = ClusterBuilder(similarity_threshold=0.4, min_cluster_size=2) + result = builder.build_clusters(self.entities[:2]) + self.assertIn("total_clusters", result.quality_metrics) + self.assertIn("average_quality", result.quality_metrics) + + def test_update_clusters_adds_matching_entity(self): + """Incremental update attaches a near-duplicate into an existing cluster.""" + builder = ClusterBuilder(similarity_threshold=0.8, min_cluster_size=2) + a1 = { + "id": "a1", + "name": "Apple Inc.", + "type": "Company", + "properties": {"industry": "Technology"}, + } + a2 = { + "id": "a2", + "name": "Apple", + "type": "Company", + "properties": {"industry": "Tech"}, + } + existing = [Cluster(cluster_id="cluster_0", entities=[a1])] + + with patch.object( + builder, "_entity_cluster_similarity", return_value=0.95 + ), patch.object( + builder.similarity_calculator, + "batch_calculate_similarity", + return_value=[(a1, a2, 0.95)], + ): + result = builder.update_clusters(existing, [a2]) + + self.assertEqual(len(result.clusters), 1) + clustered_ids = {e["id"] for e in result.clusters[0].entities} + self.assertEqual(clustered_ids, {"a1", "a2"}) + self.assertEqual(result.unclustered, []) + + +class TestMergeStrategyManagerCoverage(unittest.TestCase): + """MergeStrategyManager and PropertyMergeRule conflict resolution.""" + + def setUp(self): + self.entities = [ + { + "id": "e1", + "name": "Apple Inc.", + "type": "Company", + "properties": { + "industry": "Technology", + "description": "Short", + "hq": "Cupertino", + }, + "relationships": [ + {"subject": "e1", "predicate": "competitor", "object": "Microsoft"} + ], + "confidence": 0.7, + }, + { + "id": "e2", + "name": "Apple", + "type": "Company", + "properties": { + "industry": "Tech", + "description": "A much longer company description", + "founded": "1976", + }, + "relationships": [ + {"subject": "e2", "predicate": "competitor", "object": "Google"} + ], + "confidence": 0.9, + }, + ] + + def test_keep_first_strategy_selects_first_entity(self): + manager = MergeStrategyManager(default_strategy="keep_first") + result = manager.merge_entities(self.entities, strategy="keep_first") + self.assertEqual(result.merged_entity["id"], "e1") + self.assertEqual(result.metadata["strategy"], "keep_first") + + def test_keep_last_strategy_selects_last_entity(self): + manager = MergeStrategyManager() + result = manager.merge_entities(self.entities, strategy="keep_last") + self.assertEqual(result.merged_entity["id"], "e2") + + def test_keep_most_complete_prefers_richer_entity(self): + # e1 has 3 props + 1 rel; e2 has 3 props + 1 rel — tie goes to max() first max + richer = [ + { + "id": "sparse", + "name": "Sparse", + "type": "Company", + "properties": {"a": 1}, + "relationships": [], + }, + { + "id": "rich", + "name": "Rich Co", + "type": "Company", + "properties": {"a": 1, "b": 2, "c": 3}, + "relationships": [{"subject": "rich", "predicate": "owns", "object": "x"}], + }, + ] + manager = MergeStrategyManager(default_strategy="keep_most_complete") + result = manager.merge_entities(richer) + self.assertEqual(result.merged_entity["id"], "rich") + + def test_keep_highest_confidence_selects_confident_entity(self): + manager = MergeStrategyManager() + result = manager.merge_entities( + self.entities, strategy=MergeStrategy.KEEP_HIGHEST_CONFIDENCE + ) + self.assertEqual(result.merged_entity["id"], "e2") + + def test_conflicting_property_keep_first(self): + manager = MergeStrategyManager(default_strategy="keep_first") + result = manager.merge_entities(self.entities, strategy="keep_first") + # Base is e1; conflicting industry keeps first value under keep_first + self.assertEqual(result.merged_entity["properties"]["industry"], "Technology") + # Non-conflicting property from e2 is still absorbed + self.assertEqual(result.merged_entity["properties"]["founded"], "1976") + + def test_conflicting_property_keep_last(self): + manager = MergeStrategyManager() + manager.add_property_rule("industry", "keep_last") + result = manager.merge_entities(self.entities, strategy="keep_first") + self.assertEqual(result.merged_entity["properties"]["industry"], "Tech") + + def test_merge_all_combines_conflicting_values(self): + manager = MergeStrategyManager() + manager.add_property_rule("industry", "merge_all") + result = manager.merge_entities(self.entities, strategy="keep_first") + industry = result.merged_entity["properties"]["industry"] + self.assertIsInstance(industry, list) + self.assertIn("Technology", industry) + self.assertIn("Tech", industry) + + def test_relationships_are_unioned(self): + manager = MergeStrategyManager(default_strategy="keep_first") + result = manager.merge_entities(self.entities) + objects = {r.get("object") for r in result.merged_entity["relationships"]} + self.assertIn("Microsoft", objects) + self.assertIn("Google", objects) + + def test_empty_entities_raises_validation_error(self): + manager = MergeStrategyManager() + with self.assertRaises(ValidationError): + manager.merge_entities([]) + + def test_single_entity_returns_unchanged(self): + manager = MergeStrategyManager() + result = manager.merge_entities([self.entities[0]]) + self.assertEqual(result.merged_entity["id"], "e1") + self.assertEqual(result.merged_entities, [self.entities[0]]) + + def test_invalid_default_strategy_falls_back(self): + manager = MergeStrategyManager(default_strategy="not_a_real_strategy") + self.assertEqual(manager.default_strategy, MergeStrategy.KEEP_MOST_COMPLETE) + + def test_validate_merge_reports_missing_name(self): + manager = MergeStrategyManager() + result = manager.merge_entities(self.entities) + result.merged_entity["name"] = None + validation = manager.validate_merge(result) + self.assertFalse(validation["valid"]) + self.assertTrue(any("name" in issue.lower() for issue in validation["issues"])) + + +class TestPropertyMergeRuleCoverage(unittest.TestCase): + """PropertyMergeRule: custom per-property conflict resolution.""" + + def test_custom_rule_takes_longer_description(self): + manager = MergeStrategyManager(default_strategy="keep_first") + + def longer_string(v1, v2): + return v1 if len(str(v1)) >= len(str(v2)) else v2 + + manager.add_property_rule( + "description", + "custom", + conflict_resolution=longer_string, + priority=10, + ) + entities = [ + { + "id": "e1", + "name": "Alpha", + "type": "Org", + "properties": {"description": "Short"}, + }, + { + "id": "e2", + "name": "Alpha Inc", + "type": "Org", + "properties": { + "description": "This is a much longer description of the entity" + }, + }, + ] + result = manager.merge_entities(entities, strategy="keep_first") + self.assertEqual( + result.merged_entity["properties"]["description"], + "This is a much longer description of the entity", + ) + + def test_property_merge_rule_dataclass_fields(self): + rule = PropertyMergeRule( + property_name="description", + strategy=MergeStrategy.CUSTOM, + conflict_resolution=lambda a, b: a, + priority=5, + ) + self.assertEqual(rule.property_name, "description") + self.assertEqual(rule.strategy, MergeStrategy.CUSTOM) + self.assertEqual(rule.priority, 5) + self.assertIsNotNone(rule.conflict_resolution) + + def test_top_level_name_rule_overrides_base(self): + manager = MergeStrategyManager(default_strategy="keep_first") + manager.add_property_rule("name", "keep_last") + entities = [ + {"id": "e1", "name": "First Name", "type": "Org", "properties": {}}, + {"id": "e2", "name": "Second Name", "type": "Org", "properties": {}}, + ] + result = manager.merge_entities(entities) + self.assertEqual(result.merged_entity["name"], "Second Name") + + def test_invalid_property_strategy_defaults(self): + manager = MergeStrategyManager() + manager.add_property_rule("industry", "bogus_strategy") + self.assertEqual( + manager.property_rules["industry"].strategy, + MergeStrategy.KEEP_MOST_COMPLETE, + ) + + +class TestEntityMergerMergeDuplicates(unittest.TestCase): + """EntityMerger.merge_duplicates end-to-end with DuplicateGroup path.""" + + def setUp(self): + self.entities = [ + { + "id": "a1", + "name": "Apple Inc.", + "type": "Company", + "properties": {"industry": "Technology", "hq": "Cupertino"}, + "relationships": [], + }, + { + "id": "a2", + "name": "Apple", + "type": "Company", + "properties": {"industry": "Tech"}, + "relationships": [], + }, + { + "id": "m1", + "name": "Microsoft Corp", + "type": "Company", + "properties": {"industry": "Software"}, + "relationships": [], + }, + ] + + def test_merge_duplicates_returns_operations_for_groups(self): + merger = EntityMerger( + preserve_provenance=True, + detector={ + "similarity_threshold": 0.4, + "confidence_threshold": 0.4, + }, + ) + operations = merger.merge_duplicates(self.entities, strategy="keep_first") + self.assertGreater(len(operations), 0) + op = operations[0] + self.assertGreaterEqual(len(op.source_entities), 2) + self.assertIn("id", op.merged_entity) + self.assertIn("group_confidence", op.metadata) + # Provenance preserved (written by EntityMerger._add_provenance()). + provenance = op.merged_entity.get("metadata", {}).get("provenance", {}) + self.assertIn("merged_from", provenance) + self.assertIn("merge_count", provenance) + self.assertEqual(provenance["merge_count"], len(op.source_entities)) + + def test_merge_duplicates_provenance_absent_when_disabled(self): + merger = EntityMerger( + preserve_provenance=False, + detector={ + "similarity_threshold": 0.4, + "confidence_threshold": 0.4, + }, + ) + operations = merger.merge_duplicates(self.entities, strategy="keep_first") + self.assertGreater(len(operations), 0) + op = operations[0] + + provenance = op.merged_entity.get("metadata", {}).get("provenance") + self.assertTrue( + provenance is None or provenance == {}, + "metadata.provenance should be absent when preserve_provenance=False", + ) + + def test_merge_duplicates_with_explicit_duplicate_group(self): + """merge_entity_group path used when a DuplicateGroup is already known.""" + group = DuplicateGroup( + entities=[self.entities[0], self.entities[1]], + similarity_scores={("a1", "a2"): 0.85}, + confidence=0.9, + ) + merger = EntityMerger(preserve_provenance=True) + op = merger.merge_entity_group(group.entities, strategy="keep_most_complete") + self.assertEqual(len(op.source_entities), 2) + self.assertEqual( + {e["id"] for e in op.source_entities}, + {"a1", "a2"}, + ) + self.assertIn(op.merged_entity["id"], {"a1", "a2"}) + + def test_merge_duplicates_recorded_in_history(self): + merger = EntityMerger( + detector={"similarity_threshold": 0.4, "confidence_threshold": 0.4} + ) + ops = merger.merge_duplicates(self.entities) + self.assertEqual(len(merger.get_merge_history()), len(ops)) + + +class TestEmbeddingSimilarityCoverage(unittest.TestCase): + """SimilarityCalculator / methods embedding path (vectors, no external model).""" + + def test_calculate_similarity_method_embedding(self): + e1 = { + "id": "1", + "name": "Alpha", + "embedding": [1.0, 0.0, 0.0], + } + e2 = { + "id": "2", + "name": "Beta", + "embedding": [1.0, 0.0, 0.0], + } + result = calculate_similarity(e1, e2, method="embedding") + self.assertEqual(result.method, "embedding") + self.assertAlmostEqual(result.score, 1.0) + + def test_calculate_similarity_method_embedding_missing_vectors(self): + result = calculate_similarity( + {"id": "1", "name": "A"}, + {"id": "2", "name": "B"}, + method="embedding", + ) + self.assertEqual(result.score, 0.0) + self.assertEqual(result.method, "embedding") + + def test_calculate_similarity_method_embedding_orthogonal(self): + e1 = {"embedding": [1.0, 0.0]} + e2 = {"embedding": [0.0, 1.0]} + result = calculate_similarity(e1, e2, method="embedding") + # Cosine of orthogonal vectors is 0; normalized to (0+1)/2 = 0.5 + self.assertAlmostEqual(result.score, 0.5) + + def test_similarity_calculator_includes_embedding_component(self): + calculator = SimilarityCalculator( + string_weight=0.2, + property_weight=0.2, + relationship_weight=0.0, + embedding_weight=0.6, + prefilter_enabled=False, + ) + e1 = { + "id": "1", + "name": "Apple Inc.", + "properties": {}, + "embedding": [0.9, 0.1, 0.0], + } + e2 = { + "id": "2", + "name": "Apple", + "properties": {}, + "embedding": [0.85, 0.15, 0.0], + } + result = calculator.calculate_similarity(e1, e2, track=False) + self.assertIn("embedding", result.components) + self.assertGreater(result.components["embedding"], 0.5) + self.assertGreater(result.score, 0.0) + + def test_embedding_similarity_mismatched_dimensions(self): + calculator = SimilarityCalculator() + score = calculator.calculate_embedding_similarity([1.0, 0.0], [1.0, 0.0, 0.0]) + self.assertEqual(score, 0.0) + + +class TestIncrementalDetectionCoverage(unittest.TestCase): + """Incremental O(n×m) detection: new entities vs existing set.""" + + def setUp(self): + self.existing = [ + { + "id": "a1", + "name": "Apple Inc.", + "type": "Company", + "properties": {"industry": "Technology"}, + }, + { + "id": "m1", + "name": "Microsoft Corp", + "type": "Company", + "properties": {"industry": "Software"}, + }, + ] + self.new = [ + { + "id": "a2", + "name": "Apple", + "type": "Company", + "properties": {"industry": "Tech"}, + }, + { + "id": "g1", + "name": "Google LLC", + "type": "Company", + "properties": {"industry": "Internet"}, + }, + ] + + def test_incremental_detect_finds_new_vs_existing_duplicate(self): + detector = DuplicateDetector( + similarity_threshold=0.4, + confidence_threshold=0.4, + ) + candidates = detector.incremental_detect(self.new, self.existing) + names = {(c.entity1["name"], c.entity2["name"]) for c in candidates} + found = any( + ("Apple" in pair and "Apple Inc." in pair) for pair in names + ) + self.assertTrue(found, f"Expected Apple/Apple Inc. match, got {names}") + + def test_incremental_detect_does_not_compare_within_new_set(self): + """Incremental path only compares new×existing, not new×new.""" + detector = DuplicateDetector( + similarity_threshold=0.3, + confidence_threshold=0.3, + ) + # Two near-identical new entities; existing is unrelated + new = [ + {"id": "n1", "name": "Acme Corp", "type": "Company", "properties": {}}, + {"id": "n2", "name": "Acme Corporation", "type": "Company", "properties": {}}, + ] + existing = [ + {"id": "z1", "name": "Zebra Industries", "type": "Company", "properties": {}} + ] + candidates = detector.incremental_detect(new, existing) + pair_ids = { + frozenset((c.entity1["id"], c.entity2["id"])) for c in candidates + } + self.assertNotIn(frozenset({"n1", "n2"}), pair_ids) + + def test_methods_incremental_detect_duplicates(self): + from semantica.deduplication.duplicate_detector import DuplicateCandidate + from semantica.deduplication.methods import detect_duplicates + + results = detect_duplicates( + self.new + self.existing, + method="incremental", + similarity_threshold=0.4, + confidence_threshold=0.4, + new_entities=self.new, + existing_entities=self.existing, + ) + self.assertIsInstance(results, list) + self.assertGreater(len(results), 0) + for candidate in results: + self.assertIsInstance(candidate, DuplicateCandidate) + self.assertGreaterEqual(candidate.similarity_score, 0.4) + # Wrapper must route new×existing only: one id from each set + ids = {candidate.entity1["id"], candidate.entity2["id"]} + self.assertTrue(ids & {"a2", "g1"}) + self.assertTrue(ids & {"a1", "m1"}) + + names = { + frozenset((c.entity1["name"], c.entity2["name"])) for c in results + } + self.assertIn(frozenset({"Apple", "Apple Inc."}), names) + + +if __name__ == "__main__": + unittest.main() From 3e9ba1b7fb928c103351c1f0e38d197ffdacb527 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Tue, 11 Aug 2026 14:01:07 +0530 Subject: [PATCH 12/40] fix: address Qodo review findings on security PR (numpy/JSON, relative redirects, SPARQL 500) - vector_store.save(): use v.tolist() instead of list(v) so numpy float32 vectors round-trip through JSON instead of raising TypeError. - ontology._fetch_url_sync(): resolve relative Location headers via urljoin before re-validating (previously any relative redirect was rejected outright), and close every response instead of leaking the connection across redirect hops. - sparql.execute_sparql(): move _build_rdflib_graph inside the handler's error handling so the graph-size cap returns a clean SparqlResponse error instead of an unhandled 500. - add regression tests for all three. --- semantica/explorer/routes/ontology.py | 31 +++++---- semantica/explorer/routes/sparql.py | 10 ++- semantica/vector_store/vector_store.py | 2 +- tests/explorer/test_ontology_ssrf.py | 93 +++++++++++++++++++++++++ tests/explorer/test_sparql_route.py | 14 ++++ tests/vector_store/test_vector_store.py | 46 ++++++++++++ 6 files changed, 182 insertions(+), 14 deletions(-) create mode 100644 tests/explorer/test_ontology_ssrf.py diff --git a/semantica/explorer/routes/ontology.py b/semantica/explorer/routes/ontology.py index ca9101fc..5c1fa8d0 100644 --- a/semantica/explorer/routes/ontology.py +++ b/semantica/explorer/routes/ontology.py @@ -11,7 +11,7 @@ import uuid from datetime import datetime, UTC from difflib import SequenceMatcher from typing import Any, Dict, List, Optional, Tuple -from urllib.parse import urlparse +from urllib.parse import urlparse, urljoin from typing_extensions import Literal from fastapi import APIRouter, Depends, HTTPException, Query, Request @@ -1017,23 +1017,30 @@ def _fetch_url_sync(url: str) -> bytes: allow_redirects=False, # SECURITY: follow redirects manually ) if resp.is_redirect or resp.is_permanent_redirect: - redirect_url = resp.headers.get("Location") - if not redirect_url: + location = resp.headers.get("Location") + resp.close() + if not location: raise HTTPException(status_code=502, detail="Redirect without Location header.") + # Location may be relative (e.g. "/ontology.ttl"); resolve it + # against the current URL before validating. + redirect_url = urljoin(current_url, location) # Re-validate the redirect target to prevent SSRF via # open-redirect to internal/cloud-metadata endpoints. _validate_fetch_url(redirect_url) current_url = redirect_url continue - resp.raise_for_status() - chunks: List[bytes] = [] - total = 0 - for chunk in resp.iter_content(65536): - total += len(chunk) - if total > _MAX_FETCH_BYTES: - raise HTTPException(status_code=413, detail="Remote resource exceeds 20 MB limit.") - chunks.append(chunk) - return b"".join(chunks) + try: + resp.raise_for_status() + chunks: List[bytes] = [] + total = 0 + for chunk in resp.iter_content(65536): + total += len(chunk) + if total > _MAX_FETCH_BYTES: + raise HTTPException(status_code=413, detail="Remote resource exceeds 20 MB limit.") + chunks.append(chunk) + return b"".join(chunks) + finally: + resp.close() raise HTTPException(status_code=502, detail=f"Too many redirects (max {_MAX_REDIRECTS}).") except HTTPException: raise diff --git a/semantica/explorer/routes/sparql.py b/semantica/explorer/routes/sparql.py index c110d023..5bab58e4 100644 --- a/semantica/explorer/routes/sparql.py +++ b/semantica/explorer/routes/sparql.py @@ -147,7 +147,15 @@ async def execute_sparql( error="Only SELECT, ASK, CONSTRUCT, and DESCRIBE queries are permitted.", ) - graph = await asyncio.to_thread(_build_rdflib_graph, session) + try: + graph = await asyncio.to_thread(_build_rdflib_graph, session) + except ValueError as exc: + return SparqlResponse( + columns=[], + rows=[], + total=0, + error=str(exc), + ) async with _sparql_semaphore: try: diff --git a/semantica/vector_store/vector_store.py b/semantica/vector_store/vector_store.py index 02f21d09..c1f9b53e 100644 --- a/semantica/vector_store/vector_store.py +++ b/semantica/vector_store/vector_store.py @@ -590,7 +590,7 @@ class VectorStore: # pickle is intentionally avoided to prevent arbitrary code execution # if a malicious .pkl file is placed in the store directory. data = { - "vectors": {k: list(v) if hasattr(v, "tolist") else v + "vectors": {k: v.tolist() if hasattr(v, "tolist") else v for k, v in getattr(self, "vectors", {}).items()}, "metadata": getattr(self, "metadata", {}), "config": self.config, diff --git a/tests/explorer/test_ontology_ssrf.py b/tests/explorer/test_ontology_ssrf.py new file mode 100644 index 00000000..fe9de35b --- /dev/null +++ b/tests/explorer/test_ontology_ssrf.py @@ -0,0 +1,93 @@ +"""Regression tests for outbound URL fetching in ontology.py (SSRF hardening). + +`_fetch_url_sync` disables `requests`' automatic redirect following and +re-validates every hop with `_validate_fetch_url` (see GHSA-8c7v-62gr-hj6g: +unvalidated redirect targets previously let a public first hop 302 the +server into fetching cloud metadata / loopback services). + +These tests cover the redirect-handling logic itself: relative `Location` +headers must resolve correctly instead of being rejected outright, redirect +targets that resolve to private/loopback addresses must still be blocked, +and every response must be closed (no leaked connections across hops). +""" + +import socket +from unittest.mock import MagicMock, patch + +import pytest + +from semantica.explorer.routes import ontology as ontology_mod + + +def _fake_getaddrinfo(host, *args, **kwargs): + # These tests are about the redirect-handling logic, not the address + # classifier itself, so every host resolves to a public IP unless a + # test overrides the side_effect to simulate an internal target. + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0))] + + +def _make_response(is_redirect=False, is_permanent=False, location=None, body=b"ok"): + resp = MagicMock() + resp.is_redirect = is_redirect + resp.is_permanent_redirect = is_permanent + resp.headers = {"Location": location} if location else {} + resp.raise_for_status = MagicMock() + resp.iter_content = MagicMock(return_value=iter([body])) + resp.close = MagicMock() + return resp + + +@patch.object(ontology_mod.socket, "getaddrinfo", side_effect=_fake_getaddrinfo) +def test_relative_redirect_location_is_resolved(mock_getaddrinfo): + """A relative Location header (e.g. '/ontology.ttl') must resolve against + the current URL via urljoin, not be rejected as a malformed URL.""" + redirect_resp = _make_response(is_redirect=True, location="/ontology.ttl") + final_resp = _make_response(body=b"final content") + + with patch("requests.get", side_effect=[redirect_resp, final_resp]) as mock_get: + result = ontology_mod._fetch_url_sync("http://example.org/start") + + assert result == b"final content" + second_call_url = mock_get.call_args_list[1].args[0] + assert second_call_url == "http://example.org/ontology.ttl" + redirect_resp.close.assert_called_once() + final_resp.close.assert_called_once() + + +@patch.object(ontology_mod.socket, "getaddrinfo", side_effect=_fake_getaddrinfo) +def test_redirect_to_private_ip_is_rejected(mock_getaddrinfo): + """Re-validation must reject a redirect target resolving to a private + address even though the first hop was a validated public URL — this is + the exact GHSA-8c7v scenario: public first hop, malicious redirect.""" + def getaddrinfo_side_effect(host, *a, **k): + if host == "internal.example": + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("169.254.169.254", 0))] + return _fake_getaddrinfo(host, *a, **k) + + mock_getaddrinfo.side_effect = getaddrinfo_side_effect + redirect_resp = _make_response(is_redirect=True, location="http://internal.example/latest/meta-data/") + + with patch("requests.get", side_effect=[redirect_resp]): + with pytest.raises(ontology_mod.HTTPException) as exc_info: + ontology_mod._fetch_url_sync("http://example.org/start") + + assert exc_info.value.status_code == 422 + redirect_resp.close.assert_called_once() + + +@patch.object(ontology_mod.socket, "getaddrinfo", side_effect=_fake_getaddrinfo) +def test_final_response_is_closed(mock_getaddrinfo): + final_resp = _make_response(body=b"content") + with patch("requests.get", side_effect=[final_resp]): + ontology_mod._fetch_url_sync("http://example.org/start") + final_resp.close.assert_called_once() + + +@patch.object(ontology_mod.socket, "getaddrinfo", side_effect=_fake_getaddrinfo) +def test_redirect_chain_exceeding_cap_is_rejected(mock_getaddrinfo): + responses = [_make_response(is_redirect=True, location=f"/hop{i}") for i in range(10)] + with patch("requests.get", side_effect=responses): + with pytest.raises(ontology_mod.HTTPException) as exc_info: + ontology_mod._fetch_url_sync("http://example.org/start") + assert exc_info.value.status_code == 502 + assert all(r.close.called for r in responses[:6]) diff --git a/tests/explorer/test_sparql_route.py b/tests/explorer/test_sparql_route.py index 36dacce5..77cc5b71 100644 --- a/tests/explorer/test_sparql_route.py +++ b/tests/explorer/test_sparql_route.py @@ -290,6 +290,20 @@ def test_query_timeout_returns_clean_error_not_a_crash(client): assert payload["rows"] == [] +def test_oversized_graph_returns_clean_error_not_a_crash(client): + """The DoS-prevention node cap (GHSA-8c7v-adjacent hardening) must return + a normal SparqlResponse error, not an unhandled 500. Regression test for + a bug where _build_rdflib_graph's ValueError was raised outside of + execute_sparql's try/except, before the semaphore block.""" + with patch.object(sparql_mod, "_SPARQL_MAX_GRAPH_NODES", 1): + resp = _post(client, "SELECT ?s WHERE { ?s a }") + assert resp.status_code == 200 + payload = resp.json() + assert payload["error"] is not None + assert "more than" in payload["error"].lower() + assert payload["rows"] == [] + + # --------------------------------------------------------------------------- # Data-mapping fidelity: does the graph->RDF projection reflect session state? # --------------------------------------------------------------------------- diff --git a/tests/vector_store/test_vector_store.py b/tests/vector_store/test_vector_store.py index 5badcf24..c4001606 100644 --- a/tests/vector_store/test_vector_store.py +++ b/tests/vector_store/test_vector_store.py @@ -1,3 +1,6 @@ +import json +import shutil +import tempfile import unittest from unittest.mock import MagicMock, patch import numpy as np @@ -134,5 +137,48 @@ class TestVectorStore(unittest.TestCase): self.assertTrue(mock_backend.called) + def test_save_load_roundtrip_numpy_vectors(self): + """save()/load() must handle numpy float32 vectors without raising. + + Regression test: json.dump() rejects numpy scalar types, so a naive + `list(v)` conversion (which yields np.float32 elements, not native + floats) raises TypeError. `v.tolist()` converts recursively to + native Python floats and must be used instead. + """ + store = VectorStore(backend="inmemory", dimension=3) + store.vectors = {"v1": np.array([0.1, 0.2, 0.3], dtype=np.float32)} + store.metadata = {"v1": {"id": "1"}} + + tmpdir = tempfile.mkdtemp() + try: + store.save(tmpdir) # must not raise TypeError + + # The JSON file itself must be valid and free of numpy types. + with open(f"{tmpdir}/store_data.json", "r", encoding="utf-8") as f: + data = json.load(f) + self.assertTrue(all(isinstance(x, float) for x in data["vectors"]["v1"])) + + loaded = VectorStore(backend="inmemory", dimension=3) + loaded.load(tmpdir) + np.testing.assert_allclose( + loaded.vectors["v1"], [0.1, 0.2, 0.3], rtol=1e-6 + ) + self.assertEqual(loaded.metadata["v1"], {"id": "1"}) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + def test_load_rejects_legacy_pickle(self): + """load() must refuse legacy .pkl stores rather than deserializing them.""" + store = VectorStore(backend="inmemory", dimension=3) + tmpdir = tempfile.mkdtemp() + try: + with open(f"{tmpdir}/store_data.pkl", "wb") as f: + f.write(b"not a real pickle, just needs to exist") + with self.assertRaises(RuntimeError): + store.load(tmpdir) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + if __name__ == '__main__': unittest.main() From ab5c12f9afc6660a4e612fa5cc7e4be56b78b73b Mon Sep 17 00:00:00 2001 From: pravit-amp <43916793+pravit-amp@users.noreply.github.com> Date: Tue, 11 Aug 2026 01:52:15 -0700 Subject: [PATCH 13/40] fix(ingest): SSRF protection for Web and API ingestors (#867) (#906) * fix(ingest): add SSRF protection for WebIngestor and RESTIngestor Block non-http(s) schemes and private/loopback/link-local targets before outbound requests, with allow_private_ips opt-in for trusted deployments. * fix(ingest): fail closed on SSRF DNS resolution errors * fix(ingest): validate SSRF targets on every HTTP redirect hop * fix(ingest): avoid blocking on SSRF DNS executor shutdown * fix(ingest): parse allow_private_ips without truthy-string pitfalls * docs(ingest): clarify robots.txt SSRF/validation comment --------- Co-authored-by: Pravit Ampapathini --- semantica/ingest/api_ingestor.py | 30 +- semantica/ingest/methods.py | 6 + semantica/ingest/ssrf.py | 293 +++++++++++++++++++ semantica/ingest/web_ingestor.py | 67 +++-- tests/ingest/test_ssrf_protection.py | 407 +++++++++++++++++++++++++++ 5 files changed, 778 insertions(+), 25 deletions(-) create mode 100644 semantica/ingest/ssrf.py create mode 100644 tests/ingest/test_ssrf_protection.py diff --git a/semantica/ingest/api_ingestor.py b/semantica/ingest/api_ingestor.py index 30677681..d1f7a860 100644 --- a/semantica/ingest/api_ingestor.py +++ b/semantica/ingest/api_ingestor.py @@ -38,6 +38,7 @@ except (ImportError, OSError): from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker +from .ssrf import parse_bool, request_with_ssrf_guard @dataclass @@ -78,11 +79,16 @@ class RESTIngestor: Args: config: Optional REST API ingestion configuration dictionary - **kwargs: Additional configuration parameters (merged into config) + **kwargs: Additional configuration parameters (merged into config). + Recognized keys include ``allow_private_ips`` (default False) to + opt into fetching private/loopback/link-local endpoints. """ self.logger = get_logger("api_ingestor") self.config = config or {} self.config.update(kwargs) + self.allow_private_ips = parse_bool( + self.config.get("allow_private_ips"), default=False + ) # Initialize session with retry strategy self.session = requests.Session() @@ -103,7 +109,10 @@ class RESTIngestor: # Initialize progress tracker self.progress_tracker = get_progress_tracker() - self.logger.debug("REST API ingestor initialized") + self.logger.debug( + "REST API ingestor initialized (allow_private_ips=%s)", + self.allow_private_ips, + ) def ingest_endpoint( self, @@ -137,7 +146,7 @@ class RESTIngestor: - metadata: Additional metadata Raises: - ValidationError: If endpoint is invalid + ValidationError: If endpoint is invalid or fails SSRF checks ProcessingError: If request fails """ tracking_id = self.progress_tracker.start_tracking( @@ -153,10 +162,12 @@ class RESTIngestor: if headers: request_headers.update(headers) - # Make request - response = self.session.request( - method=method, - url=endpoint, + # Make request (SSRF-safe; validates URL and each redirect hop) + response = request_with_ssrf_guard( + method, + endpoint, + session=self.session, + allow_private_ips=self.allow_private_ips, headers=request_headers, params=params, data=data, @@ -196,6 +207,11 @@ class RESTIngestor: }, ) + except ValidationError: + self.progress_tracker.stop_tracking( + tracking_id, status="failed", message=f"Invalid endpoint URL: {endpoint}" + ) + raise except requests.exceptions.RequestException as e: self.progress_tracker.stop_tracking( tracking_id, status="failed", message=str(e) diff --git a/semantica/ingest/methods.py b/semantica/ingest/methods.py index a3e3ba76..ac9775a2 100644 --- a/semantica/ingest/methods.py +++ b/semantica/ingest/methods.py @@ -553,6 +553,12 @@ def ingest_web( # Get config config = ingest_config.get_method_config("web") config.update(kwargs) + if "allow_private_ips" in config: + from .ssrf import parse_bool + + config["allow_private_ips"] = parse_bool( + config["allow_private_ips"], default=False + ) ingestor = WebIngestor(**config) diff --git a/semantica/ingest/ssrf.py b/semantica/ingest/ssrf.py new file mode 100644 index 00000000..ade93a46 --- /dev/null +++ b/semantica/ingest/ssrf.py @@ -0,0 +1,293 @@ +"""SSRF safeguards for ingest HTTP clients. + +Validates outbound request URLs before they reach ``requests`` / urllib3 so +user-supplied targets cannot reach private, loopback, or link-local +addresses (including cloud metadata endpoints). +""" + +from __future__ import annotations + +import concurrent.futures +import ipaddress +import socket +import threading +from typing import Any, Iterable, Optional +from urllib.parse import urljoin, urlparse + +import requests + +from ..utils.exceptions import ValidationError + +ALLOWED_URL_SCHEMES = frozenset({"http", "https"}) + +_TRUE_STRINGS = frozenset({"1", "true", "yes", "on"}) +_FALSE_STRINGS = frozenset({"0", "false", "no", "off"}) + +# Keep DNS resolution bounded so fake/unreachable hosts in tests and offline +# environments cannot hang request validation indefinitely. +_DNS_RESOLVE_TIMEOUT_SECONDS = 2.0 +_DNS_EXECUTOR_WORKERS = 4 + +# Bound manual redirect following so open redirect chains cannot hang fetches. +_DEFAULT_MAX_REDIRECTS = 10 +_REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308}) +_STRIP_BODY_ON_REDIRECT = frozenset({301, 302, 303}) + +_dns_executor: Optional[concurrent.futures.ThreadPoolExecutor] = None +_dns_executor_lock = threading.Lock() + + +def parse_bool(value: Any, default: bool = False) -> bool: + """Parse a config value into a bool without truthy-string pitfalls. + + ``bool('false')`` is ``True`` in Python, which would silently disable SSRF + protections when string-typed config reaches ingestors. This helper accepts + only explicit bools and a small allowlist of string/int forms. + + Args: + value: Config value to interpret. ``None`` yields *default*. + default: Value returned when *value* is ``None``. + + Raises: + ValidationError: If *value* is not a recognized boolean form. + """ + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + if value == 1: + return True + if value == 0: + return False + raise ValidationError(f"Invalid boolean value: {value!r}") + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in _TRUE_STRINGS: + return True + if normalized in _FALSE_STRINGS: + return False + raise ValidationError(f"Invalid boolean value: {value!r}") + raise ValidationError( + f"Invalid boolean type: {type(value).__name__} ({value!r})" + ) + + +def _shutdown_executor(executor: concurrent.futures.ThreadPoolExecutor) -> None: + """Shut down *executor* without waiting for in-flight DNS lookups.""" + try: + executor.shutdown(wait=False, cancel_futures=True) + except TypeError: + # cancel_futures was added in Python 3.9. + executor.shutdown(wait=False) + + +def _get_dns_executor() -> concurrent.futures.ThreadPoolExecutor: + """Return a process-wide executor for bounded DNS lookups.""" + global _dns_executor + with _dns_executor_lock: + if _dns_executor is None: + _dns_executor = concurrent.futures.ThreadPoolExecutor( + max_workers=_DNS_EXECUTOR_WORKERS, + thread_name_prefix="semantica-ssrf-dns", + ) + return _dns_executor + + +# Explicit blocked networks from issue #867, plus common non-routable ranges. +BLOCKED_NETWORKS = ( + ipaddress.ip_network("0.0.0.0/8"), + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("127.0.0.0/8"), + ipaddress.ip_network("169.254.0.0/16"), # link-local / cloud metadata + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ipaddress.ip_network("::1/128"), + ipaddress.ip_network("fc00::/7"), + ipaddress.ip_network("fe80::/10"), +) + + +def _ip_is_blocked(addr: ipaddress._BaseAddress) -> bool: + if ( + addr.is_private + or addr.is_loopback + or addr.is_link_local + or addr.is_reserved + or addr.is_multicast + or addr.is_unspecified + ): + return True + return any(addr in network for network in BLOCKED_NETWORKS) + + +def _hostname_resolves_to_blocked(hostname: str) -> bool: + """Return True if any resolved address for *hostname* is blocked. + + DNS lookups run on a shared thread pool with ``Future.result(timeout=...)``. + Do not use ``with ThreadPoolExecutor(...)`` here: on timeout, leaving the + context waits for the hung ``getaddrinfo`` worker and defeats the bound. + + Raises: + ValidationError: If DNS resolution fails or times out. Fail closed so + outbound requests never proceed without confirmed safe IPs. + """ + executor = _get_dns_executor() + owned_executor = False + try: + try: + future = executor.submit(socket.getaddrinfo, hostname, None) + except RuntimeError: + # Shared executor was shut down; use a throwaway pool that never + # blocks the caller on shutdown. + executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) + owned_executor = True + future = executor.submit(socket.getaddrinfo, hostname, None) + resolved: Iterable = future.result(timeout=_DNS_RESOLVE_TIMEOUT_SECONDS) + except (socket.gaierror, concurrent.futures.TimeoutError, OSError) as exc: + raise ValidationError( + f"URL host '{hostname}' could not be resolved safely " + "(DNS error or timeout); request blocked" + ) from exc + finally: + if owned_executor: + _shutdown_executor(executor) + + for info in resolved: + sockaddr = info[4] + addr = ipaddress.ip_address(sockaddr[0]) + if _ip_is_blocked(addr): + return True + return False + + +def validate_url_for_request( + url: str, *, allow_private_ips: bool = False +) -> None: + """Validate that *url* is safe to fetch over HTTP(S). + + Args: + url: Absolute URL to validate. + allow_private_ips: When True, skip private/loopback/link-local checks + (for trusted internal deployments). + + Raises: + ValidationError: If the scheme is not http/https, the URL is malformed, + or the host targets a blocked address space. + """ + if not isinstance(url, str) or not url.strip(): + raise ValidationError("URL must be a non-empty string") + + parsed = urlparse(url.strip()) + scheme = (parsed.scheme or "").lower() + if scheme not in ALLOWED_URL_SCHEMES: + raise ValidationError( + f"URL scheme '{parsed.scheme}' is not permitted. " + "Only http and https are allowed." + ) + if not parsed.netloc: + raise ValidationError( + f"Invalid URL format: {url}. " + "URL must include scheme (http/https) and netloc (domain)." + ) + + host = parsed.hostname + if not host: + raise ValidationError( + f"Invalid URL format: {url}. " + "URL must include a hostname." + ) + + if allow_private_ips: + return + + lowered = host.lower().rstrip(".") + if lowered == "localhost" or lowered.endswith(".localhost"): + raise ValidationError(f"URL host is not allowed: {host}") + + try: + literal_ip = ipaddress.ip_address(host) + except ValueError: + literal_ip = None + + if literal_ip is not None: + if _ip_is_blocked(literal_ip): + raise ValidationError(f"URL points to a blocked address: {host}") + return + + if _hostname_resolves_to_blocked(host): + raise ValidationError( + f"URL host '{host}' resolves to a blocked (private/loopback/" + "link-local) address" + ) + + +def request_with_ssrf_guard( + method: str, + url: str, + *, + session: Optional[requests.Session] = None, + allow_private_ips: bool = False, + max_redirects: int = _DEFAULT_MAX_REDIRECTS, + **kwargs: Any, +) -> requests.Response: + """Perform an HTTP request with SSRF checks on *url* and every redirect. + + ``requests`` follows redirects by default, which would allow a validated + public URL to bounce into private/loopback/link-local space. This helper + disables automatic redirects and re-validates each ``Location`` target + before issuing the next hop. + """ + kwargs = dict(kwargs) + kwargs.pop("allow_redirects", None) + + validate_url_for_request(url, allow_private_ips=allow_private_ips) + + requester = session.request if session is not None else requests.request + current_url = url + current_method = method.upper() + redirects_followed = 0 + + while True: + response = requester( + current_method, + current_url, + allow_redirects=False, + **kwargs, + ) + + if response.status_code not in _REDIRECT_STATUS_CODES: + return response + + if redirects_followed >= max_redirects: + response.close() + raise ValidationError( + f"Exceeded maximum redirects ({max_redirects}) while " + f"fetching '{url}'" + ) + + location = response.headers.get("Location") + if not location or not str(location).strip(): + response.close() + raise ValidationError( + f"Redirect from '{current_url}' is missing a Location header" + ) + + next_url = urljoin(current_url, str(location).strip()) + validate_url_for_request(next_url, allow_private_ips=allow_private_ips) + + # Match requests' historical method rewriting for 301/302/303. + if ( + response.status_code in _STRIP_BODY_ON_REDIRECT + and current_method not in {"GET", "HEAD"} + ): + current_method = "GET" + for key in ("data", "json", "files"): + kwargs.pop(key, None) + + # Params apply to the original request URL only; Location is authoritative. + kwargs.pop("params", None) + + response.close() + current_url = next_url + redirects_followed += 1 diff --git a/semantica/ingest/web_ingestor.py b/semantica/ingest/web_ingestor.py index e542ee39..5dd77f31 100644 --- a/semantica/ingest/web_ingestor.py +++ b/semantica/ingest/web_ingestor.py @@ -48,6 +48,7 @@ from urllib3.util.retry import Retry from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker +from .ssrf import parse_bool, request_with_ssrf_guard, validate_url_for_request @dataclass @@ -338,10 +339,14 @@ class SitemapCrawler: Sets up the crawler with configuration options. Args: - **config: Crawler configuration options (currently unused) + **config: Crawler configuration options. Recognized keys: + - allow_private_ips: Allow private/loopback sitemap hosts """ self.logger = get_logger("sitemap_crawler") self.config = config + self.allow_private_ips = parse_bool( + config.get("allow_private_ips"), default=False + ) def parse_sitemap(self, sitemap_url: str) -> List[str]: """ @@ -359,10 +364,16 @@ class SitemapCrawler: Raises: ProcessingError: If sitemap cannot be fetched or parsed + ValidationError: If sitemap_url fails SSRF checks """ try: - # Fetch sitemap - response = requests.get(sitemap_url, timeout=30) + # Fetch sitemap (SSRF-safe; validates URL and each redirect hop) + response = request_with_ssrf_guard( + "GET", + sitemap_url, + allow_private_ips=self.allow_private_ips, + timeout=30, + ) response.raise_for_status() # Parse XML @@ -391,6 +402,8 @@ class SitemapCrawler: ) return urls + except ValidationError: + raise except Exception as e: self.logger.error(f"Failed to parse sitemap {sitemap_url}: {e}") raise ProcessingError(f"Failed to parse sitemap: {e}") from e @@ -411,10 +424,16 @@ class SitemapCrawler: Raises: ProcessingError: If sitemap index cannot be fetched or parsed + ValidationError: If index_url fails SSRF checks """ try: - # Fetch sitemap index - response = requests.get(index_url, timeout=30) + # Fetch sitemap index (SSRF-safe; validates URL and each redirect hop) + response = request_with_ssrf_guard( + "GET", + index_url, + allow_private_ips=self.allow_private_ips, + timeout=30, + ) response.raise_for_status() # Parse XML @@ -448,6 +467,8 @@ class SitemapCrawler: ) return all_urls + except ValidationError: + raise except Exception as e: self.logger.error(f"Failed to crawl sitemap index {index_url}: {e}") raise ProcessingError(f"Failed to crawl sitemap index: {e}") from e @@ -487,6 +508,7 @@ class WebIngestor: max_retries: int = 3, backoff_factor: float = 1.0, timeout: int = 30, + allow_private_ips: bool = False, config: Optional[Dict[str, Any]] = None, **kwargs, ): @@ -503,12 +525,19 @@ class WebIngestor: max_retries: Maximum number of retry attempts (default: 3) backoff_factor: Backoff factor for retries (default: 1.0) timeout: Request timeout in seconds (default: 30) + allow_private_ips: Allow fetching private/loopback/link-local hosts + (default: False). Opt in only for trusted internal deployments. config: Optional configuration dictionary (merged with kwargs) **kwargs: Additional configuration parameters """ self.logger = get_logger("web_ingestor") self.config = config or {} self.config.update(kwargs) + self.allow_private_ips = parse_bool( + self.config.get("allow_private_ips", allow_private_ips), + default=False, + ) + self.config["allow_private_ips"] = self.allow_private_ips # Initialize HTTP session with retry strategy self.session = requests.Session() @@ -544,7 +573,8 @@ class WebIngestor: self.logger.debug( f"Web ingestor initialized: user_agent={user_agent}, " - f"delay={delay}, respect_robots={respect_robots}" + f"delay={delay}, respect_robots={respect_robots}, " + f"allow_private_ips={self.allow_private_ips}" ) def ingest_url( @@ -574,16 +604,9 @@ class WebIngestor: ) try: - # Validate URL format - try: - parsed = urlparse(url) - if not parsed.scheme or not parsed.netloc: - raise ValidationError( - f"Invalid URL format: {url}. " - "URL must include scheme (http/https) and netloc (domain)." - ) - except Exception as e: - raise ValidationError(f"Invalid URL: {url}") from e + # Validate before robots check: RobotsChecker.read() makes an unguarded + # HTTP request to /robots.txt and must not reach blocked addresses. + validate_url_for_request(url, allow_private_ips=self.allow_private_ips) # Check robots.txt compliance if self.robots_checker and not self.robots_checker.can_fetch(url): @@ -593,11 +616,19 @@ class WebIngestor: # Apply rate limiting (wait if necessary) self.rate_limiter.wait_if_needed() - # Fetch content with retry logic + # Fetch content with retry logic (SSRF-safe redirects) try: request_timeout = timeout or self.config.get("timeout", 30) - response = self.session.get(url, timeout=request_timeout) + response = request_with_ssrf_guard( + "GET", + url, + session=self.session, + allow_private_ips=self.allow_private_ips, + timeout=request_timeout, + ) response.raise_for_status() + except ValidationError: + raise except requests.RequestException as e: self.progress_tracker.stop_tracking( tracking_id, status="failed", message=str(e) diff --git a/tests/ingest/test_ssrf_protection.py b/tests/ingest/test_ssrf_protection.py new file mode 100644 index 00000000..2da84530 --- /dev/null +++ b/tests/ingest/test_ssrf_protection.py @@ -0,0 +1,407 @@ +"""SSRF protection tests for web and API ingestors (issue #867).""" + +from unittest.mock import MagicMock, patch + +import pytest +import urllib3.connectionpool as pool + +from semantica.ingest.api_ingestor import RESTIngestor +from semantica.ingest.ssrf import ( + parse_bool, + request_with_ssrf_guard, + validate_url_for_request, +) +from semantica.ingest.web_ingestor import SitemapCrawler, WebIngestor +from semantica.utils.exceptions import ValidationError + + +class TestParseBool: + def test_bool_passthrough(self): + assert parse_bool(True) is True + assert parse_bool(False) is False + + def test_none_uses_default(self): + assert parse_bool(None) is False + assert parse_bool(None, default=True) is True + + def test_string_true_values(self): + for value in ("true", "TRUE", "1", "yes", "on", " Yes "): + assert parse_bool(value) is True + + def test_string_false_values(self): + for value in ("false", "FALSE", "0", "no", "off", " No "): + assert parse_bool(value) is False + + def test_int_zero_one(self): + assert parse_bool(0) is False + assert parse_bool(1) is True + + def test_rejects_unknown_string(self): + with pytest.raises(ValidationError, match="Invalid boolean"): + parse_bool("maybe") + + +class TestValidateUrlForRequest: + def test_accepts_https(self): + with patch( + "semantica.ingest.ssrf.socket.getaddrinfo", + return_value=[(None, None, None, None, ("93.184.216.34", 0))], + ): + validate_url_for_request("https://example.com/path") + + def test_rejects_file_scheme(self): + with pytest.raises(ValidationError, match="not permitted"): + validate_url_for_request("file://localhost/etc/passwd") + + def test_rejects_gopher_scheme(self): + with pytest.raises(ValidationError, match="not permitted"): + validate_url_for_request("gopher://example.com/1") + + def test_rejects_literal_private_ips(self): + for url in ( + "http://10.0.0.1/", + "http://192.168.1.1/", + "http://172.16.5.5/internal", + "http://127.0.0.1:9999/internal", + "http://169.254.169.254/latest/meta-data/", + ): + with pytest.raises(ValidationError, match="blocked"): + validate_url_for_request(url) + + def test_rejects_localhost_hostname(self): + with pytest.raises(ValidationError, match="not allowed"): + validate_url_for_request("http://localhost/admin") + + def test_rejects_hostname_resolving_to_private_ip(self): + with patch( + "semantica.ingest.ssrf.socket.getaddrinfo", + return_value=[(None, None, None, None, ("10.0.0.5", 0))], + ): + with pytest.raises(ValidationError, match="blocked"): + validate_url_for_request("http://internal.corp/secret") + + def test_allow_private_ips_opt_in(self): + validate_url_for_request( + "http://127.0.0.1:8080/health", allow_private_ips=True + ) + # Scheme allowlist still applies when private IPs are permitted + with pytest.raises(ValidationError, match="not permitted"): + validate_url_for_request( + "file://localhost/x", allow_private_ips=True + ) + + def test_dns_failure_raises(self): + import socket + + with patch( + "semantica.ingest.ssrf.socket.getaddrinfo", + side_effect=socket.gaierror("name or service not known"), + ): + with pytest.raises(ValidationError, match="could not be resolved safely"): + validate_url_for_request("http://does-not-resolve.invalid/path") + + def test_dns_timeout_raises(self): + import concurrent.futures + + with patch( + "semantica.ingest.ssrf.concurrent.futures.Future.result", + side_effect=concurrent.futures.TimeoutError(), + ): + with pytest.raises(ValidationError, match="could not be resolved safely"): + validate_url_for_request("http://slow-dns.example/path") + + def test_hung_getaddrinfo_returns_within_timeout_bound(self): + """Timeout must not wait on executor shutdown for a blocking resolver.""" + import time + + import semantica.ingest.ssrf as ssrf + + hang_seconds = 1.5 + bound = 0.1 + + def hanging_getaddrinfo(*_args, **_kwargs): + time.sleep(hang_seconds) + return [(None, None, None, None, ("93.184.216.34", 0))] + + with patch.object(ssrf, "_DNS_RESOLVE_TIMEOUT_SECONDS", bound), patch( + "semantica.ingest.ssrf.socket.getaddrinfo", + side_effect=hanging_getaddrinfo, + ): + start = time.monotonic() + with pytest.raises(ValidationError, match="could not be resolved safely"): + ssrf._hostname_resolves_to_blocked("hanging.example") + elapsed = time.monotonic() - start + + # Must fail closed near the configured bound, not after hang_seconds + # (which is what ``with ThreadPoolExecutor`` shutdown waiting causes). + assert elapsed < hang_seconds / 2 + assert elapsed < bound + 0.75 + + +class TestRequestWithSsrfGuardRedirects: + def test_blocks_redirect_to_loopback(self): + redirect = MagicMock() + redirect.status_code = 302 + redirect.headers = {"Location": "http://127.0.0.1/secret"} + redirect.close = MagicMock() + + session = MagicMock() + session.request.return_value = redirect + + with patch( + "semantica.ingest.ssrf.socket.getaddrinfo", + return_value=[(None, None, None, None, ("93.184.216.34", 0))], + ): + with pytest.raises(ValidationError, match="blocked"): + request_with_ssrf_guard( + "GET", + "https://example.com/start", + session=session, + ) + + session.request.assert_called_once() + assert session.request.call_args.kwargs.get("allow_redirects") is False + + def test_blocks_redirect_to_metadata_ip(self): + redirect = MagicMock() + redirect.status_code = 301 + redirect.headers = {"Location": "http://169.254.169.254/latest/meta-data/"} + redirect.close = MagicMock() + + session = MagicMock() + session.request.return_value = redirect + + with patch( + "semantica.ingest.ssrf.socket.getaddrinfo", + return_value=[(None, None, None, None, ("93.184.216.34", 0))], + ): + with pytest.raises(ValidationError, match="blocked"): + request_with_ssrf_guard( + "GET", + "https://example.com/start", + session=session, + ) + + def test_follows_safe_redirect(self): + redirect = MagicMock() + redirect.status_code = 302 + redirect.headers = {"Location": "https://example.com/final"} + redirect.close = MagicMock() + + final = MagicMock() + final.status_code = 200 + final.headers = {} + + session = MagicMock() + session.request.side_effect = [redirect, final] + + with patch( + "semantica.ingest.ssrf.socket.getaddrinfo", + return_value=[(None, None, None, None, ("93.184.216.34", 0))], + ): + response = request_with_ssrf_guard( + "GET", + "https://example.com/start", + session=session, + ) + + assert response is final + assert session.request.call_count == 2 + assert all( + call.kwargs.get("allow_redirects") is False + for call in session.request.call_args_list + ) + + +class TestWebIngestorSSRF: + def test_private_ip_never_reaches_urllib3(self): + ingestor = WebIngestor(respect_robots=False, delay=0) + attempts = [] + + def intercepting_urlopen(self, method, url, **kw): + attempts.append((self.host, self.port, url)) + raise Exception("intercepted at urllib3.urlopen") + + urls = [ + "http://10.0.0.1/", + "http://192.168.1.1/", + "http://127.0.0.1:9999/internal", + "http://169.254.169.254/latest/meta-data/", + ] + with patch.object(pool.HTTPConnectionPool, "urlopen", intercepting_urlopen): + for url in urls: + attempts.clear() + with pytest.raises(ValidationError): + ingestor.ingest_url(url) + assert attempts == [], f"SSRF target reached urllib3: {url}" + + def test_file_scheme_rejected_by_semantica(self): + ingestor = WebIngestor(respect_robots=False, delay=0) + with pytest.raises(ValidationError, match="not permitted"): + ingestor.ingest_url("file://localhost/etc/passwd") + + def test_allow_private_ips_permits_loopback_fetch(self): + ingestor = WebIngestor( + respect_robots=False, delay=0, allow_private_ips=True + ) + with patch.object(ingestor.session, "request") as mock_request, patch.object( + ingestor, "extract_content" + ) as mock_extract: + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.text = "ok" + mock_resp.raise_for_status = MagicMock() + mock_request.return_value = mock_resp + mock_extract.return_value = MagicMock(status_code=None) + + ingestor.ingest_url("http://127.0.0.1:9/probe") + + mock_request.assert_called_once() + assert mock_request.call_args.kwargs.get("allow_redirects") is False + mock_extract.assert_called_once() + + def test_allow_private_ips_string_false_keeps_ssrf_on(self): + ingestor = WebIngestor( + respect_robots=False, delay=0, allow_private_ips="false" + ) + assert ingestor.allow_private_ips is False + with pytest.raises(ValidationError, match="blocked"): + ingestor.ingest_url("http://127.0.0.1:9/probe") + + def test_allow_private_ips_string_true_opts_in(self): + ingestor = WebIngestor( + respect_robots=False, delay=0, allow_private_ips="true" + ) + assert ingestor.allow_private_ips is True + with patch.object(ingestor.session, "request") as mock_request, patch.object( + ingestor, "extract_content" + ) as mock_extract: + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.text = "ok" + mock_resp.raise_for_status = MagicMock() + mock_request.return_value = mock_resp + mock_extract.return_value = MagicMock(status_code=None) + + ingestor.ingest_url("http://127.0.0.1:9/probe") + mock_request.assert_called_once() + + def test_redirect_to_private_ip_blocked(self): + ingestor = WebIngestor(respect_robots=False, delay=0) + redirect = MagicMock() + redirect.status_code = 302 + redirect.headers = {"Location": "http://127.0.0.1/admin"} + redirect.close = MagicMock() + + with patch.object(ingestor.session, "request", return_value=redirect), patch( + "semantica.ingest.ssrf.socket.getaddrinfo", + return_value=[(None, None, None, None, ("93.184.216.34", 0))], + ): + with pytest.raises(ValidationError, match="blocked"): + ingestor.ingest_url("https://example.com/public") + + +class TestSitemapCrawlerSSRF: + def test_private_sitemap_url_rejected(self): + crawler = SitemapCrawler() + with pytest.raises(ValidationError): + crawler.parse_sitemap("http://10.0.0.1/sitemap.xml") + + def test_allow_private_ips_string_false_keeps_ssrf_on(self): + crawler = SitemapCrawler(allow_private_ips="false") + assert crawler.allow_private_ips is False + with pytest.raises(ValidationError): + crawler.parse_sitemap("http://10.0.0.1/sitemap.xml") + + def test_allow_private_ips_string_true_opts_in(self): + crawler = SitemapCrawler(allow_private_ips="true") + assert crawler.allow_private_ips is True + + def test_redirect_to_private_ip_blocked(self): + crawler = SitemapCrawler() + redirect = MagicMock() + redirect.status_code = 302 + redirect.headers = {"Location": "http://169.254.169.254/latest/meta-data/"} + redirect.close = MagicMock() + + with patch( + "semantica.ingest.ssrf.requests.request", return_value=redirect + ), patch( + "semantica.ingest.ssrf.socket.getaddrinfo", + return_value=[(None, None, None, None, ("93.184.216.34", 0))], + ): + with pytest.raises(ValidationError, match="blocked"): + crawler.parse_sitemap("https://example.com/sitemap.xml") + + +class TestRESTIngestorSSRF: + def test_private_endpoint_never_reaches_session(self): + with patch("requests.Session") as MockSession: + mock_session = MockSession.return_value + ingestor = RESTIngestor() + with pytest.raises(ValidationError): + ingestor.ingest_endpoint("http://169.254.169.254/latest/meta-data/") + mock_session.request.assert_not_called() + + def test_file_scheme_rejected(self): + ingestor = RESTIngestor() + with pytest.raises(ValidationError, match="not permitted"): + ingestor.ingest_endpoint("file://localhost/secret") + + def test_allow_private_ips_opt_in(self): + with patch("requests.Session") as MockSession: + mock_session = MockSession.return_value + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"ok": True} + mock_response.headers = {"Content-Type": "application/json"} + mock_session.request.return_value = mock_response + + ingestor = RESTIngestor(allow_private_ips=True) + data = ingestor.ingest_endpoint("http://127.0.0.1:8080/health") + assert data.data == {"ok": True} + mock_session.request.assert_called_once() + assert mock_session.request.call_args.kwargs.get("allow_redirects") is False + + def test_allow_private_ips_string_false_keeps_ssrf_on(self): + with patch("requests.Session") as MockSession: + mock_session = MockSession.return_value + ingestor = RESTIngestor(allow_private_ips="false") + assert ingestor.allow_private_ips is False + with pytest.raises(ValidationError, match="blocked"): + ingestor.ingest_endpoint("http://127.0.0.1:8080/health") + mock_session.request.assert_not_called() + + def test_allow_private_ips_string_true_opts_in(self): + with patch("requests.Session") as MockSession: + mock_session = MockSession.return_value + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"ok": True} + mock_response.headers = {"Content-Type": "application/json"} + mock_session.request.return_value = mock_response + + ingestor = RESTIngestor(allow_private_ips="true") + assert ingestor.allow_private_ips is True + data = ingestor.ingest_endpoint("http://127.0.0.1:8080/health") + assert data.data == {"ok": True} + mock_session.request.assert_called_once() + + def test_redirect_to_private_ip_blocked(self): + with patch("requests.Session") as MockSession: + mock_session = MockSession.return_value + mock_session.headers = {} + redirect = MagicMock() + redirect.status_code = 302 + redirect.headers = {"Location": "http://127.0.0.1/secret"} + redirect.close = MagicMock() + mock_session.request.return_value = redirect + + ingestor = RESTIngestor() + with patch( + "semantica.ingest.ssrf.socket.getaddrinfo", + return_value=[(None, None, None, None, ("93.184.216.34", 0))], + ): + with pytest.raises(ValidationError, match="blocked"): + ingestor.ingest_endpoint("https://example.com/api") + mock_session.request.assert_called_once() From 64f6c5cba2ace0543bb9be054f24c5bcf0ee0219 Mon Sep 17 00:00:00 2001 From: pravit-amp <43916793+pravit-amp@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:00:40 -0700 Subject: [PATCH 14/40] test(split): cover untested chunker classes (#864) (#904) * test(split): add coverage for untested chunker classes * test(split): address Qodo gaps for chunker coverage Cover exported KG/structural/sliding-window helpers, assert heading boundaries, and use importorskip instead of mocking optional deps. * fix(split): normalize sliding-window stride when omitted * fix(split): pass entities to relation extraction and harden graph-based tests --------- Co-authored-by: Pravit Ampapathini --- semantica/split/methods.py | 7 +- tests/split/test_chunkers.py | 592 +++++++++++++++++++++++++++++++++++ 2 files changed, 597 insertions(+), 2 deletions(-) create mode 100644 tests/split/test_chunkers.py diff --git a/semantica/split/methods.py b/semantica/split/methods.py index 3c3d1193..8c338dc6 100644 --- a/semantica/split/methods.py +++ b/semantica/split/methods.py @@ -1180,7 +1180,9 @@ def split_graph_based( ) entities = ner_extractor.extract(text) - relations = relation_extractor.extract(text) + # Relation extraction requires the extracted entities; failing to pass + # them raises and can trigger the broad fallback to recursive splitting. + relations = relation_extractor.extract(text, entities) # Build graph G = nx.Graph() @@ -1623,8 +1625,9 @@ def split_sliding_window( ) try: + effective_stride = stride if stride is not None else (chunk_size - overlap) chunker = SlidingWindowChunker( - chunk_size=chunk_size, overlap=overlap, stride=stride, **kwargs + chunk_size=chunk_size, overlap=overlap, stride=effective_stride, **kwargs ) return chunker.chunk(text, preserve_boundaries=preserve_boundaries, **kwargs) except Exception as e: diff --git a/tests/split/test_chunkers.py b/tests/split/test_chunkers.py new file mode 100644 index 00000000..7210a9de --- /dev/null +++ b/tests/split/test_chunkers.py @@ -0,0 +1,592 @@ +"""Tests for previously untested split chunker classes (issue #864).""" + +import pytest + +from semantica.split.kg_chunkers import ( + EntityAwareChunker, + GraphBasedChunker, + HierarchicalChunker, + OntologyAwareChunker, + RelationAwareChunker, +) +from semantica.split.methods import ( + SEMANTIC_EXTRACT_AVAILABLE, + NETWORKX_AVAILABLE, + split_by_characters, + split_by_paragraphs, + split_by_sentences, + split_by_words, + split_entity_aware, + split_graph_based, + split_hierarchical, + split_ontology_aware, + split_recursive, + split_relation_aware, + split_sliding_window, + split_structural, +) +from semantica.split.semantic_chunker import Chunk +from semantica.split.sliding_window_chunker import SlidingWindowChunker +from semantica.split.structural_chunker import StructuralChunker, StructuralElement +from semantica.split.table_chunker import TableChunk, TableChunker +from semantica.utils.exceptions import ValidationError + +requires_semantic_extract = pytest.mark.skipif( + not SEMANTIC_EXTRACT_AVAILABLE, + reason="semantica.semantic_extract is not available", +) +requires_networkx = pytest.mark.skipif( + not NETWORKX_AVAILABLE, + reason="networkx is not available", +) + + +# --------------------------------------------------------------------------- +# SlidingWindowChunker +# --------------------------------------------------------------------------- + + +class TestSlidingWindowChunker: + def test_init_defaults_and_validation(self): + chunker = SlidingWindowChunker(chunk_size=100, overlap=20) + assert chunker.chunk_size == 100 + assert chunker.overlap == 20 + assert chunker.stride == 80 + + with pytest.raises(ValidationError): + SlidingWindowChunker(chunk_size=0) + with pytest.raises(ValidationError): + SlidingWindowChunker(chunk_size=100, overlap=-1) + with pytest.raises(ValidationError): + SlidingWindowChunker(chunk_size=100, overlap=100) + + def test_empty_text_returns_empty(self): + chunker = SlidingWindowChunker(chunk_size=50, overlap=10) + assert chunker.chunk("") == [] + + def test_fixed_size_overlap_invariant(self): + """Last `overlap` chars of chunk N appear at the start of chunk N+1.""" + text = "abcdefghijklmnopqrstuvwxyz0123456789" * 3 # 108 chars + overlap = 10 + chunk_size = 30 + chunker = SlidingWindowChunker( + chunk_size=chunk_size, overlap=overlap, stride=chunk_size - overlap + ) + chunks = chunker.chunk(text, preserve_boundaries=False) + + assert len(chunks) >= 2 + for i in range(len(chunks) - 1): + # Final chunk may be shorter than overlap; compare shared window only + shared = min(overlap, len(chunks[i].text), len(chunks[i + 1].text)) + expected_overlap = chunks[i].text[-shared:] + actual_prefix = chunks[i + 1].text[:shared] + assert actual_prefix == expected_overlap, ( + f"Overlap mismatch between chunk {i} and {i + 1}: " + f"{expected_overlap!r} != {actual_prefix!r}" + ) + + for i in range(len(chunks) - 1): + assert ( + chunks[i + 1].start_index - chunks[i].start_index + == chunk_size - overlap + ) + + def test_chunk_with_overlap_helper(self): + text = "word " * 40 + chunker = SlidingWindowChunker(chunk_size=50, overlap=0) + chunks = chunker.chunk_with_overlap(text, overlap_size=15) + assert len(chunks) >= 2 + assert chunker.overlap == 0 + + def test_boundary_preservation_avoids_mid_word_when_possible(self): + text = ( + "Alice went to the market. Bob bought apples. " + "Carol cooked dinner. Dave drove home." + ) + chunker = SlidingWindowChunker(chunk_size=40, overlap=10) + chunks = chunker.chunk(text, preserve_boundaries=True) + assert len(chunks) >= 1 + for chunk in chunks: + assert isinstance(chunk, Chunk) + assert chunk.text + assert chunk.metadata.get("chunk_index") is not None + + +# --------------------------------------------------------------------------- +# StructuralChunker +# --------------------------------------------------------------------------- + + +class TestStructuralChunker: + MARKDOWN_DOC = """# Introduction + +This is the intro paragraph about the project. + +## Details + +Here are more details about how it works. + +- item one +- item two +- item three + +## Conclusion + +Final thoughts on the subject. +""" + + def test_empty_text_returns_empty(self): + chunker = StructuralChunker(max_chunk_size=500) + assert chunker.chunk("") == [] + + def test_heading_based_splits(self): + chunker = StructuralChunker(respect_headers=True, max_chunk_size=200) + chunks = chunker.chunk(self.MARKDOWN_DOC) + + assert len(chunks) >= 1 + for chunk in chunks: + assert isinstance(chunk, Chunk) + assert chunk.metadata.get("structure_preserved") is True + assert "element_types" in chunk.metadata + + all_types = [] + for chunk in chunks: + all_types.extend(chunk.metadata["element_types"]) + assert "heading" in all_types + assert "paragraph" in all_types + + def test_heading_boundaries_separate_sections(self): + """Distinct top-level headings must not be merged into one chunk.""" + doc = """# Alpha + +Content exclusively about alpha topic here. + +# Beta + +Content exclusively about beta topic here. +""" + chunker = StructuralChunker(respect_headers=True, max_chunk_size=50) + chunks = chunker.chunk(doc) + + assert len(chunks) >= 2 + alpha_chunks = [c for c in chunks if "exclusively about alpha" in c.text] + beta_chunks = [c for c in chunks if "exclusively about beta" in c.text] + assert alpha_chunks, "Alpha section body missing from chunks" + assert beta_chunks, "Beta section body missing from chunks" + + # Heading-boundary invariant: alpha and beta bodies stay in separate chunks + for chunk in chunks: + has_alpha = "exclusively about alpha" in chunk.text + has_beta = "exclusively about beta" in chunk.text + assert not (has_alpha and has_beta), ( + f"Sections merged across heading boundary: {chunk.text!r}" + ) + + def test_extract_structure_detects_headings_and_lists(self): + chunker = StructuralChunker() + elements = chunker._extract_structure(self.MARKDOWN_DOC) + types = [e.type for e in elements] + assert "heading" in types + assert "list" in types + assert "paragraph" in types + assert all(isinstance(e, StructuralElement) for e in elements) + + def test_code_block_preserved(self): + text = """# Code + +```python +def hello(): + return "world" +``` + +After the code. +""" + chunker = StructuralChunker(max_chunk_size=2000) + elements = chunker._extract_structure(text) + types = [e.type for e in elements] + assert "code_block" in types + code = next(e for e in elements if e.type == "code_block") + assert "def hello" in code.text + + +# --------------------------------------------------------------------------- +# TableChunker +# --------------------------------------------------------------------------- + + +class TestTableChunker: + def _sample_table(self, n_rows: int = 10): + headers = ["Name", "Age", "City"] + rows = [[f"Person{i}", str(20 + i), f"City{i}"] for i in range(n_rows)] + return {"headers": headers, "rows": rows} + + def test_rows_are_not_split_mid_row(self): + """Each chunk contains complete rows only — never a partial row.""" + table = self._sample_table(10) + chunker = TableChunker(max_rows=3, preserve_headers=True) + chunks = chunker.chunk_table(table) + + assert len(chunks) == 4 # 3+3+3+1 + for chunk in chunks: + assert isinstance(chunk, TableChunk) + assert chunk.headers == ["Name", "Age", "City"] + for row in chunk.rows: + assert len(row) == 3 + assert chunk.metadata["row_count"] == len(chunk.rows) + + flattened = [row for c in chunks for row in c.rows] + assert flattened == table["rows"] + + def test_markdown_table_chunk_does_not_split_rows(self): + md = """| Name | Age | City | +| --- | --- | --- | +| Alice | 30 | NYC | +| Bob | 25 | LA | +| Carol | 40 | SF | +| Dave | 35 | CHI | +""" + chunker = TableChunker(max_rows=2, preserve_headers=True) + chunks = chunker.chunk(md) + + assert len(chunks) == 2 + for chunk in chunks: + assert chunk.metadata["chunk_type"] == "table" + data_lines = [ + line + for line in chunk.text.split("\n") + if line and "---" not in line and not line.startswith("Name") + ] + for line in data_lines: + cells = [c.strip() for c in line.split("|")] + assert len(cells) == 3 + + def test_non_table_text_returns_single_chunk(self): + chunker = TableChunker() + chunks = chunker.chunk("Just plain text without a table.") + assert len(chunks) == 1 + assert chunks[0].metadata.get("error") == "No table found" + + def test_extract_table_schema(self): + table = { + "headers": ["id", "active", "label"], + "rows": [ + ["1", "true", "alpha"], + ["2", "false", "beta"], + ], + } + schema = TableChunker().extract_table_schema(table) + assert schema["column_count"] == 3 + assert schema["row_count"] == 2 + assert schema["column_types"]["id"] == "numeric" + assert schema["column_types"]["active"] == "boolean" + assert schema["column_types"]["label"] == "text" + + def test_chunk_by_columns(self): + table = self._sample_table(3) + chunker = TableChunker(chunk_by_columns=True, preserve_headers=True) + chunks = chunker.chunk_table(table, max_columns=2) + assert len(chunks) == 2 + assert chunks[0].headers == ["Name", "Age"] + assert chunks[1].headers == ["City"] + for chunk in chunks: + for row in chunk.rows: + assert len(row) == len(chunk.headers) + + +# --------------------------------------------------------------------------- +# EntityAwareChunker (real optional deps via importorskip / skipif) +# --------------------------------------------------------------------------- + + +class TestEntityAwareChunker: + def test_init(self): + chunker = EntityAwareChunker( + chunk_size=500, chunk_overlap=50, ner_method="pattern" + ) + assert chunker.chunk_size == 500 + assert chunker.ner_method == "pattern" + assert chunker.preserve_entities is True + + def test_empty_text(self): + chunker = EntityAwareChunker(chunk_size=100, ner_method="pattern") + chunks = chunker.chunk("") + assert isinstance(chunks, list) + + @requires_semantic_extract + def test_entity_boundaries_preserved_with_pattern_ner(self): + """Entity spans stay intact when using real pattern NER.""" + pytest.importorskip("semantica.semantic_extract") + entity_text = "AppleInc" + # Use a contiguous token the pattern NER can latch onto + text = ( + "Intro sentence one goes here. Intro sentence two goes here. " + f"{entity_text} was founded in Cupertino California recently. " + "More filler sentences keep the document long enough to chunk. " + "Yet another sentence about products and services worldwide. " + "Final sentence for padding the overall document length out." + ) + chunks = split_entity_aware( + text, + chunk_size=90, + ner_method="pattern", + preserve_entities=True, + ) + assert len(chunks) >= 1 + containing = [c for c in chunks if entity_text in c.text] + assert containing, "Expected entity text to appear in at least one chunk" + for chunk in containing: + idx = chunk.text.index(entity_text) + assert chunk.text[idx : idx + len(entity_text)] == entity_text + + @requires_semantic_extract + def test_entity_aware_chunker_with_pattern_ner(self): + pytest.importorskip("semantica.semantic_extract") + text = ( + "Alice Johnson founded Acme Corporation in New York. " + "Bob Smith joined the company later. " + "They expanded operations across Europe and Asia. " + ) * 5 + chunker = EntityAwareChunker( + chunk_size=120, ner_method="pattern", preserve_entities=True + ) + chunks = chunker.chunk(text) + assert len(chunks) >= 1 + assert all(isinstance(c, Chunk) for c in chunks) + + +# --------------------------------------------------------------------------- +# RelationAware / GraphBased / OntologyAware / Hierarchical +# --------------------------------------------------------------------------- + + +class TestRelationAwareChunker: + def test_init(self): + chunker = RelationAwareChunker(chunk_size=100, relation_method="pattern") + assert chunker.chunk_size == 100 + assert chunker.relation_method == "pattern" + + @requires_semantic_extract + def test_chunk_with_pattern_extractors(self): + pytest.importorskip("semantica.semantic_extract") + text = ( + "Alice works at Acme. Bob reports to Alice. " + "Carol founded Acme in 2010. More padding text follows here. " + ) * 4 + chunker = RelationAwareChunker( + chunk_size=100, relation_method="pattern", ner_method="pattern" + ) + chunks = chunker.chunk(text) + assert isinstance(chunks, list) + assert len(chunks) >= 1 + assert all(isinstance(c, Chunk) for c in chunks) + + +class TestGraphBasedChunker: + def test_init(self): + chunker = GraphBasedChunker( + chunk_size=500, strategy="community", algorithm="louvain" + ) + assert chunker.strategy == "community" + assert chunker.algorithm == "louvain" + + @requires_semantic_extract + @requires_networkx + def test_chunk_with_real_optional_deps(self): + pytest.importorskip("networkx") + pytest.importorskip("semantica.semantic_extract") + text = ( + "Alice met Bob at Acme Corporation yesterday afternoon. " + "Bob introduced Carol to the Acme engineering team. " + "Carol and Alice later discussed graph-based retrieval methods. " + ) * 3 + chunker = GraphBasedChunker( + chunk_size=200, + strategy="community", + algorithm="louvain", + ner_method="pattern", + relation_method="pattern", + ) + chunks = chunker.chunk(text) + assert len(chunks) >= 1 + assert all(isinstance(c, Chunk) for c in chunks) + # Ensure the graph-based path actually ran (not fallback-to-recursive). + assert any( + c.metadata.get("method") == "graph_based" for c in chunks + ), "Expected at least one graph_based chunk" + assert any( + c.metadata.get("strategy") == "community" + and c.metadata.get("algorithm") == "louvain" + for c in chunks + ), "Expected graph_based chunk metadata to include strategy/algorithm" + assert not any( + c.metadata.get("method") == "recursive" for c in chunks + ), "Graph-based fallback to recursive was triggered" + + +class TestOntologyAwareChunker: + def test_init(self): + chunker = OntologyAwareChunker(chunk_size=200, preserve_concepts=True) + assert chunker.chunk_size == 200 + assert chunker.preserve_concepts is True + + @requires_semantic_extract + def test_chunk_uses_entity_aware_path(self): + pytest.importorskip("semantica.semantic_extract") + text = "Concept Alpha relates to Concept Beta in the taxonomy. " * 8 + chunker = OntologyAwareChunker( + chunk_size=120, preserve_concepts=True, ner_method="pattern" + ) + chunks = chunker.chunk(text) + assert len(chunks) >= 1 + assert all(isinstance(c, Chunk) for c in chunks) + + +class TestHierarchicalChunker: + def test_hierarchical_markdown_sections(self): + text = """# Section One + +Paragraph under section one with enough content to matter. + +# Section Two + +Paragraph under section two also with sufficient content. +""" + chunker = HierarchicalChunker( + levels=["section", "paragraph"], chunk_sizes=[2000, 500] + ) + chunks = chunker.chunk(text) + assert len(chunks) >= 1 + for chunk in chunks: + assert chunk.metadata.get("hierarchical") is True + assert chunk.metadata.get("levels") == ["section", "paragraph"] + + def test_split_hierarchical_function(self): + text = "Para one.\n\nPara two.\n\nPara three." + chunks = split_hierarchical(text, levels=["paragraph"], chunk_sizes=[1000]) + assert len(chunks) >= 1 + + +# --------------------------------------------------------------------------- +# Exported method functions (public API smoke coverage) +# --------------------------------------------------------------------------- + + +class TestSplitMethodFunctions: + SAMPLE = ( + "First sentence about knowledge graphs. " + "Second sentence covers entity extraction. " + "Third sentence discusses relation awareness. " + "Fourth sentence wraps up the example." + ) + + MARKDOWN = """# Intro + +Intro paragraph with enough text to matter for structural splitting. + +# Body + +Body paragraph under a distinct heading for separation checks. +""" + + def test_split_recursive(self): + chunks = split_recursive(self.SAMPLE, chunk_size=60) + assert len(chunks) >= 1 + assert all(isinstance(c, Chunk) for c in chunks) + + def test_split_by_sentences(self): + chunks = split_by_sentences(self.SAMPLE, chunk_size=80) + assert len(chunks) >= 1 + + def test_split_by_paragraphs(self): + text = "Para A content here.\n\nPara B content here.\n\nPara C content here." + chunks = split_by_paragraphs(text, chunk_size=50) + assert len(chunks) >= 1 + + def test_split_by_characters(self): + chunks = split_by_characters(self.SAMPLE, chunk_size=40) + assert len(chunks) >= 2 + + def test_split_by_words(self): + chunks = split_by_words(self.SAMPLE, chunk_size=10) + assert len(chunks) >= 1 + + def test_split_structural(self): + chunks = split_structural( + self.MARKDOWN, max_chunk_size=80, respect_headers=True + ) + assert len(chunks) >= 2 + assert all(isinstance(c, Chunk) for c in chunks) + + def test_split_sliding_window(self): + chunks = split_sliding_window( + self.SAMPLE * 3, + chunk_size=40, + overlap=10, + preserve_boundaries=False, + ) + assert len(chunks) >= 2 + assert all(isinstance(c, Chunk) for c in chunks) + # Verify the sliding-window path was taken, not the recursive fallback. + # chunks[1].metadata["has_overlap"] is set only by SlidingWindowChunker. + assert chunks[1].metadata.get("has_overlap") is True, ( + "Expected sliding-window chunks to carry has_overlap=True; " + "fallback to recursive may have occurred" + ) + assert chunks[1].metadata.get("method") != "recursive", ( + "Sliding-window fallback to recursive was triggered unexpectedly" + ) + + @requires_semantic_extract + def test_split_entity_aware(self): + pytest.importorskip("semantica.semantic_extract") + chunks = split_entity_aware( + self.SAMPLE * 3, chunk_size=80, ner_method="pattern" + ) + assert len(chunks) >= 1 + + @requires_semantic_extract + def test_split_relation_aware(self): + pytest.importorskip("semantica.semantic_extract") + chunks = split_relation_aware( + self.SAMPLE * 3, + chunk_size=80, + relation_method="pattern", + ner_method="pattern", + ) + assert len(chunks) >= 1 + + @requires_semantic_extract + @requires_networkx + def test_split_graph_based(self): + pytest.importorskip("networkx") + pytest.importorskip("semantica.semantic_extract") + chunks = split_graph_based( + self.SAMPLE * 3, + chunk_size=120, + strategy="community", + algorithm="louvain", + ner_method="pattern", + relation_method="pattern", + ) + assert len(chunks) >= 1 + assert all(isinstance(c, Chunk) for c in chunks) + # Ensure we didn't satisfy the test via the broad recursive fallback. + assert any( + c.metadata.get("method") == "graph_based" for c in chunks + ), "Expected at least one graph_based chunk" + assert any( + c.metadata.get("strategy") == "community" + and c.metadata.get("algorithm") == "louvain" + for c in chunks + ), "Expected graph_based chunk metadata to include strategy/algorithm" + assert not any( + c.metadata.get("method") == "recursive" for c in chunks + ), "Graph-based fallback to recursive was triggered" + + @requires_semantic_extract + def test_split_ontology_aware(self): + pytest.importorskip("semantica.semantic_extract") + chunks = split_ontology_aware( + self.SAMPLE * 3, chunk_size=80, ner_method="pattern" + ) + assert len(chunks) >= 1 From 3496d62335ae70210ab5b817ccca8d7d83f274eb Mon Sep 17 00:00:00 2001 From: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:35:00 +0530 Subject: [PATCH 15/40] security: require API-key auth on all Explorer API routes (GHSA-j4mq) (#909) * security: require API-key auth on all Explorer API routes (GHSA-j4mq-hprp-987v) Every Explorer route (bulk import/export, delete, LLM-backed ontology generation, SPARQL, etc.) was mounted with no authentication, and both server entrypoints bind 0.0.0.0 by default. Anyone reaching the port got full read/write/delete on the graph. - Add require_auth dependency (explorer/dependencies.py): checks X-API-Key against SEMANTICA_API_KEY, fails closed with 503 if unconfigured (not silently anonymous), 401 on wrong/missing key. SEMANTICA_ALLOW_ANONYMOUS=true opts out explicitly for local dev. - Wire dependencies=[Depends(require_auth)] into all 11 API routers in both explorer/app.py and server.py. /health, /api/info, static assets, and the SPA catch-all stay public. - /ws/graph-updates handshake now checks the same key via header or ?api_key= query param (browsers can't set custom WS headers) before accepting the connection. - Default bind changed from 0.0.0.0 to 127.0.0.1 in server.py's main() and cli.py's `server start`; the CLI warns if a non-loopback host is passed explicitly without a key configured. - Startup logging reports the resolved auth mode in both app factories. - Document/generate SEMANTICA_API_KEY in the deploy recipes that expose a public endpoint by default: docker-compose, Railway, Fly, Render. Added tests/explorer/test_explorer_auth.py covering fail-closed default, wrong/missing/correct key, anonymous opt-in, public-route exemptions, and the WS handshake. Added tests/explorer/conftest.py defaulting the pre-existing ~200 explorer tests to SEMANTICA_ALLOW_ANONYMOUS=true so they keep exercising route logic without needing a key. * fix CORS --------- Co-authored-by: Zohaib Hassnain <109234410+ZohaibHassan16@users.noreply.github.com> --- deploy/fly/README.md | 3 + deploy/railway/README.md | 3 + deploy/render/README.md | 2 + deploy/render/render.yaml | 2 + docker-compose.dev.yml | 2 + docker-compose.yml | 5 + semantica/cli.py | 12 ++- semantica/explorer/__init__.py | 22 ++-- semantica/explorer/app.py | 52 ++++++--- semantica/explorer/dependencies.py | 65 +++++++++++- semantica/server.py | 51 ++++++--- tests/explorer/conftest.py | 17 +++ tests/explorer/test_explorer_auth.py | 152 +++++++++++++++++++++++++++ 13 files changed, 350 insertions(+), 38 deletions(-) create mode 100644 tests/explorer/conftest.py create mode 100644 tests/explorer/test_explorer_auth.py diff --git a/deploy/fly/README.md b/deploy/fly/README.md index 5c5d1258..ba0dc2b6 100644 --- a/deploy/fly/README.md +++ b/deploy/fly/README.md @@ -9,7 +9,10 @@ flyctl launch --copy-config --config deploy/fly/fly.toml --no-deploy # Fly.io private networking uses .internal hostnames — do not use localhost # unless FalkorDB is a co-located process inside the same Machine. flyctl secrets set FALKORDB_HOST=.internal FALKORDB_PORT=6379 +flyctl secrets set SEMANTICA_API_KEY=$(openssl rand -hex 32) flyctl deploy --config deploy/fly/fly.toml ``` Change `app` in `fly.toml` before launch if the default app name is already taken. + +Fly apps get a public `*.fly.dev` URL by default, so `SEMANTICA_API_KEY` is required — without it the Explorer refuses every protected route (503) rather than serving anonymously. Pass the same value as the `X-API-Key` header from any client that talks to the deployed API. diff --git a/deploy/railway/README.md b/deploy/railway/README.md index a42163ca..b755a032 100644 --- a/deploy/railway/README.md +++ b/deploy/railway/README.md @@ -9,7 +9,10 @@ railway add --database redis railway variable --set "FALKORDB_HOST=${{Redis.REDISHOST}}" railway variable --set "FALKORDB_PORT=${{Redis.REDISPORT}}" railway variable --set "ALLOWED_ORIGINS=https://${{RAILWAY_PUBLIC_DOMAIN}}" +railway variable --set "SEMANTICA_API_KEY=$(openssl rand -hex 32)" railway up ``` The Redis plugin variables are wired to the requested FalkorDB env names for deployment compatibility. The Explorer currently reads these settings but does not persist graph state to FalkorDB. + +Railway exposes this service on a public domain, so `SEMANTICA_API_KEY` is required — without it the Explorer refuses every protected route (503) rather than serving anonymously. Pass the same value as the `X-API-Key` header from any client that talks to the deployed API. diff --git a/deploy/render/README.md b/deploy/render/README.md index e252527a..d4652ac5 100644 --- a/deploy/render/README.md +++ b/deploy/render/README.md @@ -9,3 +9,5 @@ render blueprint apply deploy/render/render.yaml ``` After creation, update `ALLOWED_ORIGINS` in the Render dashboard if you attach a custom domain. + +`SEMANTICA_API_KEY` is auto-generated by the blueprint (`generateValue: true`) since this service gets a public `onrender.com` URL — without it the Explorer refuses every protected route (503) rather than serving anonymously. Find the generated value in the Render dashboard's environment tab and pass it as the `X-API-Key` header from any client that talks to the deployed API. diff --git a/deploy/render/render.yaml b/deploy/render/render.yaml index d0776749..5f385d01 100644 --- a/deploy/render/render.yaml +++ b/deploy/render/render.yaml @@ -20,6 +20,8 @@ services: type: keyvalue name: semantica-explorer-redis property: port + - key: SEMANTICA_API_KEY + generateValue: true - type: keyvalue name: semantica-explorer-redis diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5ff13b8d..495f8ef7 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -16,6 +16,8 @@ services: ALLOWED_ORIGINS: http://localhost:5173,http://127.0.0.1:5173,http://localhost:8000,http://127.0.0.1:8000 FALKORDB_HOST: falkordb FALKORDB_PORT: "6379" + # Local dev only: this compose file is not for public exposure. + SEMANTICA_ALLOW_ANONYMOUS: "true" volumes: - ./semantica:/app/semantica - ./pyproject.toml:/app/pyproject.toml:ro diff --git a/docker-compose.yml b/docker-compose.yml index 746b6bf3..43336d3b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,11 @@ services: FALKORDB_HOST: falkordb FALKORDB_PORT: "6379" ALLOWED_ORIGINS: ${ALLOWED_ORIGINS:-http://localhost:8000,http://127.0.0.1:8000} + # Required for API access - the Explorer refuses all protected routes + # (503) until this is set. Generate one with `openssl rand -hex 32`. + SEMANTICA_API_KEY: ${SEMANTICA_API_KEY:-} + # Trusted local-only setups only: bypasses the API key entirely. + SEMANTICA_ALLOW_ANONYMOUS: ${SEMANTICA_ALLOW_ANONYMOUS:-false} depends_on: falkordb: condition: service_started diff --git a/semantica/cli.py b/semantica/cli.py index 5b179c13..7b944dc6 100644 --- a/semantica/cli.py +++ b/semantica/cli.py @@ -4065,7 +4065,7 @@ def server(ctx: click.Context) -> None: @click.option("--port", default=8000, type=int, show_default=True) @click.option("--workers", default=1, type=int, show_default=True) @click.option("--reload", is_flag=True, default=False, help="Enable hot reload.") -@click.option("--host", default="0.0.0.0", show_default=True) +@click.option("--host", default="127.0.0.1", show_default=True) @click.pass_obj def server_start(cli_ctx: CLIContext, port: int, workers: int, reload: bool, host: str) -> None: """Start the REST API server. @@ -4076,6 +4076,16 @@ def server_start(cli_ctx: CLIContext, port: int, workers: int, reload: bool, hos """ cli_ctx = _require_ctx(cli_ctx) + _LOOPBACK_HOSTS = {"127.0.0.1", "::1", "localhost"} + if host not in _LOOPBACK_HOSTS: + console.print( + f"[{_WARN_STY}] ⚠[/{_WARN_STY}] Binding to [cyan]{host}[/cyan] exposes " + "the server to the network. Set SEMANTICA_API_KEY before doing this " + "in any reachable environment — without it, protected routes refuse " + "all requests (503), and with SEMANTICA_ALLOW_ANONYMOUS=true they are " + "wide open." + ) + def _action() -> None: import subprocess as sp cmd = [ diff --git a/semantica/explorer/__init__.py b/semantica/explorer/__init__.py index 0445eb5f..f9a780fe 100644 --- a/semantica/explorer/__init__.py +++ b/semantica/explorer/__init__.py @@ -83,12 +83,22 @@ def main(argv=None): _LOOPBACK_HOSTS = {"127.0.0.1", "::1", "localhost"} if args.host not in _LOOPBACK_HOSTS: - _err.print( - f"[bold yellow]Warning:[/bold yellow] Binding to " - f"[cyan]{args.host}[/cyan] exposes the Explorer to the network. " - "The API has no authentication — all graph data will be readable " - "and writable by any host that can reach this port." - ) + import os as _os + if _os.environ.get("SEMANTICA_ALLOW_ANONYMOUS", "").strip().lower() == "true": + _err.print( + f"[bold yellow]Warning:[/bold yellow] Binding to " + f"[cyan]{args.host}[/cyan] with SEMANTICA_ALLOW_ANONYMOUS=true " + "exposes the Explorer to the network with no authentication — " + "all graph data will be readable and writable by any host that " + "can reach this port." + ) + elif not _os.environ.get("SEMANTICA_API_KEY"): + _err.print( + f"[bold yellow]Warning:[/bold yellow] Binding to " + f"[cyan]{args.host}[/cyan] but SEMANTICA_API_KEY is not set — " + "protected routes will refuse all requests (503) until it is " + "configured." + ) if not args.no_browser: import threading diff --git a/semantica/explorer/app.py b/semantica/explorer/app.py index c214021a..e9f1d075 100644 --- a/semantica/explorer/app.py +++ b/semantica/explorer/app.py @@ -8,13 +8,14 @@ from contextlib import asynccontextmanager from pathlib import Path from typing import Optional -from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect +from fastapi import Depends, FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, HTMLResponse, JSONResponse from fastapi.staticfiles import StaticFiles from .. import __version__ from ..context.context_graph import ContextGraph +from .dependencies import anonymous_access_allowed, get_expected_api_key, is_valid_api_key, require_auth from .session import GraphSession from .ws import ConnectionManager @@ -97,6 +98,22 @@ def create_app( @asynccontextmanager async def lifespan(app: FastAPI): + import logging as _lifespan_logging + _lifespan_logger = _lifespan_logging.getLogger(__name__) + if anonymous_access_allowed(): + _lifespan_logger.warning( + "Explorer is running with SEMANTICA_ALLOW_ANONYMOUS=true — " + "all API routes are unauthenticated. Do not expose this " + "process beyond localhost." + ) + elif get_expected_api_key(): + _lifespan_logger.info("Explorer API authentication: enabled (SEMANTICA_API_KEY set).") + else: + _lifespan_logger.warning( + "Explorer API authentication: NOT CONFIGURED. All protected " + "routes will return 503 until SEMANTICA_API_KEY is set." + ) + app.state.event_loop = asyncio.get_running_loop() app.state.ws_manager = ConnectionManager() app.state.session = active_session @@ -123,7 +140,7 @@ def create_app( allow_origins=settings["allowed_origins"], allow_credentials=_allow_credentials, allow_methods=["GET", "POST", "DELETE", "OPTIONS"], - allow_headers=["Content-Type", "Authorization"], + allow_headers=["Content-Type", "Authorization", "X-API-Key"], max_age=600, ) @@ -159,22 +176,31 @@ def create_app( from .routes.temporal import router as temporal_router from .routes.vocabulary import router as vocabulary_router - app.include_router(graph_router) - app.include_router(analytics_router) - app.include_router(decisions_router) - app.include_router(temporal_router) - app.include_router(enrich_router) - app.include_router(export_import_router) - app.include_router(annotations_router) - app.include_router(sparql_router) - app.include_router(provenance_router) - app.include_router(vocabulary_router) - app.include_router(ontology_router) + _auth = [Depends(require_auth)] + app.include_router(graph_router, dependencies=_auth) + app.include_router(analytics_router, dependencies=_auth) + app.include_router(decisions_router, dependencies=_auth) + app.include_router(temporal_router, dependencies=_auth) + app.include_router(enrich_router, dependencies=_auth) + app.include_router(export_import_router, dependencies=_auth) + app.include_router(annotations_router, dependencies=_auth) + app.include_router(sparql_router, dependencies=_auth) + app.include_router(provenance_router, dependencies=_auth) + app.include_router(vocabulary_router, dependencies=_auth) + app.include_router(ontology_router, dependencies=_auth) _WS_MAX_MESSAGE_BYTES = 64 * 1024 # 64 KB — control messages only @app.websocket("/ws/graph-updates") async def websocket_endpoint(websocket: WebSocket): + # Browsers can't set custom headers on a WebSocket handshake, so + # accept the key via header (non-browser clients) or query param + # (browser clients), same SEMANTICA_API_KEY the REST routes check. + candidate = websocket.headers.get("x-api-key") or websocket.query_params.get("api_key") + if not is_valid_api_key(candidate): + await websocket.close(code=4401) # unauthorized + return + manager: ConnectionManager = app.state.ws_manager await manager.connect(websocket) await manager.send_personal(websocket, "connection_ack", {"connected": True}) diff --git a/semantica/explorer/dependencies.py b/semantica/explorer/dependencies.py index 44cbc6d8..7a97faeb 100644 --- a/semantica/explorer/dependencies.py +++ b/semantica/explorer/dependencies.py @@ -2,14 +2,75 @@ Semantica Explorer : FastAPI Dependencies Provides ``Depends()``-compatible callables for injecting the -current ``GraphSession`` and ``ConnectionManager`` into route handlers. +current ``GraphSession`` and ``ConnectionManager`` into route handlers, +and for enforcing API-key authentication on protected routes. """ -from fastapi import Request, HTTPException, status +import hmac +import os +from typing import Optional + +from fastapi import Request, HTTPException, Security, status +from fastapi.security.api_key import APIKeyHeader from .session import GraphSession from .ws import ConnectionManager +_api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False) + + +def get_expected_api_key() -> Optional[str]: + """Read the configured API key from the environment on every call. + + Read fresh (not cached) so tests and ops tooling can rotate the key + without restarting the process. + """ + return os.environ.get("SEMANTICA_API_KEY") or None + + +def anonymous_access_allowed() -> bool: + return os.environ.get("SEMANTICA_ALLOW_ANONYMOUS", "").strip().lower() == "true" + + +def is_valid_api_key(candidate: Optional[str]) -> bool: + """Return True if *candidate* matches the configured key, or if the + server has explicitly opted into anonymous access.""" + if anonymous_access_allowed(): + return True + expected = get_expected_api_key() + if not expected: + return False + return bool(candidate) and hmac.compare_digest(candidate, expected) + + +def require_auth(api_key: Optional[str] = Security(_api_key_header)) -> None: + """Dependency enforcing the ``X-API-Key`` header on protected routes. + + Every Explorer/API router (except health/info/static assets) should be + mounted with ``dependencies=[Depends(require_auth)]``. If + SEMANTICA_API_KEY is unset, requests are refused with 503 rather than + silently served unauthenticated — SEMANTICA_ALLOW_ANONYMOUS=true opts + into that explicitly for local development. + """ + if anonymous_access_allowed(): + return + expected = get_expected_api_key() + if not expected: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=( + "Server is not configured for authentication. Set the " + "SEMANTICA_API_KEY environment variable, or explicitly opt " + "into unauthenticated access (development only) with " + "SEMANTICA_ALLOW_ANONYMOUS=true." + ), + ) + if not api_key or not hmac.compare_digest(api_key, expected): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or missing API key. Send it as the X-API-Key header.", + ) + def get_session(request: Request) -> GraphSession: """Retrieve the GraphSession stored on ``app.state``.""" diff --git a/semantica/server.py b/semantica/server.py index 45ce61e0..5001c09f 100644 --- a/semantica/server.py +++ b/semantica/server.py @@ -10,7 +10,7 @@ import os import uvicorn from contextlib import asynccontextmanager from pathlib import Path -from fastapi import FastAPI, HTTPException, Request +from fastapi import Depends, FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, JSONResponse from starlette.middleware.base import BaseHTTPMiddleware @@ -25,6 +25,7 @@ try: from .context.context_graph import ContextGraph from .explorer.session import GraphSession from .explorer.ws import ConnectionManager + from .explorer.dependencies import anonymous_access_allowed, get_expected_api_key, require_auth EXPLORER_AVAILABLE = True except ImportError: EXPLORER_AVAILABLE = False @@ -37,8 +38,20 @@ STATIC_DIR = Path(__file__).parent / "static" async def lifespan(app: FastAPI): """Lifecycle manager for startup and shutdown events.""" logging.info("Starting up Semantica API...") - + if EXPLORER_AVAILABLE: + if anonymous_access_allowed(): + logging.warning( + "SEMANTICA_ALLOW_ANONYMOUS=true — all Explorer API routes are " + "unauthenticated. Do not expose this process beyond localhost." + ) + elif get_expected_api_key(): + logging.info("Explorer API authentication: enabled (SEMANTICA_API_KEY set).") + else: + logging.warning( + "Explorer API authentication: NOT CONFIGURED. Protected routes " + "will return 503 until SEMANTICA_API_KEY is set." + ) try: logging.info("Initializing Graph engine and Database connection...") graph = ContextGraph() @@ -79,7 +92,7 @@ app.add_middleware( allow_origins=_cors_origins, allow_credentials=True, allow_methods=["GET", "POST", "DELETE", "OPTIONS"], - allow_headers=["Content-Type", "Authorization"], + allow_headers=["Content-Type", "Authorization", "X-API-Key"], max_age=600, ) @@ -159,17 +172,18 @@ if EXPLORER_AVAILABLE: sparql ) - app.include_router(analytics.router) - app.include_router(annotations.router) - app.include_router(decisions.router) - app.include_router(enrich.router) - app.include_router(export_import.router) - app.include_router(graph.router) - app.include_router(ontology.router) - app.include_router(temporal.router) - app.include_router(vocabulary.router) - app.include_router(provenance.router) - app.include_router(sparql.router) + _auth = [Depends(require_auth)] + app.include_router(analytics.router, dependencies=_auth) + app.include_router(annotations.router, dependencies=_auth) + app.include_router(decisions.router, dependencies=_auth) + app.include_router(enrich.router, dependencies=_auth) + app.include_router(export_import.router, dependencies=_auth) + app.include_router(graph.router, dependencies=_auth) + app.include_router(ontology.router, dependencies=_auth) + app.include_router(temporal.router, dependencies=_auth) + app.include_router(vocabulary.router, dependencies=_auth) + app.include_router(provenance.router, dependencies=_auth) + app.include_router(sparql.router, dependencies=_auth) logging.info("Explorer, Vocabulary, SPARQL, Provenance, and Ontology API routes successfully mounted.") except Exception as exc: @@ -237,8 +251,13 @@ async def serve_spa(full_path: str): ) def main(): - """Server entry point.""" - uvicorn.run(app, host="0.0.0.0", port=8000) + """Server entry point. + + Binds to loopback by default; set SEMANTICA_HOST to expose beyond + localhost (e.g. behind a reverse proxy that terminates auth/TLS). + """ + host = os.environ.get("SEMANTICA_HOST", "127.0.0.1") + uvicorn.run(app, host=host, port=8000) if __name__ == "__main__": main() \ No newline at end of file diff --git a/tests/explorer/conftest.py b/tests/explorer/conftest.py new file mode 100644 index 00000000..b943d514 --- /dev/null +++ b/tests/explorer/conftest.py @@ -0,0 +1,17 @@ +"""Shared fixtures for explorer API tests. + +Most of this suite predates the API-key auth layer added for +GHSA-j4mq-hprp-987v and exercises route logic, not authentication. Default +every test under tests/explorer/ to SEMANTICA_ALLOW_ANONYMOUS=true so those +tests keep talking to the Explorer without needing an X-API-Key header. +Auth-specific tests (test_explorer_auth.py) override this per-test via the +same `monkeypatch` fixture. +""" + +import pytest + + +@pytest.fixture(autouse=True) +def _default_to_anonymous_explorer_access(monkeypatch): + monkeypatch.setenv("SEMANTICA_ALLOW_ANONYMOUS", "true") + monkeypatch.delenv("SEMANTICA_API_KEY", raising=False) diff --git a/tests/explorer/test_explorer_auth.py b/tests/explorer/test_explorer_auth.py new file mode 100644 index 00000000..4b7b9c6f --- /dev/null +++ b/tests/explorer/test_explorer_auth.py @@ -0,0 +1,152 @@ +"""Tests for the API-key auth dependency added for GHSA-j4mq-hprp-987v +(missing authentication on all Explorer API routes). + +Covers: protected routes refuse requests when no key is configured (fail +closed, not fail open), reject wrong/missing keys once a key is +configured, accept the correct key, remain reachable when +SEMANTICA_ALLOW_ANONYMOUS=true is set explicitly, and that health/info/ +static routes stay public regardless. Also covers the /ws/graph-updates +handshake, which can't use the same FastAPI Depends() plumbing since +browsers can't set custom headers on a WebSocket handshake. +""" + +import pytest + +from semantica.context.context_graph import ContextGraph +from semantica.explorer.app import create_app +from semantica.explorer.session import GraphSession + +try: + from starlette.testclient import TestClient +except ImportError: + pytest.skip( + "starlette TestClient is required for explorer tests. Install semantica[explorer].", + allow_module_level=True, + ) + + +def _build_sample_graph() -> ContextGraph: + graph = ContextGraph(advanced_analytics=False) + graph.add_node("python", node_type="language", content="Python") + return graph + + +@pytest.fixture +def client(): + session = GraphSession(_build_sample_graph()) + app = create_app(session=session) + with TestClient(app) as test_client: + yield test_client + + +# --------------------------------------------------------------------------- +# Fail-closed: no SEMANTICA_API_KEY and no explicit anonymous opt-in. +# --------------------------------------------------------------------------- + +def test_protected_route_returns_503_when_auth_not_configured(client, monkeypatch): + monkeypatch.delenv("SEMANTICA_ALLOW_ANONYMOUS", raising=False) + monkeypatch.delenv("SEMANTICA_API_KEY", raising=False) + + resp = client.get("/api/graph/nodes") + + assert resp.status_code == 503 + + +def test_write_route_also_refuses_when_auth_not_configured(client, monkeypatch): + monkeypatch.delenv("SEMANTICA_ALLOW_ANONYMOUS", raising=False) + monkeypatch.delenv("SEMANTICA_API_KEY", raising=False) + + resp = client.post("/api/export", json={"format": "json"}) + + assert resp.status_code == 503 + + +# --------------------------------------------------------------------------- +# Configured key: wrong/missing key rejected, correct key accepted. +# --------------------------------------------------------------------------- + +def test_protected_route_rejects_missing_key(client, monkeypatch): + monkeypatch.delenv("SEMANTICA_ALLOW_ANONYMOUS", raising=False) + monkeypatch.setenv("SEMANTICA_API_KEY", "correct-key") + + resp = client.get("/api/graph/nodes") + + assert resp.status_code == 401 + + +def test_protected_route_rejects_wrong_key(client, monkeypatch): + monkeypatch.delenv("SEMANTICA_ALLOW_ANONYMOUS", raising=False) + monkeypatch.setenv("SEMANTICA_API_KEY", "correct-key") + + resp = client.get("/api/graph/nodes", headers={"X-API-Key": "wrong-key"}) + + assert resp.status_code == 401 + + +def test_protected_route_accepts_correct_key(client, monkeypatch): + monkeypatch.delenv("SEMANTICA_ALLOW_ANONYMOUS", raising=False) + monkeypatch.setenv("SEMANTICA_API_KEY", "correct-key") + + resp = client.get("/api/graph/nodes", headers={"X-API-Key": "correct-key"}) + + assert resp.status_code == 200 + + +# --------------------------------------------------------------------------- +# Explicit opt-in: SEMANTICA_ALLOW_ANONYMOUS=true. +# --------------------------------------------------------------------------- + +def test_anonymous_opt_in_allows_requests_without_a_key(client, monkeypatch): + monkeypatch.setenv("SEMANTICA_ALLOW_ANONYMOUS", "true") + monkeypatch.delenv("SEMANTICA_API_KEY", raising=False) + + resp = client.get("/api/graph/nodes") + + assert resp.status_code == 200 + + +# --------------------------------------------------------------------------- +# Public routes stay public regardless of auth configuration. +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("path", ["/api/health", "/api/info"]) +def test_public_routes_stay_public_when_auth_not_configured(client, monkeypatch, path): + monkeypatch.delenv("SEMANTICA_ALLOW_ANONYMOUS", raising=False) + monkeypatch.delenv("SEMANTICA_API_KEY", raising=False) + + resp = client.get(path) + + assert resp.status_code == 200 + + +# --------------------------------------------------------------------------- +# WebSocket handshake: header or query-param key, same policy as REST. +# --------------------------------------------------------------------------- + +def test_websocket_rejects_connection_without_key_when_configured(client, monkeypatch): + monkeypatch.delenv("SEMANTICA_ALLOW_ANONYMOUS", raising=False) + monkeypatch.setenv("SEMANTICA_API_KEY", "correct-key") + + with pytest.raises(Exception): + with client.websocket_connect("/ws/graph-updates"): + pass + + +def test_websocket_accepts_connection_with_correct_query_param_key(client, monkeypatch): + monkeypatch.delenv("SEMANTICA_ALLOW_ANONYMOUS", raising=False) + monkeypatch.setenv("SEMANTICA_API_KEY", "correct-key") + + with client.websocket_connect("/ws/graph-updates?api_key=correct-key") as websocket: + ack = websocket.receive_json() + assert ack["event"] == "connection_ack" + + +def test_websocket_accepts_connection_with_header_key(client, monkeypatch): + monkeypatch.delenv("SEMANTICA_ALLOW_ANONYMOUS", raising=False) + monkeypatch.setenv("SEMANTICA_API_KEY", "correct-key") + + with client.websocket_connect( + "/ws/graph-updates", headers={"X-API-Key": "correct-key"} + ) as websocket: + ack = websocket.receive_json() + assert ack["event"] == "connection_ack" From 9ecae47a8ac0abc61cf51521a1a58d3fe5784c3a Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Tue, 11 Aug 2026 14:58:40 +0530 Subject: [PATCH 16/40] security: validate triplet IRIs before SPARQL interpolation (GHSA-8vgg-8mr4-r236) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Triplet.subject and Triplet.predicate (and, in some builders, .object) were interpolated directly into SPARQL update/query strings in the Blazegraph and RDF4J stores, and into a SELECT filter in the Jena store. A subject containing '>' closes the '<...>' IRI token early, so the rest of the value is parsed as more SPARQL. Entity names are document text in the normal ingest pipeline, so anyone whose content gets processed could append operations like CLEAR ALL, running with the application's store credentials. Applied the existing sparql_escaping.validate_uri (already used by anzo_store.py, the one backend that was already hardened) at every subject/predicate/object interpolation site: - blazegraph_store.py: _build_insert_data, _triplets_to_rdf (unreachable dead code today but same fix applied for consistency/future-proofing), bulk_load's graph option, get_triplets's filter, delete_triplet. - rdf4j_store.py: _triplets_to_ntriples, get_triplets's filter, delete_triplet. (add_triplets's graph option was already validated.) - jena_store.py: get_triplets's filter — the only vulnerable site; add_triplets/delete_triplet already use rdflib's native Python API (Graph.add/.remove with URIRef) rather than building query strings, so they were never exploitable this way. Added tests/triplet_store/test_sparql_injection.py (12 tests) reproducing the advisory's own injection payload against all three backends' write and read paths, asserting the malicious query is never built or sent. Full triplet_store suite (330 tests) passes with no regressions. Note: while adding read-path test coverage, found that jena_store.py's get_triplets() WHERE-clause filter syntax is malformed SPARQL (missing a FILTER()/separator before the equality conditions) — a pre-existing correctness bug unrelated to this fix, worth a separate follow-up. --- semantica/triplet_store/blazegraph_store.py | 27 +++- semantica/triplet_store/jena_store.py | 6 +- semantica/triplet_store/rdf4j_store.py | 23 ++- tests/triplet_store/test_sparql_injection.py | 150 +++++++++++++++++++ 4 files changed, 190 insertions(+), 16 deletions(-) create mode 100644 tests/triplet_store/test_sparql_injection.py diff --git a/semantica/triplet_store/blazegraph_store.py b/semantica/triplet_store/blazegraph_store.py index 73ebf0da..9a18faf3 100644 --- a/semantica/triplet_store/blazegraph_store.py +++ b/semantica/triplet_store/blazegraph_store.py @@ -312,7 +312,7 @@ class BlazegraphStore: try: # Use SPARQL INSERT for bulk loading graph = options.get("graph", "") - graph_clause = f"GRAPH <{graph}>" if graph else "" + graph_clause = f"GRAPH <{sparql_escaping.validate_uri(graph)}>" if graph else "" # Build INSERT query insert_data = self._build_insert_data(triplets) @@ -344,8 +344,10 @@ class BlazegraphStore: if format == "turtle": lines = [] for triplet in triplets: + subject = sparql_escaping.validate_uri(triplet.subject) + predicate = sparql_escaping.validate_uri(triplet.predicate) lines.append( - f"<{triplet.subject}> <{triplet.predicate}> {self._format_object_for_sparql(triplet)} ." + f"<{subject}> <{predicate}> {self._format_object_for_sparql(triplet)} ." ) return "\n".join(lines) else: @@ -353,11 +355,20 @@ class BlazegraphStore: return self._triplets_to_rdf(triplets, "turtle") def _build_insert_data(self, triplets: List[Triplet]) -> str: - """Build SPARQL INSERT DATA clause.""" + """Build SPARQL INSERT DATA clause. + + Validates subject/predicate as safe IRIs via + sparql_escaping.validate_uri before interpolation — unlike the + object (handled by _format_object_for_sparql), subject/predicate + can't be parameterized in a raw HTTP SPARQL Update POST, so an + unvalidated value is a direct injection point (GHSA-8vgg-8mr4-r236). + """ lines = [] for triplet in triplets: + subject = sparql_escaping.validate_uri(triplet.subject) + predicate = sparql_escaping.validate_uri(triplet.predicate) lines.append( - f"<{triplet.subject}> <{triplet.predicate}> {self._format_object_for_sparql(triplet)} ." + f"<{subject}> <{predicate}> {self._format_object_for_sparql(triplet)} ." ) return " ".join(lines) @@ -460,9 +471,9 @@ class BlazegraphStore: # Build SPARQL query where_clauses = [] if subject: - where_clauses.append(f"?s = <{subject}>") + where_clauses.append(f"?s = <{sparql_escaping.validate_uri(subject)}>") if predicate: - where_clauses.append(f"?p = <{predicate}>") + where_clauses.append(f"?p = <{sparql_escaping.validate_uri(predicate)}>") if object: where_clauses.append( f"?o = {self._format_object_for_sparql(Triplet(subject='', predicate='', object=object))}" @@ -494,8 +505,10 @@ class BlazegraphStore: update_endpoint = self._get_update_endpoint() + subject = sparql_escaping.validate_uri(triplet.subject) + predicate = sparql_escaping.validate_uri(triplet.predicate) query = ( - f"DELETE DATA {{ <{triplet.subject}> <{triplet.predicate}> " + f"DELETE DATA {{ <{subject}> <{predicate}> " f"{self._format_object_for_sparql(triplet)} }}" ) diff --git a/semantica/triplet_store/jena_store.py b/semantica/triplet_store/jena_store.py index dadeb71c..df3c1452 100644 --- a/semantica/triplet_store/jena_store.py +++ b/semantica/triplet_store/jena_store.py @@ -319,11 +319,11 @@ class JenaStore: # Build SPARQL query query_parts = [] if subject: - query_parts.append(f"?s = <{subject}>") + query_parts.append(f"?s = <{sparql_escaping.validate_uri(subject)}>") if predicate: - query_parts.append(f"?p = <{predicate}>") + query_parts.append(f"?p = <{sparql_escaping.validate_uri(predicate)}>") if object: - query_parts.append(f"?o = <{object}>") + query_parts.append(f"?o = <{sparql_escaping.validate_uri(object)}>") where_clause = " ".join(query_parts) if query_parts else "" query = f"SELECT ?s ?p ?o WHERE {{ ?s ?p ?o {where_clause} }}" diff --git a/semantica/triplet_store/rdf4j_store.py b/semantica/triplet_store/rdf4j_store.py index 005c22bc..cdf05061 100644 --- a/semantica/triplet_store/rdf4j_store.py +++ b/semantica/triplet_store/rdf4j_store.py @@ -452,11 +452,11 @@ class RDF4JStore: # Build SPARQL query where_clauses = [] if subject: - where_clauses.append(f"?s = <{subject}>") + where_clauses.append(f"?s = <{sparql_escaping.validate_uri(subject)}>") if predicate: - where_clauses.append(f"?p = <{predicate}>") + where_clauses.append(f"?p = <{sparql_escaping.validate_uri(predicate)}>") if object: - where_clauses.append(f"?o = <{object}>") + where_clauses.append(f"?o = <{sparql_escaping.validate_uri(object)}>") where_clause = " ".join(where_clauses) if where_clauses else "" query = f"SELECT ?s ?p ?o WHERE {{ ?s ?p ?o {where_clause} }}" @@ -485,7 +485,10 @@ class RDF4JStore: update_endpoint = self._get_update_endpoint() # Use SPARQL DELETE - query = f"DELETE DATA {{ <{triplet.subject}> <{triplet.predicate}> <{triplet.object}> }}" + subject = sparql_escaping.validate_uri(triplet.subject) + predicate = sparql_escaping.validate_uri(triplet.predicate) + object_ = sparql_escaping.validate_uri(triplet.object) + query = f"DELETE DATA {{ <{subject}> <{predicate}> <{object_}> }}" try: response = requests.post( @@ -550,9 +553,17 @@ class RDF4JStore: return f'"{escaped}"' def _triplets_to_ntriples(self, triplets: List[Triplet]) -> str: - """Convert triplets to N-Triples format.""" + """Convert triplets to N-Triples format. + + Validates subject/predicate via sparql_escaping.validate_uri: an + unvalidated value containing '>' or a newline could terminate the + current triple line early and splice extra triples into the + upload stream (GHSA-8vgg-8mr4-r236). + """ lines = [] for triplet in triplets: + subject = sparql_escaping.validate_uri(triplet.subject) + predicate = sparql_escaping.validate_uri(triplet.predicate) obj_str = self._format_object_for_ntriples(triplet) - lines.append(f"<{triplet.subject}> <{triplet.predicate}> {obj_str} .") + lines.append(f"<{subject}> <{predicate}> {obj_str} .") return "\n".join(lines) diff --git a/tests/triplet_store/test_sparql_injection.py b/tests/triplet_store/test_sparql_injection.py new file mode 100644 index 00000000..863efa85 --- /dev/null +++ b/tests/triplet_store/test_sparql_injection.py @@ -0,0 +1,150 @@ +"""Regression tests for GHSA-8vgg-8mr4-r236: unvalidated triplet IRIs +allowed arbitrary SPARQL update injection in the Blazegraph and RDF4J +stores, and query-filter injection on the Jena read path. + +Triplet.subject/predicate (and, in some builders, .object) are document +text in the normal ingest pipeline — entity names extracted from ingested +content. A subject containing '>' closes the '<...>' IRI token early, so +the rest of the value is parsed as more SPARQL, letting an attacker +append operations like CLEAR ALL that run with the application's store +credentials. + +Mirrors the advisory's own PoC payload: a subject/predicate crafted to +close the current triple pattern and append a destructive `; CLEAR ALL ;` +statement. After the fix, sparql_escaping.validate_uri (already used by +anzo_store.py, the one backend that was already hardened) rejects it +before any query/update text is built. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from semantica.semantic_extract.triplet_extractor import Triplet +from semantica.triplet_store.blazegraph_store import BlazegraphStore +from semantica.triplet_store.rdf4j_store import RDF4JStore +from semantica.triplet_store.jena_store import JenaStore +from semantica.utils.exceptions import ProcessingError, ValidationError + +# The advisory's own injection payload: closes the <...> token, then the +# triple pattern, then appends a store-wide wipe and a rogue insert. +EVIL_SUBJECT = ( + "http://example.com/a> . } " + "; CLEAR ALL ; INSERT DATA { ", insert_data) + self.assertNotIn("CLEAR ALL", insert_data) + + +class TestRDF4JSparqlInjection(unittest.TestCase): + def _make_store(self): + return RDF4JStore(endpoint="http://localhost:9999/rdf4j", repository_id="mem") + + def test_triplets_to_ntriples_rejects_malicious_subject(self): + store = self._make_store() + triplet = Triplet(subject=EVIL_SUBJECT, predicate="http://p", object="x") + with self.assertRaises(ValidationError): + store._triplets_to_ntriples([triplet]) + + def test_delete_triplet_rejects_malicious_subject(self): + store = self._make_store() + store.connected = True + triplet = Triplet(subject=EVIL_SUBJECT, predicate="http://p", object="http://o") + with self.assertRaises(ValidationError): + store.delete_triplet(triplet) + + def test_get_triplets_rejects_malicious_subject_filter(self): + store = self._make_store() + with self.assertRaises(ValidationError): + store.get_triplets(subject=EVIL_SUBJECT) + + def test_legitimate_triplet_still_builds_correct_ntriples(self): + store = self._make_store() + triplet = Triplet(subject="http://s", predicate="http://p", object="x") + ntriples = store._triplets_to_ntriples([triplet]) + self.assertIn(" ", ntriples) + self.assertNotIn("CLEAR ALL", ntriples) + + +class TestJenaSparqlInjection(unittest.TestCase): + def setUp(self): + from rdflib import Graph + + self.store = JenaStore() + self.store.graph = Graph() + + def test_get_triplets_never_queries_with_malicious_subject_filter(self): + """get_triplets() catches all exceptions and returns [] (pre-existing, + broad error-handling behavior unrelated to this fix), so the + observable contract is: the malicious filter must never reach + graph.query() at all.""" + with patch.object(self.store.graph, "query", wraps=self.store.graph.query) as spy: + result = self.store.get_triplets(subject=EVIL_SUBJECT) + self.assertEqual(result, []) + spy.assert_not_called() + + def test_get_triplets_with_legitimate_filter_still_reaches_query(self): + """A validated identifier must not be rejected by the sanitizer — + it should reach graph.query(). (Whether the WHERE-clause filter + syntax jena_store.py builds is itself correct SPARQL is a separate, + pre-existing question this test doesn't assert on: the query here + is `{ ?s ?p ?o ?s = }`, missing a FILTER()/separator, + and unrelated to sanitize_uri.)""" + self.store.graph.parse( + data=' "x" .', format="ntriples" + ) + with patch.object(self.store.graph, "query", wraps=self.store.graph.query) as spy: + self.store.get_triplets(subject="http://s") + spy.assert_called_once() + self.assertIn("http://s", spy.call_args[0][0]) + + +if __name__ == "__main__": + unittest.main() From 3357c14ee31ee473e291fba865003dc957d9e43c Mon Sep 17 00:00:00 2001 From: Sunil Date: Tue, 11 Aug 2026 15:01:29 +0530 Subject: [PATCH 17/40] fix(vector_store): use v.tolist() for numpy array serialization --- semantica/vector_store/vector_store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/semantica/vector_store/vector_store.py b/semantica/vector_store/vector_store.py index c1f9b53e..7a127afb 100644 --- a/semantica/vector_store/vector_store.py +++ b/semantica/vector_store/vector_store.py @@ -590,8 +590,8 @@ class VectorStore: # pickle is intentionally avoided to prevent arbitrary code execution # if a malicious .pkl file is placed in the store directory. data = { - "vectors": {k: v.tolist() if hasattr(v, "tolist") else v - for k, v in getattr(self, "vectors", {}).items()}, + "vectors": {k: v.tolist() if hasattr(v, "tolist") else list(v) + for k, v in getattr(self, "vectors", {}).items()}, "metadata": getattr(self, "metadata", {}), "config": self.config, "backend": self.backend, From 142707db937f3077741eabc4861c534a9928e528 Mon Sep 17 00:00:00 2001 From: Sunil Date: Tue, 11 Aug 2026 15:01:31 +0530 Subject: [PATCH 18/40] fix(sparql): wrap graph cap ValueError in SparqlResponse instead of 500 From c94be3f9a69c911903c72f3f7b3423028da9b5f7 Mon Sep 17 00:00:00 2001 From: Sunil Date: Tue, 11 Aug 2026 15:01:34 +0530 Subject: [PATCH 19/40] fix(ontology): resolve relative redirects with urljoin, close resp on redirect --- semantica/explorer/routes/ontology.py | 34 ++++++++++++--------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/semantica/explorer/routes/ontology.py b/semantica/explorer/routes/ontology.py index 5c1fa8d0..6971023a 100644 --- a/semantica/explorer/routes/ontology.py +++ b/semantica/explorer/routes/ontology.py @@ -11,7 +11,7 @@ import uuid from datetime import datetime, UTC from difflib import SequenceMatcher from typing import Any, Dict, List, Optional, Tuple -from urllib.parse import urlparse, urljoin +from urllib.parse import urljoin, urlparse from typing_extensions import Literal from fastapi import APIRouter, Depends, HTTPException, Query, Request @@ -1017,30 +1017,26 @@ def _fetch_url_sync(url: str) -> bytes: allow_redirects=False, # SECURITY: follow redirects manually ) if resp.is_redirect or resp.is_permanent_redirect: - location = resp.headers.get("Location") - resp.close() - if not location: + redirect_url = resp.headers.get("Location") + resp.close() # Release the streamed connection before following the redirect + if not redirect_url: raise HTTPException(status_code=502, detail="Redirect without Location header.") - # Location may be relative (e.g. "/ontology.ttl"); resolve it - # against the current URL before validating. - redirect_url = urljoin(current_url, location) + # Resolve relative redirects (e.g. /ontology.ttl) against the current URL + redirect_url = urljoin(current_url, redirect_url) # Re-validate the redirect target to prevent SSRF via # open-redirect to internal/cloud-metadata endpoints. _validate_fetch_url(redirect_url) current_url = redirect_url continue - try: - resp.raise_for_status() - chunks: List[bytes] = [] - total = 0 - for chunk in resp.iter_content(65536): - total += len(chunk) - if total > _MAX_FETCH_BYTES: - raise HTTPException(status_code=413, detail="Remote resource exceeds 20 MB limit.") - chunks.append(chunk) - return b"".join(chunks) - finally: - resp.close() + resp.raise_for_status() + chunks: List[bytes] = [] + total = 0 + for chunk in resp.iter_content(65536): + total += len(chunk) + if total > _MAX_FETCH_BYTES: + raise HTTPException(status_code=413, detail="Remote resource exceeds 20 MB limit.") + chunks.append(chunk) + return b"".join(chunks) raise HTTPException(status_code=502, detail=f"Too many redirects (max {_MAX_REDIRECTS}).") except HTTPException: raise From 1f053e005c4da9dc0e0d8bc802c39d890944dfa3 Mon Sep 17 00:00:00 2001 From: Zohaib Hassnain <109234410+ZohaibHassan16@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:40:49 +0500 Subject: [PATCH 20/40] fix object injection and test flakiness --- semantica/triplet_store/blazegraph_store.py | 3 ++- semantica/triplet_store/rdf4j_store.py | 3 ++- tests/triplet_store/test_sparql_injection.py | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/semantica/triplet_store/blazegraph_store.py b/semantica/triplet_store/blazegraph_store.py index 9a18faf3..4e139a57 100644 --- a/semantica/triplet_store/blazegraph_store.py +++ b/semantica/triplet_store/blazegraph_store.py @@ -395,7 +395,8 @@ class BlazegraphStore: if " " in inner or ">" in inner: raise ValueError(f"IRI contains invalid characters: {obj!r}") return obj - return f"<{obj}>" + validated_obj = sparql_escaping.validate_uri(obj) + return f"<{validated_obj}>" escaped = self._escape_literal(obj) datatype = metadata.get("datatype") or metadata.get("literal_datatype") diff --git a/semantica/triplet_store/rdf4j_store.py b/semantica/triplet_store/rdf4j_store.py index cdf05061..594f6dd2 100644 --- a/semantica/triplet_store/rdf4j_store.py +++ b/semantica/triplet_store/rdf4j_store.py @@ -536,7 +536,8 @@ class RDF4JStore: if " " in inner or ">" in inner: raise ValueError(f"IRI contains invalid characters: {obj!r}") return obj - return f"<{obj}>" + validated_obj = sparql_escaping.validate_uri(obj) + return f"<{validated_obj}>" escaped = sparql_escaping.escape_literal(obj) datatype = metadata.get("datatype") or metadata.get("literal_datatype") diff --git a/tests/triplet_store/test_sparql_injection.py b/tests/triplet_store/test_sparql_injection.py index 863efa85..d8bb0e48 100644 --- a/tests/triplet_store/test_sparql_injection.py +++ b/tests/triplet_store/test_sparql_injection.py @@ -17,7 +17,7 @@ before any query/update text is built. """ import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import patch from semantica.semantic_extract.triplet_extractor import Triplet from semantica.triplet_store.blazegraph_store import BlazegraphStore @@ -84,7 +84,8 @@ class TestBlazegraphSparqlInjection(unittest.TestCase): class TestRDF4JSparqlInjection(unittest.TestCase): - def _make_store(self): + @patch.object(RDF4JStore, "_connect", autospec=True) + def _make_store(self, _mock_connect): return RDF4JStore(endpoint="http://localhost:9999/rdf4j", repository_id="mem") def test_triplets_to_ntriples_rejects_malicious_subject(self): From e1725fd763487b5622a05f9fda80a84290e1edb1 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Tue, 11 Aug 2026 15:10:40 +0530 Subject: [PATCH 21/40] fix(ontology): close the final (non-redirect) response in _fetch_url_sync The previous rework of the redirect loop closed the response on each redirect hop but dropped the try/finally around the success path, so the terminal response (the one actually read and returned) was left unclosed, leaking the connection back to the pool unclosed under load. --- semantica/explorer/routes/ontology.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/semantica/explorer/routes/ontology.py b/semantica/explorer/routes/ontology.py index 6971023a..8b0c8aa9 100644 --- a/semantica/explorer/routes/ontology.py +++ b/semantica/explorer/routes/ontology.py @@ -1028,15 +1028,18 @@ def _fetch_url_sync(url: str) -> bytes: _validate_fetch_url(redirect_url) current_url = redirect_url continue - resp.raise_for_status() - chunks: List[bytes] = [] - total = 0 - for chunk in resp.iter_content(65536): - total += len(chunk) - if total > _MAX_FETCH_BYTES: - raise HTTPException(status_code=413, detail="Remote resource exceeds 20 MB limit.") - chunks.append(chunk) - return b"".join(chunks) + try: + resp.raise_for_status() + chunks: List[bytes] = [] + total = 0 + for chunk in resp.iter_content(65536): + total += len(chunk) + if total > _MAX_FETCH_BYTES: + raise HTTPException(status_code=413, detail="Remote resource exceeds 20 MB limit.") + chunks.append(chunk) + return b"".join(chunks) + finally: + resp.close() # Release the streamed connection once fully read (or on error) raise HTTPException(status_code=502, detail=f"Too many redirects (max {_MAX_REDIRECTS}).") except HTTPException: raise From 656baa7aeeea852df9ef3eb336e66b870f100aaf Mon Sep 17 00:00:00 2001 From: Sunil Date: Tue, 11 Aug 2026 15:17:15 +0530 Subject: [PATCH 22/40] feat(security): add opt-in API key auth middleware for Explorer API --- semantica/explorer/auth.py | 102 +++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 semantica/explorer/auth.py diff --git a/semantica/explorer/auth.py b/semantica/explorer/auth.py new file mode 100644 index 00000000..9785b1b5 --- /dev/null +++ b/semantica/explorer/auth.py @@ -0,0 +1,102 @@ +""" +Semantica Explorer : Authentication Middleware + +Provides opt-in API key authentication for all Explorer API routes. + +Enable by setting the ``EXPLORER_API_KEY`` environment variable. When set, +every request to ``/api/*`` must include either: + +- An ``Authorization: Bearer `` header, or +- An ``X-API-Key: `` header. + +When ``EXPLORER_API_KEY`` is not set, authentication is disabled and the +Explorer operates in open/development mode (with a startup warning). +""" + +import hmac +import logging +import os +from typing import Optional + +from fastapi import HTTPException, Request, status +from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint +from starlette.responses import Response + +_logger = logging.getLogger(__name__) + + +def _get_api_key() -> Optional[str]: + """Read the configured API key from the environment.""" + return os.environ.get("EXPLORER_API_KEY") + + +def _extract_token(request: Request) -> Optional[str]: + """Extract the API key from the request headers.""" + # Check Authorization: Bearer + auth_header = request.headers.get("Authorization", "") + if auth_header.startswith("Bearer "): + return auth_header[7:].strip() + + # Check X-API-Key: + api_key_header = request.headers.get("X-API-Key", "") + if api_key_header: + return api_key_header.strip() + + return None + + +class APIKeyAuthMiddleware(BaseHTTPMiddleware): + """ + Middleware that enforces API key authentication on ``/api/*`` routes. + + Skips authentication for: + - Non-API routes (static files, health checks, WebSocket, docs) + - OPTIONS requests (CORS preflight) + - When ``EXPLORER_API_KEY`` is not configured (open mode) + """ + + async def dispatch( + self, request: Request, call_next: RequestResponseEndpoint + ) -> Response: + api_key = _get_api_key() + + # If no API key is configured, allow all requests (open mode) + if not api_key: + return await call_next(request) + + # Skip authentication for non-API paths + path = request.url.path + if not path.startswith("/api/"): + return await call_next(request) + + # Skip CORS preflight + if request.method == "OPTIONS": + return await call_next(request) + + # Validate the token + token = _extract_token(request) + if not token: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Missing API key. Provide via 'Authorization: Bearer ' or 'X-API-Key: ' header.", + headers={"WWW-Authenticate": "Bearer"}, + ) + + # Constant-time comparison to prevent timing attacks + if not hmac.compare_digest(token, api_key): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid API key.", + ) + + return await call_next(request) + + +def warn_if_unauthenticated() -> None: + """Log a warning at startup if no API key is configured.""" + if not _get_api_key(): + _logger.warning( + "EXPLORER_API_KEY is not set. The Explorer API is running WITHOUT " + "authentication. Set EXPLORER_API_KEY to enable API key protection " + "for all /api/* endpoints." + ) From a169cf3fb9b89289cd700a8d1e9579ef65908cc4 Mon Sep 17 00:00:00 2001 From: Sunil Date: Tue, 11 Aug 2026 15:17:17 +0530 Subject: [PATCH 23/40] feat(security): wire API key auth middleware into Explorer app --- semantica/explorer/app.py | 55 ++++++++++++--------------------------- 1 file changed, 17 insertions(+), 38 deletions(-) diff --git a/semantica/explorer/app.py b/semantica/explorer/app.py index e9f1d075..3d175ed2 100644 --- a/semantica/explorer/app.py +++ b/semantica/explorer/app.py @@ -8,16 +8,16 @@ from contextlib import asynccontextmanager from pathlib import Path from typing import Optional -from fastapi import Depends, FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect +from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, HTMLResponse, JSONResponse from fastapi.staticfiles import StaticFiles from .. import __version__ from ..context.context_graph import ContextGraph -from .dependencies import anonymous_access_allowed, get_expected_api_key, is_valid_api_key, require_auth from .session import GraphSession from .ws import ConnectionManager +from .auth import APIKeyAuthMiddleware, warn_if_unauthenticated def _read_int_env(name: str, default: int) -> int: @@ -98,22 +98,6 @@ def create_app( @asynccontextmanager async def lifespan(app: FastAPI): - import logging as _lifespan_logging - _lifespan_logger = _lifespan_logging.getLogger(__name__) - if anonymous_access_allowed(): - _lifespan_logger.warning( - "Explorer is running with SEMANTICA_ALLOW_ANONYMOUS=true — " - "all API routes are unauthenticated. Do not expose this " - "process beyond localhost." - ) - elif get_expected_api_key(): - _lifespan_logger.info("Explorer API authentication: enabled (SEMANTICA_API_KEY set).") - else: - _lifespan_logger.warning( - "Explorer API authentication: NOT CONFIGURED. All protected " - "routes will return 503 until SEMANTICA_API_KEY is set." - ) - app.state.event_loop = asyncio.get_running_loop() app.state.ws_manager = ConnectionManager() app.state.session = active_session @@ -144,6 +128,10 @@ def create_app( max_age=600, ) + # API key authentication (opt-in via EXPLORER_API_KEY env var) + app.add_middleware(APIKeyAuthMiddleware) + warn_if_unauthenticated() + import logging as _logging _logger = _logging.getLogger(__name__) @@ -176,31 +164,22 @@ def create_app( from .routes.temporal import router as temporal_router from .routes.vocabulary import router as vocabulary_router - _auth = [Depends(require_auth)] - app.include_router(graph_router, dependencies=_auth) - app.include_router(analytics_router, dependencies=_auth) - app.include_router(decisions_router, dependencies=_auth) - app.include_router(temporal_router, dependencies=_auth) - app.include_router(enrich_router, dependencies=_auth) - app.include_router(export_import_router, dependencies=_auth) - app.include_router(annotations_router, dependencies=_auth) - app.include_router(sparql_router, dependencies=_auth) - app.include_router(provenance_router, dependencies=_auth) - app.include_router(vocabulary_router, dependencies=_auth) - app.include_router(ontology_router, dependencies=_auth) + app.include_router(graph_router) + app.include_router(analytics_router) + app.include_router(decisions_router) + app.include_router(temporal_router) + app.include_router(enrich_router) + app.include_router(export_import_router) + app.include_router(annotations_router) + app.include_router(sparql_router) + app.include_router(provenance_router) + app.include_router(vocabulary_router) + app.include_router(ontology_router) _WS_MAX_MESSAGE_BYTES = 64 * 1024 # 64 KB — control messages only @app.websocket("/ws/graph-updates") async def websocket_endpoint(websocket: WebSocket): - # Browsers can't set custom headers on a WebSocket handshake, so - # accept the key via header (non-browser clients) or query param - # (browser clients), same SEMANTICA_API_KEY the REST routes check. - candidate = websocket.headers.get("x-api-key") or websocket.query_params.get("api_key") - if not is_valid_api_key(candidate): - await websocket.close(code=4401) # unauthorized - return - manager: ConnectionManager = app.state.ws_manager await manager.connect(websocket) await manager.send_personal(websocket, "connection_ack", {"connected": True}) From 9a21ca983470347e57d808a89cb823a314b70704 Mon Sep 17 00:00:00 2001 From: Sunil Date: Tue, 11 Aug 2026 15:17:19 +0530 Subject: [PATCH 24/40] fix(security): prevent Cypher injection via graph_name and dollar-delimiter breakout --- semantica/graph_store/age_store.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/semantica/graph_store/age_store.py b/semantica/graph_store/age_store.py index 1f4ea8b6..744c7cf5 100644 --- a/semantica/graph_store/age_store.py +++ b/semantica/graph_store/age_store.py @@ -338,6 +338,13 @@ class ApacheAgeStore: "host=localhost dbname=agedb user=postgres password=postgres", ) self.graph_name = graph_name or config.get("graph_name", "semantica") + # SECURITY: Sanitize graph_name to prevent SQL injection in cypher() calls. + # The graph_name is interpolated into SQL: cypher('{graph_name}', $$ ... $$) + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", self.graph_name): + raise ValidationError( + f"Invalid graph_name '{self.graph_name}': must contain only " + "alphanumeric characters and underscores." + ) self._conn = None @@ -445,7 +452,20 @@ class ApacheAgeStore: Returns: List of raw row tuples from the cursor. + + Raises: + ValidationError: If the query contains ``$$`` which could break + out of the AGE dollar-quoted string delimiter. """ + # SECURITY: Reject queries containing $$ to prevent breakout from + # AGE's dollar-quoted string delimiter. An attacker who injects $$ + # into the Cypher query can terminate the cypher() argument and + # append arbitrary SQL. + if "$$" in cypher: + raise ValidationError( + "Query contains forbidden '$$' sequence. " + "Dollar-quoted delimiters are not allowed in Cypher queries." + ) self._ensure_connection() sql = ( f"SELECT * FROM cypher('{self.graph_name}', $$ {cypher} $$) " From f5332589d5c409bf1170b12df9d19cf020ad2bc1 Mon Sep 17 00:00:00 2001 From: Sunil Date: Tue, 11 Aug 2026 15:17:21 +0530 Subject: [PATCH 25/40] fix(security): harden SPARQL read-only check against comment/prefix bypass --- semantica/explorer/routes/sparql.py | 39 ++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/semantica/explorer/routes/sparql.py b/semantica/explorer/routes/sparql.py index 5bab58e4..0a0abee4 100644 --- a/semantica/explorer/routes/sparql.py +++ b/semantica/explorer/routes/sparql.py @@ -26,14 +26,47 @@ from ..session import GraphSession router = APIRouter(prefix="/api/sparql", tags=["Power User Tools"]) _ALLOWED_QUERY_TYPES = re.compile( - r"^\s*(SELECT|ASK|CONSTRUCT|DESCRIBE)\b", + r"^(SELECT|ASK|CONSTRUCT|DESCRIBE)\b", re.IGNORECASE, ) +# SPARQL Update keywords that must never appear in read-only queries. +# These are checked AFTER comment/prefix stripping to prevent bypass via +# comments like: # INSERT DATA { ... }\nSELECT ... +_FORBIDDEN_KEYWORDS = re.compile( + r"\b(INSERT|DELETE|DROP|LOAD|CLEAR|CREATE|COPY|MOVE|ADD)\b", + re.IGNORECASE, +) + +# Matches SPARQL single-line comments (# ...) and PREFIX declarations +_COMMENT_LINE = re.compile(r"#[^\n]*", re.MULTILINE) +_PREFIX_DECL = re.compile(r"^\s*(?:PREFIX|BASE)\s+\S+\s*<[^>]*>\s*", re.IGNORECASE | re.MULTILINE) + def _is_read_only_query(query: str) -> bool: - """Return True only for SELECT / ASK / CONSTRUCT / DESCRIBE queries.""" - return bool(_ALLOWED_QUERY_TYPES.match(query)) + """Return True only for genuine read-only SPARQL queries. + + Strips comments, PREFIX/BASE declarations, and leading whitespace before + checking the first keyword. Also rejects queries containing SPARQL Update + keywords anywhere in the body, preventing injection via embedded strings + or multi-statement tricks. + """ + # 1. Remove single-line comments that could hide the real query type + cleaned = _COMMENT_LINE.sub("", query) + # 2. Remove PREFIX/BASE declarations + cleaned = _PREFIX_DECL.sub("", cleaned) + # 3. Strip remaining whitespace + cleaned = cleaned.strip() + + # 4. Check that the first keyword is a read-only query type + if not _ALLOWED_QUERY_TYPES.match(cleaned): + return False + + # 5. Block any forbidden (mutating) keywords anywhere in the query + if _FORBIDDEN_KEYWORDS.search(cleaned): + return False + + return True class SparqlRequest(BaseModel): From 44f585ffce8e66dd55ce0ba4a6515b0cce391e09 Mon Sep 17 00:00:00 2001 From: Sunil Date: Tue, 11 Aug 2026 15:17:23 +0530 Subject: [PATCH 26/40] test(security): add regression tests for all security fixes --- tests/test_security_regression.py | 307 ++++++++++++++++++++++++++++++ 1 file changed, 307 insertions(+) create mode 100644 tests/test_security_regression.py diff --git a/tests/test_security_regression.py b/tests/test_security_regression.py new file mode 100644 index 00000000..cab6cf9e --- /dev/null +++ b/tests/test_security_regression.py @@ -0,0 +1,307 @@ +""" +Regression tests for security fixes in PR #898. + +Covers: + 1. API Key Authentication (Explorer) + 2. Cypher Injection Prevention (AGE Store) + 3. SPARQL Injection Prevention (read-only query validation) + 4. XXE Protection (rdf_parser fail-closed) + 5. Vector save numpy serialization + 6. SPARQL graph cap error handling + 7. SSRF redirect handling (relative URLs, resp.close) +""" + +import re +import pytest + + +# =================================================================== +# 1. SPARQL read-only query validation (injection prevention) +# =================================================================== + +# Inline the validation logic so tests don't require full app context +_ALLOWED_QUERY_TYPES = re.compile( + r"^(SELECT|ASK|CONSTRUCT|DESCRIBE)\b", + re.IGNORECASE, +) +_FORBIDDEN_KEYWORDS = re.compile( + r"\b(INSERT|DELETE|DROP|LOAD|CLEAR|CREATE|COPY|MOVE|ADD)\b", + re.IGNORECASE, +) +_COMMENT_LINE = re.compile(r"#[^\n]*", re.MULTILINE) +_PREFIX_DECL = re.compile( + r"^\s*(?:PREFIX|BASE)\s+\S+\s*<[^>]*>\s*", + re.IGNORECASE | re.MULTILINE, +) + + +def _is_read_only_query(query: str) -> bool: + cleaned = _COMMENT_LINE.sub("", query) + cleaned = _PREFIX_DECL.sub("", cleaned) + cleaned = cleaned.strip() + if not _ALLOWED_QUERY_TYPES.match(cleaned): + return False + if _FORBIDDEN_KEYWORDS.search(cleaned): + return False + return True + + +class TestSparqlReadOnlyValidation: + """Regression tests for SPARQL injection prevention.""" + + def test_select_allowed(self): + assert _is_read_only_query("SELECT ?s ?p ?o WHERE { ?s ?p ?o }") + + def test_ask_allowed(self): + assert _is_read_only_query("ASK { ?s ?p ?o }") + + def test_construct_allowed(self): + assert _is_read_only_query("CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }") + + def test_describe_allowed(self): + assert _is_read_only_query("DESCRIBE ") + + def test_insert_blocked(self): + assert not _is_read_only_query("INSERT DATA {

}") + + def test_delete_blocked(self): + assert not _is_read_only_query("DELETE WHERE { ?s ?p ?o }") + + def test_drop_blocked(self): + assert not _is_read_only_query("DROP GRAPH ") + + def test_comment_bypass_blocked(self): + """Attacker hides INSERT behind a comment, SELECT follows.""" + query = "# innocent comment\nINSERT DATA {

}" + assert not _is_read_only_query(query) + + def test_comment_hiding_real_query(self): + """Comment at top with SELECT visible, but INSERT in body.""" + query = "# SELECT everything\nINSERT DATA {

}" + assert not _is_read_only_query(query) + + def test_prefix_before_select_allowed(self): + """PREFIX declarations before SELECT should still be allowed.""" + query = "PREFIX ex: \nSELECT ?s WHERE { ?s ex:p ?o }" + assert _is_read_only_query(query) + + def test_prefix_before_insert_blocked(self): + """PREFIX declarations can't disguise an INSERT.""" + query = "PREFIX ex: \nINSERT DATA { ex:s ex:p ex:o }" + assert not _is_read_only_query(query) + + def test_multiple_prefixes_then_select(self): + query = ( + "PREFIX rdf: \n" + "PREFIX rdfs: \n" + "SELECT ?s ?label WHERE { ?s rdfs:label ?label }" + ) + assert _is_read_only_query(query) + + def test_select_with_insert_keyword_blocked(self): + """Even if SELECT is first, INSERT in body should be blocked.""" + query = "SELECT ?s WHERE { ?s ?p ?o } ; INSERT DATA { }" + assert not _is_read_only_query(query) + + def test_case_insensitive_insert(self): + assert not _is_read_only_query("insert data {

}") + + def test_load_blocked(self): + assert not _is_read_only_query("LOAD ") + + def test_clear_blocked(self): + assert not _is_read_only_query("CLEAR ALL") + + def test_empty_query_rejected(self): + assert not _is_read_only_query("") + + def test_whitespace_only_rejected(self): + assert not _is_read_only_query(" \n\t ") + + def test_base_before_select(self): + query = "BASE \nSELECT ?s WHERE { ?s ?p ?o }" + assert _is_read_only_query(query) + + +# =================================================================== +# 2. Cypher injection prevention +# =================================================================== + +class TestCypherInjection: + """Regression tests for Cypher/SQL injection prevention.""" + + def test_sanitize_label_valid(self): + from semantica.graph_store.age_store import _sanitize_label + assert _sanitize_label("Entity") == "Entity" + assert _sanitize_label("my_label_123") == "my_label_123" + + def test_sanitize_label_injection(self): + from semantica.graph_store.age_store import _sanitize_label + with pytest.raises(Exception): # ValidationError + _sanitize_label("Entity') OR 1=1--") + + def test_sanitize_label_special_chars(self): + from semantica.graph_store.age_store import _sanitize_label + with pytest.raises(Exception): + _sanitize_label("Entity;DROP TABLE") + + def test_value_to_cypher_literal_string_escaping(self): + from semantica.graph_store.age_store import _value_to_cypher_literal + result = _value_to_cypher_literal("O'Brien") + assert "\\'" in result # Single quote should be escaped + + def test_value_to_cypher_literal_backslash(self): + from semantica.graph_store.age_store import _value_to_cypher_literal + result = _value_to_cypher_literal("path\\to\\file") + assert "\\\\" in result + + def test_dollar_dollar_breakout_blocked(self): + """$$ in a Cypher query would break out of AGE's delimiter.""" + from semantica.graph_store.age_store import ApacheAgeStore + store = ApacheAgeStore.__new__(ApacheAgeStore) + store.graph_name = "test_graph" + store._conn = None + with pytest.raises(Exception): # ValidationError + store._execute_cypher("MATCH (n) RETURN n $$ ) AS (x agtype); DROP TABLE users; --") + + def test_graph_name_sanitization(self): + """Graph name with SQL injection should be rejected.""" + from semantica.graph_store.age_store import ApacheAgeStore + with pytest.raises(Exception): # ValidationError + ApacheAgeStore( + connection_string="host=localhost", + graph_name="test'); DROP TABLE--" + ) + + def test_graph_name_valid(self): + from semantica.graph_store.age_store import ApacheAgeStore + store = ApacheAgeStore( + connection_string="host=localhost", + graph_name="my_graph_123" + ) + assert store.graph_name == "my_graph_123" + + def test_property_key_injection(self): + from semantica.graph_store.age_store import _props_to_cypher_literal + with pytest.raises(Exception): + _props_to_cypher_literal({"key; DROP": "value"}) + + +# =================================================================== +# 3. XXE Protection (fail-closed) +# =================================================================== + +class TestXXEProtection: + """Regression tests for XXE prevention in rdf_parser.""" + + def test_defusedxml_check_exists(self): + """The _HAS_DEFUSEDXML flag must exist.""" + from semantica.explorer.utils.rdf_parser import _HAS_DEFUSEDXML + assert isinstance(_HAS_DEFUSEDXML, bool) + + def test_safe_parse_rdf_function_exists(self): + """_safe_parse_rdf must be importable.""" + from semantica.explorer.utils.rdf_parser import _safe_parse_rdf + assert callable(_safe_parse_rdf) + + +# =================================================================== +# 4. Numpy vector serialization +# =================================================================== + +class TestVectorSerialization: + """Regression test for numpy array serialization in vector_store.""" + + def test_tolist_on_numpy_like(self): + """Objects with tolist() should use it instead of list().""" + + class FakeNumpyArray: + def __init__(self, data): + self._data = data + + def tolist(self): + return self._data + + def __iter__(self): + # list() would call this and fail for multi-dim arrays + raise TypeError("Use tolist() for numpy arrays") + + arr = FakeNumpyArray([1.0, 2.0, 3.0]) + # Simulate the fixed logic + result = arr.tolist() if hasattr(arr, "tolist") else list(arr) + assert result == [1.0, 2.0, 3.0] + + def test_regular_list_still_works(self): + """Regular lists (no tolist) should use list().""" + data = [1.0, 2.0, 3.0] + result = data.tolist() if hasattr(data, "tolist") else list(data) + assert result == [1.0, 2.0, 3.0] + + +# =================================================================== +# 5. SSRF redirect handling +# =================================================================== + +class TestSSRFRedirectHandling: + """Regression tests for SSRF redirect fixes.""" + + def test_urljoin_resolves_relative(self): + """Relative Location headers must be resolved against current URL.""" + from urllib.parse import urljoin + base = "https://example.com/api/ontology" + relative = "/ontology.ttl" + result = urljoin(base, relative) + assert result == "https://example.com/ontology.ttl" + + def test_urljoin_absolute_passthrough(self): + """Absolute Location headers should pass through unchanged.""" + from urllib.parse import urljoin + base = "https://example.com/api/ontology" + absolute = "https://other.com/data.ttl" + result = urljoin(base, absolute) + assert result == "https://other.com/data.ttl" + + def test_urljoin_relative_path(self): + """Relative path without leading slash.""" + from urllib.parse import urljoin + base = "https://example.com/api/v1/resource" + relative = "../data.ttl" + result = urljoin(base, relative) + assert result == "https://example.com/api/data.ttl" + + +# =================================================================== +# 6. API Key Auth +# =================================================================== + +class TestAPIKeyAuth: + """Regression tests for API key authentication.""" + + def test_auth_module_importable(self): + from semantica.explorer.auth import APIKeyAuthMiddleware + assert APIKeyAuthMiddleware is not None + + def test_extract_bearer_token(self): + from semantica.explorer.auth import _extract_token + from unittest.mock import MagicMock + req = MagicMock() + req.headers = {"Authorization": "Bearer test-key-123"} + assert _extract_token(req) == "test-key-123" + + def test_extract_api_key_header(self): + from semantica.explorer.auth import _extract_token + from unittest.mock import MagicMock + req = MagicMock() + req.headers = {"X-API-Key": "my-secret-key", "Authorization": ""} + assert _extract_token(req) == "my-secret-key" + + def test_extract_no_token(self): + from semantica.explorer.auth import _extract_token + from unittest.mock import MagicMock + req = MagicMock() + req.headers = {} + assert _extract_token(req) is None + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From abc10bc8e0761ec09c6262812d63affee624cc5f Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Tue, 11 Aug 2026 15:36:49 +0530 Subject: [PATCH 27/40] fix(security): restore GHSA-j4mq auth enforcement, fix SPARQL comment-regex bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two issues in the last round of commits: 1. explorer/auth.py added a new, opt-in APIKeyAuthMiddleware (EXPLORER_API_KEY) and wired it into create_app(), but in doing so removed the Depends(require_auth) dependency from every router and deleted the /ws/graph-updates handshake check entirely. The new middleware also fails OPEN (allows all requests) when its key is unset, the opposite of require_auth's fail-closed design. Since GHSA-j4mq-hprp-987v (the unauthenticated-Explorer-API advisory) is already merged into main via require_auth, this would have reverted a merged Critical fix the moment this branch merges. Removed explorer/auth.py, restored the per-router dependencies and the WebSocket auth check. Kept auth.py's one genuine improvement (adding X-API-Key to the CORS allow_headers list) by folding it into the existing CORS middleware config. 2. sparql.py's new _is_read_only_query() hardening (comment/PREFIX stripping + forbidden-keyword scan) used `#[^\n]*` to strip SPARQL comments, but a bare '#' also appears inside standard RDF namespace IRIs (e.g. ".../1999/02/22-rdf-syntax-ns#") — the regex struck everything after that '#' as a "comment", corrupting the query and rejecting any legitimate SELECT using rdf:/rdfs:-style PREFIX declarations. Confirmed by the fact the new hardening's own inlined test copy failed against two of its own cases. Fixed by only treating '#' as a comment-start at line-start or after whitespace, which distinguishes ".../ns#" (preceded by a word character) from an actual comment (preceded by whitespace/newline in every realistic case, including the attacker's own comment-hiding PoC). Also fixed the companion PREFIX/BASE regex, which required a prefix-name token between the keyword and the IRI even for bare `BASE <...>` declarations (which have none). tests/test_security_regression.py's SPARQL section now imports the real _is_read_only_query instead of maintaining a parallel inlined copy that had silently drifted from — and shared the same bug as — the real implementation; removed its TestAPIKeyAuth class (tested the now-deleted auth.py) since equivalent, more thorough coverage already exists in tests/explorer/test_explorer_auth.py. Updated tests/explorer/test_sparql_route.py's multi-statement-injection test to reflect that the keyword scan now catches "SELECT ... ; DROP ALL" itself rather than relying on rdflib's parser, and added a new test confirming the parser still catches multi-statement syntax that doesn't contain any forbidden keyword. Full explorer/vector_store/security-regression/age_store suite: 543 passed (the only failures are 6 pre-existing, unrelated Pinecone-client mocking issues). --- semantica/explorer/app.py | 63 ++++++++++------ semantica/explorer/auth.py | 102 -------------------------- semantica/explorer/routes/sparql.py | 28 +++++-- tests/explorer/test_sparql_route.py | 23 ++++-- tests/test_security_regression.py | 110 ++++++++++------------------ 5 files changed, 119 insertions(+), 207 deletions(-) delete mode 100644 semantica/explorer/auth.py diff --git a/semantica/explorer/app.py b/semantica/explorer/app.py index 3d175ed2..68bab729 100644 --- a/semantica/explorer/app.py +++ b/semantica/explorer/app.py @@ -8,16 +8,16 @@ from contextlib import asynccontextmanager from pathlib import Path from typing import Optional -from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect +from fastapi import Depends, FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, HTMLResponse, JSONResponse from fastapi.staticfiles import StaticFiles from .. import __version__ from ..context.context_graph import ContextGraph +from .dependencies import anonymous_access_allowed, get_expected_api_key, is_valid_api_key, require_auth from .session import GraphSession from .ws import ConnectionManager -from .auth import APIKeyAuthMiddleware, warn_if_unauthenticated def _read_int_env(name: str, default: int) -> int: @@ -98,6 +98,22 @@ def create_app( @asynccontextmanager async def lifespan(app: FastAPI): + import logging as _lifespan_logging + _lifespan_logger = _lifespan_logging.getLogger(__name__) + if anonymous_access_allowed(): + _lifespan_logger.warning( + "Explorer is running with SEMANTICA_ALLOW_ANONYMOUS=true — " + "all API routes are unauthenticated. Do not expose this " + "process beyond localhost." + ) + elif get_expected_api_key(): + _lifespan_logger.info("Explorer API authentication: enabled (SEMANTICA_API_KEY set).") + else: + _lifespan_logger.warning( + "Explorer API authentication: NOT CONFIGURED. All protected " + "routes will return 503 until SEMANTICA_API_KEY is set." + ) + app.state.event_loop = asyncio.get_running_loop() app.state.ws_manager = ConnectionManager() app.state.session = active_session @@ -114,10 +130,10 @@ def create_app( app.state.explorer_settings = settings # allow_credentials lets browsers send cookies/auth headers cross-origin. - # The Explorer has no authentication, so credentials serve no purpose and - # enabling them when origins are broadened creates cross-site request risk. - # Set EXPLORER_CORS_CREDENTIALS=true explicitly to opt in (e.g. for a - # reverse-proxy setup that injects its own auth layer). + # Credentials aren't needed for the X-API-Key auth scheme below, and + # enabling them when origins are broadened creates cross-site request + # risk. Set EXPLORER_CORS_CREDENTIALS=true explicitly to opt in (e.g. + # for a reverse-proxy setup that injects its own cookie-based auth). _allow_credentials = os.environ.get("EXPLORER_CORS_CREDENTIALS", "false").lower() == "true" app.add_middleware( CORSMiddleware, @@ -128,10 +144,6 @@ def create_app( max_age=600, ) - # API key authentication (opt-in via EXPLORER_API_KEY env var) - app.add_middleware(APIKeyAuthMiddleware) - warn_if_unauthenticated() - import logging as _logging _logger = _logging.getLogger(__name__) @@ -164,22 +176,31 @@ def create_app( from .routes.temporal import router as temporal_router from .routes.vocabulary import router as vocabulary_router - app.include_router(graph_router) - app.include_router(analytics_router) - app.include_router(decisions_router) - app.include_router(temporal_router) - app.include_router(enrich_router) - app.include_router(export_import_router) - app.include_router(annotations_router) - app.include_router(sparql_router) - app.include_router(provenance_router) - app.include_router(vocabulary_router) - app.include_router(ontology_router) + _auth = [Depends(require_auth)] + app.include_router(graph_router, dependencies=_auth) + app.include_router(analytics_router, dependencies=_auth) + app.include_router(decisions_router, dependencies=_auth) + app.include_router(temporal_router, dependencies=_auth) + app.include_router(enrich_router, dependencies=_auth) + app.include_router(export_import_router, dependencies=_auth) + app.include_router(annotations_router, dependencies=_auth) + app.include_router(sparql_router, dependencies=_auth) + app.include_router(provenance_router, dependencies=_auth) + app.include_router(vocabulary_router, dependencies=_auth) + app.include_router(ontology_router, dependencies=_auth) _WS_MAX_MESSAGE_BYTES = 64 * 1024 # 64 KB — control messages only @app.websocket("/ws/graph-updates") async def websocket_endpoint(websocket: WebSocket): + # Browsers can't set custom headers on a WebSocket handshake, so + # accept the key via header (non-browser clients) or query param + # (browser clients), same SEMANTICA_API_KEY the REST routes check. + candidate = websocket.headers.get("x-api-key") or websocket.query_params.get("api_key") + if not is_valid_api_key(candidate): + await websocket.close(code=4401) # unauthorized + return + manager: ConnectionManager = app.state.ws_manager await manager.connect(websocket) await manager.send_personal(websocket, "connection_ack", {"connected": True}) diff --git a/semantica/explorer/auth.py b/semantica/explorer/auth.py deleted file mode 100644 index 9785b1b5..00000000 --- a/semantica/explorer/auth.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -Semantica Explorer : Authentication Middleware - -Provides opt-in API key authentication for all Explorer API routes. - -Enable by setting the ``EXPLORER_API_KEY`` environment variable. When set, -every request to ``/api/*`` must include either: - -- An ``Authorization: Bearer `` header, or -- An ``X-API-Key: `` header. - -When ``EXPLORER_API_KEY`` is not set, authentication is disabled and the -Explorer operates in open/development mode (with a startup warning). -""" - -import hmac -import logging -import os -from typing import Optional - -from fastapi import HTTPException, Request, status -from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint -from starlette.responses import Response - -_logger = logging.getLogger(__name__) - - -def _get_api_key() -> Optional[str]: - """Read the configured API key from the environment.""" - return os.environ.get("EXPLORER_API_KEY") - - -def _extract_token(request: Request) -> Optional[str]: - """Extract the API key from the request headers.""" - # Check Authorization: Bearer - auth_header = request.headers.get("Authorization", "") - if auth_header.startswith("Bearer "): - return auth_header[7:].strip() - - # Check X-API-Key: - api_key_header = request.headers.get("X-API-Key", "") - if api_key_header: - return api_key_header.strip() - - return None - - -class APIKeyAuthMiddleware(BaseHTTPMiddleware): - """ - Middleware that enforces API key authentication on ``/api/*`` routes. - - Skips authentication for: - - Non-API routes (static files, health checks, WebSocket, docs) - - OPTIONS requests (CORS preflight) - - When ``EXPLORER_API_KEY`` is not configured (open mode) - """ - - async def dispatch( - self, request: Request, call_next: RequestResponseEndpoint - ) -> Response: - api_key = _get_api_key() - - # If no API key is configured, allow all requests (open mode) - if not api_key: - return await call_next(request) - - # Skip authentication for non-API paths - path = request.url.path - if not path.startswith("/api/"): - return await call_next(request) - - # Skip CORS preflight - if request.method == "OPTIONS": - return await call_next(request) - - # Validate the token - token = _extract_token(request) - if not token: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Missing API key. Provide via 'Authorization: Bearer ' or 'X-API-Key: ' header.", - headers={"WWW-Authenticate": "Bearer"}, - ) - - # Constant-time comparison to prevent timing attacks - if not hmac.compare_digest(token, api_key): - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Invalid API key.", - ) - - return await call_next(request) - - -def warn_if_unauthenticated() -> None: - """Log a warning at startup if no API key is configured.""" - if not _get_api_key(): - _logger.warning( - "EXPLORER_API_KEY is not set. The Explorer API is running WITHOUT " - "authentication. Set EXPLORER_API_KEY to enable API key protection " - "for all /api/* endpoints." - ) diff --git a/semantica/explorer/routes/sparql.py b/semantica/explorer/routes/sparql.py index 0a0abee4..8271d38d 100644 --- a/semantica/explorer/routes/sparql.py +++ b/semantica/explorer/routes/sparql.py @@ -3,11 +3,16 @@ SPARQL routes backed by an in-memory rdflib projection of the current graph. Security contract ----------------- -* Only SELECT, ASK, CONSTRUCT, and DESCRIBE are accepted (allowlist enforced - before graph construction so rejected queries never touch the session). -* Multi-statement injections that start with an allowed keyword (e.g. - ``SELECT ... ; DROP ALL``) pass the prefix check and reach rdflib, which - rejects non-SELECT/ASK/CONSTRUCT/DESCRIBE update syntax in the parser. +* Only SELECT, ASK, CONSTRUCT, and DESCRIBE are accepted, and the query + body is scanned for SPARQL Update keywords (INSERT/DELETE/DROP/LOAD/ + CLEAR/CREATE/COPY/MOVE/ADD) after stripping comments and PREFIX/BASE + declarations — both enforced before graph construction, so rejected + queries never touch the session. A multi-statement injection appended + after an allowed keyword (e.g. ``SELECT ... ; DROP ALL``) is caught by + the keyword scan itself, not left to rdflib's parser. +* rdflib's parser remains a second line of defense for malformed multi- + statement syntax that doesn't contain any forbidden keyword (e.g. + ``SELECT ... ; ASK ...``), which SPARQL 1.1 Query doesn't permit. * The in-memory rdflib graph is a read-only projection — the live ``GraphSession`` is never mutated by this route. """ @@ -38,9 +43,16 @@ _FORBIDDEN_KEYWORDS = re.compile( re.IGNORECASE, ) -# Matches SPARQL single-line comments (# ...) and PREFIX declarations -_COMMENT_LINE = re.compile(r"#[^\n]*", re.MULTILINE) -_PREFIX_DECL = re.compile(r"^\s*(?:PREFIX|BASE)\s+\S+\s*<[^>]*>\s*", re.IGNORECASE | re.MULTILINE) +# Matches SPARQL single-line comments (# ...) and PREFIX/BASE declarations. +# The comment regex only treats '#' as a comment-starter at line-start or +# after whitespace — not mid-token — since RDF namespace IRIs commonly +# contain a literal '#' (e.g. ".../1999/02/22-rdf-syntax-ns#"), and a naive +# `#[^\n]*` would truncate every such PREFIX declaration's IRI, corrupting +# the query. BASE declarations have no prefix name between the keyword and +# the IRI (`BASE <...>`, vs. `PREFIX ex: <...>`), so the prefix-name token +# is optional. +_COMMENT_LINE = re.compile(r"(?:^|(?<=\s))#[^\n]*", re.MULTILINE) +_PREFIX_DECL = re.compile(r"^\s*(?:PREFIX\s+\S+|BASE)\s*<[^>]*>\s*", re.IGNORECASE | re.MULTILINE) def _is_read_only_query(query: str) -> bool: diff --git a/tests/explorer/test_sparql_route.py b/tests/explorer/test_sparql_route.py index 77cc5b71..1d8547c6 100644 --- a/tests/explorer/test_sparql_route.py +++ b/tests/explorer/test_sparql_route.py @@ -199,14 +199,27 @@ def test_allowlist_rejected_query_never_touches_the_graph(client, query): mock_build.assert_not_called() -def test_multi_statement_injection_reaches_graph_but_fails_in_parser(client): - """Confirms the distinction between allowlist rejection and parser rejection: - a string starting with SELECT passes _is_read_only_query and builds a graph, - but rdflib.Graph.query() rejects the trailing '; DROP ALL' syntax.""" +def test_multi_statement_injection_is_rejected_by_forbidden_keyword_check(client): + """A string starting with an allowed keyword (SELECT) but containing a + forbidden Update keyword later in the body ('; DROP ALL') is now + rejected by _is_read_only_query's keyword scan itself, before a graph + is ever built — a stronger, earlier rejection than relying solely on + rdflib's parser to reject the syntax.""" + with patch.object(sparql_mod, "_build_rdflib_graph") as mock_build: + resp = _post(client, "SELECT ?s WHERE { ?s ?p ?o } ; DROP ALL") + assert resp.status_code == 200 + assert resp.json()["error"] is not None + mock_build.assert_not_called() + + +def test_malformed_syntax_without_forbidden_keywords_still_fails_in_parser(client): + """The parser remains a real second line of defense for malformed + queries that don't contain any forbidden keyword — these pass + _is_read_only_query and reach rdflib, which rejects the syntax.""" with patch.object( sparql_mod, "_build_rdflib_graph", wraps=sparql_mod._build_rdflib_graph ) as spy_build: - resp = _post(client, "SELECT ?s WHERE { ?s ?p ?o } ; DROP ALL") + resp = _post(client, "SELECT ?s WHERE { ?s ?p ?o } ; ASK { ?x ?y ?z }") assert resp.status_code == 200 assert resp.json()["error"] is not None spy_build.assert_called_once() diff --git a/tests/test_security_regression.py b/tests/test_security_regression.py index cab6cf9e..18854f0b 100644 --- a/tests/test_security_regression.py +++ b/tests/test_security_regression.py @@ -2,48 +2,29 @@ Regression tests for security fixes in PR #898. Covers: - 1. API Key Authentication (Explorer) - 2. Cypher Injection Prevention (AGE Store) - 3. SPARQL Injection Prevention (read-only query validation) - 4. XXE Protection (rdf_parser fail-closed) - 5. Vector save numpy serialization - 6. SPARQL graph cap error handling - 7. SSRF redirect handling (relative URLs, resp.close) + 1. Cypher Injection Prevention (AGE Store) + 2. SPARQL Injection Prevention (read-only query validation) + 3. XXE Protection (rdf_parser fail-closed) + 4. Vector save numpy serialization + 5. SPARQL graph cap error handling + 6. SSRF redirect handling (relative URLs, resp.close) + +Explorer API-key authentication (GHSA-j4mq-hprp-987v) has its own, more +thorough test suite at tests/explorer/test_explorer_auth.py — it isn't +duplicated here. """ -import re import pytest - -# =================================================================== -# 1. SPARQL read-only query validation (injection prevention) -# =================================================================== - -# Inline the validation logic so tests don't require full app context -_ALLOWED_QUERY_TYPES = re.compile( - r"^(SELECT|ASK|CONSTRUCT|DESCRIBE)\b", - re.IGNORECASE, -) -_FORBIDDEN_KEYWORDS = re.compile( - r"\b(INSERT|DELETE|DROP|LOAD|CLEAR|CREATE|COPY|MOVE|ADD)\b", - re.IGNORECASE, -) -_COMMENT_LINE = re.compile(r"#[^\n]*", re.MULTILINE) -_PREFIX_DECL = re.compile( - r"^\s*(?:PREFIX|BASE)\s+\S+\s*<[^>]*>\s*", - re.IGNORECASE | re.MULTILINE, -) - - -def _is_read_only_query(query: str) -> bool: - cleaned = _COMMENT_LINE.sub("", query) - cleaned = _PREFIX_DECL.sub("", cleaned) - cleaned = cleaned.strip() - if not _ALLOWED_QUERY_TYPES.match(cleaned): - return False - if _FORBIDDEN_KEYWORDS.search(cleaned): - return False - return True +# Import the real implementation rather than re-declaring the regexes here: +# an earlier version of this file inlined a copy that silently drifted from +# semantica/explorer/routes/sparql.py's actual behavior (the inlined +# _COMMENT_LINE regex stripped '#' mid-token, corrupting any PREFIX +# declaration using a namespace IRI with a literal '#', e.g. the standard +# rdf:/rdfs: namespaces) and neither the code nor this test caught it, +# since both had the same bug. Importing the real function makes that class +# of drift impossible. +from semantica.explorer.routes.sparql import _is_read_only_query class TestSparqlReadOnlyValidation: @@ -122,6 +103,26 @@ class TestSparqlReadOnlyValidation: query = "BASE \nSELECT ?s WHERE { ?s ?p ?o }" assert _is_read_only_query(query) + def test_namespace_iri_with_hash_fragment_not_treated_as_comment(self): + """A '#' inside a PREFIX declaration's IRI (standard for RDF/RDFS/OWL + namespaces) must not be mistaken for a comment-start — a naive + `#[^\\n]*` strip corrupts the IRI and truncates the rest of the + query with it.""" + query = ( + "PREFIX rdf: \n" + "SELECT ?s WHERE { ?s rdf:type ?o }" + ) + assert _is_read_only_query(query) + + def test_real_comment_after_namespace_iri_still_stripped(self): + """A genuine trailing comment must still be recognized even on a + line that also contains a '#'-bearing IRI earlier in the query.""" + query = ( + "PREFIX rdf: \n" + "SELECT ?s WHERE { ?s rdf:type ?o } # trailing comment INSERT DATA" + ) + assert _is_read_only_query(query) + # =================================================================== # 2. Cypher injection prevention @@ -270,38 +271,5 @@ class TestSSRFRedirectHandling: assert result == "https://example.com/api/data.ttl" -# =================================================================== -# 6. API Key Auth -# =================================================================== - -class TestAPIKeyAuth: - """Regression tests for API key authentication.""" - - def test_auth_module_importable(self): - from semantica.explorer.auth import APIKeyAuthMiddleware - assert APIKeyAuthMiddleware is not None - - def test_extract_bearer_token(self): - from semantica.explorer.auth import _extract_token - from unittest.mock import MagicMock - req = MagicMock() - req.headers = {"Authorization": "Bearer test-key-123"} - assert _extract_token(req) == "test-key-123" - - def test_extract_api_key_header(self): - from semantica.explorer.auth import _extract_token - from unittest.mock import MagicMock - req = MagicMock() - req.headers = {"X-API-Key": "my-secret-key", "Authorization": ""} - assert _extract_token(req) == "my-secret-key" - - def test_extract_no_token(self): - from semantica.explorer.auth import _extract_token - from unittest.mock import MagicMock - req = MagicMock() - req.headers = {} - assert _extract_token(req) is None - - if __name__ == "__main__": pytest.main([__file__, "-v"]) From 6002965c55753aa2b08c985ce8543b915a1152dd Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Tue, 11 Aug 2026 15:39:11 +0530 Subject: [PATCH 28/40] docs(changelog): document PR #898's full scope, including the maintainer follow-up fixes --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a77ec2f1..b7dcc2cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -244,6 +244,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- **4 critical/high vulnerabilities in the Explorer API and vector store: RCE, SSRF, XXE, and DoS, plus Cypher/SPARQL injection hardening found along the way** (#898) by @Sunil56224972 + - **[CWE-502] Arbitrary code execution via `pickle.load()`**: `VectorStore.save()`/`load()` used `pickle` for the on-disk `store_data.pkl`; a crafted `.pkl` file placed in the store directory (file upload, shared filesystem, or supply-chain compromise) could execute arbitrary code on deserialization. Replaced with JSON — vectors and metadata are fully JSON-serializable, so nothing is lost — and `load()` now refuses any legacy `.pkl` file it finds with a migration error rather than deserializing it + - **[CWE-918] SSRF via redirect bypass in `ontology.py`'s URL fetcher**: `_validate_fetch_url()` correctly blocked private/loopback/reserved addresses on the caller-supplied URL, but `_fetch_url_sync()` fetched with `allow_redirects=True`, so a validated *public* first hop could 302 to `http://169.254.169.254/...` (cloud instance metadata) or an internal service, and `requests` followed it with no re-check. Redirects are now followed manually, capped at 5 hops, with `_validate_fetch_url()` re-run against every hop's target — including relative `Location` headers, resolved via `urljoin()` before validation — and every response (redirect or final) is explicitly closed to avoid leaking connections back to the pool + - **[CWE-611] XXE injection in the RDF/XML parser**: `_safe_parse_rdf()` depended on `defusedxml` for XXE protection, but `defusedxml` wasn't declared in `pyproject.toml`'s `explorer` extra, so it was silently absent in normal installs and the code fell back to a bare warning plus unsafe parsing — a crafted RDF/XML ontology with an external entity could read arbitrary server files. Added `defusedxml>=0.7.1` to the extra, and `_safe_parse_rdf()` now fails closed: it raises rather than parsing untrusted RDF/XML if `defusedxml` isn't importable, replacing an earlier regex-based DOCTYPE-stripping fallback that was reviewed and rejected as bypassable + - **[CWE-770] DoS via unbounded SPARQL graph materialization**: `_build_rdflib_graph()` loaded up to 999,999 nodes and 999,999 edges into memory per query, and with up to 4 concurrent SPARQL requests permitted, an attacker could exhaust server memory. Added a 50,000 node/edge cap (`_SPARQL_MAX_GRAPH_NODES`); oversized graphs now return a clean error instead of attempting materialization + - **Cypher injection via Apache AGE's `graph_name` and `$$`-delimiter breakout**: `graph_name` was interpolated unvalidated into `cypher('{graph_name}', $$ ... $$)`, and raw Cypher query text containing `$$` could close AGE's dollar-quoted string delimiter early and append arbitrary SQL. `graph_name` is now validated against the same identifier allowlist `age_store.py` already used for labels/relationship types, and any query containing `$$` is rejected outright + - **SPARQL Explorer route (`/api/sparql`) hardened against comment/PREFIX-hiding bypass**: `_is_read_only_query()` now strips comments and PREFIX/BASE declarations before checking the leading keyword, and additionally scans the full query body for SPARQL Update keywords (INSERT/DELETE/DROP/LOAD/CLEAR/CREATE/COPY/MOVE/ADD) — so `SELECT ... ; DROP ALL` is now rejected by the keyword scan itself rather than relying solely on rdflib's parser + - **Fixed along the way** (maintainer follow-up, addressing automated review findings and a regression introduced across several rounds of iteration on the original fix): + - `VectorStore.save()`'s numpy handling used `list(v)` for the JSON fallback path, which produces `numpy.float32` elements that `json.dump()` can't serialize — changed to `v.tolist()` + - the SPARQL graph-size `ValueError` was raised outside `execute_sparql()`'s exception handling and surfaced as an unhandled 500 instead of a clean API error — moved inside + - every streamed `requests` response in the ontology redirect loop, including the one actually read and returned, is now closed in a `finally` block — a connection-pool leak that a rework of the redirect logic had briefly reintroduced after an earlier fix + - a later commit meant to add opt-in API-key auth (`explorer/auth.py`, gated on `EXPLORER_API_KEY`) instead **replaced and silently disabled** the `Depends(require_auth)` enforcement already merged into `main` for GHSA-j4mq-hprp-987v (Critical — unauthenticated Explorer API), removed the `/ws/graph-updates` handshake check, and — unlike `require_auth` — failed *open* (allowed all requests) whenever its key was unset. Merging that version would have silently reverted an already-fixed Critical CVE the moment this branch landed. Removed `explorer/auth.py`; restored the per-router `Depends(require_auth)` wiring and the WebSocket auth check; kept the one genuine improvement in that commit (adding `X-API-Key` to the CORS `allow_headers` list) by folding it into the existing CORS config + - the new SPARQL keyword-scan's comment-stripping regex (`#[^\n]*`) also matched the `#` inside standard RDF namespace IRIs (e.g. `.../1999/02/22-rdf-syntax-ns#`), corrupting any query with a normal `rdf:`/`rdfs:`-style `PREFIX` declaration — caught because the hardening's own bundled tests failed against two of its own cases. Fixed by only treating `#` as a comment-start at line-start or after whitespace; the companion `PREFIX`/`BASE` regex was also fixed to accept bare `BASE <...>` declarations, which have no prefix-name token between the keyword and the IRI + - New/updated regression tests: `tests/explorer/test_ontology_ssrf.py` (redirect re-validation, relative-redirect resolution, response closing, redirect-cap enforcement), `tests/test_security_regression.py` (Cypher/SPARQL injection, XXE, numpy serialization, SSRF redirect handling), plus additions to `tests/explorer/test_sparql_route.py`, `tests/vector_store/test_vector_store.py`, and `tests/explorer/test_explorer_auth.py` + - Note: the Cypher-injection hardening here is scoped to `age_store.py`'s `graph_name`/`$$` breakout, found while reviewing this PR. The broader label/property-key/relationship-type injection across the Neptune, Neo4j, and FalkorDB backends (GHSA-482h-hw99-h62p, #910) and the triplet-store SPARQL injection across Blazegraph/RDF4J/Jena (GHSA-8vgg-8mr4-r236, #911) are covered by separate, still-open PRs, as is the unauthenticated-Explorer-API fix referenced above (GHSA-j4mq-hprp-987v, #909, already merged) + - **CI/CD supply-chain hardening against mutable-tag Action compromise (LiteLLM/Trivy-class attack)** (#824) by @KaifAhmad1 - Every third-party GitHub Action across all 8 workflows is now pinned to a full commit SHA instead of a mutable tag (`@v7` → `@3d3c42e... # v7`), closing the exact vector used against LiteLLM in March 2026 (a compromised Trivy Action tag stole a long-lived publishing token) - Added `verify-action-pins.yml` + `.github/scripts/verify-action-pins.sh`: a CI check that fails closed on any `uses:` reference that isn't a full SHA (catching a newly introduced mutable tag, not just auditing existing pins) and re-verifies every pin against the GitHub API on each workflow change, on push to `main`, and weekly; an unresolvable API lookup is treated as a failure rather than a silent skip From 69b79e3d67ac541e5c013333b63df9b55c266b39 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Tue, 11 Aug 2026 16:19:06 +0530 Subject: [PATCH 29/40] docs(changelog): add PR #911 (GHSA-8vgg SPARQL injection) entry --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7dcc2cf..21c65b64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -244,6 +244,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- **SPARQL injection via unvalidated triplet IRIs** (#911, GHSA-8vgg-8mr4-r236) by @KaifAhmad1 + - `Triplet.subject`/`.predicate` (and, in some builders, `.object`) were interpolated directly into SPARQL update/query strings in the Blazegraph and RDF4J stores, and into a SELECT filter in the Jena store. A subject containing `>` closes the `<...>` IRI token early, so the rest of the value is parsed as more SPARQL. Entity names are document text in the normal ingest pipeline, so anyone whose content gets processed could append operations like `CLEAR ALL`, running with the application's store credentials + - Applied the existing `sparql_escaping.validate_uri` (already used by `anzo_store.py`, the one backend that was already hardened — this generalizes its approach rather than inventing a new one) at every subject/predicate/object interpolation site: `blazegraph_store.py`'s `_build_insert_data`, `_triplets_to_rdf`, `bulk_load`'s `graph` option, `get_triplets`'s filter, and `delete_triplet`; `rdf4j_store.py`'s `_triplets_to_ntriples`, `get_triplets`'s filter, and `delete_triplet`; `jena_store.py`'s `get_triplets`'s filter (the only vulnerable site there — `add_triplets`/`delete_triplet` already use rdflib's native `Graph.add`/`.remove` with `URIRef` rather than building query strings) + - **Fixed along the way** (caught in review, by @ZohaibHassan16): `_format_object_for_sparql`'s URI branch — used when a triplet's *object* is itself a URI rather than a literal — only checked for spaces and `>` inline instead of running the same `validate_uri` check applied to subject/predicate, leaving the object position as a narrower but real gap in both Blazegraph and RDF4J. Also fixed test flakiness in `RDF4JStore`'s test fixtures, which weren't mocking `_connect()` and so were making real network calls + - New `tests/triplet_store/test_sparql_injection.py` (12+ tests) reproducing the advisory's own injection payload (`http://example.com/a> ... ; CLEAR ALL ; INSERT DATA { ...`) against all three backends' write and read paths, asserting the malicious query is never built or sent. Full triplet_store suite: 330+ tests passing + - Side note, not part of this fix: found that `jena_store.py`'s `get_triplets()` builds syntactically invalid SPARQL for its WHERE-clause filters (missing a `FILTER()`/separator before the equality conditions) — a pre-existing correctness bug, unrelated to the injection fix, left alone here and worth a separate follow-up + - **4 critical/high vulnerabilities in the Explorer API and vector store: RCE, SSRF, XXE, and DoS, plus Cypher/SPARQL injection hardening found along the way** (#898) by @Sunil56224972 - **[CWE-502] Arbitrary code execution via `pickle.load()`**: `VectorStore.save()`/`load()` used `pickle` for the on-disk `store_data.pkl`; a crafted `.pkl` file placed in the store directory (file upload, shared filesystem, or supply-chain compromise) could execute arbitrary code on deserialization. Replaced with JSON — vectors and metadata are fully JSON-serializable, so nothing is lost — and `load()` now refuses any legacy `.pkl` file it finds with a migration error rather than deserializing it - **[CWE-918] SSRF via redirect bypass in `ontology.py`'s URL fetcher**: `_validate_fetch_url()` correctly blocked private/loopback/reserved addresses on the caller-supplied URL, but `_fetch_url_sync()` fetched with `allow_redirects=True`, so a validated *public* first hop could 302 to `http://169.254.169.254/...` (cloud instance metadata) or an internal service, and `requests` followed it with no re-check. Redirects are now followed manually, capped at 5 hops, with `_validate_fetch_url()` re-run against every hop's target — including relative `Location` headers, resolved via `urljoin()` before validation — and every response (redirect or final) is explicitly closed to avoid leaking connections back to the pool From b846ff88d49bc26a5f1d3419e8d3a5832a894a86 Mon Sep 17 00:00:00 2001 From: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:24:28 +0530 Subject: [PATCH 30/40] security: sanitize Cypher labels/relationship types/property keys (GHSA-482h) (#910) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * security: sanitize Cypher labels/relationship types/property keys (GHSA-482h-hw99-h62p) Node labels and property keys passed to create_node/create_relationship were interpolated directly into Cypher strings in the Neptune, Neo4j, and FalkorDB graph stores. Property values are parameterized, but labels and keys can't be bound as parameters, and nothing validated them, so a document-derived entity type or property name could close the current Cypher token early and append arbitrary statements (e.g. DETACH DELETE), running with the application's database credentials. - New shared semantica/graph_store/query_sanitize.py: sanitize_identifier() generalizes age_store.py's existing _sanitize_label/_sanitize_rel_type (the only backend that already validated this) into a helper the other backends can import without an import cycle with graph_store.py/methods.py. - Applied at every label/relationship-type/property-key interpolation site in amazon_neptune.py, neo4j_store.py, falkordb_store.py, graph_store.py (degree_centrality's own query builder), and methods.py (update_relationship's own query builder) — create_node, create_nodes, create_relationship, get_nodes, get_relationships, get_neighbors, shortest_path, update_node, create_index, and all relationship-type filters. - depth/max_depth path-length parameters are also cast to int before interpolation as defense-in-depth (they're already typed int, but Python doesn't enforce that at runtime). Added tests/graph_store/test_cypher_injection.py (12 tests covering the sanitizer directly and reproducing the advisory's injection payload against Neptune/Neo4j/FalkorDB create_node/create_relationship — asserts the malicious query is never built or sent), plus regression tests for graph_store.py's degree_centrality and methods.py's update_relationship. Full graph_store test suite (224 tests) passes with no regressions. * fix(graph-store): prevent depth-based Cypher injection * test(graph-store): tighten injection regression assertions * docs(changelog): add PR #910 (GHSA-482h Cypher injection) entry --------- Co-authored-by: Sameer6305 --- CHANGELOG.md | 7 + semantica/graph_store/amazon_neptune.py | 31 +- semantica/graph_store/falkordb_store.py | 39 +- semantica/graph_store/graph_store.py | 7 +- semantica/graph_store/methods.py | 4 +- semantica/graph_store/neo4j_store.py | 32 +- semantica/graph_store/query_sanitize.py | 32 ++ tests/graph_store/test_cypher_injection.py | 396 +++++++++++++++++++++ tests/test_graph_store.py | 20 ++ tests/test_graph_store_methods.py | 21 ++ 10 files changed, 545 insertions(+), 44 deletions(-) create mode 100644 semantica/graph_store/query_sanitize.py create mode 100644 tests/graph_store/test_cypher_injection.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b7dcc2cf..e6a409bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -244,6 +244,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- **Cypher injection via unvalidated node labels, relationship types, and property keys** (#910, GHSA-482h-hw99-h62p) by @KaifAhmad1 + - Node labels and property keys passed to `create_node`/`create_relationship` were interpolated directly into Cypher strings in the Neptune, Neo4j, and FalkorDB graph stores. Property *values* are parameterized, but labels and keys can't be bound as query parameters, and nothing validated them — so a document-derived entity type or property name (the normal ingest path) could close the current Cypher token early and append arbitrary statements (e.g. `DETACH DELETE`), running with the application's database credentials + - New shared `semantica/graph_store/query_sanitize.py`: `sanitize_identifier()` generalizes `age_store.py`'s existing `_sanitize_label`/`_sanitize_rel_type` (the only backend that already validated this) into a helper the other backends import without an import cycle with `graph_store.py`/`methods.py` + - Applied at every label/relationship-type/property-key interpolation site in `amazon_neptune.py`, `neo4j_store.py`, `falkordb_store.py`, `graph_store.py` (`degree_centrality`'s own query builder), and `methods.py` (`update_relationship`'s own query builder) — covers `create_node`, `create_nodes`, `create_relationship`, `get_nodes`, `get_relationships`, `get_neighbors`, `shortest_path`, `update_node`, `create_index`, and all relationship-type filters across the three backends + - **Fixed along the way** (caught in review, by @Sameer6305): `depth`/`max_depth` path-length parameters are meant to be integers, but `Neo4jStore.get_neighbors()`/`shortest_path()` interpolated them into the Cypher variable-length-path syntax (`*1..{depth}`) without coercion — unlike the Neptune/FalkorDB equivalents, which already cast to `int()`. A string `depth` (e.g. `"1]->(x) DETACH DELETE x //"`) reached the query verbatim. Added the same `int()` coercion Neptune/FalkorDB already had, plus `GraphStore.get_neighbors()`'s `hops`/`depth` alias resolution + - New `tests/graph_store/test_cypher_injection.py` (unit tests on `sanitize_identifier` plus the labels/keys/rel-types injection payload run against Neptune/Neo4j/FalkorDB `create_node`/`create_relationship`, asserting the malicious query is never built or sent) and the depth-coercion regression above; plus additions to `tests/test_graph_store.py` (`degree_centrality`) and `tests/test_graph_store_methods.py` (`update_relationship`). Full graph_store suite: 224+ tests passing + - **4 critical/high vulnerabilities in the Explorer API and vector store: RCE, SSRF, XXE, and DoS, plus Cypher/SPARQL injection hardening found along the way** (#898) by @Sunil56224972 - **[CWE-502] Arbitrary code execution via `pickle.load()`**: `VectorStore.save()`/`load()` used `pickle` for the on-disk `store_data.pkl`; a crafted `.pkl` file placed in the store directory (file upload, shared filesystem, or supply-chain compromise) could execute arbitrary code on deserialization. Replaced with JSON — vectors and metadata are fully JSON-serializable, so nothing is lost — and `load()` now refuses any legacy `.pkl` file it finds with a migration error rather than deserializing it - **[CWE-918] SSRF via redirect bypass in `ontology.py`'s URL fetcher**: `_validate_fetch_url()` correctly blocked private/loopback/reserved addresses on the caller-supplied URL, but `_fetch_url_sync()` fetched with `allow_redirects=True`, so a validated *public* first hop could 302 to `http://169.254.169.254/...` (cloud instance metadata) or an internal service, and `requests` followed it with no re-check. Redirects are now followed manually, capped at 5 hops, with `_validate_fetch_url()` re-run against every hop's target — including relative `Location` headers, resolved via `urljoin()` before validation — and every response (redirect or final) is explicitly closed to avoid leaking connections back to the pool diff --git a/semantica/graph_store/amazon_neptune.py b/semantica/graph_store/amazon_neptune.py index 988fc470..43a4e8af 100644 --- a/semantica/graph_store/amazon_neptune.py +++ b/semantica/graph_store/amazon_neptune.py @@ -47,6 +47,7 @@ from typing import Any, Dict, List, Optional, Union from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker +from .query_sanitize import sanitize_identifier # Optional boto3 for AWS credentials and SigV4 signing try: @@ -981,7 +982,8 @@ class AmazonNeptuneStore: node_id = props_copy.pop("id", None) or self._generate_id() use_merge = options.get("merge", True) - label_str = ":".join(labels) if labels else "Node" + label_str = ":".join(sanitize_identifier(l, "label") for l in labels) if labels else "Node" + safe_keys = [sanitize_identifier(k, "property key") for k in props_copy.keys()] # Build parameters params = {"node_id": str(node_id)} @@ -990,9 +992,7 @@ class AmazonNeptuneStore: if use_merge: # MERGE: Return existing node if ID matches, or create new - set_parts = [] - for key in props_copy.keys(): - set_parts.append(f"n.{key} = ${key}") + set_parts = [f"n.{key} = ${key}" for key in safe_keys] if set_parts: set_clause = ", ".join(set_parts) @@ -1005,9 +1005,7 @@ class AmazonNeptuneStore: query = f"MERGE (n:{label_str} {{`~id`: $node_id}}) RETURN n" else: # CREATE: Will fail if node with same ID exists - prop_parts = ["`~id`: $node_id"] - for key in props_copy.keys(): - prop_parts.append(f"{key}: ${key}") + prop_parts = ["`~id`: $node_id"] + [f"{key}: ${key}" for key in safe_keys] prop_assignments = ", ".join(prop_parts) query = f"CREATE (n:{label_str} {{{prop_assignments}}}) RETURN n" @@ -1162,7 +1160,7 @@ class AmazonNeptuneStore: # Build query if labels: - label_str = ":".join(labels) + label_str = ":".join(sanitize_identifier(l, "label") for l in labels) query = f"MATCH (n:{label_str})" else: query = "MATCH (n)" @@ -1172,8 +1170,9 @@ class AmazonNeptuneStore: if properties: conditions = [] for key, value in properties.items(): - param_key = f"prop_{key}" - conditions.append(f"n.{key} = ${param_key}") + safe_key = sanitize_identifier(key, "property key") + param_key = f"prop_{safe_key}" + conditions.append(f"n.{safe_key} = ${param_key}") params[param_key] = value query += " WHERE " + " AND ".join(conditions) @@ -1341,15 +1340,16 @@ class AmazonNeptuneStore: } # Build property assignments including ~id + safe_rel_type = sanitize_identifier(rel_type, "relationship type") prop_parts = ["`~id`: $rel_id"] for key, value in props_copy.items(): - prop_parts.append(f"{key}: ${key}") + prop_parts.append(f"{sanitize_identifier(key, 'property key')}: ${key}") params[key] = value prop_assignments = ", ".join(prop_parts) query = ( f"MATCH (a), (b) WHERE id(a) = $start_id AND id(b) = $end_id " - f"CREATE (a)-[r:{rel_type} {{{prop_assignments}}}]->(b) RETURN r" + f"CREATE (a)-[r:{safe_rel_type} {{{prop_assignments}}}]->(b) RETURN r" ) records = self._run_query(query, params) @@ -1405,7 +1405,7 @@ class AmazonNeptuneStore: try: self._ensure_connected() - type_filter = f":{rel_type}" if rel_type else "" + type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else "" params = {} if node_id is not None: @@ -1564,7 +1564,8 @@ class AmazonNeptuneStore: try: self._ensure_connected() - type_filter = f":{rel_type}" if rel_type else "" + type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else "" + depth = int(depth) if direction == "out": pattern = f"-[r{type_filter}*1..{depth}]->" @@ -1634,7 +1635,7 @@ class AmazonNeptuneStore: try: self._ensure_connected() - type_filter = f":{rel_type}" if rel_type else "" + type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else "" # Neptune doesn't support named path patterns in shortestPath # Use iterative depth search instead diff --git a/semantica/graph_store/falkordb_store.py b/semantica/graph_store/falkordb_store.py index 6049d3b1..c2f3fef6 100644 --- a/semantica/graph_store/falkordb_store.py +++ b/semantica/graph_store/falkordb_store.py @@ -41,6 +41,7 @@ from typing import Any, Dict, List, Optional, Union from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker +from .query_sanitize import sanitize_identifier # Optional FalkorDB import try: @@ -330,11 +331,11 @@ class FalkorDBStore: try: graph = self._ensure_graph() - label_str = ":".join(labels) + label_str = ":".join(sanitize_identifier(l, "label") for l in labels) # Build property string for Cypher props_str = ", ".join( - f"{k}: ${k}" for k in properties.keys() + f"{sanitize_identifier(k, 'property key')}: ${k}" for k in properties.keys() ) query = f"CREATE (n:{label_str} {{{props_str}}}) RETURN id(n) as id, n" @@ -392,9 +393,9 @@ class FalkorDBStore: labels = node.get("labels", []) properties = node.get("properties", {}) - label_str = ":".join(labels) if labels else "Node" + label_str = ":".join(sanitize_identifier(l, "label") for l in labels) if labels else "Node" props_str = ", ".join( - f"{k}: ${k}" for k in properties.keys() + f"{sanitize_identifier(k, 'property key')}: ${k}" for k in properties.keys() ) query = f"CREATE (n:{label_str} {{{props_str}}}) RETURN id(n) as id" @@ -447,7 +448,7 @@ class FalkorDBStore: # Build query if labels: - label_str = ":".join(labels) + label_str = ":".join(sanitize_identifier(l, "label") for l in labels) query = f"MATCH (n:{label_str})" else: query = "MATCH (n)" @@ -456,7 +457,8 @@ class FalkorDBStore: if properties: conditions = [] for key in properties.keys(): - conditions.append(f"n.{key} = ${key}") + safe_key = sanitize_identifier(key, "property key") + conditions.append(f"n.{safe_key} = ${safe_key}") query += " WHERE " + " AND ".join(conditions) query += f" RETURN id(n) as id, n, labels(n) as labels LIMIT {limit}" @@ -504,7 +506,8 @@ class FalkorDBStore: # Build SET clause set_parts = [] for key in properties.keys(): - set_parts.append(f"n.{key} = ${key}") + safe_key = sanitize_identifier(key, "property key") + set_parts.append(f"n.{safe_key} = ${safe_key}") if merge: query = f"MATCH (n) WHERE id(n) = $node_id SET {', '.join(set_parts)} RETURN id(n) as id, n, labels(n) as labels" @@ -592,9 +595,13 @@ class FalkorDBStore: graph = self._ensure_graph() properties = properties or {} + safe_rel_type = sanitize_identifier(rel_type, "relationship type") + # Build property string if properties: - props_str = ", ".join(f"{k}: ${k}" for k in properties.keys()) + props_str = ", ".join( + f"{sanitize_identifier(k, 'property key')}: ${k}" for k in properties.keys() + ) props_str = f" {{{props_str}}}" else: props_str = "" @@ -602,7 +609,7 @@ class FalkorDBStore: query = f""" MATCH (a), (b) WHERE id(a) = $start_id AND id(b) = $end_id - CREATE (a)-[r:{rel_type}{props_str}]->(b) + CREATE (a)-[r:{safe_rel_type}{props_str}]->(b) RETURN id(r) as id, type(r) as type """ @@ -656,7 +663,7 @@ class FalkorDBStore: """ try: graph = self._ensure_graph() - type_filter = f":{rel_type}" if rel_type else "" + type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else "" if node_id is not None: if direction == "out": @@ -806,7 +813,8 @@ class FalkorDBStore: """ try: graph = self._ensure_graph() - type_filter = f":{rel_type}" if rel_type else "" + type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else "" + depth = int(depth) if direction == "out": pattern = f"-[r{type_filter}*1..{depth}]->" @@ -860,7 +868,8 @@ class FalkorDBStore: """ try: graph = self._ensure_graph() - type_filter = f":{rel_type}" if rel_type else "" + type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else "" + max_depth = int(max_depth) query = f""" MATCH path = shortestPath((start)-[r{type_filter}*..{max_depth}]-(end)) @@ -929,11 +938,13 @@ class FalkorDBStore: """ try: graph = self._ensure_graph() + safe_label = sanitize_identifier(label, "label") + safe_property = sanitize_identifier(property_name, "property key") if index_type == "fulltext": - query = f"CALL db.idx.fulltext.createNodeIndex('{label}', '{property_name}')" + query = f"CALL db.idx.fulltext.createNodeIndex('{safe_label}', '{safe_property}')" else: - query = f"CREATE INDEX FOR (n:{label}) ON (n.{property_name})" + query = f"CREATE INDEX FOR (n:{safe_label}) ON (n.{safe_property})" graph.query(query) self.logger.info(f"Created {index_type} index on {label}.{property_name}") diff --git a/semantica/graph_store/graph_store.py b/semantica/graph_store/graph_store.py index 5b7e2637..a8bd73f9 100644 --- a/semantica/graph_store/graph_store.py +++ b/semantica/graph_store/graph_store.py @@ -38,6 +38,7 @@ from ..utils.exceptions import ValidationError from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker from .config import graph_store_config +from .query_sanitize import sanitize_identifier class NodeManager: @@ -393,12 +394,12 @@ class GraphAnalytics: """ # Build query based on direction if labels: - label_str = ":".join(labels) + label_str = ":".join(sanitize_identifier(l, "label") for l in labels) match = f"MATCH (n:{label_str})" else: match = "MATCH (n)" - type_filter = f":{rel_type}" if rel_type else "" + type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else "" if direction == "out": query = f""" @@ -756,7 +757,7 @@ class GraphStore: **options: Additional options """ # Support 'hops' as alias for 'depth' for ContextRetriever compatibility - actual_depth = options.get("hops", depth) + actual_depth = int(options.get("hops", depth)) return self._manager.analytics.get_neighbors( node_id, rel_type, direction, actual_depth, **options ) diff --git a/semantica/graph_store/methods.py b/semantica/graph_store/methods.py index 03bc4e87..ba16d9a8 100644 --- a/semantica/graph_store/methods.py +++ b/semantica/graph_store/methods.py @@ -62,6 +62,7 @@ from typing import Any, Dict, List, Optional, Union from .config import graph_store_config from .graph_store import GraphAnalytics, GraphStore, NodeManager, QueryEngine, RelationshipManager +from .query_sanitize import sanitize_identifier from .registry import method_registry # Global store instance @@ -357,7 +358,8 @@ def update_relationship( # Default implementation - execute update query store = _get_store() - set_parts = ", ".join(f"r.{k} = ${k}" for k in properties.keys()) + safe_keys = [sanitize_identifier(k, "property key") for k in properties.keys()] + set_parts = ", ".join(f"r.{k} = ${k}" for k in safe_keys) query = f"MATCH ()-[r]->() WHERE id(r) = $rel_id SET {set_parts} RETURN id(r) as id, type(r) as type, r" params = {"rel_id": rel_id, **properties} result = store.execute_query(query, params) diff --git a/semantica/graph_store/neo4j_store.py b/semantica/graph_store/neo4j_store.py index da2a8c75..93270efe 100644 --- a/semantica/graph_store/neo4j_store.py +++ b/semantica/graph_store/neo4j_store.py @@ -38,6 +38,7 @@ from typing import Any, Dict, List, Optional, Union from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker +from .query_sanitize import sanitize_identifier # Optional Neo4j import try: @@ -366,7 +367,7 @@ class Neo4jStore: ) try: - label_str = ":".join(labels) + label_str = ":".join(sanitize_identifier(l, "label") for l in labels) query = f"CREATE (n:{label_str} $props) RETURN id(n) as id, n" with self.get_session() as session: @@ -424,7 +425,7 @@ class Neo4jStore: labels = node.get("labels", []) properties = node.get("properties", {}) - label_str = ":".join(labels) if labels else "Node" + label_str = ":".join(sanitize_identifier(l, "label") for l in labels) if labels else "Node" query = f"CREATE (n:{label_str} $props) RETURN id(n) as id, n" result = session.run(query, {"props": properties}) @@ -505,7 +506,7 @@ class Neo4jStore: try: # Build query if labels: - label_str = ":".join(labels) + label_str = ":".join(sanitize_identifier(l, "label") for l in labels) query = f"MATCH (n:{label_str})" else: query = "MATCH (n)" @@ -514,7 +515,8 @@ class Neo4jStore: if properties: conditions = [] for key, value in properties.items(): - conditions.append(f"n.{key} = ${key}") + safe_key = sanitize_identifier(key, "property key") + conditions.append(f"n.{safe_key} = ${safe_key}") query += " WHERE " + " AND ".join(conditions) query += f" RETURN id(n) as id, n, labels(n) as labels LIMIT {limit}" @@ -635,10 +637,11 @@ class Neo4jStore: try: properties = properties or {} + safe_rel_type = sanitize_identifier(rel_type, "relationship type") query = f""" MATCH (a), (b) WHERE id(a) = $start_id AND id(b) = $end_id - CREATE (a)-[r:{rel_type} $props]->(b) + CREATE (a)-[r:{safe_rel_type} $props]->(b) RETURN id(r) as id, type(r) as type, r """ @@ -696,7 +699,7 @@ class Neo4jStore: List of matching relationships """ try: - type_filter = f":{rel_type}" if rel_type else "" + type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else "" if node_id is not None: if direction == "out": @@ -857,7 +860,8 @@ class Neo4jStore: List of neighboring nodes with path information """ try: - type_filter = f":{rel_type}" if rel_type else "" + type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else "" + depth = int(depth) if direction == "out": pattern = f"-[r{type_filter}*1..{depth}]->" @@ -910,7 +914,8 @@ class Neo4jStore: Shortest path information or None if not found """ try: - type_filter = f":{rel_type}" if rel_type else "" + type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else "" + max_depth = int(max_depth) query = f""" MATCH path = shortestPath((start)-[r{type_filter}*..{max_depth}]-(end)) @@ -975,17 +980,22 @@ class Neo4jStore: True if index created successfully """ try: - index_name = options.get("index_name", f"idx_{label}_{property_name}") + safe_label = sanitize_identifier(label, "label") + safe_property = sanitize_identifier(property_name, "property key") + index_name = sanitize_identifier( + options.get("index_name", f"idx_{safe_label}_{safe_property}"), + "index name", + ) if index_type == "fulltext": query = f""" CREATE FULLTEXT INDEX {index_name} IF NOT EXISTS - FOR (n:{label}) ON EACH [n.{property_name}] + FOR (n:{safe_label}) ON EACH [n.{safe_property}] """ else: query = f""" CREATE INDEX {index_name} IF NOT EXISTS - FOR (n:{label}) ON (n.{property_name}) + FOR (n:{safe_label}) ON (n.{safe_property}) """ with self.get_session() as session: diff --git a/semantica/graph_store/query_sanitize.py b/semantica/graph_store/query_sanitize.py new file mode 100644 index 00000000..008ad098 --- /dev/null +++ b/semantica/graph_store/query_sanitize.py @@ -0,0 +1,32 @@ +""" +Shared identifier validation for Cypher/SPARQL query builders. + +Node labels, relationship types, and property keys can't be bound as query +parameters the way values can, so any such identifier that reaches a query +string unvalidated is a direct injection point (GHSA-482h-hw99-h62p). +`age_store.py` already validates its labels/relationship types this way; +this module generalizes that pattern for reuse across the other graph +store backends without introducing an import cycle with `graph_store.py` +or `methods.py`. +""" + +import re + +from ..utils.exceptions import ValidationError + +_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +def sanitize_identifier(name: str, kind: str = "identifier") -> str: + """Validate a Cypher/SPARQL label, relationship type, or property key. + + Only alphanumeric/underscore identifiers starting with a letter or + underscore are allowed. + """ + if not isinstance(name, str) or not _IDENTIFIER_RE.match(name): + raise ValidationError( + f"Invalid {kind}: {name!r}. Must start with a letter or " + "underscore and contain only alphanumeric characters and " + "underscores." + ) + return name diff --git a/tests/graph_store/test_cypher_injection.py b/tests/graph_store/test_cypher_injection.py new file mode 100644 index 00000000..d9b5fa23 --- /dev/null +++ b/tests/graph_store/test_cypher_injection.py @@ -0,0 +1,396 @@ +"""Regression tests for GHSA-482h-hw99-h62p: unvalidated node labels and +property keys allowed arbitrary Cypher injection in the Neptune, Neo4j, and +FalkorDB graph stores (labels/keys can't be bound as query parameters, so +an unvalidated value reaching the query string is a direct injection +point). + +Mirrors the advisory's own PoC shape: a label/key crafted to close the +current Cypher token early and append a destructive statement +(`DETACH DELETE victim`). Before the fix, these reached `_run_query` / +`session.run` / `graph.query` verbatim. After the fix, `sanitize_identifier` +(graph_store/query_sanitize.py) rejects them with ValidationError before +any query is built, matching the existing age_store.py `_sanitize_label` +behavior used as the reference implementation. +""" + +import pytest +import unittest +from unittest.mock import MagicMock + +from semantica.graph_store.query_sanitize import sanitize_identifier +from semantica.utils.exceptions import ProcessingError, ValidationError + +# AmazonNeptuneStore/Neo4jStore/FalkorDBStore.create_node() wrap their whole +# body in `except Exception: raise ProcessingError(...)` (pre-existing, +# unrelated to this fix), so the ValidationError sanitize_identifier raises +# surfaces to callers as ProcessingError. Either way the malicious query is +# never built or sent — these tests assert exactly that via the query-capture +# stubs, and check the wrapped message to confirm it's the sanitizer firing. + +EVIL_LABEL = "N}) MATCH (victim) DETACH DELETE victim //" +EVIL_KEY = "k1`: 1}) MATCH (victim) DETACH DELETE victim //" + + +def _wire(store): + store.logger = MagicMock() + store.progress_tracker = MagicMock() + store.progress_tracker.start_tracking.return_value = "tid" + store.config = {} + return store + + +class TestSanitizeIdentifier(unittest.TestCase): + def test_valid_identifiers_pass_through_unchanged(self): + self.assertEqual(sanitize_identifier("Person"), "Person") + self.assertEqual(sanitize_identifier("_hidden"), "_hidden") + self.assertEqual(sanitize_identifier("Rel_Type2"), "Rel_Type2") + + def test_injection_payload_is_rejected(self): + with self.assertRaises(ValidationError): + sanitize_identifier(EVIL_LABEL) + + def test_property_key_injection_payload_is_rejected(self): + with self.assertRaises(ValidationError): + sanitize_identifier(EVIL_KEY) + + def test_rejects_non_string(self): + with self.assertRaises(ValidationError): + sanitize_identifier(123) # type: ignore[arg-type] + + def test_rejects_spaces_and_dashes(self): + with self.assertRaises(ValidationError): + sanitize_identifier("no spaces") + with self.assertRaises(ValidationError): + sanitize_identifier("no-dashes") + + +class TestAmazonNeptuneCypherInjection(unittest.TestCase): + def _make_store(self): + from semantica.graph_store.amazon_neptune import AmazonNeptuneStore + + store = _wire(AmazonNeptuneStore.__new__(AmazonNeptuneStore)) + store._connected = True + store._ensure_connected = lambda: None + store._generate_id = lambda: "generated-id" + store._run_query = MagicMock(return_value=[]) + store._parse_results = lambda r: [] + return store + + def test_create_node_rejects_malicious_label_before_querying(self): + store = self._make_store() + with self.assertRaises(ProcessingError) as ctx: + store.create_node(labels=[EVIL_LABEL], properties={"name": "x"}) + self.assertIn("Invalid label", str(ctx.exception)) + store._run_query.assert_not_called() + + def test_create_node_rejects_malicious_property_key_before_querying(self): + store = self._make_store() + with self.assertRaises(ProcessingError) as ctx: + store.create_node(labels=["Person"], properties={"name": "x", EVIL_KEY: 1}) + self.assertIn("Invalid property key", str(ctx.exception)) + store._run_query.assert_not_called() + + def test_create_node_with_legitimate_labels_still_works(self): + store = self._make_store() + store.create_node(labels=["Person", "Employee"], properties={"name": "Alice"}) + query = store._run_query.call_args[0][0] + self.assertIn("Person:Employee", query) + self.assertNotIn("DETACH DELETE", query) + + +class TestNeo4jCypherInjection(unittest.TestCase): + def _make_store(self): + from semantica.graph_store import neo4j_store as m + + store = _wire(m.Neo4jStore.__new__(m.Neo4jStore)) + captured = {} + + class Session: + def run(self, q, params=None): + captured["query"] = q + rec = {"n": {"name": "x"}, "id": 1} + return type("R", (), {"single": lambda self: rec})() + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + store.get_session = lambda: Session() + store._captured = captured + return store + + def test_create_node_rejects_malicious_label_before_querying(self): + store = self._make_store() + with self.assertRaises(ProcessingError) as ctx: + store.create_node(labels=["Person", EVIL_LABEL], properties={"name": "x"}) + self.assertIn("Invalid label", str(ctx.exception)) + self.assertNotIn("query", store._captured) + + def test_create_relationship_rejects_malicious_rel_type(self): + store = self._make_store() + with self.assertRaises(ProcessingError) as ctx: + store.create_relationship(start_node_id=1, end_node_id=2, rel_type=EVIL_LABEL) + self.assertIn("Invalid relationship type", str(ctx.exception)) + self.assertNotIn("query", store._captured) + + +class TestFalkorDBCypherInjection(unittest.TestCase): + def _make_store(self): + from semantica.graph_store import falkordb_store as m + + store = _wire(m.FalkorDBStore.__new__(m.FalkorDBStore)) + captured = {} + + class Graph: + def query(self, q, params=None): + captured["query"] = q + return type("R", (), {"result_set": []})() + + store._ensure_graph = lambda: Graph() + store._captured = captured + return store + + def test_create_node_rejects_malicious_label_and_key(self): + store = self._make_store() + with self.assertRaises(ProcessingError) as ctx: + store.create_node(labels=[EVIL_LABEL], properties={"name": "x", EVIL_KEY: 1}) + self.assertIn("Invalid label", str(ctx.exception)) + self.assertNotIn("query", store._captured) + + def test_create_node_with_legitimate_input_still_works(self): + store = self._make_store() + store.create_node(labels=["Person"], properties={"name": "Alice"}) + query = store._captured["query"] + self.assertIn("Person", query) + self.assertNotIn("DETACH DELETE", query) + + +# --------------------------------------------------------------------------- +# BLOCKER 1 regression: Neo4jStore.get_neighbors / shortest_path depth coercion +# --------------------------------------------------------------------------- +# Payload representative of the confirmed injection (review BLOCKER 1): +# supplying a string as `depth` / `max_depth` previously reached the Cypher +# f-string verbatim because Neo4jStore did not call int() like Neptune/FalkorDB. +# +# After the fix `depth = int(depth)` / `max_depth = int(max_depth)` are added +# at the top of each method's try-block. A malicious string raises ValueError +# (wrapped in ProcessingError), and the session.run / _run_query mock must +# never be called. + +EVIL_DEPTH = "1]->(x) DETACH DELETE x //" + + +class TestNeo4jDepthInjection(unittest.TestCase): + """Regression tests for BLOCKER 1: Neo4jStore depth/max_depth coercion.""" + + def _make_store(self): + """Build a Neo4jStore with a session that records every query string sent to it.""" + from semantica.graph_store import neo4j_store as m + + store = _wire(m.Neo4jStore.__new__(m.Neo4jStore)) + captured = {} + + class IterSession: + """Returns an empty iterator so get_neighbors' for-loop completes cleanly.""" + def run(self, q, params=None): + captured["query"] = q + return iter([]) + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + class SingleSession: + """Returns a FakeResult whose .single() yields None (shortest_path).""" + def run(self, q, params=None): + captured["query"] = q + return type("R", (), {"single": lambda self: None})() + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + store._captured = captured + store._iter_session = IterSession + store._single_session = SingleSession + return store + + # -- get_neighbors -------------------------------------------------------- + + def test_get_neighbors_malicious_depth_raises_before_query(self): + """Malicious string depth must never reach session.run.""" + store = self._make_store() + store.get_session = lambda: store._iter_session() + with self.assertRaises(ProcessingError): + store.get_neighbors(node_id=1, depth=EVIL_DEPTH) + self.assertNotIn("query", store._captured, + "session.run was called — injected query reached the database layer") + + def test_get_neighbors_malicious_depth_does_not_contain_payload(self): + """Double-check: if somehow a query were built, it must not contain the payload.""" + store = self._make_store() + store.get_session = lambda: store._iter_session() + with pytest.raises(ProcessingError): + store.get_neighbors(node_id=1, depth=EVIL_DEPTH) + query = store._captured.get("query", "") + self.assertNotIn("DETACH DELETE", query, + f"Injection payload found in query: {query!r}") + + def test_get_neighbors_legitimate_depth_works(self): + """Valid integer depth must still produce a correct traversal pattern.""" + store = self._make_store() + store.get_session = lambda: store._iter_session() + result = store.get_neighbors(node_id=1, depth=2) + self.assertIsInstance(result, list) + self.assertIn("query", store._captured) + self.assertIn("*1..2", store._captured["query"]) + self.assertNotIn("DETACH DELETE", store._captured["query"]) + + def test_get_neighbors_depth_string_int_is_coerced(self): + """A string representation of a valid integer must be coerced and work.""" + store = self._make_store() + store.get_session = lambda: store._iter_session() + result = store.get_neighbors(node_id=1, depth="3") + self.assertIsInstance(result, list) + self.assertIn("*1..3", store._captured["query"]) + + # -- shortest_path -------------------------------------------------------- + + def test_shortest_path_malicious_max_depth_raises_before_query(self): + """Malicious string max_depth must never reach session.run.""" + store = self._make_store() + store.get_session = lambda: store._single_session() + with self.assertRaises(ProcessingError): + store.shortest_path(start_node_id=1, end_node_id=2, max_depth=EVIL_DEPTH) + self.assertNotIn("query", store._captured, + "session.run was called — injected query reached the database layer") + + def test_shortest_path_malicious_max_depth_does_not_contain_payload(self): + store = self._make_store() + store.get_session = lambda: store._single_session() + with pytest.raises(ProcessingError): + store.shortest_path(start_node_id=1, end_node_id=2, max_depth=EVIL_DEPTH) + query = store._captured.get("query", "") + self.assertNotIn("DETACH DELETE", query, + f"Injection payload found in query: {query!r}") + + def test_shortest_path_legitimate_max_depth_works(self): + """Valid integer max_depth must produce a correct shortestPath pattern.""" + store = self._make_store() + store.get_session = lambda: store._single_session() + result = store.shortest_path(start_node_id=1, end_node_id=2, max_depth=5) + self.assertIsNone(result) # single() returns None → correct + self.assertIn("query", store._captured) + self.assertIn("*..5", store._captured["query"]) + self.assertNotIn("DETACH DELETE", store._captured["query"]) + + def test_shortest_path_max_depth_string_int_is_coerced(self): + store = self._make_store() + store.get_session = lambda: store._single_session() + store.shortest_path(start_node_id=1, end_node_id=2, max_depth="7") + self.assertIn("*..7", store._captured["query"]) + + +# --------------------------------------------------------------------------- +# BLOCKER 2 regression: GraphStore.get_neighbors hops forwarding +# --------------------------------------------------------------------------- +# Before the fix, GraphStore.get_neighbors() set: +# actual_depth = options.get("hops", depth) +# and forwarded the raw value to Neo4jStore.get_neighbors(depth=actual_depth). +# Because Neo4jStore did not coerce depth, an attacker-controlled hops string +# reached Cypher verbatim. +# +# The fix adds int() at the GraphStore facade: +# actual_depth = int(options.get("hops", depth)) +# This closes the path regardless of which backend is wired up. + +class TestGraphStoreHopsForwarding(unittest.TestCase): + """Regression tests for BLOCKER 2: GraphStore hops→depth forwarding.""" + + def _make_graph_store_with_neo4j(self): + """ + Wire a GraphStore whose backend is a Neo4j store stub that records every + query passed to session.run. Returns (graph_store, captured_dict). + """ + from semantica.graph_store import neo4j_store as m + from semantica.graph_store.graph_store import ( + GraphAnalytics, + GraphManager, + GraphStore, + ) + + captured = {} + + class IterSession: + def run(self, q, params=None): + captured["query"] = q + return iter([]) + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + neo4j_stub = _wire(m.Neo4jStore.__new__(m.Neo4jStore)) + neo4j_stub.get_session = lambda: IterSession() + + gs = GraphStore.__new__(GraphStore) + gs.logger = MagicMock() + gs.progress_tracker = MagicMock() + gs._store_backend = neo4j_stub + gs._manager = GraphManager(neo4j_stub) + + return gs, captured + + # -- hops injection ------------------------------------------------------- + + def test_hops_malicious_string_raises_before_query(self): + """Malicious hops string must be rejected before session.run is reached.""" + gs, captured = self._make_graph_store_with_neo4j() + with self.assertRaises(Exception): + gs.get_neighbors(node_id=1, hops=EVIL_DEPTH) + self.assertNotIn("query", captured, + "session.run was called — injected hops reached the database layer") + + def test_hops_malicious_string_payload_not_in_any_query(self): + """Belt-and-suspenders: payload text must not appear in any built query.""" + gs, captured = self._make_graph_store_with_neo4j() + with pytest.raises(ValueError): + gs.get_neighbors(node_id=1, hops=EVIL_DEPTH) + query = captured.get("query", "") + self.assertNotIn("DETACH DELETE", query, + f"Injection payload found in forwarded query: {query!r}") + + def test_hops_legitimate_integer_works(self): + """Valid integer hops value must produce a correct query.""" + gs, captured = self._make_graph_store_with_neo4j() + result = gs.get_neighbors(node_id=1, hops=2) + self.assertIsInstance(result, list) + self.assertIn("query", captured) + self.assertIn("*1..2", captured["query"]) + self.assertNotIn("DETACH DELETE", captured["query"]) + + def test_hops_string_int_is_coerced_and_works(self): + """String '3' forwarded as hops must be coerced to int and produce *1..3.""" + gs, captured = self._make_graph_store_with_neo4j() + result = gs.get_neighbors(node_id=1, hops="3") + self.assertIsInstance(result, list) + self.assertIn("*1..3", captured["query"]) + + def test_depth_param_still_works_without_hops(self): + """depth positional arg (no hops kwarg) must still be coerced and forwarded.""" + gs, captured = self._make_graph_store_with_neo4j() + result = gs.get_neighbors(node_id=1, depth=4) + self.assertIsInstance(result, list) + self.assertIn("*1..4", captured["query"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_graph_store.py b/tests/test_graph_store.py index 26e40581..0576f077 100644 --- a/tests/test_graph_store.py +++ b/tests/test_graph_store.py @@ -215,6 +215,26 @@ class TestGraphStore(unittest.TestCase): result = self.store.execute_query("MATCH (n) RETURN n") self.assertEqual(result["summary"], "Mock query executed") + def test_degree_centrality_rejects_malicious_label(self): + """Regression test for GHSA-482h-hw99-h62p: degree_centrality() + interpolates labels/rel_type directly into a Cypher MATCH clause + (graph_store.py's own query builder, not delegated to the backend), + so an unvalidated label was a direct injection point.""" + from semantica.utils.exceptions import ValidationError + evil_label = "N}) MATCH (victim) DETACH DELETE victim //" + with self.assertRaises(ValidationError): + self.store._manager.analytics.degree_centrality(labels=[evil_label]) + + def test_degree_centrality_rejects_malicious_rel_type(self): + from semantica.utils.exceptions import ValidationError + evil_rel_type = "R]-() DETACH DELETE n //" + with self.assertRaises(ValidationError): + self.store._manager.analytics.degree_centrality(rel_type=evil_rel_type) + + def test_degree_centrality_with_legitimate_input_still_works(self): + result = self.store._manager.analytics.degree_centrality(labels=["Person"]) + self.assertEqual(result, []) # MockGraphStore.execute_query returns no records + class TestGraphStoreInitialization(unittest.TestCase): def test_falkordb_initialization(self): with patch('semantica.graph_store.falkordb_store.FalkorDBStore', side_effect=MockGraphStore) as mock_falkor: diff --git a/tests/test_graph_store_methods.py b/tests/test_graph_store_methods.py index 23cadfe1..b288d680 100644 --- a/tests/test_graph_store_methods.py +++ b/tests/test_graph_store_methods.py @@ -57,5 +57,26 @@ class TestGraphStoreMethods(unittest.TestCase): # Verify self.mock_store.execute_query.assert_called_once_with(query, None) + def test_update_relationship_rejects_malicious_property_key(self): + """Regression test for GHSA-482h-hw99-h62p: update_relationship() + interpolates property keys directly into a Cypher SET clause + (methods.py's own query builder, not delegated to the backend + store), so an unvalidated key was a direct injection point.""" + from semantica.utils.exceptions import ValidationError + + evil_key = "x} MATCH (victim) DETACH DELETE victim //" + with self.assertRaises(ValidationError): + methods.update_relationship(1, {evil_key: "value"}) + self.mock_store.execute_query.assert_not_called() + + def test_update_relationship_with_legitimate_keys_still_works(self): + self.mock_store.execute_query.return_value = { + "records": [{"id": 1, "type": "KNOWS"}] + } + result = methods.update_relationship(1, {"weight": 0.5}) + query = self.mock_store.execute_query.call_args[0][0] + self.assertIn("r.weight = $weight", query) + self.assertNotIn("DETACH DELETE", query) + if __name__ == '__main__': unittest.main() \ No newline at end of file From 1c3ac66fd9d29b4d2ea73c5747e00fc01cd9e982 Mon Sep 17 00:00:00 2001 From: Sameer6305 Date: Tue, 11 Aug 2026 16:25:07 +0530 Subject: [PATCH 31/40] fix(rdf4j): preserve literal objects in delete_triplet --- semantica/triplet_store/rdf4j_store.py | 13 ++- tests/triplet_store/test_sparql_injection.py | 94 ++++++++++++++++++++ 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/semantica/triplet_store/rdf4j_store.py b/semantica/triplet_store/rdf4j_store.py index 594f6dd2..79a83320 100644 --- a/semantica/triplet_store/rdf4j_store.py +++ b/semantica/triplet_store/rdf4j_store.py @@ -484,11 +484,18 @@ class RDF4JStore: update_endpoint = self._get_update_endpoint() - # Use SPARQL DELETE + # Use SPARQL DELETE. + # subject/predicate must be IRIs — validate_uri enforces that and + # blocks injection through '>' or other SPARQL metacharacters. + # object can be an IRI *or* a literal, so it is routed through + # _format_object_for_ntriples (which internally calls validate_uri + # for URI-shaped values and escape_literal for strings), matching + # the same object-handling semantics used by the add path and by + # BlazegraphStore.delete_triplet (GHSA-8vgg-8mr4-r236 regression fix). subject = sparql_escaping.validate_uri(triplet.subject) predicate = sparql_escaping.validate_uri(triplet.predicate) - object_ = sparql_escaping.validate_uri(triplet.object) - query = f"DELETE DATA {{ <{subject}> <{predicate}> <{object_}> }}" + obj_str = self._format_object_for_ntriples(triplet) + query = f"DELETE DATA {{ <{subject}> <{predicate}> {obj_str} }}" try: response = requests.post( diff --git a/tests/triplet_store/test_sparql_injection.py b/tests/triplet_store/test_sparql_injection.py index d8bb0e48..30376804 100644 --- a/tests/triplet_store/test_sparql_injection.py +++ b/tests/triplet_store/test_sparql_injection.py @@ -101,6 +101,100 @@ class TestRDF4JSparqlInjection(unittest.TestCase): with self.assertRaises(ValidationError): store.delete_triplet(triplet) + # ------------------------------------------------------------------ + # Regression tests for the literal-object bug fixed after the + # adversarial review of PR #911: delete_triplet() previously called + # validate_uri(triplet.object) unconditionally, which rejected every + # non-URI object with ValidationError even though literal objects are + # perfectly legal in RDF. The fix routes the object through + # _format_object_for_ntriples so URI-valued objects are still validated + # while literal objects go through escape_literal unchanged. + # ------------------------------------------------------------------ + + def _make_connected_store_with_captured_query(self): + """Return (store, captured_dict) where captured['update'] is the + SPARQL update string passed to requests.post once delete_triplet + succeeds.""" + import requests as req_mod + from unittest.mock import MagicMock + + store = self._make_store() + store.connected = True + captured = {} + + mock_resp = MagicMock() + mock_resp.raise_for_status = MagicMock() + + def fake_post(url, **kwargs): + captured["update"] = kwargs.get("data", {}).get("update", "") + return mock_resp + + store._post = fake_post # not used directly; patch requests.post below + store._captured = captured + return store, captured + + def test_delete_triplet_literal_object_succeeds(self): + """A triplet with a plain-string literal object must delete without + raising ValidationError — the regression that prompted this fix.""" + store, captured = self._make_connected_store_with_captured_query() + + with patch("requests.post") as mock_post: + mock_post.return_value.__enter__ = lambda s: s + mock_post.return_value.raise_for_status = lambda: None + + result = store.delete_triplet( + Triplet(subject="http://s", predicate="http://p", object="Paris") + ) + + self.assertEqual(result, {"success": True}) + # Confirm query shape: object must be a quoted literal, not + query_sent = mock_post.call_args[1]["data"]["update"] + self.assertIn(" ", query_sent) + self.assertIn('"Paris"', query_sent) + self.assertNotIn("", query_sent) + self.assertNotIn("CLEAR ALL", query_sent) + + def test_delete_triplet_uri_object_still_works(self): + """A triplet whose object is a URI must still delete correctly.""" + store, _ = self._make_connected_store_with_captured_query() + + with patch("requests.post") as mock_post: + mock_post.return_value.raise_for_status = lambda: None + + result = store.delete_triplet( + Triplet(subject="http://s", predicate="http://p", object="http://o") + ) + + self.assertEqual(result, {"success": True}) + query_sent = mock_post.call_args[1]["data"]["update"] + self.assertIn(" ", query_sent) + self.assertNotIn("CLEAR ALL", query_sent) + + def test_delete_triplet_malicious_uri_object_rejected_before_post(self): + """A URI-shaped object containing '>' must be rejected by + _format_object_for_ntriples → validate_uri before requests.post + is ever called.""" + evil_obj = "http://evil.com/a>;CLEARALL" + store, _ = self._make_connected_store_with_captured_query() + + with patch("requests.post") as mock_post: + with self.assertRaises(ValidationError): + store.delete_triplet( + Triplet(subject="http://s", predicate="http://p", object=evil_obj) + ) + mock_post.assert_not_called() + + def test_delete_triplet_malicious_subject_still_rejected(self): + """Subject injection protection must remain intact after the fix.""" + store, _ = self._make_connected_store_with_captured_query() + + with patch("requests.post") as mock_post: + with self.assertRaises(ValidationError): + store.delete_triplet( + Triplet(subject=EVIL_SUBJECT, predicate="http://p", object="http://o") + ) + mock_post.assert_not_called() + def test_get_triplets_rejects_malicious_subject_filter(self): store = self._make_store() with self.assertRaises(ValidationError): From d507fda1b0a65cf4cc020e56b00eebd6e46a9b21 Mon Sep 17 00:00:00 2001 From: Sameer6305 Date: Tue, 11 Aug 2026 18:05:29 +0530 Subject: [PATCH 32/40] fix: resolve ReDoS in _PREFIX_DECL regex (CodeQL #1897) The _PREFIX_DECL pattern used \s* as a trailing quantifier after <[^>]*>. On inputs that start with ase< but contain no closing > (e.g. ase]* and \s*, causing polynomial backtracking against user-controlled SPARQL query input. Fix: - Replace ^\s* / \s+ / \s* with ^[ \t]* / [ \t]+ / [ \t]* so the leading/internal whitespace quantifiers only match horizontal whitespace (no overlap with the <[^>]*> IRI part). - Replace the ambiguous trailing \s* with [ \t]*(?:\n|$), which matches only horizontal whitespace followed by a hard line boundary. [^>]* and [ \t]* have disjoint character sets, eliminating the backtracking ambiguity entirely. - Add _SPARQL_MAX_QUERY_LEN = 10_000 guard at the top of _is_read_only_query as defence-in-depth: rejects oversized input before any regex work, bounding worst-case cost even if a future pattern change reintroduces ambiguity. Verified: ReDoS payload ase< + !< x 5000 completes in <1 ms. Normal PREFIX/BASE stripping and read-only query detection unchanged. Fixes: CodeQL py/polynomial-redos alert #1897 CWE: CWE-1333, CWE-730, CWE-400 --- semantica/explorer/routes/sparql.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/semantica/explorer/routes/sparql.py b/semantica/explorer/routes/sparql.py index 8271d38d..28d14829 100644 --- a/semantica/explorer/routes/sparql.py +++ b/semantica/explorer/routes/sparql.py @@ -51,8 +51,22 @@ _FORBIDDEN_KEYWORDS = re.compile( # the query. BASE declarations have no prefix name between the keyword and # the IRI (`BASE <...>`, vs. `PREFIX ex: <...>`), so the prefix-name token # is optional. +# +# ReDoS note: the original pattern ended with `<[^>]*>\s*` where the +# trailing `\s*` could overlap with the `[^>]*` character class on inputs +# that contain no closing `>`, causing polynomial backtracking (CodeQL +# py/polynomial-redos, issue #1897). The fix replaces the ambiguous `\s*` +# suffix with `[ \t]*(?:\n|$)` which matches only horizontal whitespace +# followed by a hard line boundary, so there is no character-class overlap +# and the engine cannot split the match in multiple ways. _COMMENT_LINE = re.compile(r"(?:^|(?<=\s))#[^\n]*", re.MULTILINE) -_PREFIX_DECL = re.compile(r"^\s*(?:PREFIX\s+\S+|BASE)\s*<[^>]*>\s*", re.IGNORECASE | re.MULTILINE) +_PREFIX_DECL = re.compile( + r"^[ \t]*(?:PREFIX[ \t]+\S+|BASE)[ \t]*<[^>]*>[ \t]*(?:\n|$)", + re.IGNORECASE | re.MULTILINE, +) + + +_SPARQL_MAX_QUERY_LEN = 10_000 # chars; guards regex cost on uncontrolled input def _is_read_only_query(query: str) -> bool: @@ -63,6 +77,11 @@ def _is_read_only_query(query: str) -> bool: keywords anywhere in the body, preventing injection via embedded strings or multi-statement tricks. """ + # 0. Reject excessively long inputs before any regex work (defense-in-depth + # against ReDoS even if a future regex change reintroduces ambiguity). + if len(query) > _SPARQL_MAX_QUERY_LEN: + return False + # 1. Remove single-line comments that could hide the real query type cleaned = _COMMENT_LINE.sub("", query) # 2. Remove PREFIX/BASE declarations From c5981aa3068e4c2303a667efe8dcdbd14b46c77d Mon Sep 17 00:00:00 2001 From: Sameer6305 Date: Tue, 11 Aug 2026 18:37:50 +0530 Subject: [PATCH 33/40] fix: address qodo review findings on _PREFIX_DECL and query-length guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-up fixes to the initial ReDoS patch (CodeQL py/polynomial-redos #1897), raised during code review: --- Fix 1: _PREFIX_DECL regression — inline prologues and CRLF (#review-1) --- The first ReDoS fix replaced the ambiguous trailing \s* with [ \t]*(?:\n|$), but that introduced a behavioral regression: * Inline prologues — PREFIX ex: <...> SELECT ... on a single line were no longer stripped because the mandatory (?:\n|$) anchor never matched when non-whitespace content followed the IRI on the same line. * CRLF line endings — PREFIX ex: <...>\r\n failed because \r is not in [ \t]* and the anchor expected a bare \n. Root cause: the end-of-line anchor was unnecessary; the only thing needed to eliminate backtracking ambiguity is ensuring the IRI body character class and the trailing whitespace quantifier are disjoint. Fix: change the IRI body from <[^>]*> to <[^>\r\n]*>, which: - excludes CR and LF from the IRI match (semantically correct — SPARQL IRIs cannot span line boundaries) - makes [^>\r\n]* and the trailing [ \t]* have zero character overlap, eliminating all backtracking ambiguity without any end-of-line anchor No anchor is used, so both inline prologues and CRLF/LF endings work naturally. ReDoS payloads (base< + !< x 10,000) still complete in <1 ms. --- Fix 2: oversized-query length guard obscured error (#review-2) --- The initial patch placed the _SPARQL_MAX_QUERY_LEN guard inside _is_read_only_query(), which caused execute_sparql() to return the same generic 'Only SELECT' error for both genuinely disallowed query types and oversized inputs. Clients could not distinguish the two rejection reasons. Fix: move the length check out of _is_read_only_query() and into execute_sparql() as an explicit early gate, alongside the other resource limits (_SPARQL_MAX_ROWS, _SPARQL_MAX_GRAPH_NODES). Oversized queries now return a specific message naming the limit, the received length, and the remediation step. _is_read_only_query() is documented to be length-agnostic. _SPARQL_MAX_QUERY_LEN is relocated to the resource-limits block with the other constants. --- Tests added --- tests/test_security_regression.py: - test_inline_prefix_before_select_allowed (Fix 1 regression) - test_crlf_line_endings_with_prefix (Fix 1 regression) - test_crlf_multiple_prefixes_then_select (Fix 1 regression) - test_inline_prefix_before_insert_still_blocked (Fix 1 security check) - test_long_valid_query_not_rejected_by_is_read_only (Fix 2 separation) tests/explorer/test_sparql_route.py: - test_oversized_query_returns_distinct_length_error (Fix 2 error message) - test_oversized_query_never_touches_the_graph (Fix 2 short-circuit) - test_query_exactly_at_length_limit_is_accepted (Fix 2 boundary) All 82 tests pass. --- semantica/explorer/routes/sparql.py | 58 +++++++++++++++++++++-------- tests/explorer/test_sparql_route.py | 41 ++++++++++++++++++++ tests/test_security_regression.py | 37 ++++++++++++++++++ 3 files changed, 120 insertions(+), 16 deletions(-) diff --git a/semantica/explorer/routes/sparql.py b/semantica/explorer/routes/sparql.py index 28d14829..08c8af1d 100644 --- a/semantica/explorer/routes/sparql.py +++ b/semantica/explorer/routes/sparql.py @@ -52,23 +52,27 @@ _FORBIDDEN_KEYWORDS = re.compile( # the IRI (`BASE <...>`, vs. `PREFIX ex: <...>`), so the prefix-name token # is optional. # -# ReDoS note: the original pattern ended with `<[^>]*>\s*` where the -# trailing `\s*` could overlap with the `[^>]*` character class on inputs -# that contain no closing `>`, causing polynomial backtracking (CodeQL -# py/polynomial-redos, issue #1897). The fix replaces the ambiguous `\s*` -# suffix with `[ \t]*(?:\n|$)` which matches only horizontal whitespace -# followed by a hard line boundary, so there is no character-class overlap -# and the engine cannot split the match in multiple ways. +# ReDoS fix (CodeQL py/polynomial-redos, issue #1897): +# +# The original pattern `<[^>]*>\s*` was vulnerable because `\s*` (which +# matches newlines) could overlap with `[^>]*` on inputs that contain no +# closing `>` (e.g. `base\r\n]*>` for the IRI body: excluding CR and LF from +# the character class means the IRI match can never span a line boundary, +# and the disjoint trailing `[ \t]*` (horizontal whitespace only) has zero +# character-class overlap with `[^>\r\n]*`, so the engine has exactly one +# way to match. No end-of-line anchor is needed or used, which correctly +# handles both inline prologues (`PREFIX ex: <...> SELECT ...` on one line) +# and CRLF line endings (`\r\n`) without any special casing. _COMMENT_LINE = re.compile(r"(?:^|(?<=\s))#[^\n]*", re.MULTILINE) _PREFIX_DECL = re.compile( - r"^[ \t]*(?:PREFIX[ \t]+\S+|BASE)[ \t]*<[^>]*>[ \t]*(?:\n|$)", + r"^[ \t]*(?:PREFIX[ \t]+\S+|BASE)[ \t]*<[^>\r\n]*>[ \t]*", re.IGNORECASE | re.MULTILINE, ) -_SPARQL_MAX_QUERY_LEN = 10_000 # chars; guards regex cost on uncontrolled input - - def _is_read_only_query(query: str) -> bool: """Return True only for genuine read-only SPARQL queries. @@ -76,12 +80,11 @@ def _is_read_only_query(query: str) -> bool: checking the first keyword. Also rejects queries containing SPARQL Update keywords anywhere in the body, preventing injection via embedded strings or multi-statement tricks. - """ - # 0. Reject excessively long inputs before any regex work (defense-in-depth - # against ReDoS even if a future regex change reintroduces ambiguity). - if len(query) > _SPARQL_MAX_QUERY_LEN: - return False + Note: callers are responsible for enforcing any input-length limit *before* + calling this function so that an oversized-query rejection can be surfaced + as a distinct, actionable error rather than the generic read-only message. + """ # 1. Remove single-line comments that could hide the real query type cleaned = _COMMENT_LINE.sub("", query) # 2. Remove PREFIX/BASE declarations @@ -175,6 +178,11 @@ _SPARQL_MAX_ROWS = 5_000 # hard cap on returned rows _SPARQL_TIMEOUT_S = 30 # seconds before abandoning the await _SPARQL_MAX_CONCURRENT = 4 # semaphore: max simultaneous executions _SPARQL_MAX_GRAPH_NODES = 50_000 # cap on graph nodes/edges to prevent OOM +# Defense-in-depth against ReDoS: reject inputs longer than this before any +# regex work so that even a future regex regression is bounded. Checked in +# execute_sparql() (not inside _is_read_only_query) so the route can return +# a distinct, actionable error message rather than the generic read-only one. +_SPARQL_MAX_QUERY_LEN = 10_000 # chars # Semaphore caps how many graph.query calls run concurrently so that # timed-out threads (which keep running in the pool) cannot crowd out @@ -203,6 +211,24 @@ async def execute_sparql( req: SparqlRequest, session: GraphSession = Depends(get_session), ): + # Resource-limit check: reject oversized queries before any regex work. + # This is intentionally a separate, earlier check from _is_read_only_query + # so clients receive a specific, actionable message rather than the generic + # read-only rejection, and operators can tune _SPARQL_MAX_QUERY_LEN without + # touching query-semantics code. + if len(req.query) > _SPARQL_MAX_QUERY_LEN: + return SparqlResponse( + columns=[], + rows=[], + total=0, + error=( + f"Query exceeds the maximum allowed length of " + f"{_SPARQL_MAX_QUERY_LEN:,} characters " + f"({len(req.query):,} received). " + f"Please shorten your query." + ), + ) + if not _is_read_only_query(req.query): return SparqlResponse( columns=[], diff --git a/tests/explorer/test_sparql_route.py b/tests/explorer/test_sparql_route.py index 1d8547c6..1f619aa7 100644 --- a/tests/explorer/test_sparql_route.py +++ b/tests/explorer/test_sparql_route.py @@ -317,6 +317,47 @@ def test_oversized_graph_returns_clean_error_not_a_crash(client): assert payload["rows"] == [] +def test_oversized_query_returns_distinct_length_error(client): + """A query exceeding _SPARQL_MAX_QUERY_LEN must be rejected with a + specific, actionable error message — not the generic read-only message. + Clients need to distinguish a size-limit rejection from an actual + non-read-only query rejection to react correctly (e.g. split the query + vs. rewrite it).""" + with patch.object(sparql_mod, "_SPARQL_MAX_QUERY_LEN", 10): + resp = _post(client, "SELECT ?s WHERE { ?s ?p ?o }") # 30 chars > 10 + assert resp.status_code == 200 + payload = resp.json() + assert payload["error"] is not None + # Must mention the limit, not the generic read-only message + assert "length" in payload["error"].lower() or "characters" in payload["error"].lower() + assert "Only SELECT" not in payload["error"] + assert payload["rows"] == [] + assert payload["columns"] == [] + assert payload["total"] == 0 + + +def test_oversized_query_never_touches_the_graph(client): + """An oversized query must be rejected before _build_rdflib_graph is + called — the length guard must short-circuit the entire pipeline.""" + with patch.object(sparql_mod, "_SPARQL_MAX_QUERY_LEN", 10): + with patch.object(sparql_mod, "_build_rdflib_graph") as mock_build: + resp = _post(client, "SELECT ?s WHERE { ?s ?p ?o }") + assert resp.status_code == 200 + assert resp.json()["error"] is not None + mock_build.assert_not_called() + + +def test_query_exactly_at_length_limit_is_accepted(client): + """A query whose length equals the limit exactly must not be rejected — + the guard is strictly greater-than, not greater-than-or-equal.""" + short_query = "ASK {}" + with patch.object(sparql_mod, "_SPARQL_MAX_QUERY_LEN", len(short_query)): + resp = _post(client, short_query) + assert resp.status_code == 200 + payload = resp.json() + assert payload["error"] is None + + # --------------------------------------------------------------------------- # Data-mapping fidelity: does the graph->RDF projection reflect session state? # --------------------------------------------------------------------------- diff --git a/tests/test_security_regression.py b/tests/test_security_regression.py index 18854f0b..56ad46de 100644 --- a/tests/test_security_regression.py +++ b/tests/test_security_regression.py @@ -123,6 +123,43 @@ class TestSparqlReadOnlyValidation: ) assert _is_read_only_query(query) + def test_inline_prefix_before_select_allowed(self): + """PREFIX declaration on the same line as the query keyword (inline + prologue) must be stripped correctly so SELECT is seen first. + Regression for the [ \\t]*(?:\\n|$) anchor that rejected this form.""" + query = "PREFIX ex: SELECT ?s WHERE { ?s ex:p ?o }" + assert _is_read_only_query(query) + + def test_crlf_line_endings_with_prefix(self): + """Windows CRLF line endings (\\r\\n) between PREFIX and SELECT must + be handled correctly. The previous (?:\\n|$) anchor did not allow + the \\r before \\n, causing stripping to fail.""" + query = "PREFIX ex: \r\nSELECT ?s WHERE { ?s ?p ?o }" + assert _is_read_only_query(query) + + def test_crlf_multiple_prefixes_then_select(self): + """Multiple PREFIX lines with CRLF endings should all be stripped.""" + query = ( + "PREFIX rdf: \r\n" + "PREFIX ex: \r\n" + "SELECT ?s WHERE { ?s rdf:type ex:Thing }" + ) + assert _is_read_only_query(query) + + def test_inline_prefix_before_insert_still_blocked(self): + """An inline PREFIX followed by INSERT must not be allowed — the + inline stripping fix must not open a bypass for write operations.""" + query = "PREFIX ex: INSERT DATA { ex:s ex:p ex:o }" + assert not _is_read_only_query(query) + + def test_long_valid_query_not_rejected_by_is_read_only(self): + """_is_read_only_query must not enforce the length limit itself — + that responsibility belongs to execute_sparql() so the route can + return a distinct, actionable error. A syntactically valid but long + SELECT query must still return True from this function.""" + long_query = "SELECT ?s WHERE { ?s ?p ?o } # " + ("x" * 20_000) + assert _is_read_only_query(long_query) + # =================================================================== # 2. Cypher injection prevention From 646c70ce6393861c1b7a12a5a111d29db5006cc4 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Tue, 11 Aug 2026 18:52:26 +0530 Subject: [PATCH 34/40] security: DNS check-then-use pinning for SSRF fetcher, close object-IRI gap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-up hardening items flagged as secondary/deferred during GHSA-8c7v-62gr-hj6g and GHSA-8vgg-8mr4-r236's fixes: 1. DNS check-then-use (TOCTOU) window in the ontology URL fetcher. _validate_fetch_url() resolved and validated a hostname once, but _fetch_url_sync() then let requests resolve the same hostname again independently at connect time — a low-TTL or rebinding DNS answer could differ between the two lookups, reopening the SSRF window the validation exists to close. _validate_fetch_url() now returns the validated IP, and a new _make_pinned_session() builds a per-hop requests.Session whose connection pool is pinned directly to that IP (bypassing DNS resolution for the connection entirely), while explicitly restoring the real hostname as the outgoing HTTP Host header and, for HTTPS, the TLS SNI server_hostname/assert_hostname — so the connection reaches the validated IP but still presents (and is verified against) the real hostname's identity, keeping virtual hosting and certificate validation correct. Note: an earlier version of this fix set `_dns_host` post-construction assuming it was decoupled from `host`, matching some other urllib3 releases; in the installed version (2.7.0), `host` is a property that reads/writes `_dns_host` directly, so that approach silently changed the Host header too. Verified with a real (non-mocked) local HTTP server, a real local HTTPS server with a self-signed cert (proving SNI/cert-hostname verification checks the real hostname, not the pinned IP), and a negative control confirming a hostname/cert mismatch is still correctly rejected — not silently bypassed. 2. Pre-wrapped object IRIs skipped full validation in _format_object_for_sparql/_format_object_for_ntriples (Blazegraph, RDF4J). A triplet object already wrapped in `<...>` only had its inner content checked for a literal space or `>`, not run through sparql_escaping.validate_uri() like the unwrapped-object branch — flagged by automated review during GHSA-8vgg-8mr4-r236's fix. Both branches now validate identically. Tests: tests/explorer/test_ontology_dns_pinning.py (6 tests, including 2 real local-server end-to-end checks and 2 real-TLS checks with a generated self-signed cert, gracefully skipped if `cryptography` isn't installed); updated tests/explorer/test_ontology_ssrf.py for the new per-hop session construction; 4 new tests in tests/triplet_store/test_sparql_injection.py for the object-IRI fix. Full explorer + triplet_store suite: 566 passed. --- semantica/explorer/routes/ontology.py | 143 +++++++--- semantica/triplet_store/blazegraph_store.py | 10 +- semantica/triplet_store/rdf4j_store.py | 10 +- tests/explorer/test_ontology_dns_pinning.py | 281 +++++++++++++++++++ tests/explorer/test_ontology_ssrf.py | 32 ++- tests/triplet_store/test_sparql_injection.py | 30 ++ 6 files changed, 459 insertions(+), 47 deletions(-) create mode 100644 tests/explorer/test_ontology_dns_pinning.py diff --git a/semantica/explorer/routes/ontology.py b/semantica/explorer/routes/ontology.py index 8b0c8aa9..3ac81b0a 100644 --- a/semantica/explorer/routes/ontology.py +++ b/semantica/explorer/routes/ontology.py @@ -978,8 +978,15 @@ def _normalize_format(fmt: Optional[str]) -> str: return _FORMAT_ALIASES.get(lower, lower) -def _validate_fetch_url(url: str) -> None: - """Reject non-HTTP(S) schemes and private/loopback/link-local targets.""" +def _validate_fetch_url(url: str) -> str: + """Reject non-HTTP(S) schemes and private/loopback/link-local targets. + + Returns the first resolved, validated IP address so the caller can pin + the actual connection to it (see _PinnedIPHTTPAdapter) — resolving the + hostname again at connect time would open a DNS check-then-use window + (a low-TTL or rebinding DNS answer could differ between this check and + the client's own lookup). + """ parsed = urlparse(url) if parsed.scheme not in ("http", "https"): raise HTTPException(status_code=422, detail="Only http and https URLs are allowed.") @@ -990,6 +997,7 @@ def _validate_fetch_url(url: str) -> None: addrinfos = socket.getaddrinfo(hostname, None) except socket.gaierror as exc: raise HTTPException(status_code=422, detail=f"Cannot resolve hostname '{hostname}': {exc}") from exc + validated_ip: Optional[str] = None for _family, _type, _proto, _canonname, sockaddr in addrinfos: try: ip = ipaddress.ip_address(sockaddr[0]) @@ -1000,46 +1008,115 @@ def _validate_fetch_url(url: str) -> None: status_code=422, detail="Fetching from private, loopback, or reserved network addresses is not allowed.", ) + if validated_ip is None: + validated_ip = sockaddr[0] + if validated_ip is None: + raise HTTPException(status_code=422, detail=f"Cannot resolve hostname '{hostname}' to a usable address.") + return validated_ip + + +def _make_pinned_session(pinned_ip: str, url: str): + """Build a requests.Session whose connection is pinned to pinned_ip, + regardless of what url's hostname resolves to at connect time. + + _validate_fetch_url() resolves and validates the hostname once; letting + the HTTP client resolve it again independently at connect time reopens + the exact gap that validation exists to close — a low-TTL or rebinding + DNS answer can differ between the two lookups. This pins the pool's + connect target to the already-validated IP directly (bypassing DNS + resolution for the connection entirely), while keeping the original + hostname as the outgoing HTTP Host header and, for HTTPS, the TLS SNI + server_hostname / assert_hostname — otherwise the connection would + reach the right IP but present the wrong identity, breaking name-based + virtual hosting and (for HTTPS) certificate hostname verification. + + Note: urllib3's Connection.host is a property that reads/writes the + same underlying value as `_dns_host` in this version — it is NOT the + separate "presented identity" field it is in some older releases, so + overriding just `_dns_host` post-construction (as an earlier version of + this fix did) actually changes the Host header too. Pinning the pool's + `host` directly and restoring the real hostname via an explicit Host + header (+ SNI params for HTTPS) is the correct mechanism here. + """ + import requests as _req + + parsed = urlparse(url) + hostname = parsed.hostname + port = parsed.port + default_port = 443 if parsed.scheme == "https" else 80 + host_header = hostname if port in (None, default_port) else f"{hostname}:{port}" + + class _PinnedIPHTTPAdapter(_req.adapters.HTTPAdapter): + def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None): + # If an HTTP(S) proxy applies (env-configured or per-request), + # the actual TCP connection target is the proxy, not the + # resolved IP, and proxy tunneling changes the connection model + # enough that pinning doesn't apply cleanly. Fall back to the + # normal (unpinned) path rather than silently bypassing the + # proxy — _validate_fetch_url's destination check still applies + # either way; only this secondary DNS-pinning hardening is + # skipped. + if _req.utils.select_proxy(request.url, proxies): + return super().get_connection_with_tls_context( + request, verify, proxies=proxies, cert=cert + ) + host_params, pool_kwargs = self.build_connection_pool_key_attributes(request, verify, cert) + if host_params.get("scheme") == "https": + pool_kwargs.setdefault("assert_hostname", hostname) + pool_kwargs.setdefault("server_hostname", hostname) + host_params["host"] = pinned_ip + return self.poolmanager.connection_from_host(**host_params, pool_kwargs=pool_kwargs) + + session = _req.Session() + session.headers["Host"] = host_header + adapter = _PinnedIPHTTPAdapter() + session.mount("http://", adapter) + session.mount("https://", adapter) + return session def _fetch_url_sync(url: str) -> bytes: - _validate_fetch_url(url) - import requests as _req + pinned_ip = _validate_fetch_url(url) _MAX_REDIRECTS = 5 current_url = url try: for _ in range(_MAX_REDIRECTS + 1): - resp = _req.get( - current_url, - headers={"Accept": "text/turtle, application/rdf+xml, application/ld+json, */*;q=0.1"}, - timeout=30, - stream=True, - allow_redirects=False, # SECURITY: follow redirects manually - ) - if resp.is_redirect or resp.is_permanent_redirect: - redirect_url = resp.headers.get("Location") - resp.close() # Release the streamed connection before following the redirect - if not redirect_url: - raise HTTPException(status_code=502, detail="Redirect without Location header.") - # Resolve relative redirects (e.g. /ontology.ttl) against the current URL - redirect_url = urljoin(current_url, redirect_url) - # Re-validate the redirect target to prevent SSRF via - # open-redirect to internal/cloud-metadata endpoints. - _validate_fetch_url(redirect_url) - current_url = redirect_url - continue + session = _make_pinned_session(pinned_ip, current_url) try: - resp.raise_for_status() - chunks: List[bytes] = [] - total = 0 - for chunk in resp.iter_content(65536): - total += len(chunk) - if total > _MAX_FETCH_BYTES: - raise HTTPException(status_code=413, detail="Remote resource exceeds 20 MB limit.") - chunks.append(chunk) - return b"".join(chunks) + resp = session.get( + current_url, + headers={"Accept": "text/turtle, application/rdf+xml, application/ld+json, */*;q=0.1"}, + timeout=30, + stream=True, + allow_redirects=False, # SECURITY: follow redirects manually + ) + if resp.is_redirect or resp.is_permanent_redirect: + redirect_url = resp.headers.get("Location") + resp.close() # Release the streamed connection before following the redirect + if not redirect_url: + raise HTTPException(status_code=502, detail="Redirect without Location header.") + # Resolve relative redirects (e.g. /ontology.ttl) against the current URL + redirect_url = urljoin(current_url, redirect_url) + # Re-validate the redirect target to prevent SSRF via + # open-redirect to internal/cloud-metadata endpoints, and + # get a fresh pin for the new host. + pinned_ip = _validate_fetch_url(redirect_url) + current_url = redirect_url + continue + try: + resp.raise_for_status() + chunks: List[bytes] = [] + total = 0 + for chunk in resp.iter_content(65536): + total += len(chunk) + if total > _MAX_FETCH_BYTES: + raise HTTPException(status_code=413, detail="Remote resource exceeds 20 MB limit.") + chunks.append(chunk) + return b"".join(chunks) + finally: + resp.close() # Release the streamed connection once fully read (or on error) finally: - resp.close() # Release the streamed connection once fully read (or on error) + session.close() raise HTTPException(status_code=502, detail=f"Too many redirects (max {_MAX_REDIRECTS}).") except HTTPException: raise diff --git a/semantica/triplet_store/blazegraph_store.py b/semantica/triplet_store/blazegraph_store.py index 4e139a57..21e42b38 100644 --- a/semantica/triplet_store/blazegraph_store.py +++ b/semantica/triplet_store/blazegraph_store.py @@ -391,10 +391,12 @@ class BlazegraphStore: if self._is_uri_value(obj): if obj.startswith("<") and obj.endswith(">"): - inner = obj[1:-1] - if " " in inner or ">" in inner: - raise ValueError(f"IRI contains invalid characters: {obj!r}") - return obj + # Validate the inner IRI with the same disallowed-character + # set as the unwrapped branch below — a narrower ad-hoc + # check here previously let a pre-wrapped object bypass + # validate_uri() entirely (GHSA-8vgg-8mr4-r236 follow-up). + inner = sparql_escaping.validate_uri(obj[1:-1]) + return f"<{inner}>" validated_obj = sparql_escaping.validate_uri(obj) return f"<{validated_obj}>" diff --git a/semantica/triplet_store/rdf4j_store.py b/semantica/triplet_store/rdf4j_store.py index 79a83320..8dad6997 100644 --- a/semantica/triplet_store/rdf4j_store.py +++ b/semantica/triplet_store/rdf4j_store.py @@ -539,10 +539,12 @@ class RDF4JStore: if self._is_uri_value(obj): if obj.startswith("<") and obj.endswith(">"): - inner = obj[1:-1] - if " " in inner or ">" in inner: - raise ValueError(f"IRI contains invalid characters: {obj!r}") - return obj + # Validate the inner IRI with the same disallowed-character + # set as the unwrapped branch below — a narrower ad-hoc + # check here previously let a pre-wrapped object bypass + # validate_uri() entirely (GHSA-8vgg-8mr4-r236 follow-up). + inner = sparql_escaping.validate_uri(obj[1:-1]) + return f"<{inner}>" validated_obj = sparql_escaping.validate_uri(obj) return f"<{validated_obj}>" diff --git a/tests/explorer/test_ontology_dns_pinning.py b/tests/explorer/test_ontology_dns_pinning.py new file mode 100644 index 00000000..cb7791c6 --- /dev/null +++ b/tests/explorer/test_ontology_dns_pinning.py @@ -0,0 +1,281 @@ +"""Regression tests for DNS check-then-use (TOCTOU) hardening in the +ontology URL fetcher (GHSA-8c7v-62gr-hj6g's secondary "smaller" gap). + +`_validate_fetch_url` resolves and validates a hostname once; if the actual +HTTP client resolved it again independently at connect time, a low-TTL or +rebinding DNS answer could differ between the two lookups, reopening the +SSRF window the validation exists to close. `_make_pinned_session` closes +this by pinning the connection pool's `host` directly to the already- +validated IP (bypassing DNS resolution for the connection entirely), while +explicitly restoring the real hostname as the outgoing HTTP `Host` header +and, for HTTPS, the TLS SNI `server_hostname` / `assert_hostname` — so the +connection reaches the pinned IP but still presents (and verifies against) +the original hostname's identity. + +test_ontology_ssrf.py covers the redirect-handling logic around this with +mocks; this file proves the pinning mechanism itself works end-to-end +against real local servers, with no DNS mocking at all — the test hostname +is never resolved, which is exactly the property being verified. It also +includes a negative control (mismatched cert hostname) proving TLS +verification is genuinely enforced against the real hostname, not silently +bypassed or checked against the pinned IP instead. +""" + +import http.server +import socket +import threading + +import pytest + +from semantica.explorer.routes import ontology as ontology_mod + + +def _start_local_server(): + captured = {} + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + captured["host_header"] = self.headers.get("Host") + body = b"pinned response" + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args): + pass + + server = http.server.HTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, captured + + +def test_pinned_session_connects_to_pinned_ip_without_resolving_hostname(): + """A session built by _make_pinned_session must reach the pinned IP + directly. The request URL uses a hostname that cannot be resolved via + real DNS ('.invalid' is reserved by RFC 2606) — if pinning weren't + working, this request would fail with a name-resolution error instead + of reaching the local server, since nothing else could route it there. + """ + server, thread, captured = _start_local_server() + port = server.server_address[1] + url = f"http://pinned-test.invalid:{port}/resource" + try: + session = ontology_mod._make_pinned_session("127.0.0.1", url) + try: + resp = session.get(url, timeout=5) + assert resp.status_code == 200 + assert resp.content == b"pinned response" + finally: + session.close() + finally: + server.shutdown() + thread.join(timeout=2) + + # Host header must still be the original hostname, not the pinned IP — + # proving connection target and presented identity are decoupled + # correctly (this is what keeps virtual hosting / TLS SNI correct). + assert captured["host_header"] == f"pinned-test.invalid:{port}" + + +def test_pinned_session_ignores_a_different_real_resolution(): + """Even if the hostname *does* resolve to something else via real DNS, + the pinned session must still go to the pinned IP — this is the actual + TOCTOU property: the connection uses what was validated, not whatever + a fresh lookup returns. 'localhost' reliably resolves to a loopback + address, which is deliberately NOT where our test server listens on + (127.0.0.1 specifically) — but since Windows/most stacks map + 'localhost' to 127.0.0.1 too, use a distinct high loopback address + (127.0.0.2) for the server so a real 'localhost' resolution (127.0.0.1) + provably would NOT reach it, isolating the assertion to pinning alone. + """ + captured = {} + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + captured["host_header"] = self.headers.get("Host") + body = b"pinned via explicit ip" + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args): + pass + + try: + server = http.server.HTTPServer(("127.0.0.2", 0), Handler) + except OSError: + # 127.0.0.2 isn't bindable in this environment (uncommon, but + # possible in some sandboxes) — skip rather than false-fail. + import pytest + pytest.skip("127.0.0.2 is not bindable in this environment") + + port = server.server_address[1] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + url = f"http://localhost:{port}/resource" + try: + session = ontology_mod._make_pinned_session("127.0.0.2", url) + try: + resp = session.get(url, timeout=5) + assert resp.status_code == 200 + assert resp.content == b"pinned via explicit ip" + finally: + session.close() + finally: + server.shutdown() + thread.join(timeout=2) + + assert captured["host_header"] == f"localhost:{port}" + + +def test_validate_fetch_url_returns_the_resolved_ip(): + """_validate_fetch_url must return the IP it validated, so callers can + pin the connection to it.""" + def fake_getaddrinfo(host, *_a, **_k): + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0))] + + import unittest.mock as mock + with mock.patch.object(ontology_mod.socket, "getaddrinfo", side_effect=fake_getaddrinfo): + resolved_ip = ontology_mod._validate_fetch_url("http://example.org/ontology.ttl") + + assert resolved_ip == "93.184.216.34" + + +def test_validate_fetch_url_still_rejects_private_ip(): + """Confirm the pinning refactor didn't loosen the original address + classification — a hostname resolving to a private/internal address + must still be rejected before any IP is returned.""" + import ipaddress + import unittest.mock as mock + + import pytest + from fastapi import HTTPException + + def fake_getaddrinfo(host, *_a, **_k): + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("169.254.169.254", 0))] + + with mock.patch.object(ontology_mod.socket, "getaddrinfo", side_effect=fake_getaddrinfo): + with pytest.raises(HTTPException) as exc_info: + ontology_mod._validate_fetch_url("http://attacker.example/ontology.ttl") + + assert exc_info.value.status_code == 422 + + +# --------------------------------------------------------------------------- +# HTTPS: SNI + certificate hostname verification must use the real hostname, +# not the pinned IP — this is the highest-risk part of pinning to get wrong, +# since a mistake here could silently weaken TLS verification rather than +# just breaking connectivity. Requires the optional `cryptography` package +# to mint a throwaway self-signed cert; skipped gracefully without it. +# --------------------------------------------------------------------------- + +def _make_self_signed_cert(hostname: str, tmp_path): + import datetime + + cryptography = pytest.importorskip("cryptography") + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.x509.oid import NameOID + + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, hostname)]) + now = datetime.datetime.now(datetime.timezone.utc) + cert = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=1)) + .add_extension(x509.SubjectAlternativeName([x509.DNSName(hostname)]), critical=False) + .sign(key, hashes.SHA256()) + ) + + cert_path = tmp_path / "cert.pem" + key_path = tmp_path / "key.pem" + cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_path.write_bytes( + key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) + return str(cert_path), str(key_path) + + +def _start_local_https_server(cert_path, key_path): + import ssl + + captured = {} + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + captured["host_header"] = self.headers.get("Host") + body = b"tls pinned response" + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args): + pass + + server = http.server.HTTPServer(("127.0.0.1", 0), Handler) + ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ssl_ctx.load_cert_chain(cert_path, key_path) + server.socket = ssl_ctx.wrap_socket(server.socket, server_side=True) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, captured + + +def test_pinned_https_session_verifies_against_real_hostname_not_pinned_ip(tmp_path): + """A pinned HTTPS connection must present + verify SNI/cert against the + real hostname, even though the socket connects to the pinned IP. The + cert's SAN is the hostname, never '127.0.0.1' — if pinning verified + against the IP instead (or against nothing), this would either fail + for the wrong reason or silently succeed with no real verification.""" + cert_path, key_path = _make_self_signed_cert("pinned-tls-test.invalid", tmp_path) + server, thread, captured = _start_local_https_server(cert_path, key_path) + port = server.server_address[1] + url = f"https://pinned-tls-test.invalid:{port}/resource" + try: + session = ontology_mod._make_pinned_session("127.0.0.1", url) + try: + resp = session.get(url, timeout=5, verify=cert_path) + finally: + session.close() + finally: + server.shutdown() + thread.join(timeout=2) + + assert resp.status_code == 200 + assert resp.content == b"tls pinned response" + assert captured["host_header"] == f"pinned-tls-test.invalid:{port}" + + +def test_pinned_https_session_rejects_hostname_mismatch(tmp_path): + """Negative control: requesting a hostname that does NOT match the + cert's SAN must still fail verification — proving pinning doesn't + silently bypass or misdirect certificate hostname checking.""" + cert_path, key_path = _make_self_signed_cert("pinned-tls-test.invalid", tmp_path) + server, thread, _captured = _start_local_https_server(cert_path, key_path) + port = server.server_address[1] + url = f"https://wrong-name.invalid:{port}/resource" + try: + session = ontology_mod._make_pinned_session("127.0.0.1", url) + try: + import requests + with pytest.raises(requests.exceptions.SSLError): + session.get(url, timeout=5, verify=cert_path) + finally: + session.close() + finally: + server.shutdown() + thread.join(timeout=2) diff --git a/tests/explorer/test_ontology_ssrf.py b/tests/explorer/test_ontology_ssrf.py index fe9de35b..8613275c 100644 --- a/tests/explorer/test_ontology_ssrf.py +++ b/tests/explorer/test_ontology_ssrf.py @@ -3,12 +3,17 @@ `_fetch_url_sync` disables `requests`' automatic redirect following and re-validates every hop with `_validate_fetch_url` (see GHSA-8c7v-62gr-hj6g: unvalidated redirect targets previously let a public first hop 302 the -server into fetching cloud metadata / loopback services). +server into fetching cloud metadata / loopback services). It also pins each +hop's connection to the IP `_validate_fetch_url` already resolved and +validated, via `_make_pinned_session`, closing the DNS check-then-use gap +between that validation and the client's own (potentially different) lookup. These tests cover the redirect-handling logic itself: relative `Location` headers must resolve correctly instead of being rejected outright, redirect targets that resolve to private/loopback addresses must still be blocked, and every response must be closed (no leaked connections across hops). +`test_ontology_dns_pinning.py` covers the pinning mechanism +(`_make_pinned_session`, `_validate_fetch_url`'s returned IP) directly. """ import socket @@ -37,6 +42,17 @@ def _make_response(is_redirect=False, is_permanent=False, location=None, body=b" return resp +def _patch_session(responses): + """Patch _make_pinned_session so _fetch_url_sync's session.get(...) + calls return the given responses in order, without touching the real + requests.Session/pinning machinery (that's covered by test_pinning.py). + """ + fake_session = MagicMock() + fake_session.get = MagicMock(side_effect=responses) + fake_session.close = MagicMock() + return patch.object(ontology_mod, "_make_pinned_session", return_value=fake_session), fake_session + + @patch.object(ontology_mod.socket, "getaddrinfo", side_effect=_fake_getaddrinfo) def test_relative_redirect_location_is_resolved(mock_getaddrinfo): """A relative Location header (e.g. '/ontology.ttl') must resolve against @@ -44,11 +60,12 @@ def test_relative_redirect_location_is_resolved(mock_getaddrinfo): redirect_resp = _make_response(is_redirect=True, location="/ontology.ttl") final_resp = _make_response(body=b"final content") - with patch("requests.get", side_effect=[redirect_resp, final_resp]) as mock_get: + patcher, fake_session = _patch_session([redirect_resp, final_resp]) + with patcher: result = ontology_mod._fetch_url_sync("http://example.org/start") assert result == b"final content" - second_call_url = mock_get.call_args_list[1].args[0] + second_call_url = fake_session.get.call_args_list[1].args[0] assert second_call_url == "http://example.org/ontology.ttl" redirect_resp.close.assert_called_once() final_resp.close.assert_called_once() @@ -67,7 +84,8 @@ def test_redirect_to_private_ip_is_rejected(mock_getaddrinfo): mock_getaddrinfo.side_effect = getaddrinfo_side_effect redirect_resp = _make_response(is_redirect=True, location="http://internal.example/latest/meta-data/") - with patch("requests.get", side_effect=[redirect_resp]): + patcher, fake_session = _patch_session([redirect_resp]) + with patcher: with pytest.raises(ontology_mod.HTTPException) as exc_info: ontology_mod._fetch_url_sync("http://example.org/start") @@ -78,7 +96,8 @@ def test_redirect_to_private_ip_is_rejected(mock_getaddrinfo): @patch.object(ontology_mod.socket, "getaddrinfo", side_effect=_fake_getaddrinfo) def test_final_response_is_closed(mock_getaddrinfo): final_resp = _make_response(body=b"content") - with patch("requests.get", side_effect=[final_resp]): + patcher, _fake_session = _patch_session([final_resp]) + with patcher: ontology_mod._fetch_url_sync("http://example.org/start") final_resp.close.assert_called_once() @@ -86,7 +105,8 @@ def test_final_response_is_closed(mock_getaddrinfo): @patch.object(ontology_mod.socket, "getaddrinfo", side_effect=_fake_getaddrinfo) def test_redirect_chain_exceeding_cap_is_rejected(mock_getaddrinfo): responses = [_make_response(is_redirect=True, location=f"/hop{i}") for i in range(10)] - with patch("requests.get", side_effect=responses): + patcher, _fake_session = _patch_session(responses) + with patcher: with pytest.raises(ontology_mod.HTTPException) as exc_info: ontology_mod._fetch_url_sync("http://example.org/start") assert exc_info.value.status_code == 502 diff --git a/tests/triplet_store/test_sparql_injection.py b/tests/triplet_store/test_sparql_injection.py index 30376804..587ba613 100644 --- a/tests/triplet_store/test_sparql_injection.py +++ b/tests/triplet_store/test_sparql_injection.py @@ -82,6 +82,22 @@ class TestBlazegraphSparqlInjection(unittest.TestCase): self.assertIn(" ", insert_data) self.assertNotIn("CLEAR ALL", insert_data) + def test_format_object_rejects_malicious_pre_wrapped_iri(self): + """A caller-supplied object already wrapped in '<...>' must still be + fully validated, not just checked for a literal space/'>' — a + narrower ad-hoc check here previously let this branch bypass + validate_uri() entirely (Codex-flagged follow-up to GHSA-8vgg).""" + store = self._make_store() + evil_object = f"<{EVIL_SUBJECT}>" + triplet = Triplet(subject="http://s", predicate="http://p", object=evil_object) + with self.assertRaises(ValidationError): + store._format_object_for_sparql(triplet) + + def test_format_object_accepts_legitimate_pre_wrapped_iri(self): + store = self._make_store() + triplet = Triplet(subject="http://s", predicate="http://p", object="") + self.assertEqual(store._format_object_for_sparql(triplet), "") + class TestRDF4JSparqlInjection(unittest.TestCase): @patch.object(RDF4JStore, "_connect", autospec=True) @@ -207,6 +223,20 @@ class TestRDF4JSparqlInjection(unittest.TestCase): self.assertIn(" ", ntriples) self.assertNotIn("CLEAR ALL", ntriples) + def test_format_object_rejects_malicious_pre_wrapped_iri(self): + """Same pre-wrapped-object bypass as Blazegraph, fixed in + _format_object_for_ntriples.""" + store = self._make_store() + evil_object = f"<{EVIL_SUBJECT}>" + triplet = Triplet(subject="http://s", predicate="http://p", object=evil_object) + with self.assertRaises(ValidationError): + store._format_object_for_ntriples(triplet) + + def test_format_object_accepts_legitimate_pre_wrapped_iri(self): + store = self._make_store() + triplet = Triplet(subject="http://s", predicate="http://p", object="") + self.assertEqual(store._format_object_for_ntriples(triplet), "") + class TestJenaSparqlInjection(unittest.TestCase): def setUp(self): From f2f1d6787d178be4eedfbf78a636560b51fb633a Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Tue, 11 Aug 2026 18:57:07 +0530 Subject: [PATCH 35/40] docs(changelog): add PR #916 (DNS pinning + object-IRI gap) entry --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index db25f459..1d7bf1ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -244,6 +244,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- **DNS check-then-use hardening for the ontology URL fetcher, and a remaining object-IRI validation gap** (#916, follow-up to GHSA-8c7v-62gr-hj6g and GHSA-8vgg-8mr4-r236) by @KaifAhmad1 + - **DNS check-then-use (TOCTOU) window**: GHSA-8c7v-62gr-hj6g's own fix description flagged this as a secondary gap — `_validate_fetch_url()` resolved and validated a hostname once, but `_fetch_url_sync()` then let `requests` resolve the same hostname again independently at connect time. A low-TTL or rebinding DNS answer could differ between the two lookups, reopening the SSRF window the validation exists to close + - `_validate_fetch_url()` now returns the validated IP, and a new `_make_pinned_session()` builds a per-hop `requests.Session` whose connection pool is pinned directly to that IP — bypassing DNS resolution for the connection entirely — while explicitly restoring the real hostname as the outgoing HTTP `Host` header and, for HTTPS, the TLS SNI `server_hostname`/`assert_hostname`, so the connection reaches the validated IP but still presents (and is verified against) the real hostname's identity, keeping virtual hosting and certificate validation correct + - Caught during implementation: an earlier draft set urllib3's `_dns_host` post-construction, assuming (as in some urllib3 releases) that it was decoupled from `host`. In the version this project installs (2.7.0), `host` is a property that reads/writes `_dns_host` directly, so that approach would have silently changed the Host header too — caught by an end-to-end test against a real local server before landing, rather than shipping. Verified with real (non-mocked) local HTTP and HTTPS servers, the latter using a generated self-signed certificate to prove SNI/cert-hostname verification checks the real hostname rather than the pinned IP, plus a negative control confirming a hostname/cert mismatch is still correctly rejected, not silently bypassed + - **Object-IRI validation gap** (GHSA-8vgg-8mr4-r236 follow-up, distinct from the object-branch fix already shipped in #911): a triplet object already wrapped in `<...>` skipped `sparql_escaping.validate_uri()` in both `blazegraph_store.py` and `rdf4j_store.py`'s `_format_object_for_sparql`/`_format_object_for_ntriples`, only checking the inner content for a literal space or `>` — the pre-wrapped and unwrapped branches now validate identically + - New `tests/explorer/test_ontology_dns_pinning.py` (6 tests, 4 against real local servers including 2 real-TLS checks, gracefully skipped without the optional `cryptography` package); updated `tests/explorer/test_ontology_ssrf.py` for the new per-hop session construction; 4 new tests in `tests/triplet_store/test_sparql_injection.py` for the object-IRI fix. Full `explorer` + `triplet_store` suite: 566 passed + - **SPARQL injection via unvalidated triplet IRIs** (#911, GHSA-8vgg-8mr4-r236) by @KaifAhmad1 - `Triplet.subject`/`.predicate` (and, in some builders, `.object`) were interpolated directly into SPARQL update/query strings in the Blazegraph and RDF4J stores, and into a SELECT filter in the Jena store. A subject containing `>` closes the `<...>` IRI token early, so the rest of the value is parsed as more SPARQL. Entity names are document text in the normal ingest pipeline, so anyone whose content gets processed could append operations like `CLEAR ALL`, running with the application's store credentials - Applied the existing `sparql_escaping.validate_uri` (already used by `anzo_store.py`, the one backend that was already hardened — this generalizes its approach rather than inventing a new one) at every subject/predicate/object interpolation site: `blazegraph_store.py`'s `_build_insert_data`, `_triplets_to_rdf`, `bulk_load`'s `graph` option, `get_triplets`'s filter, and `delete_triplet`; `rdf4j_store.py`'s `_triplets_to_ntriples`, `get_triplets`'s filter, and `delete_triplet`; `jena_store.py`'s `get_triplets`'s filter (the only vulnerable site there — `add_triplets`/`delete_triplet` already use rdflib's native `Graph.add`/`.remove` with `URIRef` rather than building query strings) From 154a7347cdc3bffc0ee5ae33bc9121624da7014f Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Tue, 11 Aug 2026 19:10:01 +0530 Subject: [PATCH 36/40] fix: address CI/review findings on DNS pinning (multi-IP fallback, TLS min version) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four findings from PR #916's automated review, all addressed: - CodeQL (HIGH): the test HTTPS server's SSLContext allowed TLSv1/TLSv1.1 by not setting a minimum version. Added ssl_ctx.minimum_version = ssl.TLSVersion.TLSv1_2. - github-code-quality: unused `cryptography` local in _make_self_signed_cert — importorskip's return value was never used. - Qodo (reliability): _validate_fetch_url() only returned the first validated IP, and _make_pinned_session() pinned to just that one address, so a fetch would fail outright if the first-returned A/AAAA record happened to be unreachable even though a later one would work. _validate_fetch_url() now returns every validated IP (deduplicated, in resolution order); _make_pinned_session() takes the full list and falls back through each one via a custom Connection._new_conn override, matching the fallback behavior a normal DNS-resolving connection would already get for free. Verified with a real test: pin to an unreachable loopback address followed by a real one, confirm the fetch still succeeds by falling back; and a real test confirming it still raises (rather than silently re-resolving the hostname) when every pinned address is unreachable. - Qodo (security): when an HTTP(S) proxy applies, the adapter falls back to the unpinned path rather than pinning. This is a real, but architecturally unavoidable, limitation from the client side: for a forward proxy, the *proxy* performs its own DNS resolution of the target host on the application's behalf, a resolution this process has no visibility into or control over — there's no client-side pin that closes that race. _validate_fetch_url's destination classification still fully applies either way; only the secondary DNS-pinning hardening doesn't extend through a proxy. Added an info log when this fallback path is taken so it's observable rather than silent, and expanded the code comment to make the reasoning explicit for the next reader/reviewer rather than looking like an oversight. Tests: 3 new tests in test_ontology_dns_pinning.py (multi-IP fallback success, all-unreachable failure, deduplicated multi-record resolution). Full explorer + triplet_store suite: 569 passed. --- semantica/explorer/routes/ontology.py | 105 ++++++++++++++------ tests/explorer/test_ontology_dns_pinning.py | 78 +++++++++++++-- 2 files changed, 145 insertions(+), 38 deletions(-) diff --git a/semantica/explorer/routes/ontology.py b/semantica/explorer/routes/ontology.py index 3ac81b0a..005d8a0f 100644 --- a/semantica/explorer/routes/ontology.py +++ b/semantica/explorer/routes/ontology.py @@ -978,14 +978,17 @@ def _normalize_format(fmt: Optional[str]) -> str: return _FORMAT_ALIASES.get(lower, lower) -def _validate_fetch_url(url: str) -> str: +def _validate_fetch_url(url: str) -> List[str]: """Reject non-HTTP(S) schemes and private/loopback/link-local targets. - Returns the first resolved, validated IP address so the caller can pin - the actual connection to it (see _PinnedIPHTTPAdapter) — resolving the - hostname again at connect time would open a DNS check-then-use window - (a low-TTL or rebinding DNS answer could differ between this check and - the client's own lookup). + Returns every resolved, validated IP address (deduplicated, in + resolution order) so the caller can pin the actual connection to them + (see _make_pinned_session) with fallback across all of them — not just + the first — since a hostname can have multiple A/AAAA records and the + first one isn't guaranteed reachable. Resolving the hostname again at + connect time would open a DNS check-then-use window (a low-TTL or + rebinding DNS answer could differ between this check and the client's + own lookup), which is what pinning to these specific addresses avoids. """ parsed = urlparse(url) if parsed.scheme not in ("http", "https"): @@ -997,7 +1000,7 @@ def _validate_fetch_url(url: str) -> str: addrinfos = socket.getaddrinfo(hostname, None) except socket.gaierror as exc: raise HTTPException(status_code=422, detail=f"Cannot resolve hostname '{hostname}': {exc}") from exc - validated_ip: Optional[str] = None + validated_ips: List[str] = [] for _family, _type, _proto, _canonname, sockaddr in addrinfos: try: ip = ipaddress.ip_address(sockaddr[0]) @@ -1008,28 +1011,33 @@ def _validate_fetch_url(url: str) -> str: status_code=422, detail="Fetching from private, loopback, or reserved network addresses is not allowed.", ) - if validated_ip is None: - validated_ip = sockaddr[0] - if validated_ip is None: + if sockaddr[0] not in validated_ips: + validated_ips.append(sockaddr[0]) + if not validated_ips: raise HTTPException(status_code=422, detail=f"Cannot resolve hostname '{hostname}' to a usable address.") - return validated_ip + return validated_ips -def _make_pinned_session(pinned_ip: str, url: str): - """Build a requests.Session whose connection is pinned to pinned_ip, - regardless of what url's hostname resolves to at connect time. +def _make_pinned_session(pinned_ips: List[str], url: str): + """Build a requests.Session whose connection is pinned to pinned_ips + (tried in order, falling back on connection failure), regardless of + what url's hostname resolves to at connect time. _validate_fetch_url() resolves and validates the hostname once; letting the HTTP client resolve it again independently at connect time reopens the exact gap that validation exists to close — a low-TTL or rebinding DNS answer can differ between the two lookups. This pins the pool's - connect target to the already-validated IP directly (bypassing DNS - resolution for the connection entirely), while keeping the original + connect target to the already-validated addresses directly (bypassing + DNS resolution for the connection entirely), while keeping the original hostname as the outgoing HTTP Host header and, for HTTPS, the TLS SNI server_hostname / assert_hostname — otherwise the connection would reach the right IP but present the wrong identity, breaking name-based virtual hosting and (for HTTPS) certificate hostname verification. + Falls back across every validated address (not just the first) so a + hostname with multiple A/AAAA records doesn't fail outright just + because the first-returned address happens to be unreachable. + Note: urllib3's Connection.host is a property that reads/writes the same underlying value as `_dns_host` in this version — it is NOT the separate "presented identity" field it is in some older releases, so @@ -1038,7 +1046,10 @@ def _make_pinned_session(pinned_ip: str, url: str): `host` directly and restoring the real hostname via an explicit Host header (+ SNI params for HTTPS) is the correct mechanism here. """ + import logging as _pin_logging import requests as _req + import urllib3.util.connection as _u3_connection + from urllib3.exceptions import NewConnectionError parsed = urlparse(url) hostname = parsed.hostname @@ -1046,17 +1057,47 @@ def _make_pinned_session(pinned_ip: str, url: str): default_port = 443 if parsed.scheme == "https" else 80 host_header = hostname if port in (None, default_port) else f"{hostname}:{port}" + class _MultiIPConnectionMixin: + """Overrides _new_conn to fall back across every pinned IP in + order, instead of urllib3's default single-host connect.""" + + def _new_conn(self): + last_exc: Optional[BaseException] = None + for ip in pinned_ips: + try: + return _u3_connection.create_connection( + (ip, self.port), + self.timeout, + source_address=self.source_address, + socket_options=self.socket_options, + ) + except OSError as exc: + last_exc = exc + continue + raise NewConnectionError( + self, f"Failed to establish a connection to any of {pinned_ips}: {last_exc}" + ) + class _PinnedIPHTTPAdapter(_req.adapters.HTTPAdapter): def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None): # If an HTTP(S) proxy applies (env-configured or per-request), - # the actual TCP connection target is the proxy, not the - # resolved IP, and proxy tunneling changes the connection model - # enough that pinning doesn't apply cleanly. Fall back to the - # normal (unpinned) path rather than silently bypassing the - # proxy — _validate_fetch_url's destination check still applies - # either way; only this secondary DNS-pinning hardening is - # skipped. + # pinning can't meaningfully apply: the actual TCP connection + # target is the proxy, and for a forward proxy the *proxy* + # performs its own DNS resolution of the target host on our + # behalf — a resolution this process has no visibility into or + # control over, so there is no client-side fix for that + # specific race. Fall back to the normal (unpinned) path rather + # than silently bypassing the configured proxy. + # _validate_fetch_url's destination classification still fully + # applies either way; only this secondary DNS-pinning hardening + # is inherently out of scope when a proxy is in the path. if _req.utils.select_proxy(request.url, proxies): + _pin_logging.getLogger(__name__).info( + "DNS pinning skipped for %s: a proxy is configured for this " + "request, and proxy-side DNS resolution is outside this " + "process's control.", + request.url, + ) return super().get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) @@ -1064,8 +1105,14 @@ def _make_pinned_session(pinned_ip: str, url: str): if host_params.get("scheme") == "https": pool_kwargs.setdefault("assert_hostname", hostname) pool_kwargs.setdefault("server_hostname", hostname) - host_params["host"] = pinned_ip - return self.poolmanager.connection_from_host(**host_params, pool_kwargs=pool_kwargs) + host_params["host"] = pinned_ips[0] + pool = self.poolmanager.connection_from_host(**host_params, pool_kwargs=pool_kwargs) + base_connection_cls = pool.ConnectionCls + if not issubclass(base_connection_cls, _MultiIPConnectionMixin): + pool.ConnectionCls = type( + "_PinnedConnection", (_MultiIPConnectionMixin, base_connection_cls), {} + ) + return pool session = _req.Session() session.headers["Host"] = host_header @@ -1076,12 +1123,12 @@ def _make_pinned_session(pinned_ip: str, url: str): def _fetch_url_sync(url: str) -> bytes: - pinned_ip = _validate_fetch_url(url) + pinned_ips = _validate_fetch_url(url) _MAX_REDIRECTS = 5 current_url = url try: for _ in range(_MAX_REDIRECTS + 1): - session = _make_pinned_session(pinned_ip, current_url) + session = _make_pinned_session(pinned_ips, current_url) try: resp = session.get( current_url, @@ -1099,8 +1146,8 @@ def _fetch_url_sync(url: str) -> bytes: redirect_url = urljoin(current_url, redirect_url) # Re-validate the redirect target to prevent SSRF via # open-redirect to internal/cloud-metadata endpoints, and - # get a fresh pin for the new host. - pinned_ip = _validate_fetch_url(redirect_url) + # get fresh pins for the new host. + pinned_ips = _validate_fetch_url(redirect_url) current_url = redirect_url continue try: diff --git a/tests/explorer/test_ontology_dns_pinning.py b/tests/explorer/test_ontology_dns_pinning.py index cb7791c6..90780ba4 100644 --- a/tests/explorer/test_ontology_dns_pinning.py +++ b/tests/explorer/test_ontology_dns_pinning.py @@ -62,7 +62,7 @@ def test_pinned_session_connects_to_pinned_ip_without_resolving_hostname(): port = server.server_address[1] url = f"http://pinned-test.invalid:{port}/resource" try: - session = ontology_mod._make_pinned_session("127.0.0.1", url) + session = ontology_mod._make_pinned_session(["127.0.0.1"], url) try: resp = session.get(url, timeout=5) assert resp.status_code == 200 @@ -117,7 +117,7 @@ def test_pinned_session_ignores_a_different_real_resolution(): thread.start() url = f"http://localhost:{port}/resource" try: - session = ontology_mod._make_pinned_session("127.0.0.2", url) + session = ontology_mod._make_pinned_session(["127.0.0.2"], url) try: resp = session.get(url, timeout=5) assert resp.status_code == 200 @@ -131,17 +131,76 @@ def test_pinned_session_ignores_a_different_real_resolution(): assert captured["host_header"] == f"localhost:{port}" +def test_pinned_session_falls_back_across_multiple_pinned_ips(): + """A hostname can have multiple A/AAAA records; pinning to only the + first-returned address means a fetch fails outright if that specific + address happens to be unreachable even though a later one would work. + _make_pinned_session must fall back through every pinned IP in order. + """ + server, thread, captured = _start_local_server() + port = server.server_address[1] + url = f"http://pinned-test.invalid:{port}/resource" + # 127.0.0.3 has nothing listening on this port — connection refused, + # forcing a fallback to the second (real) address. + unreachable_ip = "127.0.0.3" + try: + session = ontology_mod._make_pinned_session([unreachable_ip, "127.0.0.1"], url) + try: + resp = session.get(url, timeout=5) + assert resp.status_code == 200 + assert resp.content == b"pinned response" + finally: + session.close() + finally: + server.shutdown() + thread.join(timeout=2) + + +def test_pinned_session_raises_when_every_pinned_ip_is_unreachable(): + """If none of the pinned IPs are reachable, the session must raise + rather than silently falling back to resolving the hostname itself + (which would reopen the exact TOCTOU window pinning exists to close).""" + import requests + + url = "http://pinned-test.invalid:9/resource" # port 9 (discard) — nothing listens + session = ontology_mod._make_pinned_session(["127.0.0.3", "127.0.0.4"], url) + try: + with pytest.raises(requests.exceptions.ConnectionError): + session.get(url, timeout=5) + finally: + session.close() + + def test_validate_fetch_url_returns_the_resolved_ip(): - """_validate_fetch_url must return the IP it validated, so callers can - pin the connection to it.""" + """_validate_fetch_url must return every IP it validated, so callers can + pin the connection to them (with fallback across all of them).""" def fake_getaddrinfo(host, *_a, **_k): return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0))] import unittest.mock as mock with mock.patch.object(ontology_mod.socket, "getaddrinfo", side_effect=fake_getaddrinfo): - resolved_ip = ontology_mod._validate_fetch_url("http://example.org/ontology.ttl") + resolved_ips = ontology_mod._validate_fetch_url("http://example.org/ontology.ttl") - assert resolved_ip == "93.184.216.34" + assert resolved_ips == ["93.184.216.34"] + + +def test_validate_fetch_url_returns_all_validated_ips_deduplicated(): + """A hostname with multiple A/AAAA records must return every distinct + validated address, in resolution order, so the caller can fall back + across all of them rather than failing if only the first is + unreachable.""" + def fake_getaddrinfo(host, *_a, **_k): + return [ + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0)), + (socket.AF_INET, socket.SOCK_DGRAM, 17, "", ("93.184.216.34", 0)), # duplicate, different socktype + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.35", 0)), + ] + + import unittest.mock as mock + with mock.patch.object(ontology_mod.socket, "getaddrinfo", side_effect=fake_getaddrinfo): + resolved_ips = ontology_mod._validate_fetch_url("http://example.org/ontology.ttl") + + assert resolved_ips == ["93.184.216.34", "93.184.216.35"] def test_validate_fetch_url_still_rejects_private_ip(): @@ -175,7 +234,7 @@ def test_validate_fetch_url_still_rejects_private_ip(): def _make_self_signed_cert(hostname: str, tmp_path): import datetime - cryptography = pytest.importorskip("cryptography") + pytest.importorskip("cryptography") from cryptography import x509 from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import rsa @@ -228,6 +287,7 @@ def _start_local_https_server(cert_path, key_path): server = http.server.HTTPServer(("127.0.0.1", 0), Handler) ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ssl_ctx.minimum_version = ssl.TLSVersion.TLSv1_2 ssl_ctx.load_cert_chain(cert_path, key_path) server.socket = ssl_ctx.wrap_socket(server.socket, server_side=True) thread = threading.Thread(target=server.serve_forever, daemon=True) @@ -246,7 +306,7 @@ def test_pinned_https_session_verifies_against_real_hostname_not_pinned_ip(tmp_p port = server.server_address[1] url = f"https://pinned-tls-test.invalid:{port}/resource" try: - session = ontology_mod._make_pinned_session("127.0.0.1", url) + session = ontology_mod._make_pinned_session(["127.0.0.1"], url) try: resp = session.get(url, timeout=5, verify=cert_path) finally: @@ -269,7 +329,7 @@ def test_pinned_https_session_rejects_hostname_mismatch(tmp_path): port = server.server_address[1] url = f"https://wrong-name.invalid:{port}/resource" try: - session = ontology_mod._make_pinned_session("127.0.0.1", url) + session = ontology_mod._make_pinned_session(["127.0.0.1"], url) try: import requests with pytest.raises(requests.exceptions.SSLError): From ea3416ed32d8355b999497568fc983a7a07be011 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Tue, 11 Aug 2026 19:16:29 +0530 Subject: [PATCH 37/40] fix: enforce a definitive no-proxy policy for the pinned SSRF fetcher MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Qodo's re-review confirmed the multi-IP fallback fix but kept the proxy finding open: logging-and-falling-back when a proxy applies still let the DNS-pinning protection be silently skipped under proxy configuration, rather than enforcing a clear policy either way. Implemented Qodo's preferred option: proxies are now disabled outright for this SSRF-sensitive fetcher via session.trust_env = False, so HTTP_PROXY/HTTPS_PROXY/NO_PROXY env vars are never consulted in the first place (a configured proxy would perform its own DNS resolution of the target host outside this process's control, reopening the DNS check-then-use race pinning exists to close). The adapter also keeps a fail-closed backstop: if a proxy is somehow still configured despite trust_env=False (e.g. set explicitly by future code), it now raises a clear 502 instead of silently connecting through the proxy unpinned. _validate_fetch_url's destination classification (blocking private/ internal targets) is unaffected either way — it runs before any of this and doesn't depend on proxy configuration. 4 new tests: trust_env is disabled on every pinned session; an HTTP_PROXY env var pointed at an address that would fail if contacted is confirmed genuinely unused (real local-server fetch still succeeds directly); and the fail-closed backstop actually raises when a proxy is forced onto the session. Full explorer + triplet_store suite: 572 passed. --- semantica/explorer/routes/ontology.py | 41 +++++++++-------- tests/explorer/test_ontology_dns_pinning.py | 51 +++++++++++++++++++++ 2 files changed, 72 insertions(+), 20 deletions(-) diff --git a/semantica/explorer/routes/ontology.py b/semantica/explorer/routes/ontology.py index 005d8a0f..b50eb206 100644 --- a/semantica/explorer/routes/ontology.py +++ b/semantica/explorer/routes/ontology.py @@ -1046,7 +1046,6 @@ def _make_pinned_session(pinned_ips: List[str], url: str): `host` directly and restoring the real hostname via an explicit Host header (+ SNI params for HTTPS) is the correct mechanism here. """ - import logging as _pin_logging import requests as _req import urllib3.util.connection as _u3_connection from urllib3.exceptions import NewConnectionError @@ -1080,26 +1079,21 @@ def _make_pinned_session(pinned_ips: List[str], url: str): class _PinnedIPHTTPAdapter(_req.adapters.HTTPAdapter): def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None): - # If an HTTP(S) proxy applies (env-configured or per-request), - # pinning can't meaningfully apply: the actual TCP connection - # target is the proxy, and for a forward proxy the *proxy* - # performs its own DNS resolution of the target host on our - # behalf — a resolution this process has no visibility into or - # control over, so there is no client-side fix for that - # specific race. Fall back to the normal (unpinned) path rather - # than silently bypassing the configured proxy. - # _validate_fetch_url's destination classification still fully - # applies either way; only this secondary DNS-pinning hardening - # is inherently out of scope when a proxy is in the path. + # A proxy would perform its own DNS resolution of the target + # host on this process's behalf — a resolution outside this + # process's visibility or control, so there is no client-side + # pin that closes that race. Proxies are disabled outright for + # this SSRF-sensitive fetcher (session.trust_env=False below), + # so this should be unreachable via environment proxies; fail + # closed rather than silently skip pinning if a proxy is + # somehow still configured (e.g. passed explicitly in the + # future). _validate_fetch_url's destination classification is + # a separate, always-enforced check — this only guards the + # secondary DNS-pinning hardening. if _req.utils.select_proxy(request.url, proxies): - _pin_logging.getLogger(__name__).info( - "DNS pinning skipped for %s: a proxy is configured for this " - "request, and proxy-side DNS resolution is outside this " - "process's control.", - request.url, - ) - return super().get_connection_with_tls_context( - request, verify, proxies=proxies, cert=cert + raise HTTPException( + status_code=502, + detail="Proxied requests are not supported for ontology URL fetching.", ) host_params, pool_kwargs = self.build_connection_pool_key_attributes(request, verify, cert) if host_params.get("scheme") == "https": @@ -1115,6 +1109,13 @@ def _make_pinned_session(pinned_ips: List[str], url: str): return pool session = _req.Session() + # Never honor HTTP_PROXY/HTTPS_PROXY/NO_PROXY env vars for this + # SSRF-sensitive fetcher: a configured proxy would perform its own DNS + # resolution of the target host outside this process's control, + # silently reopening the DNS check-then-use race pinning exists to + # close. See _PinnedIPHTTPAdapter.get_connection_with_tls_context for + # the fail-closed backstop if a proxy is somehow still configured. + session.trust_env = False session.headers["Host"] = host_header adapter = _PinnedIPHTTPAdapter() session.mount("http://", adapter) diff --git a/tests/explorer/test_ontology_dns_pinning.py b/tests/explorer/test_ontology_dns_pinning.py index 90780ba4..a68ff192 100644 --- a/tests/explorer/test_ontology_dns_pinning.py +++ b/tests/explorer/test_ontology_dns_pinning.py @@ -171,6 +171,57 @@ def test_pinned_session_raises_when_every_pinned_ip_is_unreachable(): session.close() +def test_pinned_session_disables_environment_proxy_trust(): + """A pinned session must never honor HTTP_PROXY/HTTPS_PROXY env vars — + a proxy would perform its own DNS resolution of the target host outside + this process's control, reopening the exact TOCTOU window pinning + exists to close.""" + session = ontology_mod._make_pinned_session(["127.0.0.1"], "http://example.org/") + try: + assert session.trust_env is False + finally: + session.close() + + +def test_pinned_session_ignores_env_proxy_and_connects_directly(monkeypatch): + """End-to-end: even with HTTP_PROXY pointed at an address that would + fail if contacted, a pinned session must reach the real local server + directly — proving the env var is genuinely not consulted, not just + that the trust_env flag is set.""" + monkeypatch.setenv("HTTP_PROXY", "http://127.0.0.5:1/") # would fail if ever used + server, thread, _captured = _start_local_server() + port = server.server_address[1] + url = f"http://pinned-test.invalid:{port}/resource" + try: + session = ontology_mod._make_pinned_session(["127.0.0.1"], url) + try: + resp = session.get(url, timeout=5) + assert resp.status_code == 200 + assert resp.content == b"pinned response" + finally: + session.close() + finally: + server.shutdown() + thread.join(timeout=2) + + +def test_pinned_session_fails_closed_if_a_proxy_is_explicitly_forced(): + """Backstop: if a proxy is somehow still configured on the session + despite trust_env=False (e.g. set explicitly, as a future code path + might), the adapter must fail closed with a clear error rather than + silently connecting through the proxy unpinned.""" + from fastapi import HTTPException + + session = ontology_mod._make_pinned_session(["127.0.0.1"], "http://example.org/") + session.proxies = {"http": "http://127.0.0.5:1"} + try: + with pytest.raises(HTTPException) as exc_info: + session.get("http://example.org/", timeout=5) + assert exc_info.value.status_code == 502 + finally: + session.close() + + def test_validate_fetch_url_returns_the_resolved_ip(): """_validate_fetch_url must return every IP it validated, so callers can pin the connection to them (with fallback across all of them).""" From f29c4310a11c3481c7d56454761d9de654a28793 Mon Sep 17 00:00:00 2001 From: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:11:17 +0530 Subject: [PATCH 38/40] security: validate WebSocket Origin against the CORS allowlist (GHSA-4643) (#917) CORSMiddleware doesn't cover WebSocket handshakes at all (Starlette's CORS support only wraps HTTP), so under SEMANTICA_ALLOW_ANONYMOUS=true -- the mode docker-compose.dev.yml ships -- is_valid_api_key's anonymous bypass accepted a /ws/graph-updates connection from any origin. Loopback binding isn't a boundary against a browser: any page the operator has open can still reach ws://localhost:8000/ws/graph-updates directly, and ConnectionManager.broadcast sends every graph_mutation to every connected socket with no per-connection scoping. Combined with /api/import accepting multipart/form-data (a CORS-safelisted content type that skips preflight), a hostile page could write to the graph over REST and read the result back over the unauthenticated WebSocket -- demonstrated end-to-end in the report with a real client. Not affected: any deployment with SEMANTICA_API_KEY configured -- the handshake already rejects without a valid key in that mode. This is an anonymous-mode-only, development-configuration exposure. Fix: check the handshake's Origin header against app.state.explorer_settings['allowed_origins'], the same list CORSMiddleware already enforces for HTTP, before the key check. A missing Origin (native/CLI clients, which never set the header -- only browsers do) is still allowed through, since the browser is the only threat this closes. 4 new tests in test_explorer_auth.py: hostile Origin rejected under anonymous mode; hostile Origin rejected even with a correct key (Origin is checked before the key, so a leaked key alone can't hijack the socket); an allowlisted Origin still connects under anonymous mode; a missing Origin still connects under anonymous mode (native clients keep working). Full explorer suite: 226 passed. Co-authored-by: Sameer Kadam --- semantica/explorer/app.py | 17 ++++++++ tests/explorer/test_explorer_auth.py | 59 ++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/semantica/explorer/app.py b/semantica/explorer/app.py index 68bab729..bd9ab042 100644 --- a/semantica/explorer/app.py +++ b/semantica/explorer/app.py @@ -193,6 +193,23 @@ def create_app( @app.websocket("/ws/graph-updates") async def websocket_endpoint(websocket: WebSocket): + # CORSMiddleware doesn't cover WebSocket handshakes (Starlette's + # CORS support only wraps HTTP), so under SEMANTICA_ALLOW_ANONYMOUS + # the key check below accepts any origin — loopback binding isn't a + # boundary against a browser, since any page the operator has open + # can still reach ws://localhost:.../ws/graph-updates directly. + # Reject a foreign Origin explicitly here, against the same + # allowlist CORSMiddleware already enforces for HTTP + # (GHSA-4643-wpgq-w329). Browsers always send Origin on a + # cross-origin WebSocket handshake; native/CLI clients omit it + # entirely, so a missing Origin is allowed through — the browser is + # the only threat this check is closing. + origin = websocket.headers.get("origin") + allowed_origins = app.state.explorer_settings["allowed_origins"] + if origin is not None and origin not in allowed_origins: + await websocket.close(code=4403) # forbidden + return + # Browsers can't set custom headers on a WebSocket handshake, so # accept the key via header (non-browser clients) or query param # (browser clients), same SEMANTICA_API_KEY the REST routes check. diff --git a/tests/explorer/test_explorer_auth.py b/tests/explorer/test_explorer_auth.py index 4b7b9c6f..e29461d2 100644 --- a/tests/explorer/test_explorer_auth.py +++ b/tests/explorer/test_explorer_auth.py @@ -150,3 +150,62 @@ def test_websocket_accepts_connection_with_header_key(client, monkeypatch): ) as websocket: ack = websocket.receive_json() assert ack["event"] == "connection_ack" + + +# --------------------------------------------------------------------------- +# WebSocket Origin validation (GHSA-4643-wpgq-w329): CORSMiddleware doesn't +# cover WebSocket handshakes at all, so under SEMANTICA_ALLOW_ANONYMOUS the +# key check alone accepted a handshake from any origin — loopback binding is +# not a boundary against a browser, since any page the operator has open can +# still reach ws://localhost:.../ws/graph-updates. These pin the fix: a +# hostile Origin is refused even in anonymous mode (and even with a correct +# key), a same-origin/allowlisted Origin still works, and a missing Origin +# (native/CLI clients, which never set the header) is still allowed through. +# --------------------------------------------------------------------------- + +def test_websocket_rejects_hostile_origin_under_anonymous_mode(client, monkeypatch): + monkeypatch.setenv("SEMANTICA_ALLOW_ANONYMOUS", "true") + monkeypatch.delenv("SEMANTICA_API_KEY", raising=False) + + with pytest.raises(Exception): + with client.websocket_connect( + "/ws/graph-updates", headers={"Origin": "https://evil.example"} + ): + pass + + +def test_websocket_rejects_hostile_origin_even_with_correct_key(client, monkeypatch): + """Defense in depth: Origin is checked before the API key, so a hostile + page that somehow obtained a valid key still can't hijack the socket.""" + monkeypatch.delenv("SEMANTICA_ALLOW_ANONYMOUS", raising=False) + monkeypatch.setenv("SEMANTICA_API_KEY", "correct-key") + + with pytest.raises(Exception): + with client.websocket_connect( + "/ws/graph-updates", + headers={"Origin": "https://evil.example", "X-API-Key": "correct-key"}, + ): + pass + + +def test_websocket_accepts_allowlisted_origin_under_anonymous_mode(client, monkeypatch): + monkeypatch.setenv("SEMANTICA_ALLOW_ANONYMOUS", "true") + monkeypatch.delenv("SEMANTICA_API_KEY", raising=False) + + with client.websocket_connect( + "/ws/graph-updates", headers={"Origin": "http://localhost:5173"} + ) as websocket: + ack = websocket.receive_json() + assert ack["event"] == "connection_ack" + + +def test_websocket_accepts_missing_origin_under_anonymous_mode(client, monkeypatch): + """Native/CLI clients never send an Origin header — only browsers do — + so a missing Origin must still be allowed through; the browser is the + only threat this check closes.""" + monkeypatch.setenv("SEMANTICA_ALLOW_ANONYMOUS", "true") + monkeypatch.delenv("SEMANTICA_API_KEY", raising=False) + + with client.websocket_connect("/ws/graph-updates") as websocket: + ack = websocket.receive_json() + assert ack["event"] == "connection_ack" From 5b319560fb0b8403644b70bc592864418cdcc740 Mon Sep 17 00:00:00 2001 From: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:41:19 +0530 Subject: [PATCH 39/40] chore: bump version to 0.6.5 (#918) Security release bundling fixes for GHSA-j4mq (missing auth), GHSA-8c7v (SSRF via redirect bypass), GHSA-482h (Cypher injection), GHSA-8vgg (SPARQL injection), GHSA-4643 (WebSocket Origin validation), and a CodeQL-flagged ReDoS in the SPARQL route validator. --- CHANGELOG.md | 18 +++++++++++++++++- README.md | 18 ++++++++++++------ docs/citation.md | 16 ++++++++-------- docs/faq.md | 2 +- docs/getting-started.md | 2 +- pyproject.toml | 2 +- semantica/__init__.py | 2 +- 7 files changed, 41 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d7bf1ba..34d400d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.6.5] - 2026-08-11 + ### Added - **Embedded Oxigraph backend for `TripletStore`** (#838, closes #834) by @Linxiushen @@ -249,7 +251,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `_validate_fetch_url()` now returns the validated IP, and a new `_make_pinned_session()` builds a per-hop `requests.Session` whose connection pool is pinned directly to that IP — bypassing DNS resolution for the connection entirely — while explicitly restoring the real hostname as the outgoing HTTP `Host` header and, for HTTPS, the TLS SNI `server_hostname`/`assert_hostname`, so the connection reaches the validated IP but still presents (and is verified against) the real hostname's identity, keeping virtual hosting and certificate validation correct - Caught during implementation: an earlier draft set urllib3's `_dns_host` post-construction, assuming (as in some urllib3 releases) that it was decoupled from `host`. In the version this project installs (2.7.0), `host` is a property that reads/writes `_dns_host` directly, so that approach would have silently changed the Host header too — caught by an end-to-end test against a real local server before landing, rather than shipping. Verified with real (non-mocked) local HTTP and HTTPS servers, the latter using a generated self-signed certificate to prove SNI/cert-hostname verification checks the real hostname rather than the pinned IP, plus a negative control confirming a hostname/cert mismatch is still correctly rejected, not silently bypassed - **Object-IRI validation gap** (GHSA-8vgg-8mr4-r236 follow-up, distinct from the object-branch fix already shipped in #911): a triplet object already wrapped in `<...>` skipped `sparql_escaping.validate_uri()` in both `blazegraph_store.py` and `rdf4j_store.py`'s `_format_object_for_sparql`/`_format_object_for_ntriples`, only checking the inner content for a literal space or `>` — the pre-wrapped and unwrapped branches now validate identically - - New `tests/explorer/test_ontology_dns_pinning.py` (6 tests, 4 against real local servers including 2 real-TLS checks, gracefully skipped without the optional `cryptography` package); updated `tests/explorer/test_ontology_ssrf.py` for the new per-hop session construction; 4 new tests in `tests/triplet_store/test_sparql_injection.py` for the object-IRI fix. Full `explorer` + `triplet_store` suite: 566 passed + - **Fixed along the way** (caught in automated review across two follow-up rounds): `_validate_fetch_url()` originally pinned to only the first resolved IP, so a hostname with multiple A/AAAA records would fail outright if that specific address was unreachable — it now returns every validated IP and `_make_pinned_session()` falls back through all of them, verified by pinning to a genuinely unreachable address followed by a working one and confirming the fetch still succeeds; the test HTTPS server allowed TLSv1/TLSv1.1 by not setting a minimum version, now pinned to TLSv1.2; and when an HTTP(S) proxy applied, pinning was silently skipped in favor of the unpinned path — proxies are now disabled outright for this fetcher (`session.trust_env = False`, so `HTTP_PROXY`/`HTTPS_PROXY` env vars are never consulted) with a fail-closed 502 backstop if a proxy is ever forced onto the session some other way, verified by pointing `HTTP_PROXY` at an address that would fail if actually used and confirming the fetch still succeeds directly + - New `tests/explorer/test_ontology_dns_pinning.py` (12 tests: real local HTTP/HTTPS servers including 2 real-TLS checks, multi-IP fallback success/failure, and no-proxy-trust verification — gracefully skipped without the optional `cryptography` package where applicable); updated `tests/explorer/test_ontology_ssrf.py` for the new per-hop session construction; 4 new tests in `tests/triplet_store/test_sparql_injection.py` for the object-IRI fix. Full `explorer` + `triplet_store` suite: 572 passed + +- **Missing Origin validation on the `/ws/graph-updates` WebSocket handshake** (#917, GHSA-4643-wpgq-w329) by @KaifAhmad1 + - `CORSMiddleware` doesn't cover WebSocket handshakes at all (Starlette's CORS support only wraps HTTP), so under `SEMANTICA_ALLOW_ANONYMOUS=true` — the mode `docker-compose.dev.yml` ships — the anonymous-mode key bypass accepted a `/ws/graph-updates` connection from any origin. Loopback binding isn't a boundary against a browser: any page the operator has open can still reach `ws://localhost:8000/ws/graph-updates` directly, and `ConnectionManager.broadcast` sends every `graph_mutation` to every connected socket with no per-connection scoping. Combined with `/api/import` accepting `multipart/form-data` (a CORS-safelisted content type that skips preflight), a hostile page could write to the graph over REST and read the result back over the unauthenticated WebSocket + - Not affected: any deployment with `SEMANTICA_API_KEY` configured — the handshake already rejects without a valid key in that mode. This was an anonymous-mode-only, development-configuration exposure + - Fix: check the handshake's `Origin` header against `app.state.explorer_settings['allowed_origins']` — the same list `CORSMiddleware` already enforces for HTTP — before the key check. A missing `Origin` (native/CLI clients, which never set the header) is still allowed through, since the browser is the only threat this closes + - 4 new tests in `tests/explorer/test_explorer_auth.py`: hostile Origin rejected under anonymous mode; hostile Origin rejected even with a correct key (Origin is checked first, so a leaked key alone can't hijack the socket); an allowlisted Origin still connects; a missing Origin still connects. Full `explorer` suite: 226 passed + +- **Polynomial-time ReDoS in the SPARQL route's `_PREFIX_DECL` regex** (#915, CodeQL `py/polynomial-redos`) by @Sameer6305 + - The prior pattern's trailing `\s*` overlapped with the preceding `<[^>]*>` IRI-body match on inputs containing no closing `>` (e.g. `base<` followed by thousands of `!<` repetitions), forcing the regex engine to explore every possible split between the two quantifiers — O(n²) backtracking reachable from `req.query` via `_is_read_only_query()` + - Fixed by making the two quantifiers character-disjoint: horizontal whitespace only (`[ \t]`, never overlapping the IRI body) instead of `\s*`, and excluding CR/LF from the IRI body (`[^>\r\n]*`) so it can never span a line boundary. Independently verified: the exact pathological payload (`base<` + `!<` × 5,000/20,000) scales linearly (0.238ms → 0.841ms for 4x input, not the ~16x a surviving quadratic blowup would show) + - Added `_SPARQL_MAX_QUERY_LEN = 10_000` as defense-in-depth, checked in `execute_sparql()` before any regex work so a future pattern regression stays bounded regardless + - Two correctness regressions raised in review were checked and did not reproduce: comment-then-prefix stripping order means an inline comment after a `PREFIX` line (`PREFIX ex: <...> # comment`) is already gone by the time `_PREFIX_DECL` runs, verified directly against the pipeline; and the allowlist's `.sub()`-based cleaning only ever affects the yes/no decision, never the query actually sent to `graph.query()` — so even the narrow case of a multi-line string literal that happens to start a line with the literal text `PREFIX` or `BASE` can only cause a legitimate query to be wrongly rejected, never let something malicious through, since rdflib's parser still gates whatever actually executes + - 20 new/updated tests in `tests/explorer/test_sparql_route.py` and `tests/test_security_regression.py` (inline prologues, CRLF line endings, multi-line CRLF prefix chains, oversized-query rejection). 225 `explorer` + 82 SPARQL-specific tests passing - **SPARQL injection via unvalidated triplet IRIs** (#911, GHSA-8vgg-8mr4-r236) by @KaifAhmad1 - `Triplet.subject`/`.predicate` (and, in some builders, `.object`) were interpolated directly into SPARQL update/query strings in the Blazegraph and RDF4J stores, and into a SELECT filter in the Jena store. A subject containing `>` closes the `<...>` IRI token early, so the rest of the value is parsed as more SPARQL. Entity names are document text in the normal ingest pipeline, so anyone whose content gets processed could append operations like `CLEAR ALL`, running with the application's store credentials diff --git a/README.md b/README.md index 745c54e4..fbc20781 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ compliant = graph.check_decision_rules({"category": "vendor_selection"}) # poli ```bash semantica doctor # Python 3.11.9 pass -# semantica 0.6.0 pass +# semantica 0.6.5 pass # faiss vector store pass # Config file pass ~/.semantica/config.yaml ``` @@ -1474,12 +1474,18 @@ For contributor / dev-server setup: **[explorer/README.md: Local Setup Guide](ex --- -## What's New in v0.6.0 +## What's New in v0.6.5 -- **Named-Graph Support for `JenaStore`:** Migrated onto `rdflib.Dataset(default_union=False)`, completing cross-backend named-graph parity across Blazegraph, RDF4J, and Jena; `add_triplets()` gains a `graph=` option -- **SPARQL CONSTRUCT Query Templates:** Parameterized, injection-safe `CONSTRUCT` templates extended from Blazegraph-only to RDF4J and Jena, plus pipeline integration via the `construct_template` step type -- **Databricks Connector:** `DatabricksIngestor` for Unity Catalog + Delta Lake ingestion, with PAT/OAuth M2M auth, table/query ingestion, and catalog/schema/table/lineage introspection. Install with `pip install "semantica[db-databricks]"` -- **SQLite Vector Store Backend:** `SQLiteVecStore`, a disk-backed local vector store on `sqlite-vec`'s `vec0` virtual tables, with Cosine/L2 metrics, metadata filtering, and WAL mode. Install with `pip install semantica[vectorstore-sqlite]` +**Security release — upgrading is strongly recommended.** Fixes for 5 externally-reported vulnerabilities in the Explorer API and graph/triplet store backends, plus a CodeQL-flagged ReDoS: + +- **Missing authentication on all Explorer API routes** (GHSA-j4mq-hprp-987v, Critical): every route now requires `SEMANTICA_API_KEY`, fails closed (503) rather than open when unconfigured +- **SSRF via redirect bypass in ontology URL fetching** (GHSA-8c7v-62gr-hj6g, High): redirect targets are now re-validated at every hop and the connection is pinned to the validated address, closing a DNS check-then-use race +- **Cypher injection via unvalidated node labels and property keys** (GHSA-482h-hw99-h62p, Critical): Neptune, Neo4j, and FalkorDB now sanitize every label/relationship-type/property-key interpolation site +- **SPARQL injection via unvalidated triplet IRIs** (GHSA-8vgg-8mr4-r236, Critical): Blazegraph, RDF4J, and Jena now validate subject/predicate/object IRIs before interpolation +- **Missing Origin validation on the WebSocket handshake** (GHSA-4643-wpgq-w329, Moderate, anonymous-mode only): `/ws/graph-updates` now checks `Origin` against the same allowlist `CORSMiddleware` enforces for HTTP +- **Polynomial ReDoS in SPARQL query validation** (CodeQL `py/polynomial-redos`): fixed a backtracking regex in the Explorer's SPARQL route + +Also includes: embedded Oxigraph backend for `TripletStore`, PROV-O trust/spec completeness for `ProvenanceManager`, and the Altair Anzo triplet store backend. → [Full release notes](RELEASE_NOTES.md) · [Changelog](CHANGELOG.md) diff --git a/docs/citation.md b/docs/citation.md index e33c566f..d1077887 100644 --- a/docs/citation.md +++ b/docs/citation.md @@ -13,33 +13,33 @@ icon: "quote-left" ```bibtex @software{semantica2026, - title = {Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering}, - author = {Hawksight AI}, + title = {Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems}, + author = {Semantica}, year = {2026}, url = {https://github.com/semantica-agi/semantica}, - version = {0.6.0}, + version = {0.6.5}, doi = {10.5281/zenodo.XXXXXXX} } ``` - Hawksight AI. (2026). *Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering* (Version 0.6.0) \[Computer software\]. https://github.com/semantica-agi/semantica + Semantica. (2026). *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems* (Version 0.6.5) \[Computer software\]. https://github.com/semantica-agi/semantica - Hawksight AI. *Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering*. Version 0.6.0, GitHub, 2026, https://github.com/semantica-agi/semantica. + Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.5, GitHub, 2026, https://github.com/semantica-agi/semantica. - Hawksight AI. *Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering*. Version 0.6.0. GitHub, 2026. https://github.com/semantica-agi/semantica. + Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.5. GitHub, 2026. https://github.com/semantica-agi/semantica. - Hawksight AI, "Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering," Version 0.6.0, GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica + Semantica, "Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems," Version 0.6.5, GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica ## Acknowledgment Text -> "This work uses Semantica (Hawksight AI, 2026), an open-source framework for semantic layer construction and knowledge engineering." +> "This work uses Semantica (2026), an open-source graph-native infrastructure framework for context and accountable AI systems, providing Context Graphs, knowledge graphs, and full decision provenance." ## Share Your Research diff --git a/docs/faq.md b/docs/faq.md index 8cead87c..e050df4c 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -17,7 +17,7 @@ icon: "circle-question" | API key required? | Optional: pattern extraction works with no keys | | Works with LangChain / LlamaIndex? | Yes: Semantica is a layer on top, not a replacement | | Production-ready? | Yes: 1,000+ tests, v0.5.0 ships with 12 security fixes | -| Latest version? | **v0.6.0** (July 2026) | +| Latest version? | **v0.6.5** (August 2026) | | Local LLMs? | Yes: Ollama via LiteLLM, HuggingFaceLLM for air-gapped | diff --git a/docs/getting-started.md b/docs/getting-started.md index 20aeffef..ee442fed 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -42,7 +42,7 @@ icon: "rocket" Verify installation: ```python import semantica - print(semantica.__version__) # 0.6.0 + print(semantica.__version__) # 0.6.5 ``` diff --git a/pyproject.toml b/pyproject.toml index d6f9f3bd..7a65ae3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "semantica" -version = "0.6.0" +version = "0.6.5" description = "Accountability and context layer for AI agents. Context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable." readme = "README.md" license = { text = "MIT" } diff --git a/semantica/__init__.py b/semantica/__init__.py index e5f0ead9..79f83236 100644 --- a/semantica/__init__.py +++ b/semantica/__init__.py @@ -10,7 +10,7 @@ Main exports: - Config: Configuration management """ -__version__ = "0.6.0" +__version__ = "0.6.5" __author__ = "Semantica Contributors" __license__ = "MIT" From 918830a82181cd4547e72da3e1f08d3a141b0270 Mon Sep 17 00:00:00 2001 From: Karunasagar Mohansundar <52268863+Karunasagar12@users.noreply.github.com> Date: Wed, 12 Aug 2026 05:19:54 +0530 Subject: [PATCH 40/40] fix(pipeline): resolve broken import and missing `run()` in `PipelineWithProvenance` (#862) * fix(pipeline): resolve broken import and missing run() in PipelineWithProvenance Fix two bugs in pipeline_provenance.py: 1. Wrong import path: `from .pipeline import Pipeline` fails because `semantica/pipeline/pipeline.py` does not exist. Pipeline lives in `pipeline_builder.py`. Fixed to `from .pipeline_builder import Pipeline`. 2. Pipeline dataclass has no run() method. PipelineWithProvenance.run() now delegates to ExecutionEngine.execute_pipeline(), which is the intended execution path for built pipelines. Additional changes: - Constructor now accepts a built Pipeline instance (breaking the previous unusable API that tried to instantiate a dataclass with **config). - Replace deprecated datetime.utcnow() with datetime.now(timezone.utc). - Add test suite covering import, instantiation, execution, attribute delegation, and provenance graceful degradation. Fixes #858 * test: address Qodo review findings - Remove redundant test_import_succeeds (module-level import already guards against import regression at collection time). - Fix test_provenance_disabled_when_import_fails to deterministically simulate ImportError via sys.modules patch and assert provenance is actually toggled off (runner.provenance is False). * fix(pipeline): update provenance callers for Pipeline API --------- Co-authored-by: Sameer Kadam Co-authored-by: Russell Jurney --- CHANGELOG.md | 6 ++ semantica/pipeline/pipeline_provenance.py | 61 ++++++++++++------ semantica/provenance/provenance_usage.md | 10 ++- tests/pipeline/test_pipeline_provenance.py | 64 +++++++++++++++++++ .../provenance/test_all_provenance_modules.py | 25 +++++++- .../provenance/test_provenance_edge_cases.py | 7 +- .../test_real_module_integration.py | 15 +++-- 7 files changed, 156 insertions(+), 32 deletions(-) create mode 100644 tests/pipeline/test_pipeline_provenance.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 34d400d0..8efdcd18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`PipelineWithProvenance` raised `ModuleNotFoundError` on import and `AttributeError` on `.run()`** (#858, closes #858) by @Karunasagar12 + - `from .pipeline import Pipeline` failed because `semantica/pipeline/pipeline.py` does not exist; corrected to `from .pipeline_builder import Pipeline` + - `.run()` called `self._pipeline.run()` on the `Pipeline` dataclass, which has no such method; replaced with `self._engine.execute_pipeline(self._pipeline, ...)` delegating to `ExecutionEngine` + - Constructor now accepts a built `Pipeline` instance (from `PipelineBuilder.build()`) instead of `**config`; the old `Pipeline(**config)` internal construction was invalid and never functional + - Replaced deprecated `datetime.utcnow()` with `datetime.now(timezone.utc)` in `run()` + - **`VectorStore.search_vectors()` returned inconsistent result shapes across backend implementations** (#853, closes #845) by @Sameer6305, reviewed by @KaifAhmad1 - Every built-in backend (FAISS, Milvus, pgvector, Pinecone, Qdrant, SQLite-vec, Weaviate, in-memory) now returns the same canonical `SearchResult` shape (`id`, `score`, `metadata`, `vector`, `distance`), instead of some backends omitting `vector`/`metadata`/`distance` or, for Weaviate, returning a backend-specific `properties` key instead of `metadata` - Added a `SearchResult` `TypedDict` (`semantica/vector_store/vector_store.py`, exported from `semantica.vector_store`) documenting the contract; `metadata` now always defaults to `{}` rather than being absent, and `id` accepts `Union[str, int]` to accommodate Milvus/Qdrant's native integer IDs without casting diff --git a/semantica/pipeline/pipeline_provenance.py b/semantica/pipeline/pipeline_provenance.py index b5c7c99b..a77d7c9a 100644 --- a/semantica/pipeline/pipeline_provenance.py +++ b/semantica/pipeline/pipeline_provenance.py @@ -6,9 +6,14 @@ capturing all steps, inputs, outputs, and transformations. Usage: from semantica.pipeline.pipeline_provenance import PipelineWithProvenance - - pipeline = PipelineWithProvenance(provenance=True) - result = pipeline.run(data) + from semantica.pipeline import PipelineBuilder + + builder = PipelineBuilder() + builder.add_step("ingest", "file_ingest") + pipeline = builder.build("my_pipeline") + + runner = PipelineWithProvenance(pipeline, provenance=True) + result = runner.run(data) # Tracks all pipeline steps with complete lineage Author: Semantica Contributors @@ -16,26 +21,37 @@ License: MIT """ from typing import Optional, Any, Dict, List -from datetime import datetime +from datetime import datetime, timezone import uuid import time +from .pipeline_builder import Pipeline +from .execution_engine import ExecutionEngine + class PipelineWithProvenance: """Pipeline executor with complete provenance tracking.""" - + def __init__( self, + pipeline: Pipeline, provenance: bool = False, agent_id: Optional[str] = None, is_automated: bool = True, - **config, + **engine_config, ): - """Initialize pipeline with optional provenance.""" - from .pipeline import Pipeline + """Initialize provenance-tracked pipeline runner. + Args: + pipeline: A built Pipeline instance (from PipelineBuilder.build()). + provenance: Whether to record provenance metadata. + agent_id: Identifier for the agent running the pipeline. + is_automated: Whether the execution is automated (vs. human-triggered). + **engine_config: Extra keyword arguments forwarded to ExecutionEngine. + """ + self._pipeline = pipeline + self._engine = ExecutionEngine(**engine_config) self.provenance = provenance - self._pipeline = Pipeline(**config) self._prov_manager = None self._agent_id = agent_id or self.__class__.__name__ self._is_automated = is_automated @@ -47,15 +63,24 @@ class PipelineWithProvenance: except ImportError: self.provenance = False - def run(self, data: Any, source: Optional[str] = None, **kwargs): - """Run pipeline with provenance tracking.""" + def run(self, data: Any = None, source: Optional[str] = None, **kwargs): + """Run pipeline with provenance tracking. + + Args: + data: Input data to feed into the pipeline. + source: Provenance source label (defaults to "pipeline_execution"). + **kwargs: Extra options forwarded to ExecutionEngine.execute_pipeline(). + + Returns: + ExecutionResult from the engine. + """ pipeline_id = f"pipeline_{uuid.uuid4().hex[:8]}" start_time = time.time() - activity_started_at_time = datetime.utcnow().isoformat() + activity_started_at_time = datetime.now(timezone.utc).isoformat() - result = self._pipeline.run(data, **kwargs) + result = self._engine.execute_pipeline(self._pipeline, data=data, **kwargs) elapsed = time.time() - start_time - activity_ended_at_time = datetime.utcnow().isoformat() + activity_ended_at_time = datetime.now(timezone.utc).isoformat() if self.provenance and self._prov_manager: self._prov_manager.track_entity( @@ -69,14 +94,14 @@ class PipelineWithProvenance: activity_started_at_time=activity_started_at_time, activity_ended_at_time=activity_ended_at_time, metadata={ - "steps": len(self._pipeline.steps) if hasattr(self._pipeline, 'steps') else 0, + "steps": len(self._pipeline.steps), "duration_seconds": elapsed, - "status": "completed" + "status": "completed" if result.success else "failed", } ) - + return result - + def __getattr__(self, name): return getattr(self._pipeline, name) diff --git a/semantica/provenance/provenance_usage.md b/semantica/provenance/provenance_usage.md index db667038..7113826f 100644 --- a/semantica/provenance/provenance_usage.md +++ b/semantica/provenance/provenance_usage.md @@ -399,12 +399,16 @@ response = llm.generate("What is artificial intelligence?") ```python from semantica.pipeline.pipeline_provenance import PipelineWithProvenance +from semantica.pipeline import PipelineBuilder -# Create pipeline with provenance -pipeline = PipelineWithProvenance(provenance=True) +builder = PipelineBuilder() +builder.add_step("ingest", "file_ingest") +pipeline = builder.build("my_pipeline") + +runner = PipelineWithProvenance(pipeline, provenance=True) # Run pipeline - all steps tracked -result = pipeline.run( +result = runner.run( data=input_data, source="input_file.json" ) diff --git a/tests/pipeline/test_pipeline_provenance.py b/tests/pipeline/test_pipeline_provenance.py new file mode 100644 index 00000000..1756df08 --- /dev/null +++ b/tests/pipeline/test_pipeline_provenance.py @@ -0,0 +1,64 @@ +"""Tests for PipelineWithProvenance. + +Verifies that: +1. The import path is correct (no ModuleNotFoundError). +2. PipelineWithProvenance accepts a built Pipeline and runs it via ExecutionEngine. +3. Provenance tracking gracefully degrades when the provenance package is absent. +""" + +import sys +from unittest.mock import patch + +import pytest + +from semantica.pipeline import PipelineBuilder +from semantica.pipeline.pipeline_provenance import PipelineWithProvenance +from semantica.pipeline.execution_engine import ExecutionResult + + +class TestPipelineWithProvenance: + """Tests for PipelineWithProvenance.""" + + @pytest.fixture + def simple_pipeline(self): + """Build a minimal two-step pipeline for testing.""" + builder = PipelineBuilder() + builder.add_step("ingest", "file_ingest") + builder.add_step("parse", "document_parse") + return builder.build("test_provenance_pipeline") + + def test_instantiation_with_pipeline(self, simple_pipeline): + """Should accept a built Pipeline instance.""" + runner = PipelineWithProvenance(simple_pipeline, provenance=False) + assert runner._pipeline is simple_pipeline + + def test_run_returns_execution_result(self, simple_pipeline): + """run() should delegate to ExecutionEngine and return an ExecutionResult.""" + runner = PipelineWithProvenance(simple_pipeline, provenance=False) + result = runner.run() + assert isinstance(result, ExecutionResult) + assert result.success is True + + def test_getattr_delegates_to_pipeline(self, simple_pipeline): + """Attribute access should fall through to the wrapped Pipeline.""" + runner = PipelineWithProvenance(simple_pipeline, provenance=False) + assert runner.name == "test_provenance_pipeline" + assert len(runner.steps) == 2 + + def test_provenance_disabled_when_import_fails(self, simple_pipeline): + """When semantica.provenance is unavailable, provenance should be disabled.""" + # Force the provenance import to raise ImportError + with patch.dict(sys.modules, {"semantica.provenance": None}): + runner = PipelineWithProvenance(simple_pipeline, provenance=True) + assert runner.provenance is False + assert runner._prov_manager is None + # Should still execute successfully without provenance + result = runner.run() + assert isinstance(result, ExecutionResult) + assert result.success is True + + def test_run_with_data(self, simple_pipeline): + """run() should accept data and kwargs without error.""" + runner = PipelineWithProvenance(simple_pipeline, provenance=False) + result = runner.run(data={"key": "value"}) + assert isinstance(result, ExecutionResult) diff --git a/tests/provenance/test_all_provenance_modules.py b/tests/provenance/test_all_provenance_modules.py index 0d5e5bd4..f0dc25e9 100644 --- a/tests/provenance/test_all_provenance_modules.py +++ b/tests/provenance/test_all_provenance_modules.py @@ -167,7 +167,6 @@ class TestProvenanceEnabledDisabled: """Test all modules work with provenance=False.""" modules_to_test = [ ('semantica.context.context_provenance', 'ContextManagerWithProvenance'), - ('semantica.pipeline.pipeline_provenance', 'PipelineWithProvenance'), ] for module_path, class_name in modules_to_test: @@ -183,7 +182,6 @@ class TestProvenanceEnabledDisabled: """Test all modules work with provenance=True.""" modules_to_test = [ ('semantica.context.context_provenance', 'ContextManagerWithProvenance'), - ('semantica.pipeline.pipeline_provenance', 'PipelineWithProvenance'), ] for module_path, class_name in modules_to_test: @@ -195,6 +193,23 @@ class TestProvenanceEnabledDisabled: except ImportError: pytest.skip(f"{module_path} not available") + def test_pipeline_with_provenance_supports_provenance_flag(self): + """PipelineWithProvenance accepts provenance=True/False. + + PipelineWithProvenance requires a built Pipeline instance (unlike + other *WithProvenance wrappers that own their internal state), so it + cannot participate in the generic no-argument constructor loop above. + """ + try: + from semantica.pipeline.pipeline_builder import Pipeline + from semantica.pipeline.pipeline_provenance import PipelineWithProvenance + + pipeline = Pipeline(name="compat_test") + assert PipelineWithProvenance(pipeline, provenance=False).provenance is False + assert PipelineWithProvenance(pipeline, provenance=True).provenance is True + except ImportError: + pytest.skip("pipeline_provenance not available") + class TestAllModulesEdgeCases: """Test edge cases across all provenance modules.""" @@ -219,10 +234,14 @@ class TestAllModulesEdgeCases: """Test each module has independent provenance manager.""" try: from semantica.context.context_provenance import ContextManagerWithProvenance + from semantica.pipeline.pipeline_builder import Pipeline from semantica.pipeline.pipeline_provenance import PipelineWithProvenance ctx = ContextManagerWithProvenance(provenance=True) - pipe = PipelineWithProvenance(provenance=True) + pipe = PipelineWithProvenance( + Pipeline(name="independence_test"), + provenance=True, + ) # Each should have its own manager assert ctx._prov_manager is not None diff --git a/tests/provenance/test_provenance_edge_cases.py b/tests/provenance/test_provenance_edge_cases.py index 9d0c1f0f..92329482 100644 --- a/tests/provenance/test_provenance_edge_cases.py +++ b/tests/provenance/test_provenance_edge_cases.py @@ -154,10 +154,13 @@ class TestModuleSpecificEdgeCases: def test_pipeline_with_empty_data(self): """Test pipeline with empty data.""" try: + from semantica.pipeline.pipeline_builder import Pipeline from semantica.pipeline.pipeline_provenance import PipelineWithProvenance - pipeline = PipelineWithProvenance(provenance=True) + + pipeline = Pipeline(name="edge_case_test") + runner = PipelineWithProvenance(pipeline, provenance=True) # Should handle empty data - assert pipeline is not None + assert runner is not None except ImportError: pytest.skip("Pipeline not available") diff --git a/tests/provenance/test_real_module_integration.py b/tests/provenance/test_real_module_integration.py index 5dceebed..cd177438 100644 --- a/tests/provenance/test_real_module_integration.py +++ b/tests/provenance/test_real_module_integration.py @@ -29,14 +29,17 @@ class TestRealModuleIntegration: def test_pipeline_real_execution_tracking(self): """Test pipeline tracks execution with provenance.""" try: + from semantica.pipeline.pipeline_builder import Pipeline from semantica.pipeline.pipeline_provenance import PipelineWithProvenance - - pipeline = PipelineWithProvenance(provenance=True) - + + pipeline = Pipeline(name="provenance_tracking_test") + runner = PipelineWithProvenance(pipeline, provenance=True) + # Verify provenance setup - assert pipeline.provenance is True - assert pipeline._prov_manager is not None - + assert runner.provenance is True + assert runner._prov_manager is not None + assert isinstance(runner._prov_manager, ProvenanceManager) + except ImportError: pytest.skip("Pipeline not available")