mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Compare commits
186
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
943be0c10f | ||
|
|
3c00ffb019 | ||
|
|
7a6f1d0417 | ||
|
|
6c2ccfd3af | ||
|
|
cf6c9b7b9c | ||
|
|
52ba7b6890 | ||
|
|
1f868b9779 | ||
|
|
8d5479d22f | ||
|
|
14107e51c3 | ||
|
|
ea9b1f5d4a | ||
|
|
e63bad310e | ||
|
|
d79f2cfb8f | ||
|
|
48c2d2a7ed | ||
|
|
fdafffa980 | ||
|
|
abe1bc8f3e | ||
|
|
a4bcfade7f | ||
|
|
2b077c6d0e | ||
|
|
f6b31925c6 | ||
|
|
4820185924 | ||
|
|
7e1d2550b9 | ||
|
|
f124df4229 | ||
|
|
a47954c19a | ||
|
|
1ee2ae88a7 | ||
|
|
93e4b97517 | ||
|
|
48f219cd5c | ||
|
|
331c857672 | ||
|
|
727b0383cc | ||
|
|
248ae57694 | ||
|
|
ba3737c878 | ||
|
|
e3b24ef872 | ||
|
|
b891902d6d | ||
|
|
9123dcc0bd | ||
|
|
6cbe0ae438 | ||
|
|
fe3baad67c | ||
|
|
7efc66d0e3 | ||
|
|
50f2f82b95 | ||
|
|
d3f37f798e | ||
|
|
5cd79b6436 | ||
|
|
3c99f447e6 | ||
|
|
8dcbee386d | ||
|
|
e2f850e9e4 | ||
|
|
2d976963ab | ||
|
|
283b7ada0c | ||
|
|
483f53aaa6 | ||
|
|
14091d21fb | ||
|
|
58125a0a93 | ||
|
|
394ce5fe61 | ||
|
|
8e9f7c5526 | ||
|
|
d4fdc1f0d3 | ||
|
|
729f4fe932 | ||
|
|
5c6b40f36c | ||
|
|
6390652303 | ||
|
|
1b21fc4cbb | ||
|
|
719efa4794 | ||
|
|
b64e4b6600 | ||
|
|
db0a9e8bfd | ||
|
|
46451ae2e1 | ||
|
|
54bae5dffe | ||
|
|
5b01949dd8 | ||
|
|
3b710c79d2 | ||
|
|
4b312ca2fa | ||
|
|
363a9ad641 | ||
|
|
b7af18a70a | ||
|
|
560ffef59f | ||
|
|
a279e74468 | ||
|
|
6653cbe879 | ||
|
|
0dc26350f9 | ||
|
|
3063bf8096 | ||
|
|
cbb0a6dd8e | ||
|
|
a46be971e6 | ||
|
|
e3405ebc23 | ||
|
|
0ee38c2d99 | ||
|
|
a3074ec454 | ||
|
|
5e40d6e4ce | ||
|
|
96f60e6114 | ||
|
|
a2a8d776a3 | ||
|
|
4801ff3492 | ||
|
|
cfccdab8ed | ||
|
|
c5d382ee81 | ||
|
|
54c274e02c | ||
|
|
1273c4fb1e | ||
|
|
898a92062a | ||
|
|
556e786fd5 | ||
|
|
d83b21ba77 | ||
|
|
8f6948f85d | ||
|
|
60eb595d62 | ||
|
|
861b2bf757 | ||
|
|
78fc9028a8 | ||
|
|
6b7625ef9b | ||
|
|
48a05b00a6 | ||
|
|
58b77ddcf5 | ||
|
|
5d554ec586 | ||
|
|
a7ebec8fe5 | ||
|
|
71cffb15e9 | ||
|
|
d7ee22cf1f | ||
|
|
efdfa39c15 | ||
|
|
66e3333e41 | ||
|
|
9ca83d397f | ||
|
|
d5dc4eabac | ||
|
|
f60ca6a529 | ||
|
|
05c21af117 | ||
|
|
981c9d9208 | ||
|
|
c30ec14858 | ||
|
|
e5c5cf0efa | ||
|
|
e03212cd66 | ||
|
|
83c04a57d6 | ||
|
|
75b026c6dd | ||
|
|
2a303cf4da | ||
|
|
7595bad28f | ||
|
|
2d75952476 | ||
|
|
2ac3eaffd7 | ||
|
|
51c12d5c8f | ||
|
|
e55c03bd39 | ||
|
|
4d3259df31 | ||
|
|
4a886d970e | ||
|
|
e1092ac507 | ||
|
|
b77e3e8c3c | ||
|
|
68f7ae3807 | ||
|
|
56609ab3fc | ||
|
|
7b2b2efe6b | ||
|
|
08a6e7c053 | ||
|
|
5a3bdc393d | ||
|
|
e6b159e5c5 | ||
|
|
7db2e2f46b | ||
|
|
3fbe3cfd2d | ||
|
|
75bc6255d4 | ||
|
|
a96f1590f1 | ||
|
|
063f447202 | ||
|
|
a1194a155d | ||
|
|
17d878cbf3 | ||
|
|
488e381247 | ||
|
|
4acd2f9c33 | ||
|
|
7cac8a6bc3 | ||
|
|
6c77594ea6 | ||
|
|
5109c6fab2 | ||
|
|
12b9694a8e | ||
|
|
49707729ad | ||
|
|
430020c7c4 | ||
|
|
43b207c1c5 | ||
|
|
55bde673c9 | ||
|
|
0f308b2078 | ||
|
|
5c2901ae27 | ||
|
|
dae21166a1 | ||
|
|
67be421533 | ||
|
|
baf8f01f85 | ||
|
|
c58686b4ec | ||
|
|
c3a0078bfd | ||
|
|
04602a0e0e | ||
|
|
eedf1425ca | ||
|
|
d4cb14c1fb | ||
|
|
a8194dfc60 | ||
|
|
c7415f2e92 | ||
|
|
3331df28ad | ||
|
|
de5e20dc55 | ||
|
|
0f252ab355 | ||
|
|
0b77e5fe94 | ||
|
|
893b6db3c3 | ||
|
|
b8297b8077 | ||
|
|
4d37920007 | ||
|
|
6416fbb669 | ||
|
|
4b6cc09585 | ||
|
|
83649f6821 | ||
|
|
2f04bc01a3 | ||
|
|
eaf51b3383 | ||
|
|
616f5ca9b9 | ||
|
|
d42af280e8 | ||
|
|
13915297d2 | ||
|
|
1cee3e7cb9 | ||
|
|
e7ce092ccf | ||
|
|
92dc3304f8 | ||
|
|
f737f72675 | ||
|
|
828179e115 | ||
|
|
b3e107de8f | ||
|
|
55f7eba389 | ||
|
|
a3d8064f3d | ||
|
|
1b9bb4c345 | ||
|
|
20781e8a9e | ||
|
|
6148975e83 | ||
|
|
0ca7b8d489 | ||
|
|
fb7845240b | ||
|
|
d769bf1c39 | ||
|
|
310ac7bd85 | ||
|
|
aeb1752c83 | ||
|
|
dec05b907d | ||
|
|
c77ce9394a | ||
|
|
c7174e9852 |
@@ -69,5 +69,5 @@ If you have ideas on how this could be implemented, please share.
|
||||
|
||||
---
|
||||
|
||||
**Note**: For feature requests that are ready to be implemented, consider creating a [Feature Request issue](https://github.com/Hawksight-AI/semantica/issues/new?template=feature_request.md) instead.
|
||||
**Note**: For feature requests that are ready to be implemented, consider creating a [Feature Request issue](https://github.com/semantica-agi/semantica/issues/new?template=feature_request.md) instead.
|
||||
|
||||
|
||||
@@ -46,8 +46,8 @@ If applicable, paste any error messages or describe unexpected behavior:
|
||||
|
||||
## Checklist
|
||||
|
||||
- [ ] I have searched existing [discussions](https://github.com/Hawksight-AI/semantica/discussions) and [issues](https://github.com/Hawksight-AI/semantica/issues)
|
||||
- [ ] I have checked the [documentation](https://github.com/Hawksight-AI/semantica/tree/main/docs) and [FAQ](https://github.com/Hawksight-AI/semantica/blob/main/docs/faq.md)
|
||||
- [ ] I have searched existing [discussions](https://github.com/semantica-agi/semantica/discussions) and [issues](https://github.com/semantica-agi/semantica/issues)
|
||||
- [ ] I have checked the [documentation](https://github.com/semantica-agi/semantica/tree/main/docs) and [FAQ](https://github.com/semantica-agi/semantica/blob/main/docs/faq.md)
|
||||
- [ ] I have provided a minimal code example (if applicable)
|
||||
- [ ] I have included error messages (if applicable)
|
||||
- [ ] I have provided environment details
|
||||
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
# Funding options for Semantica
|
||||
github: Hawksight-AI
|
||||
github: semantica-agi
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
blank_issues_enabled: true
|
||||
contact_links:
|
||||
- name: 📚 Documentation
|
||||
url: https://github.com/Hawksight-AI/semantica/tree/main/docs
|
||||
url: https://github.com/semantica-agi/semantica/tree/main/docs
|
||||
about: Browse the documentation
|
||||
- name: 💬 Discussions
|
||||
url: https://github.com/Hawksight-AI/semantica/discussions
|
||||
url: https://github.com/semantica-agi/semantica/discussions
|
||||
about: Ask questions and discuss with the community
|
||||
|
||||
+9
-9
@@ -3,31 +3,31 @@
|
||||
## Getting Help
|
||||
|
||||
### 📚 Documentation
|
||||
Check the [docs folder](https://github.com/Hawksight-AI/semantica/tree/main/docs) and [README](https://github.com/Hawksight-AI/semantica/blob/main/README.md) for guides and examples.
|
||||
Check the [docs folder](https://github.com/semantica-agi/semantica/tree/main/docs) and [README](https://github.com/semantica-agi/semantica/blob/main/README.md) for guides and examples.
|
||||
|
||||
### 💬 Community Support
|
||||
- **GitHub Discussions**: [Ask questions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
- **GitHub Discussions**: [Ask questions](https://github.com/semantica-agi/semantica/discussions)
|
||||
- **Discord**: Join our [Discord server](https://discord.gg/sV34vps5hH) for real-time chat
|
||||
|
||||
### 💭 Discussions
|
||||
Join the conversation on [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions):
|
||||
Join the conversation on [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions):
|
||||
- **Q&A**: Ask questions and get help from the community
|
||||
- **Ideas**: Share feature requests and suggestions
|
||||
- **Show and Tell**: Showcase your projects and use cases
|
||||
- **General**: General discussions about Semantica
|
||||
|
||||
### 🐛 Bug Reports
|
||||
Found a bug? [Create an issue](https://github.com/Hawksight-AI/semantica/issues/new/choose)
|
||||
Found a bug? [Create an issue](https://github.com/semantica-agi/semantica/issues/new/choose)
|
||||
|
||||
### 📖 Resources
|
||||
- [Quick Start Guide](https://github.com/Hawksight-AI/semantica/blob/main/docs/quickstart.md)
|
||||
- [FAQ](https://github.com/Hawksight-AI/semantica/blob/main/docs/faq.md)
|
||||
- [Cookbook Examples](https://github.com/Hawksight-AI/semantica/tree/main/cookbook)
|
||||
- [Quick Start Guide](https://github.com/semantica-agi/semantica/blob/main/docs/quickstart.md)
|
||||
- [FAQ](https://github.com/semantica-agi/semantica/blob/main/docs/faq.md)
|
||||
- [Cookbook Examples](https://github.com/semantica-agi/semantica/tree/main/cookbook)
|
||||
|
||||
## Commercial Support
|
||||
|
||||
For enterprise support, custom development, or consulting services:
|
||||
- Contact us through [GitHub Issues](https://github.com/Hawksight-AI/semantica/issues)
|
||||
- Contact us through [GitHub Issues](https://github.com/semantica-agi/semantica/issues)
|
||||
- Include "Commercial Support" in the title
|
||||
|
||||
## Sponsorship
|
||||
@@ -35,7 +35,7 @@ For enterprise support, custom development, or consulting services:
|
||||
### Sponsor this project
|
||||
|
||||
Support Semantica development:
|
||||
- [GitHub Sponsors](https://github.com/sponsors/Hawksight-AI)
|
||||
- [GitHub Sponsors](https://github.com/sponsors/semantica-agi)
|
||||
|
||||
Your sponsorship helps us:
|
||||
- Maintain and improve the framework
|
||||
|
||||
+155
-3
@@ -9,9 +9,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.6.6] - 2026-08-20
|
||||
|
||||
### Added
|
||||
|
||||
- **First-class CrewAI integration** (#962)
|
||||
- **Semantica RDF vocabulary, and deterministic entity/relationship IRIs** (#1109, closes #1107, closes #1101) by @fabio-rovai, reviewed by @KaifAhmad1
|
||||
- Every RDF/JSON-LD export mints terms in `https://semantica.dev/ns#`, and until now nothing declared what those terms meant — the namespace 404s and no vocabulary shipped with the package, so a consumer receiving an export had no way to tell `sem:text` from a typo of it, and no closed-world checker could validate an export at all
|
||||
- `semantica/ontology/vocabulary/semantica-ns.ttl` declares the terms the exporters actually emit — drawn from the emitting call sites in `export/rdf_exporter.py`, `export/json_exporter.py` and `provenance/manager.py`, not from what a vocabulary "ought" to contain. Ships inside the package (`from semantica.ontology.vocabulary import vocabulary_turtle`) so it loads without a network round trip, and is the same document intended to be served at the namespace IRI once hosting/content-negotiation is sorted
|
||||
- `tests/ontology/test_vocabulary.py` ties the document to the code: every term a serializer can write must be declared, so adding a term to an exporter without declaring it fails the build
|
||||
- The missing-id fallback minted entity/relationship IRIs from Python's builtin `hash()`, randomised per process (`PYTHONHASHSEED`), so the same entity got a different IRI on every run and exports couldn't be diffed, deduplicated, or joined to an earlier provenance record. It also wrote `<semantica:entity_N>`, an IRI in the scheme `semantica` rather than the expansion of the declared prefix, so those nodes never joined with anything written through it. Minting now uses SHA-256 and writes a full IRI in the declared namespace; the same fix applies to the default entity/relationship types in the Turtle path
|
||||
- **Fixed during review** (Qodo): the temporal fallback minted from `source_id` only, while the main serializer accepts `source_id` or `source` — relationships using the second form hashed two empty strings, which the previous randomised `hash()` masked by making the IRI unstable anyway; once deterministic, unrelated relationships at the same list index collided on one IRI across exports. Endpoints are now resolved the same way `serialize_to_turtle` resolves them, before minting. `sem:confidence` also lost its declared `xsd:decimal` range: the N-Triples serializer types the same value `xsd:float`, and the two are disjoint, so declaring either contradicted one of the exporters (tracked in #1100) — a new `test_declared_ranges_do_not_contradict_what_the_exporters_emit` guards the whole class of that mistake
|
||||
- **Fixed in follow-up**: `serialize_to_rdfxml`'s default entity type still wrote the bare string `"semantica:Entity"` into an `rdf:resource` attribute, which (unlike a Turtle angle-bracket or an XML element name) is not namespace-expanded — the exact #1101 failure mode, just on the untested RDF/XML path. `json_exporter.py`'s `semantica:format` and `@type: "semantica:KnowledgeGraph"` were emitted but absent from both the vocabulary and the test's `EMITTED_TERMS` guard set, so the "undeclared terms fail the build" claim didn't actually cover them — both are now declared and guarded. `MANIFEST.in` didn't mirror the `pyproject.toml` package-data addition, so a source-distribution install could omit the vocabulary file. The cross-process minting-stability test replaced the subprocess's entire environment with a POSIX-only `PATH`, breaking it on Windows; now overrides only `PYTHONHASHSEED` on top of the inherited environment
|
||||
- **Also fixed, on the JSON-LD paths**: the first fix covered the Turtle, N-Triples and RDF/XML serializers, and left both JSON-LD writers interpolating the entity's own text into `f"semantica:entity/{text}"` and the endpoints into `f"semantica:rel/{source}_{target}"`. Three consequences, all live in 0.6.5: an entity whose text contained a space produced an invalid IRI, and a JSON-LD parser dropped that node in full rather than reporting it, so the entity disappeared from the export; every relationship carrying `source`/`target` rather than `source_id`/`target_id` minted the identical `semantica:rel/_`, collapsing all of them onto one node whose types and endpoints merged; and the JSON-LD `@id` disagreed with the Turtle IRI for the same entity, so the two serializations of one knowledge graph were two different graphs. Both JSON-LD writers now use `mint_entity_iri`/`mint_relationship_iri`, and `JSONExporter.export_entities`/`export_relationships` declare the `semantica` prefix their `@context` was already writing `semantica:entities` against — without it a processor reads that as an IRI in the scheme `semantica`, which is the original #1101 defect on a third path
|
||||
- `tests/export/test_jsonld_iri_minting.py` parses each export with a real JSON-LD processor and asserts the entity survives, the relationships stay distinct, no term expands into the `semantica` scheme, and the JSON-LD `@id` equals the Turtle IRI
|
||||
- 236 export and ontology tests pass
|
||||
|
||||
- **First-class CrewAI integration** (#988, closes #962) by @Shindevrp
|
||||
- New `pip install semantica[crewai]` extra (`crewai>=0.80.0`) — crewai core provides `BaseTool`/`BaseKnowledgeSource`, so `crewai-tools` is intentionally not included, and the extra is intentionally **not** part of the `all` bundle: crewai hard-requires `chromadb~=1.1.0`, which is affected by the unpatched pre-auth code-injection CVE-2026-45829 (see `integrations/crewai/README.md`)
|
||||
- `integrations/crewai/SemanticaKGTool` — a CrewAI `BaseTool` exposing 5 KG actions (`extract_entities`, `extract_relations`, `add_to_graph`, `query_graph`, `find_related`) backed by `NERExtractor` / `RelationExtractor` / `ContextGraph`; supports both sync `run()` and async `arun()`
|
||||
- `integrations/crewai/SemanticaDecisionTool` — a CrewAI `BaseTool` wrapping `AgentContext` with 5 decision-intelligence actions (`record_decision`, `find_precedents`, `trace_causal_chain`, `analyze_impact`, `check_policy`)
|
||||
@@ -41,6 +54,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- New `tests/export/test_distance_exporter_metric_errors.py`: 6 tests covering success, single/multiple failures, opt-out, the no-path-vs-error distinction, and default-schema stability; existing `tests/export/test_distance_exporter.py` updated for the new tuple return type
|
||||
- Full `tests/export/` suite: 77 passed
|
||||
|
||||
- **`ContextGraph.to_kg_dict()`: an adapter converting a `ContextGraph`'s internal `nodes`/`edges`/`source` shape into the canonical `entities`/`relationships`/`source_id` shape `RDFExporter` and `TemporalGraphQuery` consume** (#1081) by @cxzg007
|
||||
- Previously there was no supported way to feed a `ContextGraph` into those consumers without hand-rolling the field remapping; `to_kg_dict()` does it once, with an `entities_only` option that drops relationships left dangling by the filter
|
||||
- **Fixed during review** (Qodo): null `properties`/`metadata` on a node loaded from JSON raised `TypeError` when copied — both are now guarded with `or {}`; entity ids are coerced to `str(node_id)` to match `ContextEdge`'s already-str-coerced endpoints, so valid relationships were no longer dropped by `entities_only` filtering
|
||||
- `RDFExporter`'s validator and `TemporalGraphQuery` now also accept `source_id`/`target_id` endpoints, the shape `to_kg_dict()` emits
|
||||
|
||||
### Changed
|
||||
|
||||
- **`GraphBuilder`'s 6 public methods now have Google-style docstrings** (#878, closes #876) by @cakeni
|
||||
@@ -50,7 +68,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- **Corrected during review**: `add_temporal_edge`/`create_temporal_snapshot` docstrings overclaimed numeric-timestamp support; `_parse_time()` only special-cases `str` and `datetime`, falling back to a bare `str()` cast for anything else (not true numeric parsing). Narrowed to "datetime or ISO-formatted string"
|
||||
- **Fixed along the way**: `build()`'s `**options` documented a default only for `extract`; `extract_relations`, `extract_triplets`, `ner_method`, `relation_method`, and `triplet_method` all have concrete defaults in `_extract_from_text()` (`True`, `True`, `"llm"`, `"llm"`, `"llm"`) that were left unstated, inconsistent with CONTRIBUTING.md's own docstring example of noting defaults inline
|
||||
- `python -m pytest tests/kg/test_kg.py tests/kg/test_graph_builder_external.py -q`: 45 passed
|
||||
- **`GraphBuilder` raw-text extraction now defaults to local extractors instead of LLM extraction** (closes #930) by @dex0shubham
|
||||
- **`GraphBuilder` raw-text extraction now defaults to local extractors instead of LLM extraction** (#941, closes #930) by @dex0shubham
|
||||
- `GraphBuilder._extract_from_text()` defaulted `ner_method`, `relation_method`, and `triplet_method` to `"llm"`, and ran relation extraction unconditionally (`extract_relations` defaulted to `True`) — all four contradicting the defaults documented in the `build()` docstring at the time (`"ml"` / `"pattern"` / `False`), and diverging from the standalone extractors (`NERExtractor` defaults to `method="ml"`, `RelationExtractor` and `TripletExtractor` to `method="pattern"`). The practical effect was that any raw-text `build()` call silently required a configured provider, an API key, and network access
|
||||
- Defaults are now `ner_method="ml"`, `relation_method="pattern"`, `triplet_method="pattern"`, and `extract_relations=False`, matching the docstring. LLM extraction remains fully available and is now opt-in
|
||||
- **To restore the previous behaviour**, pass the methods explicitly:
|
||||
@@ -71,8 +89,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- New regression coverage in `tests/kg/test_graph_builder_extraction_defaults.py` pinning all four defaults, verifying that no default resolves to `"llm"`, confirming explicit LLM opt-in still routes correctly, asserting extractors are constructed once across repeated texts, covering fallback method lists (e.g. `ner_method=["pattern", "ml"]`) for all three extractors, asserting relations are forwarded to triplet extraction (and that `None` is forwarded when relation extraction is disabled or fails), and running the real default path end to end with no provider mocked. Verified to fail against the pre-fix code
|
||||
- Full `kg` suite: 473 passed
|
||||
|
||||
- **Explorer graph canvas now renders edge labels** (#1013, closes #1009) by @yzxcj797
|
||||
- `GraphCanvas.tsx` had no edge-label rendering path at all; Sigma's edge-label renderer draws `data.label`, but the graph state stored the relationship type under `edgeType`, so simply enabling the renderer would have left every edge blank. `graphSceneState`'s edge reducer now maps `edgeType` onto `label` (suppressed for hidden edges)
|
||||
- Rendering is gated behind a new `edgeLabelsEnabled` entry in the Effects panel (default on), wired through the existing `GraphEffectToggle`/`GraphEffectsState` plumbing, so dense graphs can still turn labels off
|
||||
- **Fixed during review** (Qodo): two follow-up passes closed gaps the first cut left — label rendering wasn't wired through `explorationEffectsPluginPhaseC.tsx`'s Phase C variant, and toggling the effect off mid-session didn't clear already-rendered labels
|
||||
- New coverage in `explorer/tests/graphSceneState.display.test.ts`
|
||||
|
||||
- **Removed `GraphWorkspaceShell.tsx`, `GraphRuntimeStage.tsx`, and `useGraphData.ts` — a second, unused implementation of the graph-loading/error-handling logic already fixed in `GraphWorkspace.tsx`** (#984, resolves the cleanup tracked in #981 by #980's review note) by @lakshayxi
|
||||
- 1,564 lines removed; the surviving `GraphWorkspace` path is now the only implementation, so the "two copies that drifted apart" root cause #980 fixed can't recur in the copy nobody was maintaining
|
||||
|
||||
- **Explorer README and `docs/explorer-setup.md` corrected to describe the authentication 0.6.5 actually shipped**, plus a documented `/ws/graph-updates` auth note (#1040, fixes #1028) by @Kyou12138
|
||||
- Both docs still claimed the Explorer API had no built-in authentication after v0.6.5 added mandatory `SEMANTICA_API_KEY` enforcement with a `503` fail-closed default; corrected to describe the actual behavior, including that only protected routes require the key (`/api/health`/`/api/info` stay open), the non-loopback-bind CLI warning only fires in anonymous mode or when the key is unset, and `SEMANTICA_API_KEY`/`SEMANTICA_ALLOW_ANONYMOUS` are documented in the environment-variable table
|
||||
|
||||
- **CI: pinned `github/codeql-action` to current v4** (#986) by @ZohaibHassan16, and **pinned Python dependencies in `requirements-ci.txt` for reproducible CI runs** (#945) by @yunaremaia, closing the gap where an unpinned CI dependency could silently change behavior between runs
|
||||
|
||||
- **README now states up front that Semantica's explainability is system-level, not foundation-model-internal** (#1033, #1034) by @KaifAhmad1
|
||||
- Nothing in the README previously scoped what "explainable" meant, leaving readers to assume Semantica could expose or reconstruct an LLM's internal reasoning. A callout now states explicitly that Semantica explains and audits what the AI *system* did — context fed in, decisions produced, provenance, relationships, policies applied — not the model's private internal reasoning, and moved the note near the top of the README rather than leaving it implicit
|
||||
|
||||
### Fixed
|
||||
|
||||
- **The temporal-evolution `stability` metric was a hardcoded placeholder, not a duration**
|
||||
- `TemporalGraphQuery.analyze_evolution()` documents `stability` as a "relationship duration/stability measure", but the implementation appended a constant `1` for every relationship with both `valid_from` and `valid_until` set (`durations.append(1) # Placeholder`). The reported stability was therefore always `1.0` when any bounded relationship existed and `0` otherwise — it never reflected how long relationships actually stayed valid, so it could not distinguish a graph of decade-long relationships from one of one-second relationships
|
||||
- `stability` now computes the mean valid-time duration in seconds (`(valid_until - valid_from).total_seconds()`) across relationships that have both bounds set. Relationships with a missing or open `valid_from`/`valid_until` are skipped (their duration is unbounded), and non-positive intervals are clamped to `0`; an empty set still reports `0`
|
||||
- New tests in `tests/kg/test_kg.py` assert the mean-duration result, the skipping of unbounded/half-open intervals, and the empty-graph zero case
|
||||
|
||||
- **Every timestamp an export or a provenance record wrote was timezone-naive** (closes #1114) by @fabio-rovai
|
||||
- `semantica/export/` stamped with `datetime.now().isoformat()`, which reads the machine's **local** clock; `semantica/provenance/` stamped with `datetime.utcnow().isoformat()`, which reads **UTC**. Both produce a naive value and both serialize identically, so nothing downstream can tell which zone a given timestamp belongs to — the same string means two different instants depending on which module wrote it
|
||||
- In RDF the consequence is silent rather than loud. Under XSD 1.1 a value with no timezone compared against one with a timezone is indeterminate whenever the two fall inside the ±14 hour window; SPARQL turns an indeterminate comparison into an error, and `FILTER` discards errors as non-matches. A timezone-qualified query over an Oxigraph store returns an answer with every Semantica-written record quietly absent from it, which is a poor property for `prov:generatedAtTime`, `prov:startedAtTime`, `prov:endedAtTime` and `prov:atTime` to have
|
||||
- New `utc_now()`/`utc_now_iso()` in `semantica/utils/helpers.py`, exported from `semantica.utils`, and used at all 29 call sites in `export/` (`json_exporter`, `yaml_exporter`, `report_generator`, `export_provenance`) and `provenance/` (`manager`, `schemas`, `bridge_axiom`). Values now read `2026-08-19T14:19:04.229937+00:00`: one unambiguous instant, comparable against any correctly stamped value, and valid `xsd:dateTimeStamp`. `sem:exportedAt`'s range in `semantica/ontology/vocabulary/semantica-ns.ttl` is tightened from `xsd:dateTime` accordingly, and its comment no longer has to explain why the weaker range was necessary
|
||||
- `datetime.utcnow()` is deprecated as of Python 3.12 and scheduled for removal; constructing a `ProvenanceEntry` under `-W error::DeprecationWarning` on 3.13 raised, and no longer does
|
||||
- New `tests/export/test_timestamp_timezones.py` and `tests/provenance/test_timestamp_timezones.py`: offset presence on every export and provenance path, PROV-O literals valid as `xsd:dateTimeStamp`, comparison against a timezone-aware instant without `TypeError`, the Oxigraph filter that dropped the naive value (with a bound inside the indeterminate window, so the test cannot pass by accident), and the document `@id` remaining a valid IRI with `+00:00` in it. 11 of the 13 fail on the parent commit
|
||||
- **Fixed during review** (Qodo): once new entries carry `+00:00` and stored ones do not, `ProvenanceManager.query_recorded_between` and `audit_log` compared ISO timestamps as raw strings, so they ordered by spelling rather than by instant — an inclusive naive bound naming a stored offset-bearing timestamp sorted *below* it and dropped the record, and a bound written in another offset landed wherever its digits fell (`19:45+05:30` is 14:15Z, but sorted after 14:19Z). Both now compare instants through a new `to_utc_datetime()` helper that reads a missing offset as UTC, which is what the values written before this change actually were; a bound that cannot be read as a timestamp keeps the historical string comparison rather than raising on a call that used to work
|
||||
- The remaining 147 naive call sites are in `context/`, `vector_store/`, `seed/` and elsewhere, where timestamps are compared against values parsed from previously stored naive strings. Converting those without a read-side migration would raise `TypeError: can't compare offset-naive and offset-aware datetimes` on existing data, so they are deliberately left for a separate change
|
||||
|
||||
- **`split`/chunking paths bypassed the centralized spaCy model cache, reloading the model on every call** (#1042, closes #998) by @Accute9, reviewed by @Sameer6305
|
||||
- `semantica/split/methods.py`'s `split_by_sentences()` and `semantica/split/semantic_chunker.py`'s `SemanticChunker.__init__` each called `spacy.load()` directly instead of reusing the process-level cache added in #889/`semantic_extract/methods.py`'s `load_spacy_model()` — every call/construction re-paid the ~120ms model-load cost independently of `NERExtractor`, which already used the cache
|
||||
- Both now route through `load_spacy_model()`, sharing one cached `Language` instance per model name across `split_by_sentences()`, `SemanticChunker`, and `NERExtractor`; a missing model still falls back to regex/paragraph chunking without poisoning the cache for a later successful load
|
||||
- **Fixed during review** (@Sameer6305): `NERExtractor.__init__()` still had a direct `spacy.load()` call site with the same cache-bypass issue, outside the two files named in #998 but sharing the same root cause; routed through the cache alongside stale test patch targets and a strengthened cache-configuration assertion
|
||||
- **Fixed during review** (@KaifAhmad1): `SemanticChunker.__init__` only caught `OSError` around `load_spacy_model()`, while the sibling fix to `NERExtractor` in this same PR added a broader `except Exception` for a model that is installed but fails at runtime (e.g. a config incompatible with the installed spaCy version). A broken-but-present model crashed `SemanticChunker()` outright instead of degrading to fallback chunking like every other path in this PR. Added the matching `except Exception` branch, leaving `self.nlp` as `None`; new `test_semantic_chunker_falls_back_when_spacy_runtime_is_broken` mirrors the existing `NERExtractor` regression test for the same scenario
|
||||
- New `tests/split/test_spacy_model_cache.py`: cache reuse across repeated calls/instances, shared cache between `split_by_sentences()`/`SemanticChunker`/`NERExtractor`, distinct model names loading separately, missing-model fallback without poisoning the cache, and the broken-runtime fallback added above
|
||||
- `pytest tests/split/test_spacy_model_cache.py tests/split/test_splitter.py tests/split/test_chunkers.py`: all passing (3 pre-existing, unrelated `tests/test_ner_configurations.py` failures confirmed present on `main` before this PR)
|
||||
|
||||
- **`export_yaml` raised a raw `AttributeError` on list input, silently wrote empty exports for unrecognized dict keys, and graph payloads were reconciled differently by every exporter** (#958, closes #956, #952, #953) by @pravit-amp, reviewed by @Sameer6305
|
||||
- Graph payloads circulate under two vocabularies, `entities`/`relationships` and `nodes`/`edges`, and each exporter reconciled them locally with a different idiom — `LPGExporter` in particular dropped every entity whenever `nodes` was present but empty, the exact shape `JSONExporter` emits. A new `normalize_graph_payload()` in `utils/helpers.py` centralizes that decision once, adopted by `LPGExporter`, `ArangoAQLExporter`, `Neo4jCSVExporter`, and both YAML exporters; `ContextGraph.to_dict()` now round-trips through YAML correctly as a result
|
||||
- `export_yaml(records, path)` on a bare list previously failed with `AttributeError` from inside the exporter; it and the other YAML methods now reject non-mapping input with an actionable `ProcessingError` naming the expected keys, since these formats distinguish entities/relationships/triplets and guessing which one a list represents would mislabel the records
|
||||
@@ -166,8 +223,86 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- New regression coverage in `tests/export/test_distance_exporter.py`: warnings fire on exception for all four helpers, exported sentinel values/shape stay unchanged, and the legitimate "no KG backend" `None` path still logs nothing
|
||||
- Full `tests/export/` suite: 71 passed
|
||||
|
||||
- **`explain_violations` rendered hardcoded placeholders (`min_count=1`, `max_count=1`) instead of the SHACL shape's real constraint values, and misused the violation message text as the datatype/class value** (#1094) by @cxzg007
|
||||
- `_run_pyshacl` never read `sh:minCount`/`sh:maxCount`/`sh:datatype`/`sh:class` back from the violation's `sh:sourceShape`, so every plain-English explanation was wrong regardless of what the shape actually declared. `SHACLViolation` now carries those four fields (also exposed via `to_dict()`), populated by back-referencing `sh:sourceShape`; `explain_violations` renders the real values, falling back to `"?"` when a value is genuinely absent
|
||||
- **Known limitation**: `sh:qualifiedMinCount`/`sh:qualifiedMaxCount` are not handled yet and still fall back to the `"?"` placeholder
|
||||
- New regression tests cover both the rendering path and the `sh:sourceShape` back-reference (skipped when `pyshacl`/`rdflib` are absent)
|
||||
|
||||
- **Entity merging silently dropped `entity_id` aliases, and exact-match entity resolution had three correctness gaps** (#1086, #1026) by @T1mn
|
||||
- `entity_merger.py`/`merge_strategy.py`/`entity_resolver.py` used inconsistent logic for extracting an entity's id across the merge path, so a merged entity could lose the `entity_id` aliases that let later lookups find it under its old identity. A new `semantica/utils/entity_ids.py` unifies id extraction across all three call sites
|
||||
- `EntityResolver`'s exact-match path is now honored rather than silently falling through to fuzzy matching in some cases; entities with no identifier are preserved instead of being dropped, and blank exact-match names are ignored rather than matching every other blank name
|
||||
- New/expanded coverage in `tests/kg/test_entity_pipeline.py` and `tests/kg/test_entity_resolver_exact.py`
|
||||
|
||||
- **`flatten_dict()` silently collided keys when a flattened path from one branch matched a literal key already present at the target depth** (#1062) by @shahzaib-ahmadcs
|
||||
- Two differently-shaped inputs could flatten to the same output key, with the second write silently overwriting the first — no error, no warning, just a dropped value. Collisions are now detected and handled explicitly instead of overwriting
|
||||
|
||||
- **`ExcelParser.__init__` raised `NameError` on every instantiation — `get_progress_tracker()` was called but never imported** (#1016, closes #1014) by @pravit-amp
|
||||
- Same defect as the one fixed for `SimilarityCalculator` in #530, this time in `semantica/parse/excel_parser.py`; the existing test imported the class but never constructed it, so nothing caught the missing import. Added construction coverage for every parser exported from `semantica.parse`, driven off `__all__` so future additions are covered automatically, living outside `test_parse_comprehensive.py` (whose `setUp` mocks `get_progress_tracker` into each module and would mock away the exact interaction under test)
|
||||
|
||||
- **Graph analytics (`centrality_calculator.py`, `community_detector.py`, `connectivity_analyzer.py`) dropped isolated nodes and diverged on how each computed its working view of the graph** (#1011) by @T1mn
|
||||
- Each analyzer had its own ad hoc logic for building the node/edge set it operated over, and none of them included nodes with no edges — a node with zero connections simply vanished from centrality scores, community assignments, and connectivity reports instead of appearing with a zero/singleton value. A new shared `semantica/kg/_graph_view.py` centralizes graph-view construction (including node fallbacks and community payload shaping) for all three analyzers, which are now ~250 lines lighter combined
|
||||
- New `tests/kg/test_analytics_node_scope.py` covering isolated-node presence across all three analyzers
|
||||
|
||||
- **Explorer fired temporal-bounds and snapshot requests before the graph itself had loaded, tripling failed requests when the backend was down and leaving the timeline scrubber with nothing to scrub** (#1003) by @lakshayxi
|
||||
- Two new predicate functions gate the temporal effects on the graph having actually loaded (an empty graph still counts as loaded); confirmed against a downed backend that this cuts three failing requests per page load down to one
|
||||
|
||||
- **`SeedDataManager.load_from_database()` never actually reached the database, and connection failures were mislabeled as a missing optional dependency** (#995, closes #973) by @yzxcj797
|
||||
- `DBIngestor.execute_query`/`export_table` need the connection string as their first positional argument; `load_from_database()` only passed it into the constructor's config dict, which those methods never read, so every call raised `TypeError` before connecting. Also split the combined `except (ImportError, OSError)` handling apart — a genuine connection failure was reported as `"module not available"`, sending debugging in the wrong direction; `OSError` now propagates as an actual failure, chained via `from e`
|
||||
|
||||
- **SPARQL `CONSTRUCT` detection matched inside a leading `#`-comment, misclassifying `SELECT`/`ASK` queries as `CONSTRUCT` across all four SPARQL backends** (#951) by @pravit-amp
|
||||
- `CONSTRUCT_QUERY_RE` skipped comments with a bare `\#[^\n]*`, whose backtracking `*` let a `# CONSTRUCT ...` comment line "swallow" the real query-form keyword on the next line for a query like `# CONSTRUCT ...\nSELECT ...`. The mistaken `CONSTRUCT` classification sent `Accept: text/turtle` and tried to parse a SELECT/ASK response body as Turtle, failing with a misleading parse error. The regex now requires a comment to reach a line terminator (LF or CR, per the SPARQL grammar) before matching
|
||||
|
||||
- **`k_shortest_paths` mutated caller-visible graph state during traversal and ignored direction when excluding already-used edges** (#1000) by @T1mn
|
||||
- `semantica/kg/path_finder.py`'s search left side effects behind after returning, and edge exclusion during Yen's-algorithm-style path removal didn't respect the traversal direction of directed graphs, letting a later search see edges that should have been available. Both fixed; new coverage in `tests/kg/test_path_finder.py`
|
||||
|
||||
- **`trace_decision_causality()` ignored explicitly recorded causal edges, inferring causes only from shared NER entities plus timestamp ordering** (#983) by @hsd2514
|
||||
- A `CAUSED`/`INFLUENCED`/`PRECEDENT_FOR` edge added via `add_causal_relationship()` had no effect on the trace — when entity extraction found nothing in common between two decisions, `trace_decision_chain()` came back empty even with an explicit edge stored in the graph. Explicit causal edges are now traversed first as ground truth, with entity/timestamp inference kept as an additive fallback for pairs with no explicit link; edges whose source has no decision record (e.g. a graph restored via `from_dict`) are skipped so a stale edge can't abort the trace
|
||||
|
||||
- **`RepoIngestor`'s module-level DNS resolve cache had no lock, raising `RuntimeError: OrderedDict mutated during iteration` under concurrent `ingest_repository()` calls** (#979) by @manjunathbhaskar
|
||||
- `_REPO_HOST_RESOLVE_CACHE` is a shared `OrderedDict` read, written, and pruned by every thread with no synchronization — reliably reproduced with 32 threads hammering resolution under a low TTL and small cache cap. Now guarded by a lock
|
||||
|
||||
- **`GraphBuilder` didn't remap relationship endpoints after entity resolution merged nodes, leaving relationships pointing at ids that no longer existed in the resolved graph** (#978) by @T1mn
|
||||
- New coverage in `tests/kg/test_graph_builder_external.py`; a follow-up commit hardens the remapping against edge cases found during review
|
||||
|
||||
- **Explorer's dev server esbuild target didn't match the browser targets the production build declares**, occasionally producing dev-only syntax errors on older browsers (#966) by @le-czs
|
||||
- `explorer/vite.config.ts` now sets the dev esbuild target explicitly to match
|
||||
|
||||
- **`normalize`'s number normalizer accepted currency symbols without validating them against the surrounding text, and an earlier fix's currency-code matching wasn't token-bounded** (#940) by @Mr-Neutr0n, reviewed by @ZohaibHassan16
|
||||
- Symbol currencies are now validated before being accepted; currency codes are matched on token boundaries so a code embedded inside a longer token no longer false-positives
|
||||
|
||||
- **`ContextGraph.to_dict()` was the one reader on the class that didn't hold `self._lock`, raising `RuntimeError: dictionary changed size during iteration` under a concurrent writer and risking a torn snapshot otherwise** (#929) by @pravit-amp
|
||||
- Every other reader (`stats()`, `density()`, `find_nodes()`, `find_edges()`, `get_neighbors()`, `get_nodes_by_label()`, `state_at()`, `save_to_file()`) already took the lock after it was introduced; `to_dict()` predated that change and was missed. `save_to_file()` was safe only incidentally, since it builds its payload inline under its own lock rather than delegating to `to_dict()`
|
||||
|
||||
- **`PipelineWithProvenance` had a broken import and no working `run()` method** (#862) by @Karunasagar12
|
||||
- `from .pipeline import Pipeline` failed because `Pipeline` lives in `pipeline_builder.py`, not a nonexistent `pipeline.py` — fixed to `from .pipeline_builder import Pipeline`. The class also had no `run()`; it now delegates to `ExecutionEngine.execute_pipeline()`, the intended execution path for a built `Pipeline`. The constructor now accepts a built `Pipeline` instance directly
|
||||
|
||||
### Security
|
||||
|
||||
- **Tarball restore path traversal, latent SQL injection, DNS-rebinding TOCTOU in the shared SSRF guard, stored XSS in report generation, and unvalidated SPARQL object IRIs in AnzoStore** (#1079) by @KaifAhmad1
|
||||
- `semantica backup restore`'s tar extraction (`cli.py`) stripped only the literal `semantica-backup/` prefix and called `tar.extract()` with no path-containment check, no symlink/hardlink validation, and (on Python <3.12) no extraction filter — a crafted archive member (`../../<file>`, or a symlink pointing outside the restore root) could write arbitrary files above the restore directory. Every member is now validated for resolved-path containment before extraction, symlink/hardlink targets are rejected both lexically (absolute path, `..` segments) and by resolution, and `filter="data"` is applied on Python ≥3.12
|
||||
- `DataExporter.export_table_data()` (`db_ingestor.py`) was missing the `text` import from `sqlalchemy` — a `NameError` that made the method non-functional, but latently: the query it built from raw f-string interpolation of `table_name`/`schema`/`where`/`order_by` was already injectable, so fixing the import alone (without also fixing the injection) would have silently armed it. Both are fixed together: the import is restored, `table_name`/`schema` are now validated against a strict identifier allowlist, and `where`/`order_by` are checked against a blocklist (statement separators, comments, UNION, DDL/DML keywords, time-based blind-injection primitives, schema-enumeration terms). This is a blocklist, not a grammar — it closes the concrete UNION-exfiltration path and common injection primitives, but a boolean-blind subquery using none of the blocked keywords could still get through; `where`/`order_by` must be treated as trusted/operator input, not exposed to untrusted end users, and the docstrings now say so explicitly
|
||||
- `request_with_ssrf_guard()` (`ssrf.py`) validated a hostname's resolved IPs, then let the underlying HTTP client re-resolve the same hostname independently at connect time — a low-TTL or DNS-rebinding answer could differ between the two lookups, so a hostname that validated as public could still connect to a private/internal address. Ported the IP-pinning pattern already used by `explorer/routes/ontology.py`'s `_make_pinned_session` into the shared ingest guard: the one resolution that decides accept/reject is now also the one the connection is pinned to, via a custom `HTTPAdapter` that presents the real hostname over TLS SNI / Host header while connecting only to the validated IPs. Also closes the RFC 6598 Carrier-Grade NAT gap noted as a known limitation in #905/#868: `100.64.0.0/10` is now in `BLOCKED_NETWORKS`
|
||||
- `ReportGenerator._generate_html()` (`export/report_generator.py`) f-string-interpolated report title/summary/metrics into HTML with no escaping — an ingested entity or document whose content flowed into a report (e.g. `<img src=x onerror=...>`) executed as stored XSS when the report was opened. All interpolated values are now `html.escape()`d
|
||||
- `AnzoStore._format_object_for_sparql()` (`triplet_store/anzo_store.py`) validated the subject/predicate of a triplet via `sparql_escaping.validate_uri()` before interpolating them into a SPARQL `INSERT DATA` clause, but delegated the **object** position to a separate formatter that wrapped it as `<{obj}>` without the same validation — an object value containing `>`/`}`/`{`/`"` could close the intended `<...>` token early and inject additional SPARQL Update operations. The Blazegraph/RDF4J backends were hardened for the equivalent gap previously; Anzo's object position now goes through the same `validate_uri()` check
|
||||
- Also hardened in the same pass: Apache AGE's `create_index()` `index_type` parameter is now allowlisted (was interpolated raw into a `USING` clause); Neo4j's `limit` is now explicitly validated (raises `ValidationError` for non-integer input instead of falling through to a generic `ProcessingError`); the `ffprobe` metadata-extraction subprocess call is guarded against a filename starting with `-` being parsed as an option; the MCP server no longer echoes raw exception text to JSON-RPC clients, logging full details server-side and returning a generic message plus the exception class name instead
|
||||
- **Fixed during review** (@KaifAhmad1): the SSRF IP-pinning change introduced a connection-pool leak of its own — `requests.Session.mount()` silently drops whatever adapter it replaces without closing it, so a multi-hop redirect chain on a reused session leaked one pooled connection per hop. Pinned adapters are now tagged and explicitly closed before being replaced, both per-hop and on final restore
|
||||
- **Fixed during review** (@KaifAhmad1): mounting a pinned adapter and setting a Host header on a caller-supplied `Session` is not inherently thread-safe — two guarded calls sharing the same session from different threads could interleave their mount/restore cycles. Added a per-session lock (`_get_session_lock`) so concurrent guarded calls on the same session now serialize instead of racing; verified with a two-thread test showing correct serialization and zero cross-contamination of per-request Host headers
|
||||
- **Fixed during automated PR review** (Qodo): `export_table_data()`'s new identifier/fragment validation raised `ValidationError` from inside a `try` whose blanket `except Exception` re-wrapped it as `ProcessingError`, masking the distinction between "bad input" and "the export itself failed" that callers rely on elsewhere in this module. Added the `except ValidationError: raise` guard already used by its sibling methods
|
||||
- **Fixed during automated PR review** (Qodo): on a hop where IP pinning doesn't apply (`allow_private_ips=True`), `_apply_connection_pin()` unconditionally popped the session's `Host` header instead of restoring whatever it was before pinning touched it — a caller-supplied session carrying its own legitimate `Host` override (e.g. fronting a private endpoint under a different name) had that override silently dropped for the in-flight request, only reappearing afterward via the outer `finally` restore. It now restores the session's own pre-call header state (set back if present, popped only if it was truly absent) instead of always popping
|
||||
- **Fixed during automated PR review** (Qodo): the `where`/`order_by` blocklist matched keywords/punctuation inside properly quoted string literals and identifiers too, so legitimate data like `status = 'union'` or `name = 'a--b'` was rejected as if it were SQL syntax. The blocklist now runs against a copy with quoted-literal contents masked out (`_mask_sql_literals`) — a malformed/unterminated quote sequence doesn't match the masking pattern and is left fully exposed to the blocklist, so this closes false positives without opening a masking-based bypass; the fragment actually used in the query is unchanged
|
||||
- Re-ran each finding's proof-of-concept (or an equivalent adversarial test) against the fix and confirmed it is blocked: tar path/symlink traversal (both lexical and resolved-path forms), SQL UNION exfiltration and identifier breakout, DNS-rebinding TOCTOU (including under a configured `HTTP_PROXY`, which the pinning adapter also rejects outright since a proxy would resolve DNS itself), stored XSS, and the AnzoStore SPARQL injection
|
||||
- `pytest tests/ingest/`: 266 passed, 2 skipped (10 pre-existing failures unrelated to this change — identical failure set confirmed on unmodified `main`); full regression sweep across `graph_store`, `export`, `triplet_store`, `parse`, and backup/restore: 313 passed
|
||||
|
||||
- **`Authorization`/`Proxy-Authorization` credentials could leak to a different origin across HTTP redirects, and several ingest paths bypassed the shared SSRF/redirect guard entirely** (#1067, closes #947) by @Sameer6305, reviewed by @KaifAhmad1
|
||||
- `request_with_ssrf_guard()` previously only stripped sensitive headers from per-request `kwargs["headers"]` on a cross-origin redirect; session-level `Authorization`/`Proxy-Authorization` headers, `session.auth`, and `session.trust_env` (`.netrc` lookup) could all still resurrect credentials on the hop to a foreign origin. All five credential sources are now stripped case-insensitively, kept stripped for the remainder of a multi-hop redirect chain (no resurrection even if a later hop returns to the original host), and unconditionally restored via `finally` — including on exceptions and redirect-limit errors
|
||||
- `MCPClient._send_request_http()` and `PublicAPIIngestor.detect_public_api()`/`ingest_public_api()` called `httpx.post()`/`requests.post()`/`session.request()` directly, bypassing `request_with_ssrf_guard()` entirely. Both now route through the shared guard, including when `validate_no_auth=False`
|
||||
- `SeedDataManager.load_from_api()` mutated the caller-supplied `headers` dict in place when adding an API-key `Authorization` header, silently leaking the key back into a dict the caller might reuse elsewhere. Now copies before modifying
|
||||
- **Fixed during review** (@KaifAhmad1): `allow_private_ips=True` (used to let MCP servers run on localhost/internal networks) was applied to every redirect hop, not just the operator-configured host — a compromised or malicious MCP server could 302-redirect to an internal address (e.g. `169.254.169.254` cloud metadata) and the guard would follow it unchecked, defeating the SSRF protection this PR otherwise adds. Added `allow_private_ips_on_redirect` to `request_with_ssrf_guard()`: a redirect target inherits the original host's private-IP trust only when it matches that host; any other host falls back to strict validation. `MCPClient` now pins `allow_private_ips_on_redirect=False`, so only same-host redirects on a trusted MCP server keep working — a cross-host hop into private address space is blocked
|
||||
- **Fixed during review** (@KaifAhmad1): `detect_public_api()` only caught `requests.exceptions.RequestException`, but `request_with_ssrf_guard()` raises `ValidationError` (a disjoint hierarchy) for SSRF-blocked hosts, blocked redirect targets, missing `Location`, or exceeded redirect limits — unlike its sibling `ingest_public_api()`, which already caught it. Callers (including `is_public_api()`) got an undocumented raw `ValidationError` instead of `ProcessingError`, and the error-logging call was skipped. Now catches `(ValidationError, ProcessingError)` and re-raises, matching the sibling method
|
||||
- **Fixed during review** (@KaifAhmad1): `detect_public_api()`/`ingest_public_api()` forwarded `**options` into `request_with_ssrf_guard(..., session=self.session, allow_private_ips=self.allow_private_ips, **request_options)` without stripping `session`/`allow_private_ips` from `request_options` first — a caller passing either through the per-call `**options` (a plausible mistake, since `allow_private_ips` is also a documented constructor-level knob) got a raw `TypeError: got multiple values for keyword argument`. Both are now popped from `request_options` before the call
|
||||
- New regression coverage added during review: `TestAllowPrivateIpsOnRedirect` (cross-host redirect into private space blocked, same-host redirect trust preserved, default behavior unchanged for existing callers that don't pass the new kwarg) and `TestMCPClientAuthRedirect::test_redirect_to_private_ip_is_blocked`/`test_same_host_redirect_on_private_mcp_server_is_not_blocked` in `tests/ingest/test_auth_header_redirect_security.py`; `test_detect_public_api_propagates_ssrf_validation_error` and duplicate-kwarg regression tests for both methods in `tests/ingest/test_public_api_ingestor.py`
|
||||
- `pytest tests/ingest/test_auth_header_redirect_security.py tests/ingest/test_public_api_ingestor.py tests/test_seed_manager.py tests/ingest/test_submodules.py tests/ingest/test_cookbook_integration.py`: 111 passed
|
||||
|
||||
- **`FeedIngestor`/`FeedMonitor` (RSS/Atom feed ingestion) had no SSRF protection, allowing requests to internal/private network targets** (#928, closes #927) by @ZohaibHassan16
|
||||
- `FeedIngestor.ingest_feed()`, `discover_feeds()` (link-tag fetch, common-path HEAD probe, and feed-validation GET), and `FeedMonitor.check_updates()` all called `requests.get()`/`requests.head()` directly with default redirect-following and no scheme allowlist or private/loopback/link-local IP validation — despite `semantica/ingest/ssrf.py`'s `request_with_ssrf_guard()` already existing and being used by `web_ingestor.py`/`api_ingestor.py`. `ingest_feed()`'s own URL check only verified `urlparse(url).scheme`/`.netloc` were non-empty, never that the scheme was http/https or that the resolved target IP was safe. Reachable via the public `ingest_feed()`/`ingest()` entry points with any caller-supplied feed URL
|
||||
- All 5 call sites now route through `request_with_ssrf_guard()`, which validates scheme (http/https only) and resolved IP before the request, and re-validates every redirect `Location` before following it — closing both the direct-IP and redirect-chain SSRF paths. Added an `allow_private_ips` config option to both `FeedIngestor` and `FeedMonitor`, consistent with the other ingestors
|
||||
@@ -201,6 +336,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- **Caught by the new gate on its first run**: `python -m pip install -e ".[all]"` pulled in `setuptools==79.0.1`, vulnerable to CVE-2026-59890/GHSA-h35f-9h28-mq5c/PYSEC-2026-3447 (Unicode-normalization bypass of `MANIFEST.in` exclude/prune patterns on macOS APFS/HFS+, letting excluded files leak into a built sdist), fixed in `83.0.0`. `[build-system] requires` had the exact same too-permissive-floor pattern this whole entry is about (`setuptools>=61.0`), and `actions/setup-python`'s baked-in `setuptools` isn't governed by that pin at all since it's outside any isolated build. Bumped `[build-system] requires` to `setuptools>=83.0.0`, and the `Security` workflow now runs `pip install --upgrade pip setuptools` before auditing so the scanned environment can't have a stale ambient copy regardless of what governs it
|
||||
- Full `explorer` suite: 241 passed
|
||||
|
||||
- **`SeedDataManager.load_from_api()` made unguarded HTTP requests, with no SSRF protection at all** (#942) by @ZohaibHassan16
|
||||
- `load_from_api()` called `requests.get()` directly instead of going through `semantica/ingest/ssrf.py`'s `request_with_ssrf_guard()`, unlike every other ingestor in this module — a caller-supplied `api_url` could target internal/private network addresses with no validation. Now routes through the shared guard, gaining redirect validation and bounded DNS resolution for free
|
||||
- **Follow-up** (#959, closes #943) by @yunaremaia: added an `allow_private_ips` opt-in (parsed via the shared `parse_bool` helper) for trusted internal deployments that legitimately need to load from a private-network API, while keeping the guard's block-by-default behavior for everyone else
|
||||
|
||||
## [0.6.5] - 2026-08-11
|
||||
|
||||
### Added
|
||||
@@ -261,8 +400,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Entities and relationships round-trip as memory-local provenance only — Markdown import intentionally does not write into `ContextGraph`, matching the MVP scope agreed on in #765
|
||||
- Documented the file contract and workflow in `docs/reference/context.md`; 43 new tests in `tests/context/test_agent_memory_markdown.py` cover round-trip losslessness, idempotency, validation errors, rollback on failure, and vector-store sync ordering
|
||||
|
||||
- **Markdown directory round trips for `ContextGraph`** (#852) by @SaurabhScripts
|
||||
- `ContextGraph.save_to_file(..., format="markdown")` and `load_from_file(..., format="markdown")` persist a deterministic `graph.md` relationship manifest plus one human-editable Markdown file per node, preserving graph, node, edge, family, temporal, and cross-graph link identities
|
||||
- Imports validate the complete directory before replacing graph state, rebuild indexes and analytics state atomically, create JSON-compatible stub nodes for dangling edge endpoints, and emit the same granular node/edge audit events as JSON loading
|
||||
- Existing exports are replaced atomically only after their complete canonical layout is validated; untracked files, renamed node files, symlinks, Windows directory junctions, and other reparse points cause a fail-closed error instead of authorizing directory deletion
|
||||
- Added 30 focused tests covering deterministic round trips, manual edits, validation rollback, managed-directory identity, publish rollback, audit-manager compatibility, stale-cache clearing, mocked and real Windows junctions, and missing-path behavior
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Markdown import followed filesystem links even though Markdown export already refused to overwrite them** (#851, follow-up to #765, #786) by @SaurabhScripts
|
||||
- `AgentMemory._read_markdown_path()` now rejects symlink files, broken symlinks, symlinked directories, Windows directory junctions, and other Windows reparse points supplied directly; linked entries discovered inside an otherwise valid directory are safely skipped, preserving the current directory-import contract
|
||||
- `_read_markdown_file_content()` re-checks the file and parent directory immediately before and after opening, uses `O_NOFOLLOW` where available, and verifies the resulting descriptor is a regular file via `fstat`/`S_ISREG`, so link swaps are rejected rather than silently followed
|
||||
- Junction detection uses `os.path.isjunction()` where available and falls back to the Windows reparse-point file attribute on older Python versions; export applies the same link check before replacing a Markdown file
|
||||
- Documented the import restriction in `docs/reference/context.md`; added 11 tests to `tests/context/test_agent_memory_markdown.py` covering file/directory/broken-symlink rejection, simulated open races, mocked and real Windows junctions, and the reparse-point fallback
|
||||
- Any additional review follow-up commits land in this same PR/entry rather than as a separate changelog item
|
||||
|
||||
- **`PipelineWithProvenance` raised `ModuleNotFoundError` on import and `AttributeError` on `.run()`** (#858, closes #858) by @Karunasagar12
|
||||
- `from .pipeline import Pipeline` failed because `semantica/pipeline/pipeline.py` does not exist; corrected to `from .pipeline_builder import Pipeline`
|
||||
- `.run()` called `self._pipeline.run()` on the `Pipeline` dataclass, which has no such method; replaced with `self._engine.execute_pipeline(self._pipeline, ...)` delegating to `ExecutionEngine`
|
||||
@@ -1379,4 +1531,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
---
|
||||
|
||||
For detailed release notes, see [GitHub Releases](https://github.com/Hawksight-AI/semantica/releases).
|
||||
For detailed release notes, see [GitHub Releases](https://github.com/semantica-agi/semantica/releases).
|
||||
|
||||
+1
-1
@@ -58,7 +58,7 @@ representative at an online or offline event.
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported to the community leaders responsible for enforcement through
|
||||
[GitHub Issues](https://github.com/Hawksight-AI/semantica/issues) with "[CoC]" prefix.
|
||||
[GitHub Issues](https://github.com/semantica-agi/semantica/issues) with "[CoC]" prefix.
|
||||
All complaints will be reviewed and investigated promptly and fairly.
|
||||
|
||||
All community leaders are obligated to respect the privacy and security of the
|
||||
|
||||
+3
-3
@@ -44,7 +44,7 @@ We recognize all types of contributions:
|
||||
All contributors are recognized in:
|
||||
|
||||
- This contributors list
|
||||
- [GitHub contributors page](https://github.com/Hawksight-AI/semantica/graphs/contributors)
|
||||
- [GitHub contributors page](https://github.com/semantica-agi/semantica/graphs/contributors)
|
||||
- Release notes for significant contributions
|
||||
- Community appreciation
|
||||
|
||||
@@ -54,7 +54,7 @@ All contributors are recognized in:
|
||||
|
||||
### Automatic Recognition
|
||||
|
||||
If you've made a commit, you'll automatically appear in [GitHub's contributors graph](https://github.com/Hawksight-AI/semantica/graphs/contributors).
|
||||
If you've made a commit, you'll automatically appear in [GitHub's contributors graph](https://github.com/semantica-agi/semantica/graphs/contributors).
|
||||
|
||||
### Using All-Contributors Bot
|
||||
|
||||
@@ -111,4 +111,4 @@ Every contribution, no matter how small, helps make Semantica better. Thank you
|
||||
|
||||
**Want to contribute?**
|
||||
|
||||
⭐ Give us a Star • 🍴 [Fork us](https://github.com/Hawksight-AI/semantica/fork) • Check out our [Contributing Guide](CONTRIBUTING.md) to get started!
|
||||
⭐ Give us a Star • 🍴 [Fork us](https://github.com/semantica-agi/semantica/fork) • Check out our [Contributing Guide](CONTRIBUTING.md) to get started!
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ RUN npm ci
|
||||
COPY explorer/ ./
|
||||
RUN mkdir -p /app/semantica && npm run build
|
||||
|
||||
FROM python:3.14-slim AS runtime
|
||||
FROM python:3.13-slim AS runtime
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Hawksight AI
|
||||
Copyright (c) 2026 Semantica
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
recursive-include semantica/static *
|
||||
recursive-include semantica/ontology/vocabulary *.ttl
|
||||
|
||||
@@ -2,7 +2,15 @@
|
||||
|
||||
<img src="Semantica Logo.png" alt="Semantica" width="420"/>
|
||||
|
||||
<a href="https://trendshift.io/repositories/18986?utm_source=repository-badge&utm_medium=badge&utm_campaign=badge-repository-18986" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/18986" alt="semantica-agi%2Fsemantica | Trendshift" width="250" height="55"/></a>
|
||||
<div style="display:flex; gap:10px; align-items:center; flex-wrap:wrap;">
|
||||
<a href="https://trendshift.io/repositories/18986?utm_source=repository-badge&utm_medium=badge&utm_campaign=badge-repository-18986" target="_blank" rel="noopener noreferrer">
|
||||
<img src="https://trendshift.io/api/badge/repositories/18986" alt="semantica-agi/semantica | Trendshift" width="250" height="55"/>
|
||||
</a>
|
||||
|
||||
<a href="https://trendshift.io/repositories/18986?utm_source=trendshift-badge&utm_medium=badge&utm_campaign=badge-trendshift-18986" target="_blank" rel="noopener noreferrer">
|
||||
<img src="https://trendshift.io/api/badge/trendshift/repositories/18986/weekly?language=Python" alt="semantica-agi/semantica | Trendshift" width="250" height="55"/>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
### Graph-Native Infrastructure for Context and Accountable AI Systems
|
||||
|
||||
@@ -52,6 +60,8 @@ Most AI agents act without a trail. They store embeddings, not meaning: context
|
||||
|
||||
Semantica sits underneath your LLM, vector store, and agent framework as a deterministic infrastructure layer: no LLM required for graph construction, reasoning, or provenance.
|
||||
|
||||
> ⚠️ **System-level explainability, not foundation-model explainability.** Semantica does not expose or reconstruct what happens *inside* the LLM — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. Semantica explains what's *outside* the model: the context and data fed in, the decision produced, its provenance, relevant relationships, applied policies, and the full execution trail.
|
||||
|
||||
**Who it's for:**
|
||||
|
||||
- **AI/ML platform teams** shipping agents that make consequential decisions and need structured, queryable context built from fragmented raw data, not just a vector index
|
||||
@@ -132,7 +142,7 @@ compliant = graph.check_decision_rules({"category": "vendor_selection"}) # poli
|
||||
```bash
|
||||
semantica doctor
|
||||
# Python 3.11.9 pass
|
||||
# semantica 0.6.5 pass
|
||||
# semantica 0.6.6 pass
|
||||
# faiss vector store pass
|
||||
# Config file pass ~/.semantica/config.yaml
|
||||
```
|
||||
@@ -293,17 +303,10 @@ graph.add_causal_relationship(d1, d2, relationship_type="CAUSED")
|
||||
prov.track_entity("patient_P4821", source="ehr/medication_orders_2024.json",
|
||||
metadata={"extractor": "NamedEntityRecognizer"})
|
||||
|
||||
# Export W3C PROV-O for regulator submission - RDFExporter expects
|
||||
# {"entities": [...], "relationships": [...]}, so map ContextGraph.to_dict()'s
|
||||
# {"nodes": [...], "edges": [...]} shape onto it first
|
||||
graph_dict = graph.to_dict()
|
||||
kg = {
|
||||
"entities": [{"id": n["id"], "type": n["type"], "text": n["content"]} for n in graph_dict["nodes"]],
|
||||
"relationships": [
|
||||
{"source_id": e["source"], "target_id": e["target"], "type": e["type"]}
|
||||
for e in graph_dict["edges"]
|
||||
],
|
||||
}
|
||||
# Export W3C PROV-O for regulator submission - to_kg_dict() is the official
|
||||
# adapter that emits the {"entities": [...], "relationships": [...]} /
|
||||
# source_id shape RDFExporter expects, so no manual field mapping is needed
|
||||
kg = graph.to_kg_dict()
|
||||
RDFExporter().export(kg, "audit_trail.ttl", format="turtle")
|
||||
```
|
||||
|
||||
@@ -877,20 +880,14 @@ fact = BiTemporalFact(
|
||||
recorded_at=datetime(2024, 3, 5),
|
||||
)
|
||||
|
||||
# Query facts valid within a time window - query_time_range() expects
|
||||
# {"relationships": [...]} with source_id/target_id keys, which differs from
|
||||
# ContextGraph.to_dict()'s {"nodes", "edges"} shape, so map it first
|
||||
graph_dict = graph.to_dict()
|
||||
kg_relationships = {
|
||||
"relationships": [
|
||||
{**e, "source_id": e["source"], "target_id": e["target"]}
|
||||
for e in graph_dict["edges"]
|
||||
]
|
||||
}
|
||||
# Query facts valid within a time window - to_kg_dict() is the official
|
||||
# adapter that emits {"entities", "relationships"} with source_id/target_id
|
||||
# keys, the shape query_time_range() expects (no manual mapping required)
|
||||
kg = graph.to_kg_dict()
|
||||
|
||||
tq = TemporalGraphQuery()
|
||||
facts_in_window = tq.query_time_range(
|
||||
kg_relationships, query="valid_facts", start_time="2024-01-01", end_time="2024-12-31"
|
||||
kg, query="valid_facts", start_time="2024-01-01", end_time="2024-12-31"
|
||||
)
|
||||
|
||||
# Normalize natural language temporal expressions - returns a (start, end) range
|
||||
@@ -1469,18 +1466,18 @@ For contributor / dev-server setup: **[explorer/README.md: Local Setup Guide](ex
|
||||
|
||||
---
|
||||
|
||||
## What's New in v0.6.5
|
||||
## What's New in v0.6.6
|
||||
|
||||
**Security release — upgrading is strongly recommended.** Fixes for 5 externally-reported vulnerabilities in the Explorer API and graph/triplet store backends, plus a CodeQL-flagged ReDoS:
|
||||
**Security release — upgrading is strongly recommended.** Fixes for a privately disclosed batch of vulnerabilities spanning backup/restore, database export, outbound requests, and triplet-store backends, plus SSRF hardening across ingestion:
|
||||
|
||||
- **Missing authentication on all Explorer API routes** (GHSA-j4mq-hprp-987v, Critical): every route now requires `SEMANTICA_API_KEY`, fails closed (503) rather than open when unconfigured
|
||||
- **SSRF via redirect bypass in ontology URL fetching** (GHSA-8c7v-62gr-hj6g, High): redirect targets are now re-validated at every hop and the connection is pinned to the validated address, closing a DNS check-then-use race
|
||||
- **Cypher injection via unvalidated node labels and property keys** (GHSA-482h-hw99-h62p, Critical): Neptune, Neo4j, and FalkorDB now sanitize every label/relationship-type/property-key interpolation site
|
||||
- **SPARQL injection via unvalidated triplet IRIs** (GHSA-8vgg-8mr4-r236, Critical): Blazegraph, RDF4J, and Jena now validate subject/predicate/object IRIs before interpolation
|
||||
- **Missing Origin validation on the WebSocket handshake** (GHSA-4643-wpgq-w329, Moderate, anonymous-mode only): `/ws/graph-updates` now checks `Origin` against the same allowlist `CORSMiddleware` enforces for HTTP
|
||||
- **Polynomial ReDoS in SPARQL query validation** (CodeQL `py/polynomial-redos`): fixed a backtracking regex in the Explorer's SPARQL route
|
||||
- **Tarball restore path traversal**: `semantica backup restore` now validates every archive member for path containment and rejects symlink/hardlink escapes before extraction
|
||||
- **Latent SQL injection in `DataExporter.export_table_data()`**: table/schema names are now identifier-allowlisted and `where`/`order_by` fragments are blocklist-checked
|
||||
- **DNS-rebinding TOCTOU in the shared SSRF guard**: the resolved IP that passes validation is now the one the connection is pinned to, closing the check-then-use race (also closes the `100.64.0.0/10` CGNAT gap)
|
||||
- **Stored XSS in HTML report generation** and **unvalidated SPARQL object IRIs in AnzoStore** (SPARQL injection): both now escape/validate before interpolation
|
||||
- **`Authorization`/`Proxy-Authorization` credential leakage across redirects**, plus **SSRF gaps in `FeedIngestor`/`FeedMonitor`, `RepoIngestor`, and the MCP/public-API ingest paths**: all now route through the shared, redirect-safe SSRF guard
|
||||
- **HTTP response header injection and an unbounded-memory DoS** in the Explorer API, and a **`fastapi`/`python-multipart` ReDoS** (PYSEC-2024-38): floors raised, inputs sanitized, candidate pools capped
|
||||
|
||||
Also includes: embedded Oxigraph backend for `TripletStore`, PROV-O trust/spec completeness for `ProvenanceManager`, and the Altair Anzo triplet store backend.
|
||||
Also ships: **first-class CrewAI integration** (`semantica[crewai]`, extraction/decision tools + a knowledge source), **`ContextGraph` retraction and purge** (GDPR-style erasure without a full `clear()`), a declared **Semantica RDF vocabulary with deterministic entity/relationship IRIs** (stable, diffable exports), and **timezone-aware timestamps** across `export/` and `provenance/`.
|
||||
|
||||
→ [Full release notes](RELEASE_NOTES.md) · [Changelog](CHANGELOG.md)
|
||||
|
||||
@@ -1498,6 +1495,8 @@ Semantica is designed for environments where AI outputs must be explainable, aud
|
||||
- **Cybersecurity:** Threat attribution, incident response timelines, and IOC provenance tracking
|
||||
- **Autonomous Systems:** Decision logs, safety validation, and explainable AI for certification
|
||||
|
||||
> ⚠️ **This is system-level explainability, not foundation-model explainability.** Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. What Semantica explains is *outside* the model: the context and data fed in, the decision produced, its provenance, the relevant relationships, the policies applied, and the full execution trail. In short, Semantica explains and audits what the AI system did, not the LLM's private internal reasoning.
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
@@ -1595,6 +1594,23 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for full guidelines.
|
||||
|
||||
---
|
||||
|
||||
## Cite Us
|
||||
|
||||
If you use Semantica in your research or production systems, please cite it as:
|
||||
|
||||
```bibtex
|
||||
@software{semantica2026,
|
||||
title = {Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems},
|
||||
author = {Semantica},
|
||||
year = {2026},
|
||||
url = {https://github.com/semantica-agi/semantica}
|
||||
}
|
||||
```
|
||||
|
||||
All citation formats (APA, MLA, Chicago, IEEE) live on the [Citation](https://docs.getsemantica.ai/citation) page — every format attributes authorship to **Semantica**, not individual contributors.
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
|
||||
MIT License · Built by [Semantica](https://github.com/semantica-agi)
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/01_Advanced_Extraction.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/01_Advanced_Extraction.ipynb)\n",
|
||||
"\n",
|
||||
"# Advanced Extraction\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/03_Complete_Visualization_Suite.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/03_Complete_Visualization_Suite.ipynb)\n",
|
||||
"\n",
|
||||
"# Complete Visualization Suite\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/05_Multi_Format_Export.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/05_Multi_Format_Export.ipynb)\n",
|
||||
"\n",
|
||||
"# Advanced Multi-Format Export\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/08_Reasoning_and_Inference.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/08_Reasoning_and_Inference.ipynb)\n",
|
||||
"\n",
|
||||
"# Reasoning and Inference\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/09_Semantic_Layer_Construction.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/09_Semantic_Layer_Construction.ipynb)\n",
|
||||
"\n",
|
||||
"# Semantic Layer Construction\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb)\n",
|
||||
"\n",
|
||||
"# Deep Dive: Temporal Knowledge Graphs\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/12_Unstructured_to_Ontology.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/12_Unstructured_to_Ontology.ipynb)\n",
|
||||
"\n",
|
||||
"# Unstructured Text to Ontology\n",
|
||||
"\n",
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
"id": "cell-0",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/13_Manual_Ontology_Snowflake_Mapping.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/13_Manual_Ontology_Snowflake_Mapping.ipynb)\n",
|
||||
"\n",
|
||||
"# Manual Ontology + Snowflake Mapping\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/14_Datalog_Style_Reasoning.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/14_Datalog_Style_Reasoning.ipynb)\n",
|
||||
"\n",
|
||||
"# Datalog-Style Reasoning\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb)\n",
|
||||
"\n",
|
||||
"# Advanced Vector Store - Made Easy\n",
|
||||
"\n",
|
||||
@@ -352,7 +352,7 @@
|
||||
"- Build a multi-user application\n",
|
||||
"- Explore the [introduction notebook](../introduction/13_Vector_Store.ipynb) for more basics\n",
|
||||
"\n",
|
||||
"**Need Help?** Check our [documentation](https://semantica.readthedocs.io) or ask on [GitHub](https://github.com/Hawksight-AI/semantica)."
|
||||
"**Need Help?** Check our [documentation](https://semantica.readthedocs.io) or ask on [GitHub](https://github.com/semantica-agi/semantica)."
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)\n",
|
||||
"\n",
|
||||
"Semantica is a **semantic intelligence and knowledge engineering framework**. It helps you:\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb)\n",
|
||||
"\n",
|
||||
"# Data Ingestion - Comprehensive Guide\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/04_Document_Parsing.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/03_Document_Parsing.ipynb)\n",
|
||||
"\n",
|
||||
"# Document Parsing\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/05_Data_Normalization.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/04_Data_Normalization.ipynb)\n",
|
||||
"\n",
|
||||
"# Data Normalization\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/05_Entity_Extraction.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/05_Entity_Extraction.ipynb)\n",
|
||||
"\n",
|
||||
"# Entity Extraction - Comprehensive Guide\n",
|
||||
"\n",
|
||||
@@ -622,7 +622,7 @@
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/semantica-agi/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/06_Relation_Extraction.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/06_Relation_Extraction.ipynb)\n",
|
||||
"\n",
|
||||
"# Relation Extraction - Comprehensive Guide\n",
|
||||
"\n",
|
||||
@@ -599,7 +599,7 @@
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/semantica-agi/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Building_Knowledge_Graphs.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb)\n",
|
||||
"\n",
|
||||
"# Building Knowledge Graphs\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/09_Your_First_Knowledge_Graph.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)\n",
|
||||
"\n",
|
||||
"# 🚀 Your First Knowledge Graph\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/11_Graph_Analytics.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/10_Graph_Analytics.ipynb)\n",
|
||||
"\n",
|
||||
"# Graph Analytics\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/11_Chunking_and_Splitting.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/11_Chunking_and_Splitting.ipynb)\n",
|
||||
"\n",
|
||||
"# Chunking and Splitting - Comprehensive Guide\n",
|
||||
"\n",
|
||||
@@ -817,7 +817,7 @@
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/semantica-agi/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/13_Embedding_Generation.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/12_Embedding_Generation.ipynb)\n",
|
||||
"\n",
|
||||
"# Embedding Generation\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/13_Vector_Store.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/13_Vector_Store.ipynb)\n",
|
||||
"\n",
|
||||
"# Vector Store - Comprehensive Guide\n",
|
||||
"\n",
|
||||
@@ -492,7 +492,7 @@
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/semantica-agi/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/14_Ontology.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/14_Ontology.ipynb)\n",
|
||||
"\n",
|
||||
"# Ontology Generation \n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/15_Export.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/15_Export.ipynb)\n",
|
||||
"\n",
|
||||
"# Export Module - Comprehensive Guide\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/17_Visualization.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/16_Visualization.ipynb)\n",
|
||||
"\n",
|
||||
"# Visualization\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/18_Deduplication.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/18_Deduplication.ipynb)\n",
|
||||
"\n",
|
||||
"# Deduplication in Semantica\n",
|
||||
"\n",
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
"id": "c21e9c8d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/19_Context_Module.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/19_Context_Module.ipynb)\n",
|
||||
"\n",
|
||||
"# Context Module — Practical Guide\n",
|
||||
"\n",
|
||||
|
||||
+9
-10
@@ -13,26 +13,25 @@ icon: "quote-left"
|
||||
<Tab title="BibTeX">
|
||||
```bibtex
|
||||
@software{semantica2026,
|
||||
title = {Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems},
|
||||
author = {Semantica},
|
||||
year = {2026},
|
||||
url = {https://github.com/semantica-agi/semantica},
|
||||
version = {0.6.5},
|
||||
doi = {10.5281/zenodo.XXXXXXX}
|
||||
title = {Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems},
|
||||
author = {Semantica},
|
||||
year = {2026},
|
||||
url = {https://github.com/semantica-agi/semantica},
|
||||
doi = {10.5281/zenodo.XXXXXXX}
|
||||
}
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="APA">
|
||||
Semantica. (2026). *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems* (Version 0.6.5) \[Computer software\]. https://github.com/semantica-agi/semantica
|
||||
Semantica. (2026). *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems* \[Computer software\]. https://github.com/semantica-agi/semantica
|
||||
</Tab>
|
||||
<Tab title="MLA">
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.5, GitHub, 2026, https://github.com/semantica-agi/semantica.
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. GitHub, 2026, https://github.com/semantica-agi/semantica.
|
||||
</Tab>
|
||||
<Tab title="Chicago">
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.5. GitHub, 2026. https://github.com/semantica-agi/semantica.
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. GitHub, 2026. https://github.com/semantica-agi/semantica.
|
||||
</Tab>
|
||||
<Tab title="IEEE">
|
||||
Semantica, "Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems," Version 0.6.5, GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica
|
||||
Semantica, "Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems," GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
|
||||
@@ -16,6 +16,9 @@ At its core, Semantica adds a **context and accountability layer** on top of you
|
||||
- **Accountability Layer** — Provenance tracking, decision intelligence, conflict detection, and W3C PROV-O compliance make every claim in your AI stack auditable and explainable.
|
||||
- **Extension Layer** — `PluginRegistry` and `MethodRegistry` let you replace or augment any component: ingestors, extractors, reasoning engines, backends: without changing framework code.
|
||||
|
||||
<Warning>
|
||||
**This is system-level explainability, not foundation-model explainability.** Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. What Semantica explains is *outside* the model: the context and data fed in, the decision produced, its provenance, the relevant relationships, the policies applied, and the full execution trail. In short, Semantica explains and audits *what the AI system did*, not the foundation model's private internal reasoning.
|
||||
</Warning>
|
||||
|
||||
## Knowledge Graphs
|
||||
|
||||
|
||||
+1
-1
@@ -80,6 +80,6 @@ Deep dive into advanced features, customization, and complex workflows.
|
||||
You can also run the cookbook using Docker:
|
||||
|
||||
```bash
|
||||
docker run -p 8888:8888 hawksight/semantica-cookbook
|
||||
docker run -p 8888:8888 semantica/semantica-cookbook
|
||||
```
|
||||
</Tip>
|
||||
|
||||
@@ -162,7 +162,7 @@ semantica-explorer --graph my_graph.json --no-browser
|
||||
```
|
||||
|
||||
<Warning>
|
||||
`--host 0.0.0.0` makes Explorer reachable on every network interface. The server has no built-in authentication. Only use this on a trusted private network.
|
||||
`--host 0.0.0.0` makes Explorer reachable on every network interface. Since v0.6.5 the Explorer API requires `SEMANTICA_API_KEY` (sent as the `X-API-Key` header) and fails closed with `503` when unconfigured; unauthenticated access is only possible when `SEMANTICA_ALLOW_ANONYMOUS=true` is set explicitly. Only use this on a trusted private network.
|
||||
</Warning>
|
||||
|
||||
|
||||
|
||||
+11
-1
@@ -17,7 +17,7 @@ icon: "circle-question"
|
||||
| API key required? | Optional: pattern extraction works with no keys |
|
||||
| Works with LangChain / LlamaIndex? | Yes: Semantica is a layer on top, not a replacement |
|
||||
| Production-ready? | Yes: 1,000+ tests, v0.5.0 ships with 12 security fixes |
|
||||
| Latest version? | **v0.6.5** (August 2026) |
|
||||
| Latest version? | **v0.6.6** (August 2026) |
|
||||
| Local LLMs? | Yes: Ollama via LiteLLM, HuggingFaceLLM for air-gapped |
|
||||
|
||||
|
||||
@@ -52,6 +52,16 @@ Semantica works alongside these frameworks, not against them.
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Does Semantica explain an LLM's internal reasoning or chain-of-thought?" icon="triangle-exclamation">
|
||||
|
||||
No. This is **system-level explainability, not foundation-model explainability**. Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system.
|
||||
|
||||
What Semantica explains is *outside* the model: what context and data were used, what decision was produced, the provenance behind it, the relevant relationships, the policies applied, and the resulting decision trail.
|
||||
|
||||
In short: Semantica explains and audits *what the AI system did* — not the foundation model's private internal reasoning.
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Is Semantica free?" icon="tag">
|
||||
|
||||
Yes: MIT licensed, no vendor lock-in, no paywalled features. Some capabilities require third-party API keys (e.g., OpenAI embeddings, Groq inference), but Semantica itself is always free and open source.
|
||||
|
||||
@@ -42,7 +42,7 @@ icon: "rocket"
|
||||
Verify installation:
|
||||
```python
|
||||
import semantica
|
||||
print(semantica.__version__) # 0.6.5
|
||||
print(semantica.__version__) # 0.6.6
|
||||
```
|
||||
</Check>
|
||||
</Step>
|
||||
|
||||
+2
-2
@@ -4,12 +4,12 @@ description: "Project governance model: roles, decision process, release cadence
|
||||
icon: "scale-balanced"
|
||||
---
|
||||
|
||||
> Semantica is maintained by Hawksight AI with community contributions under an open governance model.
|
||||
> Semantica is maintained by the Semantica team with community contributions under an open governance model.
|
||||
|
||||
|
||||
## Roles
|
||||
|
||||
- **Maintainers** — Hawksight AI team: review and merge PRs, manage releases and code quality, set project direction and community standards.
|
||||
- **Maintainers** — Semantica team: review and merge PRs, manage releases and code quality, set project direction and community standards.
|
||||
- **Contributors** — Submit code, documentation, and bug reports. Help with issues and reviews. Recognized in [CONTRIBUTORS.md](https://github.com/semantica-agi/semantica/blob/main/CONTRIBUTORS.md).
|
||||
- **Community Members** — Use Semantica, provide feedback, share use cases, and participate in GitHub Discussions and Discord.
|
||||
|
||||
|
||||
@@ -436,6 +436,35 @@ d = graph.to_dict()
|
||||
# d["statistics"] → {"node_count": int, "edge_count": int}
|
||||
```
|
||||
|
||||
For a human-editable, version-control-friendly representation, save a Markdown
|
||||
directory instead:
|
||||
|
||||
```python
|
||||
graph.save_to_file("context_graph/", format="markdown")
|
||||
|
||||
restored = ContextGraph(advanced_analytics=True)
|
||||
restored.load_from_file("context_graph/", format="markdown")
|
||||
```
|
||||
|
||||
The directory contains a versioned `graph.md` manifest for graph identity,
|
||||
relationships, and cross-graph link descriptors, plus one file per node under
|
||||
`nodes/`. A node's content is its Markdown body; its ID, type, properties,
|
||||
metadata, and temporal validity are YAML frontmatter. Node, edge, family, graph,
|
||||
and cross-graph link IDs are preserved across round trips.
|
||||
|
||||
Markdown loading uses replacement semantics, like `from_dict()`: it parses and
|
||||
validates the complete directory before replacing the current graph. Invalid YAML,
|
||||
duplicate IDs, unsupported versions, and unsafe filesystem links fail without
|
||||
partially mutating the graph. As with JSON loading, an edge endpoint without a node
|
||||
file creates an `entity` stub node. Symlinks, Windows directory junctions, and other
|
||||
Windows reparse points are rejected.
|
||||
|
||||
Re-exporting to an existing managed directory atomically replaces it, removing stale
|
||||
node files. Before replacement, Semantica validates the complete canonical export
|
||||
layout, not just the manifest header. Untracked files, assets, extra directories, or
|
||||
renamed node files therefore cause the export to fail closed instead of being deleted.
|
||||
Keep attachments and hand-written indexes outside the managed export directory.
|
||||
|
||||
If the graph had cross-graph links created with `link_graph()`, call `resolve_links()` after loading to restore live navigation — object references cannot be serialized, so they must be reconnected manually:
|
||||
|
||||
```python
|
||||
|
||||
@@ -269,7 +269,7 @@ print("Loaded {} facts from graph".format(count))
|
||||
|
||||
## Step 5 — SPARQL queries over enriched working memory
|
||||
|
||||
After forward chaining has derived new facts, `SPARQLReasoner` lets you query the enriched working memory using SPARQL triple-pattern matching with optional inference expansion:
|
||||
After forward chaining has derived new facts, `SPARQLReasoner` prepares SPARQL queries over the enriched working memory with optional inference expansion:
|
||||
|
||||
```python
|
||||
from semantica.reasoning import SPARQLReasoner
|
||||
@@ -288,22 +288,13 @@ query = """
|
||||
}
|
||||
"""
|
||||
|
||||
# execute_query() runs: expansion → inference → deduplication
|
||||
result = sparql.execute_query(query)
|
||||
|
||||
for binding in result.bindings:
|
||||
print("Actor: {:15s} CVE: {}".format(
|
||||
binding.get("actor", "?"),
|
||||
binding.get("cve", "?"),
|
||||
))
|
||||
|
||||
# metadata shows how many results came from inference vs ground facts
|
||||
print("Original: {} Inferred: {}".format(
|
||||
result.metadata.get("original_count", 0),
|
||||
result.metadata.get("inferred_count", 0),
|
||||
))
|
||||
# expand_query() applies inference rules to the query text:
|
||||
expanded = sparql.expand_query(query)
|
||||
print(expanded)
|
||||
```
|
||||
|
||||
`execute_query()` is not implemented yet: no triplet-store execution path exists, so it raises `NotImplementedError` rather than returning an empty result set that callers would misread as "no matches". Until execution lands, run the expanded query against your RDF store directly (for example with `rdflib`).
|
||||
|
||||
Inspect the expanded query before running it:
|
||||
|
||||
```python
|
||||
|
||||
@@ -8,7 +8,7 @@ icon: "shield-check"
|
||||
|
||||
SHACL (Shapes Constraint Language) is a standard for validating graph-based data. While an ontology defines the conceptual *schema* (the "what" exists in your domain), SHACL defines the structural *rules and constraints* (the "how" it should be structured).
|
||||
|
||||
In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and `_run_pyshacl` evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated.
|
||||
In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and the public `run_shacl_validation` function evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated. The historical `_run_pyshacl` name remains available as a compatibility alias.
|
||||
|
||||
## Why Use SHACL Validation?
|
||||
|
||||
@@ -55,7 +55,7 @@ Let's look at a simple, universally understood example: ensuring every `Employee
|
||||
```python
|
||||
from semantica.context import ContextGraph
|
||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
# 1. Prepare your data graph
|
||||
graph = ContextGraph()
|
||||
@@ -95,7 +95,7 @@ data_ttl = """
|
||||
"""
|
||||
|
||||
# 5. Run Validation
|
||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
||||
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||
|
||||
# 6. Analyze the Report
|
||||
print(f"Graph conforms: {report.conforms}")
|
||||
@@ -265,10 +265,10 @@ cve_id_shape = NodeShape(
|
||||
|
||||
## Step 4 — Run validation and read the report
|
||||
|
||||
Serialize the graph to RDF, then run `_run_pyshacl` against the shapes.
|
||||
Serialize the graph to RDF, then run `run_shacl_validation` against the shapes.
|
||||
|
||||
```python
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
# Prepare your RDF data string (since export_rdf primarily exports structural metadata,
|
||||
# you typically serialize your custom data graph to Turtle using rdflib or similar).
|
||||
@@ -281,7 +281,7 @@ data_ttl = """
|
||||
"""
|
||||
|
||||
# Run SHACL validation
|
||||
report = _run_pyshacl(
|
||||
report = run_shacl_validation(
|
||||
data_ttl,
|
||||
shacl_ttl,
|
||||
data_graph_format="turtle",
|
||||
@@ -366,8 +366,8 @@ print(f"Malware nodes missing 'family': {len(missing_family)}")
|
||||
# e.g. graph.update_node(node_id, {"family": "UNKNOWN — requires triage"})
|
||||
|
||||
# After remediation, re-run validation to confirm the fix
|
||||
# (re-export the patched graph to Turtle first, then call _run_pyshacl again)
|
||||
report2 = _run_pyshacl(patched_data_ttl, shacl_ttl)
|
||||
# (re-export the patched graph to Turtle first, then call run_shacl_validation again)
|
||||
report2 = run_shacl_validation(patched_data_ttl, shacl_ttl)
|
||||
print(f"Violations after remediation: {report2.violation_count}")
|
||||
# Violations after remediation: 0
|
||||
```
|
||||
@@ -377,10 +377,49 @@ print(f"Violations after remediation: {report2.violation_count}")
|
||||
## Common Pitfalls
|
||||
|
||||
- **Assuming the ontology automatically enforces data quality**: `SHACLGenerator` generates shapes based on what it observes in the data. If your data is missing a field, the generator won't know it was mandatory unless you explicitly inject the constraint (as shown in Step 3).
|
||||
- **Passing `ContextGraph` directly to SHACL validators**: The `_run_pyshacl` function expects an RDF string (like Turtle format), not a raw Python dictionary or `ContextGraph` object.
|
||||
- **Passing `ContextGraph` directly to SHACL validators**: The `run_shacl_validation` function expects an RDF string (like Turtle format), not a raw Python dictionary or `ContextGraph` object.
|
||||
- **Forgetting RDF serialization**: You must serialize your graph (often via a temporary file using `export_rdf`) before validating it.
|
||||
- **Treating validation as a one-time step**: Validation should be integrated as an automated step in your CI/CD pipeline or data ingestion flow, acting as a recurring gatekeeper rather than a one-off script.
|
||||
- **Ignoring validation reports**: A graph that does not conform must be remediated. Failing to review the `violation_count` and address the issues negates the purpose of SHACL validation.
|
||||
- **Validating `sh:class`/`sh:node` range checks on a property that declares `rdfs:range` with RDFS entailment on**: RDFS is an entailment rule, not a constraint. When pyshacl runs with `inference="rdfs"`, it infers the range class onto every object of the property, so class-based constraints on that property can never fail — the report says `conforms: True` on data that does not conform:
|
||||
|
||||
```python
|
||||
from pyshacl import validate
|
||||
from rdflib import Graph
|
||||
|
||||
data = Graph()
|
||||
data.parse(
|
||||
data="""
|
||||
@prefix ex: <https://example.org/ns#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
ex:contains rdfs:domain ex:Container ; rdfs:range ex:Item .
|
||||
ex:box a ex:Container ; ex:contains ex:notAnItem .
|
||||
ex:notAnItem a ex:Fish .
|
||||
""",
|
||||
format="turtle",
|
||||
)
|
||||
|
||||
shapes = Graph()
|
||||
shapes.parse(
|
||||
data="""
|
||||
@prefix ex: <https://example.org/ns#> .
|
||||
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
||||
ex:ContainerShape a sh:NodeShape ;
|
||||
sh:targetClass ex:Container ;
|
||||
sh:property [ sh:path ex:contains ; sh:class ex:Item ] .
|
||||
""",
|
||||
format="turtle",
|
||||
)
|
||||
|
||||
for inference in ("none", "rdfs"):
|
||||
conforms, _, _ = validate(data, shacl_graph=shapes, inference=inference)
|
||||
print(inference, conforms)
|
||||
# none False <- correct: notAnItem is a Fish, not an Item
|
||||
# rdfs True <- the entailment manufactured the type
|
||||
```
|
||||
|
||||
Mitigations: prefer not to declare `rdfs:range` on properties you intend to constrain with `sh:class`; when class membership is the thing under test, run validation without RDFS entailment (`inference="none"`); or express the check as a constraint the entailment cannot satisfy (for example a literal property constraint). Note the trade-off: with entailment off, `sh:targetClass` no longer reaches subclasses, so subclass hierarchies need explicit typing or inference-aware target selection. Semantica's own `run_shacl_validation` wrapper already calls pyshacl with `inference="none"`, so this pitfall only bites when calling `pyshacl.validate` directly with entailment enabled.
|
||||
- **Trusting `conforms: True` without checking the inference mode**: an inference-enabled run can hide the exact violations the shapes were written to catch (see above). Record which inference mode validation ran under alongside the result, and re-run shape sets that contain `sh:class`/`sh:node` with entailment off before treating a pass as authoritative.
|
||||
|
||||
---
|
||||
|
||||
@@ -396,7 +435,7 @@ A DoD CTI team enforces STIX-compatible constraints on a threat graph before sha
|
||||
from semantica.context import AgentContext, ContextGraph
|
||||
from semantica.vector_store import VectorStore
|
||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
graph = ContextGraph()
|
||||
ctx = AgentContext(
|
||||
@@ -448,7 +487,7 @@ data_ttl = """
|
||||
<http://example.org/hammertoss> a ex:Malware .
|
||||
"""
|
||||
|
||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
||||
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||
print(f"CTI graph conforms : {report.conforms}")
|
||||
print(f"Violations : {report.violation_count}")
|
||||
print(f"Warnings : {report.warning_count}")
|
||||
@@ -469,7 +508,7 @@ A SOC team validates zero-trust policy nodes before publishing them to the polic
|
||||
```python
|
||||
from semantica.context import ContextGraph
|
||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
graph = ContextGraph()
|
||||
graph.add_node("policy-001", "Policy", "MFA Required for Tier-1 Resources",
|
||||
@@ -516,7 +555,7 @@ data_ttl = """
|
||||
<http://example.org/policy-002> a ex:Policy .
|
||||
"""
|
||||
|
||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
||||
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||
print(f"Policy graph conforms: {report.conforms}")
|
||||
# Policy graph conforms: False
|
||||
|
||||
@@ -534,7 +573,7 @@ A clinical informatics team validates trial ontology nodes before loading them i
|
||||
|
||||
```python
|
||||
from semantica.ontology import LLMOntologyGenerator, SHACLGenerator, PropertyShape
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
from semantica.export import export_rdf
|
||||
import tempfile, os
|
||||
|
||||
@@ -586,7 +625,7 @@ with open(tmp.name) as f:
|
||||
data_ttl = f.read()
|
||||
os.unlink(tmp.name)
|
||||
|
||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
||||
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||
print(f"Trial data conforms: {report.conforms}")
|
||||
print(f"Warnings : {report.warning_count}")
|
||||
```
|
||||
@@ -600,7 +639,7 @@ A credit risk team validates every `LoanApplication` node against Basel III CRE2
|
||||
```python
|
||||
from semantica.context import ContextGraph
|
||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
graph = ContextGraph()
|
||||
graph.add_node("loan-001", "LoanApplication", "Prime mortgage APP-2025-88421",
|
||||
@@ -645,7 +684,7 @@ data_ttl = """
|
||||
ex:ltv "0.65" .
|
||||
"""
|
||||
|
||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
||||
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||
print(f"Loan portfolio conforms: {report.conforms}")
|
||||
# Loan portfolio conforms: False
|
||||
|
||||
@@ -675,14 +714,14 @@ Call this function as a pre-publish gate; exit code 1 blocks the pipeline.
|
||||
```python
|
||||
import sys
|
||||
from semantica.ontology import OntologyGenerator, SHACLGenerator
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
def validate_before_publish(data_graph_str: str, ontology: dict) -> None:
|
||||
shacl_gen = SHACLGenerator(base_uri="https://example.org/shapes/")
|
||||
shacl_graph = shacl_gen.generate(ontology)
|
||||
shacl_ttl = shacl_gen.serialize(shacl_graph, format="turtle")
|
||||
|
||||
report = _run_pyshacl(data_graph_str, shacl_ttl)
|
||||
report = run_shacl_validation(data_graph_str, shacl_ttl)
|
||||
|
||||
if not report.conforms:
|
||||
print(f"Graph validation FAILED — {report.violation_count} violation(s)")
|
||||
@@ -700,7 +739,6 @@ def validate_before_publish(data_graph_str: str, ontology: dict) -> None:
|
||||
|
||||
- [Ontology Management](ontology) — generate the OWL ontology that SHACL shapes are derived from
|
||||
- [Reasoning & Rules](reasoning) — complement SHACL structural constraints with logical inference rules
|
||||
- [Export & Serialization](export) — serialize graph data to Turtle/RDF/XML for `_run_pyshacl` input
|
||||
- [Export & Serialization](export) — serialize graph data to Turtle/RDF/XML for `run_shacl_validation` input
|
||||
- [Conflict Resolution](conflict-resolution) — detect and resolve data conflicts before SHACL validation
|
||||
- [Change Management](change-management) — version-gate SHACL shapes alongside ontology versions
|
||||
|
||||
|
||||
+5
-1
@@ -192,7 +192,11 @@ decision_id = context.record_decision(
|
||||
|
||||
## Built for Where Mistakes Have Consequences
|
||||
|
||||
Semantica was designed for domains where every decision must be explainable and every fact must be traceable:
|
||||
Semantica was designed for domains where every decision must be explainable and every fact must be traceable.
|
||||
|
||||
<Warning>
|
||||
**This is system-level explainability, not foundation-model explainability.** Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. What Semantica explains is *outside* the model: the context and data fed in, the decision produced, its provenance, the relevant relationships, the policies applied, and the full execution trail. See [Core Concepts](concepts) for the full scope note.
|
||||
</Warning>
|
||||
|
||||
**Healthcare & Life Sciences**
|
||||
- Clinical decision support with full audit trails
|
||||
|
||||
@@ -12,7 +12,7 @@ icon: "file-contract"
|
||||
```
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Hawksight AI
|
||||
Copyright (c) 2026 Semantica
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -435,8 +435,8 @@ print("Nodes: {}, Edges: {}".format(stats["node_count"], stats["edge_count"]))
|
||||
| `query(query, skip, limit)` | `List[Dict]` | Full-text search over node content |
|
||||
| `stats()` | `Dict` | Node/edge counts, type breakdowns, graph density |
|
||||
| `density()` | `float` | Graph density score |
|
||||
| `save_to_file(path)` | `None` | Persist graph to JSON |
|
||||
| `load_from_file(path)` | `None` | Load graph from JSON |
|
||||
| `save_to_file(path, format="json")` | `None` | Persist graph as JSON or a Markdown directory |
|
||||
| `load_from_file(path, format="json")` | `None` | Replace graph state from JSON or a Markdown directory |
|
||||
| `build_from_conversations(conversations, link_entities)` | `Dict` | Build graph from conversation data |
|
||||
| `link_graph(other_graph, source_node_id, target_node_id, link_type)` | `str` | Create cross-graph navigation link; returns `link_id` |
|
||||
| `navigate_to(link_id)` | `Tuple` | Follow a cross-graph link to `(target_graph, target_node_id)` |
|
||||
@@ -625,8 +625,10 @@ malformed or duplicate fields before changing memory, and re-importing unchanged
|
||||
files is idempotent. Memory-local `entities` and `relationships` are preserved as
|
||||
provenance but are not applied to `ContextGraph` by Markdown import. Use a dedicated
|
||||
export directory: matching files are overwritten, but unrelated or stale Markdown
|
||||
files are not deleted automatically. Export refuses to overwrite symbolic links and
|
||||
uses atomic file replacement. Timestamp offsets are preserved in Markdown and
|
||||
files are not deleted automatically. Export refuses to overwrite filesystem links and
|
||||
uses atomic file replacement; import also refuses symlinks, Windows directory
|
||||
junctions, and other Windows reparse points.
|
||||
Timestamp offsets are preserved in Markdown and
|
||||
normalized to UTC only for comparisons, so aware and local-naive records can be
|
||||
queried together safely. Vector-store writes are deferred until the in-memory import
|
||||
commits; adapter synchronization remains best-effort and logs failures.
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
# Graph storage backends and feature matrix
|
||||
|
||||
Semantica separates graph modeling from physical storage. LPG backends are accessed through `graph_store` adapters; RDF backends are accessed through `triplet_store` adapters.
|
||||
|
||||
This page is intentionally conservative: it distinguishes between an adapter existing, a feature being generally available with that model, and a backend needing user-supplied wiring.
|
||||
|
||||
## Status labels
|
||||
|
||||
- `built-in`: adapter implementation exists in Semantica core.
|
||||
- `tested`: covered by automated integration fixtures or tests.
|
||||
- `example-only`: usable example exists, but support is not asserted by integration tests.
|
||||
- `interface/BYO`: interface or integration point exists; bring your own backend wiring.
|
||||
|
||||
## Adapter inventory
|
||||
|
||||
| Backend | Model | Adapter | Status | Reference |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Neo4j | LPG | `semantica.graph_store.Neo4jStore` | built-in | `cookbook/introduction/09_Graph_Store.ipynb` |
|
||||
| FalkorDB | LPG | `semantica.graph_store.FalkorDBStore` | built-in | `docs/reference/graph_store.md` |
|
||||
| Amazon Neptune | LPG | `semantica.graph_store.AmazonNeptuneStore` | built-in | `cookbook/introduction/21_Amazon_Neptune_Store.ipynb` |
|
||||
| Apache AGE | LPG | `semantica.graph_store.ApacheAgeStore` | built-in | `docs/graph_stores/apache_age.md` |
|
||||
| RDF4J | RDF | `semantica.triplet_store.RDF4JStore` | built-in | `cookbook/introduction/20_Triplet_Store.ipynb` |
|
||||
| Apache Jena | RDF | `semantica.triplet_store.JenaStore` | built-in | `cookbook/introduction/20_Triplet_Store.ipynb` |
|
||||
| Blazegraph | RDF | `semantica.triplet_store.BlazegraphStore` | built-in | `cookbook/introduction/20_Triplet_Store.ipynb` |
|
||||
| Anzo | RDF | `semantica.triplet_store.AnzoStore` | built-in | `cookbook/introduction/20_Triplet_Store.ipynb` |
|
||||
| Oxigraph | RDF | `semantica.triplet_store.OxigraphStore` | built-in | `docs/reference/triplet_store.md` |
|
||||
|
||||
## Feature matrix
|
||||
|
||||
`Yes` means the capability is expected to work with the adapter and graph model. `Partial` means the capability works with model-specific constraints. `BYO` means the user must supply or validate wiring for the backend.
|
||||
|
||||
| Backend | Model | Ingestion | Context graph construction | Reasoning/analytics | Provenance | Known limitations |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Neo4j | LPG | Yes | Yes | Yes | Partial | Provenance and context metadata are stored as node and edge properties; relationship properties and stable node identifiers are required. |
|
||||
| FalkorDB | LPG | Yes | Yes | Partial | Partial | Redis-based; provenance depends on node/edge properties, and multi-graph isolation depends on the selected graph name. |
|
||||
| Amazon Neptune | LPG | Yes | Yes | Partial | Partial | Use the property-graph endpoint; AWS auth, VPC, and endpoint configuration can affect local tests. Provenance depends on node/edge properties. |
|
||||
| Apache AGE | LPG | Yes | Yes | Partial | Partial | Runs through PostgreSQL/AGE; Cypher compatibility and property handling can differ from standalone LPG engines. |
|
||||
| RDF4J | RDF | Yes | Partial | Partial | Partial | Context separation relies on named graphs; triple-level provenance may require reification or graph-level metadata. `RDF4JStore(repository_id=...)` currently has no effect — the constructor always connects to the `"default"` repository regardless of the value passed; track a fix separately. |
|
||||
| Apache Jena | RDF | Yes | Partial | Partial | Partial | Named graphs are needed for context separation; backend configuration and transaction behavior matter. |
|
||||
| Blazegraph | RDF | Yes | Partial | Partial | Partial | Use quads/named graphs for context; IRI stability and graph naming matter for provenance. |
|
||||
| Anzo | RDF | Yes | Partial | Partial | Partial | Anzo deployments are environment-specific; validate `dataset_uri`/graphmart naming, named-graph support, and provenance mapping. |
|
||||
| Oxigraph | RDF | Yes | Partial | Partial | Partial | Embedded, single-process store (in-memory or on-disk); named graphs are supported, but there is no separate server process to scale independently. |
|
||||
|
||||
## RDF and LPG differences
|
||||
|
||||
- LPG backends store context and provenance as graph elements and properties. If a backend does not support relationship properties, some provenance patterns may be degraded.
|
||||
- RDF backends rely on IRIs, named graphs, and optional reification. Context graphs and provenance are easiest to preserve when the store supports named graphs/quads.
|
||||
- Ingestion works across both models, but the physical representation differs: LPG stores nodes/edges directly, while RDF stores subject-predicate-object statements.
|
||||
- Reasoning and analytics should be validated against the adapter's query capabilities, especially for path traversal, property filters, and named-graph queries.
|
||||
|
||||
## Minimal connection examples
|
||||
|
||||
Prefer the referenced notebook cells for a working setup. The examples below show the intended adapter entrypoints, not a universal connection DSL.
|
||||
|
||||
### Neo4j
|
||||
|
||||
```python
|
||||
import os
|
||||
from semantica.graph_store import Neo4jStore
|
||||
|
||||
store = Neo4jStore(
|
||||
uri='bolt://localhost:7687',
|
||||
user='neo4j',
|
||||
password=os.environ['NEO4J_PASSWORD']
|
||||
)
|
||||
```
|
||||
|
||||
### FalkorDB
|
||||
|
||||
```python
|
||||
from semantica.graph_store import FalkorDBStore
|
||||
|
||||
store = FalkorDBStore(
|
||||
host='localhost',
|
||||
port=6379,
|
||||
graph_name='semantica'
|
||||
)
|
||||
```
|
||||
|
||||
### Amazon Neptune
|
||||
|
||||
```python
|
||||
from semantica.graph_store import AmazonNeptuneStore
|
||||
|
||||
store = AmazonNeptuneStore(
|
||||
endpoint='your-neptune-cluster-endpoint',
|
||||
port=8182,
|
||||
region='us-east-1'
|
||||
)
|
||||
```
|
||||
|
||||
### Apache AGE
|
||||
|
||||
```python
|
||||
from semantica.graph_store import ApacheAgeStore
|
||||
|
||||
store = ApacheAgeStore(
|
||||
connection_string='host=localhost dbname=agedb user=postgres password=postgres',
|
||||
graph_name='semantica'
|
||||
)
|
||||
```
|
||||
|
||||
### RDF4J
|
||||
|
||||
```python
|
||||
from semantica.triplet_store import RDF4JStore
|
||||
|
||||
store = RDF4JStore(
|
||||
endpoint='http://localhost:8080/rdf4j-server',
|
||||
repository_id='semantica' # currently has no effect; connects to "default" (see Known limitations)
|
||||
)
|
||||
```
|
||||
|
||||
### Apache Jena
|
||||
|
||||
```python
|
||||
from semantica.triplet_store import JenaStore
|
||||
|
||||
store = JenaStore(
|
||||
endpoint='http://localhost:3030/ds'
|
||||
)
|
||||
```
|
||||
|
||||
### Blazegraph
|
||||
|
||||
```python
|
||||
from semantica.triplet_store import BlazegraphStore
|
||||
|
||||
store = BlazegraphStore(
|
||||
endpoint='http://localhost:9999/blazegraph/sparql'
|
||||
)
|
||||
```
|
||||
|
||||
### Anzo
|
||||
|
||||
```python
|
||||
from semantica.triplet_store import AnzoStore
|
||||
|
||||
store = AnzoStore(
|
||||
endpoint='http://anzo-host:8080',
|
||||
dataset_uri='http://cambridgesemantics.com/Graphmart/your-graphmart-id'
|
||||
)
|
||||
```
|
||||
|
||||
### Oxigraph
|
||||
|
||||
```python
|
||||
from semantica.triplet_store import OxigraphStore
|
||||
|
||||
# Omit `path` for an in-memory store; pass a directory for on-disk persistence.
|
||||
store = OxigraphStore(path='./semantica-oxigraph-data')
|
||||
```
|
||||
|
||||
Replace hostnames, ports, repositories, graphs, and credentials with values from your environment. For regulated or self-hosted deployments, keep credentials in environment variables or secret storage rather than source code.
|
||||
+10
-1
@@ -63,7 +63,9 @@ semantica-explorer --graph my_graph.json --no-browser
|
||||
python -m semantica.explorer --graph my_graph.json
|
||||
```
|
||||
|
||||
> **Security note:** The Explorer API has no built-in authentication. The default `--host 127.0.0.1` binds to localhost only, so it is not reachable from other machines on your network. If you bind to `0.0.0.0`, all graph data is readable and writable by any host that can reach the port. The CLI will print a warning in that case.
|
||||
> **Security note:** Since v0.6.5 the Explorer API requires an API key on protected routes. Set the `SEMANTICA_API_KEY` environment variable and send it as the `X-API-Key` header; without a configured key, protected routes fail closed with `503` rather than serving anonymously. To opt into unauthenticated access for local development only, set `SEMANTICA_ALLOW_ANONYMOUS=true` explicitly. (`/api/health` and `/api/info` are intentionally unauthenticated.)
|
||||
>
|
||||
> The default `--host 127.0.0.1` binds to localhost only, so it is not reachable from other machines on your network. If you bind to `0.0.0.0`, all graph data is readable and writable by any host that can reach the port (subject to API-key auth). The CLI prints a warning when binding to a non-loopback host in anonymous mode or when `SEMANTICA_API_KEY` is unset.
|
||||
|
||||
---
|
||||
|
||||
@@ -148,6 +150,8 @@ This writes the compiled assets to `../semantica/static/`. The Python server the
|
||||
| --- | --- | --- |
|
||||
| `EXPLORER_CORS_ORIGINS` | `http://localhost:5173,http://127.0.0.1:5173` | Comma-separated list of allowed CORS origins |
|
||||
| `EXPLORER_CORS_CREDENTIALS` | `false` | Set to `true` to allow credentialed cross-origin requests (only needed behind an authenticating reverse proxy) |
|
||||
| `SEMANTICA_API_KEY` | *(unset)* | API key required on protected routes since v0.6.5; send it as the `X-API-Key` header. When unset, protected routes fail closed with `503`. |
|
||||
| `SEMANTICA_ALLOW_ANONYMOUS` | `false` | Set to `true` to opt into unauthenticated access (local development only). |
|
||||
|
||||
---
|
||||
|
||||
@@ -251,6 +255,11 @@ Vite automatically tries the next available port and prints the actual URL in th
|
||||
- Confirm the backend exposes the `/ws/graph-updates` WebSocket endpoint.
|
||||
- Check DevTools → Network → WS tab for the connection status and error code.
|
||||
- Ensure the backend version matches the frontend — mixing major versions can cause protocol mismatches.
|
||||
- **Authentication:** `/ws/graph-updates` enforces the same API key as the REST routes. Browsers cannot set custom headers on a WebSocket handshake, so pass the key as a query parameter instead:
|
||||
```
|
||||
ws://127.0.0.1:8000/ws/graph-updates?api_key=<your-key>
|
||||
```
|
||||
Non-browser clients (native apps, scripts) may send it as the `X-API-Key` header. A missing or incorrect key results in close code `4401`; if `SEMANTICA_API_KEY` is unset and `SEMANTICA_ALLOW_ANONYMOUS` is not `true`, the connection is also rejected. Note that API keys in URLs appear in server logs — prefer the header for non-browser clients.
|
||||
|
||||
---
|
||||
|
||||
|
||||
Generated
+1489
-14
File diff suppressed because it is too large
Load Diff
@@ -9,7 +9,7 @@
|
||||
"lint": "eslint .",
|
||||
"preview": "vite preview",
|
||||
"test:graph-store": "node --test tests/graphStore.multi-edge.test.mjs",
|
||||
"test:graph-workspace": "node --import tsx --test tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts",
|
||||
"test:graph-workspace": "node --import tsx --test tests/markdownContentViewer.test.ts tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts",
|
||||
"test:plugin-registry": "node --import tsx --test tests/pluginRegistry.temporal.test.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
@@ -29,6 +29,8 @@
|
||||
"react-arborist": "^3.4.3",
|
||||
"react-dom": "^19.2.4",
|
||||
"react-dropzone": "^15.0.0",
|
||||
"react-markdown": "^10.1.0",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"sigma": "^3.0.2",
|
||||
"vis-data": "^8.0.3",
|
||||
"vis-timeline": "^8.5.0"
|
||||
|
||||
@@ -162,7 +162,11 @@ const SIGMA_SETTINGS = {
|
||||
hideLabelsOnMove: true,
|
||||
hideEdgesOnMove: true,
|
||||
enableEdgeEvents: true,
|
||||
renderEdgeLabels: false,
|
||||
// #1009: edge labels (the edge `type` — "works_for", "leads", ...) were
|
||||
// hardcoded off, so edge text never rendered regardless of data. The
|
||||
// labelDensity / labelGridCellSize / labelRenderedSizeThreshold settings
|
||||
// below already throttle label density for both nodes and edges.
|
||||
renderEdgeLabels: true,
|
||||
labelDensity: 0.7,
|
||||
labelGridCellSize: 140,
|
||||
zIndex: true,
|
||||
@@ -741,6 +745,12 @@ function buildEffectAvailability(
|
||||
? { enabled: true, available: true, reason: "Panel enabled" }
|
||||
: { enabled: false, available: false, reason: "Disabled by toggle" };
|
||||
|
||||
// #1009: edge labels are immediately available once the graph is loaded —
|
||||
// they have no async analytics or zoom-tier dependency.
|
||||
const edgeLabels = effectsState.edgeLabelsEnabled
|
||||
? { enabled: true, available: true, reason: "Ready" }
|
||||
: { enabled: false, available: false, reason: "Disabled by toggle" };
|
||||
|
||||
const diagnostics = !GRAPH_THEME.effects.diagnostics.enabledInDev
|
||||
? { enabled: false, available: false, reason: "Disabled in production" }
|
||||
: effectsState.diagnosticsEnabled
|
||||
@@ -758,6 +768,7 @@ function buildEffectAvailability(
|
||||
communities,
|
||||
centrality,
|
||||
legend,
|
||||
edgeLabels,
|
||||
diagnostics,
|
||||
};
|
||||
}
|
||||
@@ -1211,6 +1222,12 @@ function applySceneState(
|
||||
size: resolvedStyle.size,
|
||||
zIndex: resolvedStyle.zIndex,
|
||||
curvature: resolvedStyle.curvature,
|
||||
// #1009: Sigma's edge label renderer draws data.label — the graph
|
||||
// stores the relationship type in edgeType, which the renderer never
|
||||
// saw, so enabling renderEdgeLabels alone left edges blank.
|
||||
// Use || rather than ?? so that an empty-string edgeType (possible
|
||||
// when the API returns type: "") does not produce a blank label.
|
||||
label: resolvedStyle.hidden ? undefined : String(attrs.edgeType || data.label || ""),
|
||||
};
|
||||
});
|
||||
|
||||
@@ -1295,6 +1312,9 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
const onEdgeClickRef = useRef(onEdgeClick);
|
||||
const onSceneRuntimeChangeRef = useRef(onSceneRuntimeChange);
|
||||
const onCameraStateChangeRef = useRef(onCameraStateChange);
|
||||
// #1009: tracked as a ref so the Sigma creation effect always reads the
|
||||
// current value without needing effectsState in its dependency array.
|
||||
const effectsStateRef = useRef(effectsState);
|
||||
const [hoveredNodeId, setHoveredNodeId] = useState<string | null>(null);
|
||||
const [zoomTier, setZoomTier] = useState<GraphZoomTier>("overview");
|
||||
const [analyticsSnapshot, setAnalyticsSnapshot] = useState<GraphAnalyticsSnapshot | null>(null);
|
||||
@@ -1323,6 +1343,7 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
onEdgeClickRef.current = onEdgeClick;
|
||||
onSceneRuntimeChangeRef.current = onSceneRuntimeChange;
|
||||
onCameraStateChangeRef.current = onCameraStateChange;
|
||||
effectsStateRef.current = effectsState;
|
||||
|
||||
const behaviors = useMemo<GraphBehavior[]>(
|
||||
() => [
|
||||
@@ -1835,7 +1856,13 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
return;
|
||||
}
|
||||
|
||||
const sigma = new Sigma(displayGraphRef.current, containerRef.current, SIGMA_SETTINGS);
|
||||
const sigma = new Sigma(displayGraphRef.current, containerRef.current, {
|
||||
...SIGMA_SETTINGS,
|
||||
// #1009: initialize with the current toggle value rather than the
|
||||
// static default so that a user who disabled Edge Labels before
|
||||
// graph/Sigma initialization sees the correct state after mount.
|
||||
renderEdgeLabels: effectsStateRef.current.edgeLabelsEnabled,
|
||||
});
|
||||
sigmaRef.current = sigma;
|
||||
appliedGraphVersionRef.current = graphVersionRef.current;
|
||||
|
||||
@@ -1937,6 +1964,17 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
});
|
||||
}, [behaviors, dispatchToBehaviors, getBehaviorContext, graphReady, syncCameraState]);
|
||||
|
||||
// #1009: renderEdgeLabels follows the Effects-panel toggle instead of
|
||||
// staying hardcoded — dense graphs get their label-free edges back.
|
||||
useEffect(() => {
|
||||
const sigma = sigmaRef.current;
|
||||
if (!sigma) {
|
||||
return;
|
||||
}
|
||||
sigma.setSetting("renderEdgeLabels", effectsState.edgeLabelsEnabled);
|
||||
sigma.scheduleRefresh();
|
||||
}, [effectsState.edgeLabelsEnabled]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
const sigma = sigmaRef.current;
|
||||
|
||||
@@ -3,6 +3,7 @@ import { Loader2 } from "lucide-react";
|
||||
import { graph } from "../../store/graphStore";
|
||||
import { GRAPH_THEME, withAlpha } from "./graphTheme";
|
||||
import type { GraphSelectedNodeKind } from "./types";
|
||||
import { MarkdownContentViewer } from "./MarkdownContentViewer";
|
||||
|
||||
export type LinkPrediction = {
|
||||
target: string;
|
||||
@@ -364,6 +365,11 @@ export function GraphInspectorPanel({
|
||||
([key]) =>
|
||||
!["x","y","valid_from","valid_until","content","source","source_url","pmid","pmids","evidence","provenance","confidence"].includes(key),
|
||||
);
|
||||
const nodeContent = (typeof attributes?.content === "string" && attributes.content)
|
||||
? attributes.content
|
||||
: (typeof properties.content === "string" && properties.content)
|
||||
? properties.content
|
||||
: "";
|
||||
|
||||
return (
|
||||
<aside style={{ padding: 24, display: "flex", flexDirection: "column", gap: 18 }}>
|
||||
@@ -408,6 +414,20 @@ export function GraphInspectorPanel({
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{/* Content Section — only rendered when the node carries actual content.
|
||||
This matches the existing inspector convention: sections that have no
|
||||
data for the current node are either hidden (temporal bounds) or closed
|
||||
by default (Source Attribution, Properties). Always showing an open
|
||||
empty panel would add noise for every relationship/predicate node. */}
|
||||
{nodeContent && (
|
||||
<details className="node-panel-collapse" open>
|
||||
<summary className="node-panel-summary">Content</summary>
|
||||
<div className="node-panel-body" style={{ marginTop: 8 }}>
|
||||
<MarkdownContentViewer content={nodeContent} />
|
||||
</div>
|
||||
</details>
|
||||
)}
|
||||
|
||||
{/* Actions */}
|
||||
<section style={sectionStyle}>
|
||||
<div style={sectionTitleStyle}>Actions</div>
|
||||
|
||||
@@ -148,6 +148,7 @@ const DEFAULT_EFFECTS_STATE: GraphEffectsState = {
|
||||
communitiesEnabled: false,
|
||||
centralityEnabled: false,
|
||||
legendEnabled: false,
|
||||
edgeLabelsEnabled: true,
|
||||
diagnosticsEnabled: false,
|
||||
lensMode: "neighborhood",
|
||||
effectQuality: "bounded",
|
||||
|
||||
@@ -0,0 +1,403 @@
|
||||
import { useState, useRef, useEffect, type CSSProperties } from "react";
|
||||
import ReactMarkdown from "react-markdown";
|
||||
import remarkGfm from "remark-gfm";
|
||||
import { Check, Copy, Code2, Eye, ExternalLink, Image as ImageIcon } from "lucide-react";
|
||||
import { GRAPH_THEME } from "./graphTheme";
|
||||
|
||||
export interface MarkdownContentViewerProps {
|
||||
content?: string | null;
|
||||
className?: string;
|
||||
defaultMode?: "preview" | "source";
|
||||
}
|
||||
|
||||
export function isSafeUrl(url?: string): boolean {
|
||||
if (!url) return false;
|
||||
const trimmed = url.trim();
|
||||
// Reject whitespace-only strings — new URL("", base) would resolve to the base
|
||||
// protocol and produce a false positive. This guards direct callers of the exported
|
||||
// function; markdown parsers normalise whitespace-only destinations to "" which
|
||||
// already fails the !url check above.
|
||||
if (!trimmed) return false;
|
||||
if (trimmed.startsWith("//")) return false;
|
||||
if (trimmed.startsWith("#")) return true;
|
||||
if (trimmed.startsWith("/")) return true;
|
||||
try {
|
||||
const parsed = new URL(trimmed, "http://localhost");
|
||||
return ["http:", "https:", "mailto:"].includes(parsed.protocol);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function MarkdownContentViewer({
|
||||
content,
|
||||
className,
|
||||
defaultMode = "preview",
|
||||
}: MarkdownContentViewerProps) {
|
||||
const [activeMode, setActiveMode] = useState<"preview" | "source">(defaultMode);
|
||||
const [copied, setCopied] = useState(false);
|
||||
// Track the content value for which the copied indicator is valid.
|
||||
// When content changes (i.e. the user selects a different node), reset the
|
||||
// copied indicator inline during render rather than in a useEffect — this
|
||||
// avoids a cascading-render lint error and is the React-recommended pattern
|
||||
// for resetting derived visual state on prop changes.
|
||||
const [copiedForContent, setCopiedForContent] = useState<string | null | undefined>(content);
|
||||
if (copiedForContent !== content) {
|
||||
setCopiedForContent(content);
|
||||
if (copied) {
|
||||
// Clear the stale indicator synchronously so the new node's copy button
|
||||
// never shows "Copied" from the previous selection.
|
||||
setCopied(false);
|
||||
}
|
||||
}
|
||||
|
||||
const copyTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
// Clean up any outstanding timeout on unmount.
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (copyTimeoutRef.current) {
|
||||
clearTimeout(copyTimeoutRef.current);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const rawContent = typeof content === "string" ? content : "";
|
||||
const hasContent = rawContent.trim().length > 0;
|
||||
|
||||
const handleCopy = async () => {
|
||||
if (!hasContent) return;
|
||||
try {
|
||||
await navigator.clipboard.writeText(rawContent);
|
||||
if (copyTimeoutRef.current) {
|
||||
clearTimeout(copyTimeoutRef.current);
|
||||
}
|
||||
setCopied(true);
|
||||
copyTimeoutRef.current = setTimeout(() => setCopied(false), 1500);
|
||||
} catch {
|
||||
// Clipboard write unavailable
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className={className} style={viewerContainerStyle}>
|
||||
<div style={viewerHeaderStyle}>
|
||||
<div style={{ display: "flex", gap: 4 }} role="tablist">
|
||||
<button
|
||||
type="button"
|
||||
role="tab"
|
||||
aria-selected={activeMode === "preview"}
|
||||
onClick={() => setActiveMode("preview")}
|
||||
style={{ ...tabBtnStyle, ...(activeMode === "preview" ? activeTabBtnStyle : {}) }}
|
||||
>
|
||||
<Eye size={12} style={{ marginRight: 5 }} />
|
||||
Preview
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
role="tab"
|
||||
aria-selected={activeMode === "source"}
|
||||
onClick={() => setActiveMode("source")}
|
||||
style={{ ...tabBtnStyle, ...(activeMode === "source" ? activeTabBtnStyle : {}) }}
|
||||
>
|
||||
<Code2 size={12} style={{ marginRight: 5 }} />
|
||||
Source
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{hasContent && (
|
||||
<button type="button" onClick={() => void handleCopy()} style={copyBtnStyle} title="Copy raw content">
|
||||
{copied ? (
|
||||
<>
|
||||
<Check size={12} color="#3fb950" style={{ marginRight: 4 }} />
|
||||
<span style={{ color: "#3fb950", fontSize: 11 }}>Copied</span>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Copy size={12} style={{ marginRight: 4 }} />
|
||||
<span style={{ fontSize: 11 }}>Copy</span>
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div style={viewerBodyStyle}>
|
||||
{!hasContent ? (
|
||||
<div style={emptyTextStyle}>No content available for this node.</div>
|
||||
) : activeMode === "source" ? (
|
||||
<pre style={sourcePreStyle}>
|
||||
<code style={sourceCodeStyle}>{rawContent}</code>
|
||||
</pre>
|
||||
) : (
|
||||
<div style={previewStyle}>
|
||||
<ReactMarkdown
|
||||
remarkPlugins={[remarkGfm]}
|
||||
components={{
|
||||
// C-1: react-markdown passes a HAST `node` prop (the raw AST
|
||||
// Element) to every custom component override via passNode:true.
|
||||
// In React 19 any unknown prop spreads onto a native element are
|
||||
// serialised as HTML attributes, producing node="[object Object]"
|
||||
// on every rendered link. Fix: destructure `node` by name so it
|
||||
// is explicitly discarded, then spread `...rest` to preserve all
|
||||
// other legitimate HAST/remark-gfm attributes — e.g. the `id`,
|
||||
// `aria-describedby`, `aria-label`, `data-footnote-ref`,
|
||||
// `data-footnote-backref`, and `class` attrs that GFM footnotes
|
||||
// require for correct in-page navigation and accessibility.
|
||||
//
|
||||
// C-2: fragment links (#anchor, GFM footnote backlinks) must
|
||||
// navigate within the current document. External links continue
|
||||
// to use target="_blank" with noopener noreferrer.
|
||||
//
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
a: ({ href, children, title, node: _node, ...rest }) => {
|
||||
if (!isSafeUrl(href)) {
|
||||
return <span style={{ color: GRAPH_THEME.ui.text.muted, textDecoration: "line-through" }}>{children}</span>;
|
||||
}
|
||||
// isSafeUrl returning true guarantees href is a non-empty string.
|
||||
const safeHref = href ?? "";
|
||||
// Fragment links (#section, footnote backlinks like
|
||||
// #user-content-fnref-1) are in-document anchors. Opening them
|
||||
// in a new tab would break GFM footnote back-navigation.
|
||||
const isFragment = safeHref.startsWith("#");
|
||||
if (isFragment) {
|
||||
return (
|
||||
<a href={safeHref} title={title} style={linkStyle} {...rest}>
|
||||
{children}
|
||||
</a>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<a href={safeHref} title={title} target="_blank" rel="noopener noreferrer" style={linkStyle} {...rest}>
|
||||
{children}
|
||||
<ExternalLink size={10} style={{ marginLeft: 3, verticalAlign: "middle", display: "inline" }} />
|
||||
</a>
|
||||
);
|
||||
},
|
||||
img: ({ src, alt }) => (
|
||||
<span style={imageBadgeStyle} title={src || "Image"}>
|
||||
<ImageIcon size={12} style={{ marginRight: 5 }} />
|
||||
<span>Image: {alt || src || "unlabeled"}</span>
|
||||
</span>
|
||||
),
|
||||
h1: ({ children }) => <h1 style={h1Style}>{children}</h1>,
|
||||
h2: ({ children }) => <h2 style={h2Style}>{children}</h2>,
|
||||
h3: ({ children }) => <h3 style={h3Style}>{children}</h3>,
|
||||
h4: ({ children }) => <h4 style={h4Style}>{children}</h4>,
|
||||
p: ({ children }) => <p style={{ margin: "0 0 8px 0" }}>{children}</p>,
|
||||
ul: ({ children }) => <ul style={{ margin: "0 0 8px 0", paddingLeft: 18 }}>{children}</ul>,
|
||||
ol: ({ children }) => <ol style={{ margin: "0 0 8px 0", paddingLeft: 18 }}>{children}</ol>,
|
||||
li: ({ children }) => <li style={{ marginBottom: 3 }}>{children}</li>,
|
||||
blockquote: ({ children }) => <blockquote style={blockquoteStyle}>{children}</blockquote>,
|
||||
hr: () => <hr style={{ border: "none", borderTop: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}`, margin: "10px 0" }} />,
|
||||
table: ({ children }) => (
|
||||
<div style={{ width: "100%", overflowX: "auto", margin: "8px 0", borderRadius: 6, border: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}` }}>
|
||||
<table style={{ width: "100%", borderCollapse: "collapse", fontSize: 12 }}>{children}</table>
|
||||
</div>
|
||||
),
|
||||
thead: ({ children }) => <thead style={{ background: "rgba(255, 255, 255, 0.04)" }}>{children}</thead>,
|
||||
tbody: ({ children }) => <tbody>{children}</tbody>,
|
||||
tr: ({ children }) => <tr style={{ borderBottom: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}` }}>{children}</tr>,
|
||||
th: ({ children }) => <th style={{ padding: "6px 8px", textAlign: "left", fontWeight: 700, color: GRAPH_THEME.ui.text.strong, borderRight: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}` }}>{children}</th>,
|
||||
td: ({ children }) => <td style={{ padding: "6px 8px", color: GRAPH_THEME.ui.text.body, borderRight: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}` }}>{children}</td>,
|
||||
pre: ({ children }) => <pre style={preBlockStyle}>{children}</pre>,
|
||||
// C-1: discard `node` here too — code elements are custom components
|
||||
// and would otherwise receive node="[object Object]" in the DOM.
|
||||
code: ({ className: codeClass, children }) => {
|
||||
const isInline = !codeClass && typeof children === "string" && !children.includes("\n");
|
||||
return (
|
||||
<code style={isInline ? inlineCodeStyle : blockCodeStyle}>
|
||||
{children}
|
||||
</code>
|
||||
);
|
||||
},
|
||||
}}
|
||||
>
|
||||
{rawContent}
|
||||
</ReactMarkdown>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/* ─── Styles ──────────────────────────────────────────────────────── */
|
||||
|
||||
const viewerContainerStyle: CSSProperties = {
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
background: "rgba(255, 255, 255, 0.025)",
|
||||
border: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}`,
|
||||
borderRadius: 12,
|
||||
overflow: "hidden",
|
||||
};
|
||||
|
||||
const viewerHeaderStyle: CSSProperties = {
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "space-between",
|
||||
padding: "6px 10px",
|
||||
background: "rgba(0, 0, 0, 0.2)",
|
||||
borderBottom: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}`,
|
||||
};
|
||||
|
||||
const tabBtnStyle: CSSProperties = {
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
padding: "4px 9px",
|
||||
borderRadius: 6,
|
||||
border: "1px solid transparent",
|
||||
background: "transparent",
|
||||
color: GRAPH_THEME.ui.text.muted,
|
||||
fontSize: 12,
|
||||
fontWeight: 600,
|
||||
cursor: "pointer",
|
||||
transition: "all 150ms ease",
|
||||
};
|
||||
|
||||
const activeTabBtnStyle: CSSProperties = {
|
||||
background: GRAPH_THEME.ui.timeline.playheadSoft,
|
||||
border: `1px solid ${GRAPH_THEME.ui.control.activeBorder}`,
|
||||
color: GRAPH_THEME.ui.timeline.playhead,
|
||||
};
|
||||
|
||||
const copyBtnStyle: CSSProperties = {
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
padding: "3px 8px",
|
||||
borderRadius: 6,
|
||||
border: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}`,
|
||||
background: "rgba(255, 255, 255, 0.04)",
|
||||
color: GRAPH_THEME.ui.text.subtle,
|
||||
fontSize: 11,
|
||||
cursor: "pointer",
|
||||
};
|
||||
|
||||
const viewerBodyStyle: CSSProperties = {
|
||||
padding: 12,
|
||||
maxHeight: 380,
|
||||
overflowY: "auto",
|
||||
};
|
||||
|
||||
const emptyTextStyle: CSSProperties = {
|
||||
color: GRAPH_THEME.ui.text.muted,
|
||||
fontSize: 12,
|
||||
lineHeight: 1.5,
|
||||
fontStyle: "italic",
|
||||
};
|
||||
|
||||
const sourcePreStyle: CSSProperties = {
|
||||
margin: 0,
|
||||
padding: 10,
|
||||
borderRadius: 8,
|
||||
background: "rgba(0, 0, 0, 0.3)",
|
||||
border: "1px solid rgba(255, 255, 255, 0.05)",
|
||||
overflowX: "auto",
|
||||
};
|
||||
|
||||
const sourceCodeStyle: CSSProperties = {
|
||||
fontFamily: "'JetBrains Mono', 'Fira Code', monospace",
|
||||
fontSize: 12,
|
||||
lineHeight: 1.6,
|
||||
color: GRAPH_THEME.ui.text.strong,
|
||||
whiteSpace: "pre-wrap",
|
||||
wordBreak: "break-word",
|
||||
userSelect: "text",
|
||||
};
|
||||
|
||||
const previewStyle: CSSProperties = {
|
||||
color: GRAPH_THEME.ui.text.body,
|
||||
fontSize: 13,
|
||||
lineHeight: 1.6,
|
||||
wordBreak: "break-word",
|
||||
};
|
||||
|
||||
const h1Style: CSSProperties = {
|
||||
fontSize: 16,
|
||||
fontWeight: 700,
|
||||
color: GRAPH_THEME.ui.text.strong,
|
||||
marginTop: 8,
|
||||
marginBottom: 6,
|
||||
paddingBottom: 3,
|
||||
borderBottom: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}`,
|
||||
};
|
||||
|
||||
const h2Style: CSSProperties = {
|
||||
fontSize: 14,
|
||||
fontWeight: 700,
|
||||
color: GRAPH_THEME.ui.text.strong,
|
||||
marginTop: 8,
|
||||
marginBottom: 4,
|
||||
};
|
||||
|
||||
const h3Style: CSSProperties = {
|
||||
fontSize: 13,
|
||||
fontWeight: 600,
|
||||
color: GRAPH_THEME.ui.text.strong,
|
||||
marginTop: 6,
|
||||
marginBottom: 4,
|
||||
};
|
||||
|
||||
const h4Style: CSSProperties = {
|
||||
fontSize: 12,
|
||||
fontWeight: 600,
|
||||
color: GRAPH_THEME.ui.text.strong,
|
||||
marginTop: 4,
|
||||
marginBottom: 2,
|
||||
};
|
||||
|
||||
const blockquoteStyle: CSSProperties = {
|
||||
margin: "8px 0",
|
||||
padding: "6px 12px",
|
||||
borderLeft: `3px solid ${GRAPH_THEME.ui.timeline.playhead}`,
|
||||
background: "rgba(98, 226, 205, 0.05)",
|
||||
borderRadius: "0 6px 6px 0",
|
||||
color: GRAPH_THEME.ui.text.body,
|
||||
fontStyle: "italic",
|
||||
};
|
||||
|
||||
const linkStyle: CSSProperties = {
|
||||
color: "#79c0ff",
|
||||
textDecoration: "underline",
|
||||
textUnderlineOffset: "3px",
|
||||
wordBreak: "break-all",
|
||||
};
|
||||
|
||||
const imageBadgeStyle: CSSProperties = {
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
padding: "3px 7px",
|
||||
background: "rgba(255, 255, 255, 0.04)",
|
||||
border: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}`,
|
||||
borderRadius: 6,
|
||||
color: GRAPH_THEME.ui.text.muted,
|
||||
fontSize: 11,
|
||||
margin: "3px 0",
|
||||
};
|
||||
|
||||
const inlineCodeStyle: CSSProperties = {
|
||||
fontFamily: "'JetBrains Mono', monospace",
|
||||
fontSize: 12,
|
||||
padding: "2px 5px",
|
||||
borderRadius: 4,
|
||||
background: "rgba(255, 255, 255, 0.07)",
|
||||
color: "#e6edf3",
|
||||
border: "1px solid rgba(255, 255, 255, 0.08)",
|
||||
};
|
||||
|
||||
const preBlockStyle: CSSProperties = {
|
||||
margin: "8px 0",
|
||||
padding: 10,
|
||||
borderRadius: 8,
|
||||
background: "rgba(0, 0, 0, 0.35)",
|
||||
border: "1px solid rgba(255, 255, 255, 0.08)",
|
||||
overflowX: "auto",
|
||||
};
|
||||
|
||||
const blockCodeStyle: CSSProperties = {
|
||||
fontFamily: "'JetBrains Mono', monospace",
|
||||
fontSize: 12,
|
||||
lineHeight: 1.5,
|
||||
color: "#e6edf3",
|
||||
};
|
||||
@@ -2099,6 +2099,15 @@ function createCollapsedNeighborhoodGraph(
|
||||
return collapsedGraph;
|
||||
}
|
||||
|
||||
// Normalize an edge relationship type: empty string, null, and undefined all
|
||||
// fall back to the project-wide default used consistently across every
|
||||
// aggregation path. Keep this local — it exists only to guarantee that the
|
||||
// three code paths (single-entry, multi-entry, community-grouped) produce the
|
||||
// same semantics and do not diverge again.
|
||||
function normalizeEdgeType(value: string | null | undefined): string {
|
||||
return value || "related_to";
|
||||
}
|
||||
|
||||
function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAttributes> {
|
||||
const aggregated = new Graph<NodeAttributes, EdgeAttributes>({
|
||||
type: "directed",
|
||||
@@ -2124,10 +2133,13 @@ function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAt
|
||||
const [{ edgeId, attrs }] = entries;
|
||||
aggregated.mergeDirectedEdgeWithKey(edgeId, sourceId, targetId, {
|
||||
...attrs,
|
||||
// #1009: normalize empty/null/undefined edgeType so Sigma's label
|
||||
// renderer never receives a blank string on the single-entry path.
|
||||
edgeType: normalizeEdgeType(attrs.edgeType),
|
||||
dominantEdgeType: normalizeEdgeType(attrs.dominantEdgeType ?? attrs.edgeType),
|
||||
rawEdgeIds: collectRawEdgeIds(attrs, edgeId),
|
||||
isAggregated: isAggregatedEdgeAttributes(attrs),
|
||||
aggregateCount: attrs.aggregateCount ?? collectRawEdgeIds(attrs, edgeId).length,
|
||||
dominantEdgeType: attrs.dominantEdgeType ?? attrs.edgeType,
|
||||
representativeWeight: attrs.representativeWeight ?? Number(attrs.weight ?? 1),
|
||||
});
|
||||
return;
|
||||
@@ -2150,10 +2162,11 @@ function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAt
|
||||
const rawEdgeIds = entries.flatMap(({ edgeId, attrs }) => collectRawEdgeIds(attrs, edgeId));
|
||||
const typeCounts = new Map<string, number>();
|
||||
entries.forEach(({ attrs }) => {
|
||||
const edgeType = String(attrs.edgeType ?? "related_to");
|
||||
const edgeType = normalizeEdgeType(attrs.edgeType);
|
||||
typeCounts.set(edgeType, (typeCounts.get(edgeType) ?? 0) + 1);
|
||||
});
|
||||
const dominantEdgeType = [...typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0] ?? representative.attrs.edgeType ?? "related_to";
|
||||
const dominantEdgeType = [...typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0]
|
||||
?? normalizeEdgeType(representative.attrs.edgeType);
|
||||
const reverseKey = `${targetId}→${sourceId}`;
|
||||
const isBidirectionalBundle = groupedEdges.has(reverseKey);
|
||||
const syntheticEdgeId = `${AGGREGATED_EDGE_PREFIX}${sourceId}::${targetId}`;
|
||||
@@ -2167,10 +2180,10 @@ function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAt
|
||||
rawEdgeIds,
|
||||
isAggregated: true,
|
||||
aggregateCount: rawEdgeIds.length,
|
||||
dominantEdgeType: String(dominantEdgeType),
|
||||
dominantEdgeType: dominantEdgeType,
|
||||
representativeWeight: Number(representative.attrs.weight ?? 1),
|
||||
weight: Number(representative.attrs.weight ?? 1),
|
||||
edgeType: String(representative.attrs.edgeType ?? dominantEdgeType ?? "related_to"),
|
||||
edgeType: representative.attrs.edgeType || dominantEdgeType,
|
||||
parallelCount: rawEdgeIds.length,
|
||||
familySize: rawEdgeIds.length,
|
||||
bundleKind: isBidirectionalBundle ? "bidirectional" : "parallel",
|
||||
@@ -2280,7 +2293,7 @@ function buildCommunityGroupedGraph(): GraphDisplayResult {
|
||||
};
|
||||
bucket.rawEdgeIds.push(String(edgeId));
|
||||
bucket.weight = Math.max(bucket.weight, Number((attrs as EdgeAttributes).weight ?? 1));
|
||||
const edgeType = String((attrs as EdgeAttributes).edgeType ?? "related_to");
|
||||
const edgeType = normalizeEdgeType((attrs as EdgeAttributes).edgeType);
|
||||
bucket.typeCounts.set(edgeType, (bucket.typeCounts.get(edgeType) ?? 0) + 1);
|
||||
groupedEdges.set(key, bucket);
|
||||
});
|
||||
@@ -2396,7 +2409,8 @@ function buildCommunityGroupedGraph(): GraphDisplayResult {
|
||||
if (!visibleGroupedEdgeKeys.has(key)) {
|
||||
return;
|
||||
}
|
||||
const dominantEdgeType = [...bundle.typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0] ?? "related_to";
|
||||
const dominantEdgeType = [...bundle.typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0]
|
||||
?? "related_to";
|
||||
const reverseKey = `${bundle.targetId}→${bundle.sourceId}`;
|
||||
const syntheticEdgeId = `${AGGREGATED_EDGE_PREFIX}${key}`;
|
||||
const aggregateCount = bundle.rawEdgeIds.length;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { CSSProperties } from "react";
|
||||
|
||||
import type {
|
||||
GraphDiagnosticsSnapshot,
|
||||
GraphEffectAvailability,
|
||||
GraphEffectToggle,
|
||||
} from "../types";
|
||||
@@ -30,6 +31,11 @@ const EFFECT_ROWS: EffectRowConfig[] = [
|
||||
label: "Neighborhood Lens",
|
||||
description: "Local emphasis around the hovered or selected node.",
|
||||
},
|
||||
{
|
||||
key: "edgeLabelsEnabled",
|
||||
label: "Edge Labels",
|
||||
description: "Draw the relationship type on graph edges. Off restores label-free edges on dense graphs.",
|
||||
},
|
||||
{
|
||||
key: "legendEnabled",
|
||||
label: "Semantic Legend",
|
||||
@@ -37,6 +43,17 @@ const EFFECT_ROWS: EffectRowConfig[] = [
|
||||
},
|
||||
];
|
||||
|
||||
// Maps the effect toggle keys rendered by this plugin to their corresponding
|
||||
// availability keys in GraphDiagnosticsSnapshot["effectAvailability"]. Kept
|
||||
// local because this plugin only renders a subset of all effects.
|
||||
const EFFECT_AVAILABILITY_KEYS: Partial<Record<GraphEffectToggle, keyof GraphDiagnosticsSnapshot["effectAvailability"]>> = {
|
||||
pathPulseEnabled: "pathPulse",
|
||||
pathFlowEnabled: "pathFlow",
|
||||
lensEnabled: "lens",
|
||||
edgeLabelsEnabled: "edgeLabels",
|
||||
legendEnabled: "legend",
|
||||
};
|
||||
|
||||
function renderAvailabilityText(availability: GraphEffectAvailability) {
|
||||
if (availability.available) {
|
||||
if (typeof availability.visibleSegments === "number" && typeof availability.segmentCap === "number") {
|
||||
@@ -139,15 +156,9 @@ export const explorationEffectsPlugin: GraphPlugin = {
|
||||
description={row.description}
|
||||
checked={effectsState[row.key]}
|
||||
availability={
|
||||
availability?.[
|
||||
row.key === "pathPulseEnabled"
|
||||
? "pathPulse"
|
||||
: row.key === "pathFlowEnabled"
|
||||
? "pathFlow"
|
||||
: row.key === "lensEnabled"
|
||||
? "lens"
|
||||
: "legend"
|
||||
] ?? {
|
||||
(EFFECT_AVAILABILITY_KEYS[row.key] !== undefined
|
||||
? availability?.[EFFECT_AVAILABILITY_KEYS[row.key]!]
|
||||
: undefined) ?? {
|
||||
enabled: effectsState[row.key],
|
||||
available: false,
|
||||
reason: "Waiting for graph runtime",
|
||||
|
||||
@@ -47,6 +47,11 @@ const SCENE_EFFECT_ROWS: EffectRowConfig[] = [
|
||||
label: "Contours",
|
||||
description: "Low-contrast density halos around the strongest visible anchors.",
|
||||
},
|
||||
{
|
||||
key: "edgeLabelsEnabled",
|
||||
label: "Edge Labels",
|
||||
description: "Draw the relationship type on graph edges. Off restores label-free edges on dense graphs.",
|
||||
},
|
||||
{
|
||||
key: "legendEnabled",
|
||||
label: "Regions Summary",
|
||||
@@ -83,6 +88,7 @@ const AVAILABILITY_KEYS: Record<GraphEffectToggle, keyof GraphDiagnosticsSnapsho
|
||||
communitiesEnabled: "communities",
|
||||
centralityEnabled: "centrality",
|
||||
legendEnabled: "legend",
|
||||
edgeLabelsEnabled: "edgeLabels",
|
||||
diagnosticsEnabled: "diagnostics",
|
||||
};
|
||||
|
||||
|
||||
@@ -103,6 +103,7 @@ export type GraphEffectToggle =
|
||||
| "communitiesEnabled"
|
||||
| "centralityEnabled"
|
||||
| "legendEnabled"
|
||||
| "edgeLabelsEnabled"
|
||||
| "diagnosticsEnabled";
|
||||
|
||||
export interface GraphEffectsState {
|
||||
@@ -113,6 +114,7 @@ export interface GraphEffectsState {
|
||||
semanticRegionsEnabled: boolean;
|
||||
contoursEnabled: boolean;
|
||||
pathfindingEnabled: boolean;
|
||||
edgeLabelsEnabled: boolean;
|
||||
communitiesEnabled: boolean;
|
||||
centralityEnabled: boolean;
|
||||
legendEnabled: boolean;
|
||||
@@ -186,6 +188,7 @@ export interface GraphDiagnosticsSnapshot {
|
||||
communities: GraphEffectAvailability;
|
||||
centrality: GraphEffectAvailability;
|
||||
legend: GraphEffectAvailability;
|
||||
edgeLabels: GraphEffectAvailability;
|
||||
diagnostics: GraphEffectAvailability;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1061,3 +1061,198 @@ test("checkGroupedViewAvailability returns available when communities exist", ()
|
||||
assert.equal(result.reason, null);
|
||||
});
|
||||
|
||||
|
||||
// ── #1009: edge label data-path regression tests ─────────────────────────────
|
||||
|
||||
test("resolveDisplayGraph parallel-bundle preserves edgeType on aggregated edge", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
batchMergeEdges([
|
||||
{ id: "e1", source: "a", target: "b", attributes: { edgeType: "causes", weight: 1, properties: {} } },
|
||||
{ id: "e2", source: "a", target: "b", attributes: { edgeType: "causes", weight: 2, properties: {} } },
|
||||
]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string; isAggregated?: boolean };
|
||||
assert.equal(attrs.isAggregated, true);
|
||||
// The aggregated representative must carry the relationship text through to
|
||||
// the edgeReducer's label assignment.
|
||||
assert.equal(typeof attrs.edgeType, "string");
|
||||
assert.ok((attrs.edgeType ?? "").length > 0, "aggregated edge must have a non-empty edgeType");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph parallel-bundle picks dominant edgeType across mixed types", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
batchMergeEdges([
|
||||
{ id: "e1", source: "a", target: "b", attributes: { edgeType: "inhibits", weight: 1, properties: {} } },
|
||||
{ id: "e2", source: "a", target: "b", attributes: { edgeType: "inhibits", weight: 1, properties: {} } },
|
||||
{ id: "e3", source: "a", target: "b", attributes: { edgeType: "activates", weight: 1, properties: {} } },
|
||||
]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string; dominantEdgeType?: string };
|
||||
// "inhibits" appears twice so it must be the dominant type.
|
||||
assert.equal(attrs.edgeType, "inhibits");
|
||||
assert.equal(attrs.dominantEdgeType, "inhibits");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph grouped view community edges carry non-empty edgeType", () => {
|
||||
const left = ["g1", "g2", "g3", "g4"];
|
||||
const right = ["h1", "h2", "h3", "h4"];
|
||||
[...left, ...right].forEach((nodeId, index) => addNode(nodeId, index < left.length ? "left" : "right"));
|
||||
|
||||
let edgeIndex = 0;
|
||||
for (let i = 0; i < left.length; i += 1) {
|
||||
for (let j = 0; j < left.length; j += 1) {
|
||||
if (i !== j) {
|
||||
batchMergeEdges([{
|
||||
id: `lg-${edgeIndex++}`,
|
||||
source: left[i],
|
||||
target: left[j],
|
||||
attributes: { edgeType: "co_occurs", weight: 3, properties: {} },
|
||||
}]);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < right.length; i += 1) {
|
||||
for (let j = 0; j < right.length; j += 1) {
|
||||
if (i !== j) {
|
||||
batchMergeEdges([{
|
||||
id: `rg-${edgeIndex++}`,
|
||||
source: right[i],
|
||||
target: right[j],
|
||||
attributes: { edgeType: "co_occurs", weight: 3, properties: {} },
|
||||
}]);
|
||||
}
|
||||
}
|
||||
}
|
||||
batchMergeEdges([{ id: "bridge-g", source: "g1", target: "h1", attributes: { edgeType: "interacts_with", weight: 0.1, properties: {} } }]);
|
||||
|
||||
const { graph: displayGraph, state } = resolveDisplayGraph("", [], [], "grouped", { aggregationEnabled: true });
|
||||
assert.equal(state.groupedViewAvailable, true);
|
||||
|
||||
const communityEdges = displayGraph.edges().filter((edgeId) => {
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { bundleKind?: string };
|
||||
return attrs.bundleKind === "community";
|
||||
});
|
||||
assert.ok(communityEdges.length > 0, "expected at least one community bundle edge");
|
||||
|
||||
for (const edgeId of communityEdges) {
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string };
|
||||
assert.equal(typeof attrs.edgeType, "string");
|
||||
assert.ok((attrs.edgeType ?? "").length > 0, `community edge ${edgeId} must have a non-empty edgeType`);
|
||||
}
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph raw edge preserves exact edgeType string for label rendering", () => {
|
||||
addNode("src");
|
||||
addNode("tgt");
|
||||
batchMergeEdges([{
|
||||
id: "raw-1",
|
||||
source: "src",
|
||||
target: "tgt",
|
||||
attributes: { edgeType: "works_for", weight: 1, properties: {} },
|
||||
}]);
|
||||
|
||||
// In full view without aggregation the edge passes through unchanged.
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: false });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string };
|
||||
assert.equal(attrs.edgeType, "works_for");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph does not produce empty-string edgeType on aggregated edges when source has empty type", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
// Simulate an API response where type is empty string — the aggregation
|
||||
// path must not propagate a blank label.
|
||||
batchMergeEdges([
|
||||
{ id: "e-empty-1", source: "a", target: "b", attributes: { edgeType: "", weight: 1, properties: {} } },
|
||||
{ id: "e-empty-2", source: "a", target: "b", attributes: { edgeType: "", weight: 1, properties: {} } },
|
||||
]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as {
|
||||
edgeType?: string;
|
||||
isAggregated?: boolean;
|
||||
};
|
||||
assert.equal(attrs.isAggregated, true);
|
||||
// The aggregation falls back to "related_to" when all source edgeTypes are
|
||||
// empty, so the rendered label should never be an empty string.
|
||||
assert.equal(attrs.edgeType, "related_to");
|
||||
});
|
||||
|
||||
test("resolveEdgeElementStyle hidden class produces hidden:true for suppressed edges", () => {
|
||||
// Verify the data condition the edgeReducer relies on: hidden-classified
|
||||
// edges must have hidden:true so that the label assignment sets undefined.
|
||||
const style = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"overview",
|
||||
"inactive",
|
||||
{
|
||||
edgeType: "causes",
|
||||
weight: 1,
|
||||
properties: {},
|
||||
edgeVariant: "line",
|
||||
visualPriority: 0.05,
|
||||
baseSize: 0.3,
|
||||
},
|
||||
"source",
|
||||
"target",
|
||||
"full",
|
||||
"inactive-edge",
|
||||
"hidden",
|
||||
);
|
||||
assert.equal(style.hidden, true);
|
||||
});
|
||||
|
||||
// ── #1009 maintainer-blocking regression: single-edge empty edgeType ─────────
|
||||
|
||||
test("resolveDisplayGraph single-edge normalizes empty-string edgeType to related_to", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
// One edge only — exercises the entries.length === 1 path in aggregateDisplayGraph.
|
||||
batchMergeEdges([{
|
||||
id: "e-single-empty",
|
||||
source: "a",
|
||||
target: "b",
|
||||
attributes: { edgeType: "", weight: 1, properties: {} },
|
||||
}]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string; dominantEdgeType?: string };
|
||||
assert.equal(attrs.edgeType, "related_to",
|
||||
"single-edge path must normalize empty edgeType to the canonical fallback");
|
||||
assert.equal(attrs.dominantEdgeType, "related_to",
|
||||
"single-edge dominantEdgeType must also be normalized");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph single-edge preserves a valid non-empty edgeType unchanged", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
batchMergeEdges([{
|
||||
id: "e-single-valid",
|
||||
source: "a",
|
||||
target: "b",
|
||||
attributes: { edgeType: "works_for", weight: 1, properties: {} },
|
||||
}]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string };
|
||||
assert.equal(attrs.edgeType, "works_for",
|
||||
"single-edge path must not alter a valid relationship type");
|
||||
});
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import React from "react";
|
||||
import { renderToString } from "react-dom/server";
|
||||
|
||||
(globalThis as any).React = React;
|
||||
|
||||
import { isSafeUrl, MarkdownContentViewer } from "../src/workspaces/GraphWorkspace/MarkdownContentViewer.tsx";
|
||||
|
||||
test("isSafeUrl permits safe http, https, and mailto URLs and relative paths", () => {
|
||||
assert.equal(isSafeUrl("https://example.com"), true);
|
||||
assert.equal(isSafeUrl("http://localhost:8000"), true);
|
||||
assert.equal(isSafeUrl("mailto:user@example.com"), true);
|
||||
assert.equal(isSafeUrl("#section-1"), true);
|
||||
assert.equal(isSafeUrl("/relative/path"), true);
|
||||
});
|
||||
|
||||
test("isSafeUrl rejects protocol-relative URLs and dangerous schemes", () => {
|
||||
// Protocol-relative URLs (must be blocked)
|
||||
assert.equal(isSafeUrl("//evil.com"), false);
|
||||
assert.equal(isSafeUrl("//localhost:8000"), false);
|
||||
assert.equal(isSafeUrl("//"), false);
|
||||
|
||||
// Dangerous schemes
|
||||
assert.equal(isSafeUrl("javascript:alert('xss')"), false);
|
||||
assert.equal(isSafeUrl("JAVASCRIPT:alert(1)"), false);
|
||||
assert.equal(isSafeUrl("data:text/html;base64,PHNjcmlwdD4="), false);
|
||||
assert.equal(isSafeUrl("vbscript:MsgBox(1)"), false);
|
||||
assert.equal(isSafeUrl(""), false);
|
||||
assert.equal(isSafeUrl(undefined), false);
|
||||
});
|
||||
|
||||
// ─── C URL contract: whitespace-only strings ────────────────────────────────
|
||||
// The CommonMark parser normalises whitespace-only link destinations to "" so
|
||||
// these values are unreachable through normal markdown rendering. However, the
|
||||
// function is exported and its direct-call contract must be correct.
|
||||
test("isSafeUrl rejects whitespace-only strings (contract correctness)", () => {
|
||||
assert.equal(isSafeUrl(" "), false, "single space must be rejected");
|
||||
assert.equal(isSafeUrl("\t"), false, "tab must be rejected");
|
||||
assert.equal(isSafeUrl("\n"), false, "newline must be rejected");
|
||||
assert.equal(isSafeUrl(" "), false, "multiple spaces must be rejected");
|
||||
assert.equal(isSafeUrl(" \t\n "), false, "mixed whitespace must be rejected");
|
||||
});
|
||||
|
||||
test("renders Preview mode with formatted Markdown elements and tabs", () => {
|
||||
const markdown = `# Main Title\n\n**Bold Statement**\n\n* Item A\n* Item B`;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content: markdown, defaultMode: "preview" }));
|
||||
|
||||
// Tab buttons are present
|
||||
assert.equal(html.includes("Preview"), true);
|
||||
assert.equal(html.includes("Source"), true);
|
||||
assert.equal(html.includes("Copy"), true);
|
||||
|
||||
// Formatted preview elements
|
||||
assert.equal(html.includes("Main Title"), true);
|
||||
assert.equal(html.includes("Bold Statement"), true);
|
||||
assert.equal(html.includes("<strong>Bold Statement</strong>"), true);
|
||||
assert.equal(html.includes("Item A"), true);
|
||||
assert.equal(html.includes("Item B"), true);
|
||||
});
|
||||
|
||||
test("renders Source mode with exact unmodified text inside pre/code", () => {
|
||||
const markdown = `# Title 🚀\n\n * Indented item\n\n\`\`\`python\ndef test():\n return "α + β"\n\`\`\``;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content: markdown, defaultMode: "source" }));
|
||||
|
||||
assert.equal(html.includes("<pre"), true);
|
||||
assert.equal(html.includes("<code"), true);
|
||||
assert.equal(html.includes("# Title 🚀"), true);
|
||||
assert.equal(html.includes(" * Indented item"), true);
|
||||
assert.equal(html.includes('return "α + β"'), true);
|
||||
});
|
||||
|
||||
test("renders raw HTML safely as escaped text without executing elements", () => {
|
||||
const dangerousHtml = `<script>alert("XSS")</script><iframe src="https://evil.com"></iframe>`;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content: dangerousHtml, defaultMode: "preview" }));
|
||||
|
||||
// Script and iframe tags must NOT be rendered as active DOM tags
|
||||
assert.equal(html.includes("<script>"), false);
|
||||
assert.equal(html.includes("<iframe"), false);
|
||||
// Content is escaped as text
|
||||
assert.equal(html.includes("<script>"), true);
|
||||
});
|
||||
|
||||
// ─── C-1: HAST node prop must not reach the DOM ─────────────────────────────
|
||||
// react-markdown passes a HAST `node` (Element) object to custom component
|
||||
// overrides. Before this fix, ...props spread caused React 19 to serialise it
|
||||
// as node="[object Object]" on every <a> and <code> element.
|
||||
test("rendered links do not expose the HAST node object as a DOM attribute", () => {
|
||||
const content = `[Example](https://example.com)\n\nInline \`code\` here.`;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content, defaultMode: "preview" }));
|
||||
|
||||
// The rendered HTML must not contain the serialised HAST object
|
||||
assert.equal(html.includes("node="), false, "node= attribute must not appear in rendered HTML");
|
||||
assert.equal(html.includes("[object Object]"), false, "serialised HAST object must not appear in rendered HTML");
|
||||
|
||||
// The link must still render correctly with the right href
|
||||
assert.equal(html.includes('href="https://example.com"'), true, "href must be present");
|
||||
});
|
||||
|
||||
// ─── C-2: Fragment links must not open in a new tab ─────────────────────────
|
||||
// Links to in-document anchors such as #section or GFM footnote backlinks like
|
||||
// #user-content-fn-1 must stay in the current document. Only external links
|
||||
// use target="_blank".
|
||||
test("fragment links render in the current document without target blank", () => {
|
||||
const content = `[Jump to section](#introduction)\n\n[External](https://example.com)`;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content, defaultMode: "preview" }));
|
||||
|
||||
// Fragment link must have the href
|
||||
assert.equal(html.includes('href="#introduction"'), true, "fragment href must be present");
|
||||
|
||||
// Confirm no target=_blank attribute appears anywhere near the fragment link.
|
||||
// We check that the output contains a fragment href WITHOUT target="_blank"
|
||||
// by verifying the two strings are not both present (the external link has
|
||||
// target blank; the fragment link must not).
|
||||
const fragmentLinkIdx = html.indexOf('href="#introduction"');
|
||||
assert.notEqual(fragmentLinkIdx, -1, "fragment link must be rendered");
|
||||
// Inspect the 80 chars around the fragment href — should not contain target
|
||||
const fragmentContext = html.slice(Math.max(0, fragmentLinkIdx - 10), fragmentLinkIdx + 90);
|
||||
assert.equal(fragmentContext.includes('target="_blank"'), false, "fragment link must not have target=_blank");
|
||||
|
||||
// External link must still have target blank
|
||||
assert.equal(html.includes('href="https://example.com"'), true, "external href must be present");
|
||||
assert.equal(html.includes('target="_blank"'), true, "external link must have target=_blank");
|
||||
assert.equal(html.includes('rel="noopener noreferrer"'), true, "external link must have rel");
|
||||
});
|
||||
|
||||
test("GFM footnote backlinks render without target blank", () => {
|
||||
// GFM footnote syntax: footnote ref in text + definition below
|
||||
const content = `See the note[^1] for more.\n\n[^1]: This is the footnote text.`;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content, defaultMode: "preview" }));
|
||||
|
||||
// The footnote reference link (#user-content-fn-1) and backlink
|
||||
// (#user-content-fnref-1) are fragment links and must not open in a new tab.
|
||||
// We verify no fragment href is paired with target=_blank.
|
||||
// Extract all href="#..." occurrences and confirm none is adjacent to target=_blank.
|
||||
const anchorMatches = [...html.matchAll(/href="#[^"]*"/g)];
|
||||
assert.ok(anchorMatches.length > 0, "GFM footnotes must produce fragment links");
|
||||
for (const match of anchorMatches) {
|
||||
const start = match.index ?? 0;
|
||||
const context = html.slice(Math.max(0, start - 10), start + 120);
|
||||
assert.equal(
|
||||
context.includes('target="_blank"'),
|
||||
false,
|
||||
`fragment link ${match[0]} must not have target=_blank`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// ─── C-1-R: GFM footnote attributes must be preserved (regression test) ─────
|
||||
// The C-1 fix (removing the HAST `node` prop) must NOT silently drop other
|
||||
// legitimate HAST attributes. remark-gfm generates the following on footnote
|
||||
// links that are required for correct in-page navigation and accessibility:
|
||||
//
|
||||
// Footnote reference anchor:
|
||||
// id="user-content-fnref-1" ← backlink target
|
||||
// data-footnote-ref="true"
|
||||
// aria-describedby="footnote-label"
|
||||
//
|
||||
// Footnote back-link anchor:
|
||||
// data-footnote-backref=""
|
||||
// aria-label="Back to reference 1" ← screen-reader label
|
||||
// class="data-footnote-backref"
|
||||
//
|
||||
// If these are absent, clicking the ↩ back-link cannot scroll back to the
|
||||
// in-text reference, and screen readers cannot announce the backlink purpose.
|
||||
test("GFM footnote links preserve generated id, aria, and class attributes", () => {
|
||||
const content = `See the note[^1] for more.\n\n[^1]: This is the footnote text.`;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content, defaultMode: "preview" }));
|
||||
|
||||
// The HAST `node` object must not appear serialised as a DOM attribute.
|
||||
assert.equal(html.includes("node="), false, "node= attribute must not appear in HTML");
|
||||
assert.equal(html.includes("[object Object]"), false, "serialised HAST object must not appear in HTML");
|
||||
|
||||
// Footnote reference anchor must retain its id so the backlink can navigate to it.
|
||||
assert.equal(
|
||||
html.includes('id="user-content-fnref-1"'),
|
||||
true,
|
||||
"footnote reference anchor must retain id for back-navigation",
|
||||
);
|
||||
|
||||
// Footnote backlink must retain its aria-label for screen-reader accessibility.
|
||||
assert.equal(
|
||||
html.includes('aria-label="Back to reference 1"'),
|
||||
true,
|
||||
"footnote backlink must retain aria-label for accessibility",
|
||||
);
|
||||
|
||||
// Footnote backlink must retain its class attribute.
|
||||
assert.equal(
|
||||
html.includes('class="data-footnote-backref"'),
|
||||
true,
|
||||
"footnote backlink must retain class attribute",
|
||||
);
|
||||
});
|
||||
|
||||
test("renders safe links as <a> with target blank and unclickable span for unsafe links", () => {
|
||||
const content = `[Safe Link](https://getsemantica.ai)\n\n[Unsafe Scheme](javascript:alert(1))\n\n[Protocol Relative](//evil.com)`;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content, defaultMode: "preview" }));
|
||||
|
||||
// Safe link renders as <a> with security attributes
|
||||
assert.equal(html.includes('href="https://getsemantica.ai"'), true);
|
||||
assert.equal(html.includes('target="_blank"'), true);
|
||||
assert.equal(html.includes('rel="noopener noreferrer"'), true);
|
||||
|
||||
// Unsafe links do NOT render as <a> tags
|
||||
assert.equal(html.includes('href="javascript:alert(1)"'), false);
|
||||
assert.equal(html.includes('href="//evil.com"'), false);
|
||||
assert.equal(html.includes("Unsafe Scheme"), true);
|
||||
assert.equal(html.includes("Protocol Relative"), true);
|
||||
});
|
||||
|
||||
test("renders remote images as safe placeholder badges instead of <img> tags", () => {
|
||||
const content = ``;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content, defaultMode: "preview" }));
|
||||
|
||||
// No <img> tag rendered
|
||||
assert.equal(html.includes("<img"), false);
|
||||
// Image placeholder badge rendered
|
||||
assert.equal(html.includes("Image:"), true);
|
||||
assert.equal(html.includes("System Diagram"), true);
|
||||
});
|
||||
|
||||
test("renders clear empty-state message when content is empty or null", () => {
|
||||
const emptyHtml = renderToString(React.createElement(MarkdownContentViewer, { content: "" }));
|
||||
assert.equal(emptyHtml.includes("No content available for this node."), true);
|
||||
|
||||
const nullHtml = renderToString(React.createElement(MarkdownContentViewer, { content: null }));
|
||||
assert.equal(nullHtml.includes("No content available for this node."), true);
|
||||
});
|
||||
|
||||
test("renders plain text cleanly without requiring Markdown formatting", () => {
|
||||
const plainText = "Plain entity summary text without markdown formatting.";
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content: plainText, defaultMode: "preview" }));
|
||||
|
||||
assert.equal(html.includes(plainText), true);
|
||||
});
|
||||
|
||||
test("handles very large Markdown content without failure", () => {
|
||||
const largeContent = `# Large Knowledge Node\n\n` + "Structured observation paragraph. ".repeat(400);
|
||||
assert.equal(largeContent.length > 10000, true);
|
||||
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content: largeContent, defaultMode: "preview" }));
|
||||
assert.equal(html.includes("Large Knowledge Node"), true);
|
||||
});
|
||||
|
||||
// ─── H-2: Stale copied state lifecycle (SSR-compatible portion) ─────────────
|
||||
// Full state-transition testing (Node A → copy → Node B) requires an interactive
|
||||
// framework. The lifecycle correctness is guaranteed by the render-phase
|
||||
// previous-prop synchronisation pattern: a `copiedForContent` state value tracks
|
||||
// the content for which the copied indicator was set; when `content` changes, the
|
||||
// mismatch is detected during render and `copied` is reset to false in the same
|
||||
// React batch, before the new node's UI is painted. What we CAN verify in SSR
|
||||
// is that the initial render for any content value shows the Copy button (not the
|
||||
// Copied indicator), which confirms the initial state is always clean.
|
||||
test("copy button always starts in un-copied state on initial render", () => {
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, {
|
||||
content: "# Some Node\n\nDescription text.",
|
||||
defaultMode: "preview",
|
||||
}));
|
||||
|
||||
// Initial render must show 'Copy', never 'Copied'
|
||||
assert.equal(html.includes("Copy"), true, "Copy button must be present on initial render");
|
||||
assert.equal(html.includes("Copied"), false, "Copied indicator must NOT be present on initial render");
|
||||
});
|
||||
+12
-2
@@ -93,7 +93,14 @@ def _handle_tools_call(req_id: Any, params: dict) -> dict:
|
||||
result = tool["_handler"](args)
|
||||
except Exception as exc:
|
||||
log.exception("Tool %s raised an exception", name)
|
||||
return _err(req_id, _INTERNAL_ERROR, str(exc))
|
||||
# The exception's class name (e.g. "ValidationError", "TimeoutError")
|
||||
# is safe to surface — unlike str(exc), it never carries paths,
|
||||
# connection strings, or other internal detail — and lets the
|
||||
# client distinguish failure kinds without a full message.
|
||||
return _err(
|
||||
req_id, _INTERNAL_ERROR,
|
||||
f"Tool '{name}' failed ({type(exc).__name__}). See server logs for details.",
|
||||
)
|
||||
|
||||
# MCP spec: content must be a list of content items
|
||||
return _ok(req_id, {
|
||||
@@ -171,7 +178,10 @@ class SemanticaMCPServer:
|
||||
log.exception("Unhandled error in method %s", method)
|
||||
if req_id is None:
|
||||
return None
|
||||
return _err(req_id, _INTERNAL_ERROR, str(exc))
|
||||
return _err(
|
||||
req_id, _INTERNAL_ERROR,
|
||||
f"Method '{method}' failed ({type(exc).__name__}). See server logs for details.",
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def run(self) -> None:
|
||||
|
||||
@@ -53,7 +53,7 @@ plugins/
|
||||
## Prerequisites
|
||||
|
||||
```bash
|
||||
git clone https://github.com/Hawksight-AI/semantica.git
|
||||
git clone https://github.com/semantica-agi/semantica.git
|
||||
cd semantica
|
||||
pip install semantica # Python 3.10+
|
||||
```
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"name": "semantica-local",
|
||||
"owner": {
|
||||
"name": "Hawksight AI",
|
||||
"url": "https://github.com/Hawksight-AI/semantica"
|
||||
"name": "Semantica",
|
||||
"url": "https://github.com/semantica-agi/semantica"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
+6
-5
@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "semantica"
|
||||
version = "0.6.5"
|
||||
description = "Accountability and context layer for AI agents. Context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable."
|
||||
version = "0.6.6"
|
||||
description = "Graph-Native Infrastructure for Context and Accountable AI Systems: context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable."
|
||||
readme = "README.md"
|
||||
license = { text = "MIT" }
|
||||
|
||||
@@ -85,7 +85,8 @@ dependencies = [
|
||||
"loguru>=0.7.3",
|
||||
"structlog>=22.1.0",
|
||||
"gensim>=4.4.0",
|
||||
"httpx<0.29.0"
|
||||
"httpx<0.29.0",
|
||||
"pyarrow>=14.0.0"
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
@@ -103,7 +104,7 @@ Discord = "https://discord.gg/sV34vps5hH"
|
||||
llm-openai = ["openai>=1.0.0"]
|
||||
llm-groq = ["groq>=0.4.0"]
|
||||
llm-gemini = ["google-genai>=0.1.0"]
|
||||
llm-anthropic = ["anthropic>=0.18.0"]
|
||||
llm-anthropic = ["anthropic>=0.122.0"]
|
||||
llm-ollama = ["ollama>=0.1.0"]
|
||||
llm-deepseek = ["openai>=1.0.0"]
|
||||
llm-litellm = ["litellm>=1.83.9"]
|
||||
@@ -271,7 +272,7 @@ include = ["semantica*", "integrations*"]
|
||||
[tool.setuptools.package-data]
|
||||
# Explicit patterns are more reliable than **/* across setuptools versions.
|
||||
# static/* covers index.html / favicon; static/assets/* covers all JS/CSS chunks.
|
||||
"semantica" = ["static/*", "static/assets/*"]
|
||||
"semantica" = ["static/*", "static/assets/*", "ontology/vocabulary/*.ttl"]
|
||||
|
||||
[tool.black]
|
||||
line-length = 88
|
||||
|
||||
+12
-12
@@ -6,9 +6,9 @@ accelerate==1.14.0 \
|
||||
# via
|
||||
# docling-ibm-models
|
||||
# docling-slim
|
||||
agno==2.8.7 \
|
||||
--hash=sha256:6a2763eb469163f7b79ab1da6ca2f22d8619f6b9d614574f975d9c12bb4323ea \
|
||||
--hash=sha256:d49396a2062ee6994ca82695b9bd1e1b95667fec432c544afa38133e564bf090
|
||||
agno==2.9.0 \
|
||||
--hash=sha256:7777674b3931b341fad4fcf02a61b185a08588c509101348facf87feb2144c0c \
|
||||
--hash=sha256:7d9c134703e3c2798023cd57dcb9caa8e1174f6914813f9b130becfc3521a46f
|
||||
# via semantica (pyproject.toml)
|
||||
agnoctl==0.1.3 \
|
||||
--hash=sha256:6fce1d2482b1f2e0a3d14b0a7c12fbd49d8df4f0bf0a4fd9fd91753cbff5efdc \
|
||||
@@ -159,9 +159,9 @@ annotated-types==0.8.0 \
|
||||
--hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \
|
||||
--hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0
|
||||
# via pydantic
|
||||
anthropic==0.121.0 \
|
||||
--hash=sha256:6048713fa441e59e1cba8363171cd2a86273b25bd213e9c7ac70a523af88b011 \
|
||||
--hash=sha256:e79d6e08ab3376602fc9a70d4d5ea3540817c76cf7e16658bed790834e1833d6
|
||||
anthropic==0.122.0 \
|
||||
--hash=sha256:45ec906452ffae6b5f7f0c53d01f50bfb7e4ce878d7ae8e4309d13171e557e67 \
|
||||
--hash=sha256:ffec56ae96657c8d19fa575ec96f140f380c353a07ab7d61b92eb18ee6536601
|
||||
# via semantica (pyproject.toml)
|
||||
antlr4-python3-runtime==4.9.3 \
|
||||
--hash=sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b
|
||||
@@ -403,9 +403,9 @@ boto3==1.43.69 \
|
||||
--hash=sha256:4eb494d05b2bd08a7eee61b8ac4c34745c99e9bbce435c91f8d15d372dd8c2db \
|
||||
--hash=sha256:76297a0b415849c63575ae08a4f1661b2dc8ee0100f104b86f98aa69b47fa2c7
|
||||
# via semantica (pyproject.toml)
|
||||
botocore==1.43.69 \
|
||||
--hash=sha256:5caa46b740d9a886137146ffbb69edb691f702bfe74c64e85621947ae00181fd \
|
||||
--hash=sha256:b1f0e01c53d6b84ee9c184ebf3636c3b3aef85e0ae8498c74afb8734ff224f87
|
||||
botocore==1.43.73 \
|
||||
--hash=sha256:068433028e011ccbeab1dd7c46b1090c24e378397693c66e67ca571176498daa \
|
||||
--hash=sha256:0fa1e63c24b3531be3e1bc1687a88b3be9e63a430153f24edd93efc162bb1c51
|
||||
# via
|
||||
# boto3
|
||||
# s3transfer
|
||||
@@ -1731,9 +1731,9 @@ google-crc32c==1.8.0 \
|
||||
# via
|
||||
# google-cloud-storage
|
||||
# google-resumable-media
|
||||
google-genai==2.17.0 \
|
||||
--hash=sha256:6b640a2390c82b4a240873eddb9f518c6d2c33244b2de16ed3d14526a6da7f57 \
|
||||
--hash=sha256:a4835563c60aee646c9c4b261c507aa4a624710d25017012d20dc65abf3d9a54
|
||||
google-genai==2.18.1 \
|
||||
--hash=sha256:36a5949233e64a60f6cc4521bff7a76b7c569d0aa227bbe9fa642213b8a3a3b2 \
|
||||
--hash=sha256:a1e2be75c16234adc6641afd1ad4dd44218c9eec005d938bdc428585a048918a
|
||||
# via semantica (pyproject.toml)
|
||||
google-resumable-media==2.10.1 \
|
||||
--hash=sha256:224975032ddb73f7ed9e2f0f4cc08ed1b06874c52d48cc8533e3eb72980b21a0 \
|
||||
|
||||
@@ -10,7 +10,7 @@ Main exports:
|
||||
- Config: Configuration management
|
||||
"""
|
||||
|
||||
__version__ = "0.6.5"
|
||||
__version__ = "0.6.6"
|
||||
__author__ = "Semantica Contributors"
|
||||
__license__ = "MIT"
|
||||
|
||||
|
||||
@@ -1039,6 +1039,6 @@ manager = TemporalVersionManager(storage_path="large_data.db")
|
||||
## Support
|
||||
|
||||
For questions or issues:
|
||||
- GitHub Issues: https://github.com/Hawksight-AI/semantica/issues
|
||||
- GitHub Issues: https://github.com/semantica-agi/semantica/issues
|
||||
- Documentation: https://semantica.readthedocs.io
|
||||
- Community: https://discord.gg/sV34vps5hH
|
||||
|
||||
+162
-7
@@ -20,7 +20,7 @@ if sys.platform == "win32":
|
||||
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
||||
|
||||
from dataclasses import asdict, dataclass, field, is_dataclass
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PurePosixPath, PureWindowsPath
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
import yaml
|
||||
@@ -773,10 +773,22 @@ def changelog(cli_ctx: CLIContext, local_json: bool) -> None:
|
||||
_run_with_error_handling(_action)
|
||||
|
||||
|
||||
class _DeepEmbeddingFailure(Exception):
|
||||
"""A deep-probe failure from doctor's embedding checks.
|
||||
|
||||
Marks failures that happened AFTER the backend imported cleanly — model
|
||||
load, probe, or runtime problems — so the check's hint can point at the
|
||||
real remediation instead of `pip install`.
|
||||
"""
|
||||
|
||||
|
||||
@main.command()
|
||||
@click.option("--json", "local_json", is_flag=True, default=False)
|
||||
@click.option("--deep-embeddings", "deep_embeddings", is_flag=True, default=False,
|
||||
help="Also instantiate the local embedding backends and embed a probe "
|
||||
"text (catches backends that import cleanly but cannot load).")
|
||||
@click.pass_obj
|
||||
def doctor(cli_ctx: CLIContext, local_json: bool) -> None:
|
||||
def doctor(cli_ctx: CLIContext, local_json: bool, deep_embeddings: bool) -> None:
|
||||
"""Run a health check on all Semantica components and backends."""
|
||||
import importlib.metadata
|
||||
cli_ctx = _require_ctx(cli_ctx)
|
||||
@@ -787,6 +799,16 @@ def doctor(cli_ctx: CLIContext, local_json: bool) -> None:
|
||||
try:
|
||||
note = fn()
|
||||
return label, "ok", note, None
|
||||
except _DeepEmbeddingFailure as exc:
|
||||
# A deep-probe failure means the package IMPORTED fine: the pip
|
||||
# hint would be the wrong remediation for what is actually a
|
||||
# runtime/model-load problem (broken torch, failed model
|
||||
# download, missing shared libs).
|
||||
return label, "fail", str(exc), (
|
||||
"runtime/model-load failure — reinstalling the package usually "
|
||||
"does not help; check the warnings above (torch install, model "
|
||||
"download, disk space)"
|
||||
)
|
||||
except Exception as exc:
|
||||
return label, "fail", str(exc), hint
|
||||
|
||||
@@ -827,6 +849,50 @@ def doctor(cli_ctx: CLIContext, local_json: bool) -> None:
|
||||
return f"{backend} importable"
|
||||
checks.append(_check("Vector store", _vector, hint="pip install semantica[vectorstore-…]"))
|
||||
|
||||
# Embedding backends (#994): `doctor` used to report all green while
|
||||
# every local embedding backend was non-functional — import success
|
||||
# says nothing about model loading. Default checks stay cheap
|
||||
# (import + version); --deep-embeddings (or
|
||||
# SEMANTICA_DOCTOR_DEEP_EMBEDDINGS=1) instantiates the backend through
|
||||
# TextEmbedder and embeds a probe, which is the only level that
|
||||
# catches a backend that imports cleanly but cannot actually load.
|
||||
deep = deep_embeddings or os.environ.get("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", "").strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
def _embedding_backend(method: str) -> str:
|
||||
if method == "sentence_transformers":
|
||||
import sentence_transformers # noqa: F401
|
||||
note = f"importable ({importlib.metadata.version('sentence-transformers')})"
|
||||
else:
|
||||
import fastembed # noqa: F401
|
||||
note = f"importable ({importlib.metadata.version('fastembed')})"
|
||||
if not deep:
|
||||
return note
|
||||
try:
|
||||
from .embeddings import TextEmbedder
|
||||
embedder = TextEmbedder(method=method)
|
||||
if embedder.model is None and embedder.fastembed_model is None:
|
||||
raise RuntimeError(
|
||||
"model failed to load — the hash fallback is active "
|
||||
"(see warnings above); embedding quality is degraded"
|
||||
)
|
||||
probe = embedder.embed_text("semantica doctor embedding probe")
|
||||
except _DeepEmbeddingFailure:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise _DeepEmbeddingFailure(str(exc)) from exc
|
||||
return f"{note}; deep probe ok ({len(probe)}-dim)"
|
||||
|
||||
checks.append(_check(
|
||||
"Embeddings (sentence-transformers)",
|
||||
lambda: _embedding_backend("sentence_transformers"),
|
||||
hint="pip install sentence-transformers",
|
||||
))
|
||||
checks.append(_check(
|
||||
"Embeddings (fastembed)",
|
||||
lambda: _embedding_backend("fastembed"),
|
||||
hint="pip install fastembed",
|
||||
))
|
||||
|
||||
# LLM provider keys
|
||||
for provider, var in [("OpenAI", "OPENAI_API_KEY"), ("Anthropic", "ANTHROPIC_API_KEY"),
|
||||
("Groq", "GROQ_API_KEY")]:
|
||||
@@ -1708,7 +1774,43 @@ def embed_generate(cli_ctx: CLIContext, input_path: str, model: str,
|
||||
except ImportError as exc:
|
||||
raise click.ClickException(f"Embeddings module not available: {exc}") from exc
|
||||
if output:
|
||||
Path(output).write_text(json.dumps(result, default=str), encoding="utf-8")
|
||||
output_path = Path(output)
|
||||
suffix = output_path.suffix.lower()
|
||||
try:
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
arr = np.asarray(result)
|
||||
if arr.ndim == 1:
|
||||
arr = arr[np.newaxis, :]
|
||||
if arr.ndim != 2:
|
||||
raise click.ClickException(
|
||||
f"embed generate --output expects a 1-D or 2-D array, "
|
||||
f"got {arr.ndim}-D (shape {arr.shape})"
|
||||
)
|
||||
rows = [list(row) for row in arr]
|
||||
if suffix == ".parquet":
|
||||
# Schema: single 'embedding' column (list[float] per row).
|
||||
# embed index detects vector columns via
|
||||
# isinstance(df[c].iloc[0], (list, np.ndarray)).
|
||||
df = pd.DataFrame({"embedding": rows})
|
||||
df.to_parquet(output_path, index=False)
|
||||
elif suffix in (".json", ".jsonl"):
|
||||
df = pd.DataFrame({"embedding": rows})
|
||||
df.to_json(
|
||||
output_path,
|
||||
orient="records",
|
||||
lines=(suffix == ".jsonl"),
|
||||
)
|
||||
else:
|
||||
raise click.ClickException(
|
||||
f"Unsupported output format '{suffix}'. "
|
||||
"Use .parquet, .json, or .jsonl"
|
||||
)
|
||||
except ImportError as exc:
|
||||
raise click.ClickException(
|
||||
f"Missing dependency for --output: {exc}. "
|
||||
"Install pyarrow with: pip install pyarrow"
|
||||
) from exc
|
||||
_ok(cli_ctx, f"Wrote {output}")
|
||||
elif _is_json(cli_ctx, local_json):
|
||||
_jecho(result if isinstance(result, dict) else {"status": "ok"})
|
||||
@@ -3933,15 +4035,68 @@ def backup_restore(cli_ctx: CLIContext, source: str, local_dry: bool) -> None:
|
||||
|
||||
try:
|
||||
if _tf.is_tarfile(str(work_path)):
|
||||
restore_root = Path.cwd()
|
||||
restore_root = Path.cwd().resolve()
|
||||
with _tf.open(str(work_path), "r:*") as tar:
|
||||
# Dry-run listing was already handled above; extract now
|
||||
for member in tar.getmembers():
|
||||
# Strip the leading "semantica-backup/" prefix
|
||||
member.name = member.name.replace("semantica-backup/", "", 1)
|
||||
if member.name:
|
||||
tar.extract(member, path=str(restore_root))
|
||||
console.print(f" restored: {member.name}")
|
||||
if not member.name:
|
||||
continue
|
||||
|
||||
# Reject members whose resolved path escapes the
|
||||
# restore root (path traversal / absolute paths),
|
||||
# regardless of the "semantica-backup/" prefix.
|
||||
member_path = (restore_root / member.name).resolve()
|
||||
try:
|
||||
member_path.relative_to(restore_root)
|
||||
except ValueError:
|
||||
raise click.ClickException(
|
||||
f"Refusing to restore '{member.name}': "
|
||||
"path escapes the restore directory."
|
||||
)
|
||||
|
||||
# Reject symlink/hardlink members whose target
|
||||
# escapes the restore root. Checked two ways:
|
||||
# lexically (linkname itself, so an absolute path or
|
||||
# a literal ".." segment is rejected outright, with
|
||||
# no dependence on what else does or doesn't already
|
||||
# exist on disk) and by resolution (catches any
|
||||
# remaining traversal the lexical check misses).
|
||||
if member.issym() or member.islnk():
|
||||
linkname = member.linkname or ""
|
||||
linkname_parts = PurePosixPath(
|
||||
linkname.replace("\\", "/")
|
||||
).parts
|
||||
if (
|
||||
not linkname
|
||||
or os.path.isabs(linkname)
|
||||
or PureWindowsPath(linkname).is_absolute()
|
||||
or ".." in linkname_parts
|
||||
):
|
||||
raise click.ClickException(
|
||||
f"Refusing to restore '{member.name}': "
|
||||
"link target is absolute or traverses "
|
||||
"out of the archive."
|
||||
)
|
||||
link_target = (
|
||||
member_path.parent / linkname
|
||||
).resolve()
|
||||
try:
|
||||
link_target.relative_to(restore_root)
|
||||
except ValueError:
|
||||
raise click.ClickException(
|
||||
f"Refusing to restore '{member.name}': "
|
||||
"link target escapes the restore directory."
|
||||
)
|
||||
|
||||
extract_kwargs: Dict[str, Any] = {"path": str(restore_root)}
|
||||
if hasattr(_tf, "data_filter"):
|
||||
# Python >=3.12: also reject device files, and
|
||||
# further harden the traversal/ownership checks.
|
||||
extract_kwargs["filter"] = "data"
|
||||
tar.extract(member, **extract_kwargs)
|
||||
console.print(f" restored: {member.name}")
|
||||
elif src.is_dir():
|
||||
restore_root = Path.cwd()
|
||||
for f in src.rglob("*"):
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Filesystem safety helpers for human-editable Markdown persistence."""
|
||||
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def is_filesystem_link(path: Path) -> bool:
|
||||
"""Return whether *path* is a symlink, junction, or Windows reparse point."""
|
||||
if path.is_symlink():
|
||||
return True
|
||||
|
||||
isjunction = getattr(os.path, "isjunction", None)
|
||||
if isjunction is not None and isjunction(path):
|
||||
return True
|
||||
|
||||
try:
|
||||
attributes = getattr(os.lstat(path), "st_file_attributes", 0)
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return False
|
||||
|
||||
reparse_point = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
|
||||
return bool(attributes & reparse_point)
|
||||
|
||||
|
||||
def find_filesystem_link(path: Path) -> Optional[Path]:
|
||||
"""Return the first linked component in *path*, including its ancestors."""
|
||||
for candidate in (path, *path.parents):
|
||||
if is_filesystem_link(candidate):
|
||||
return candidate
|
||||
return None
|
||||
@@ -77,6 +77,7 @@ import yaml
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from ..utils.types import EntityDict, RelationshipDict
|
||||
from ._markdown_filesystem import find_filesystem_link
|
||||
|
||||
|
||||
class _UniqueKeySafeLoader(yaml.SafeLoader):
|
||||
@@ -1742,9 +1743,10 @@ class AgentMemory:
|
||||
@staticmethod
|
||||
def _write_markdown_file(file_path: Path, document: str) -> None:
|
||||
"""Atomically replace a Markdown file without following output symlinks."""
|
||||
if file_path.is_symlink():
|
||||
if find_filesystem_link(file_path) is not None:
|
||||
raise ValueError(
|
||||
f"Refusing to overwrite Markdown symbolic link: {file_path}"
|
||||
"Refusing to overwrite Markdown symbolic link or junction: "
|
||||
f"{file_path}"
|
||||
)
|
||||
|
||||
temporary_path = None
|
||||
@@ -1867,7 +1869,8 @@ class AgentMemory:
|
||||
if "\n" not in data and "\r" not in data:
|
||||
candidate = Path(data)
|
||||
try:
|
||||
candidate_exists = candidate.exists()
|
||||
candidate_is_link = find_filesystem_link(candidate) is not None
|
||||
candidate_exists = candidate_is_link or candidate.exists()
|
||||
except OSError as exc:
|
||||
error_message = (
|
||||
"Failed to inspect possible Markdown import "
|
||||
@@ -1909,62 +1912,71 @@ class AgentMemory:
|
||||
return memories
|
||||
|
||||
def _read_markdown_file_content(self, file_path: Path) -> str:
|
||||
if file_path.is_symlink():
|
||||
raise ValueError(f"Symlink Markdown import paths are rejected: {file_path}")
|
||||
if find_filesystem_link(file_path) is not None:
|
||||
raise ValueError(
|
||||
"Symlink Markdown import paths are rejected; symbolic links and "
|
||||
f"junctions are unsafe: {file_path}"
|
||||
)
|
||||
|
||||
flags = os.O_RDONLY
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
# On POSIX, O_NOFOLLOW makes os.open() fail with ELOOP if the
|
||||
# final path component is a symlink, atomically closing the TOCTOU
|
||||
# window between the is_symlink() check above and the open call.
|
||||
# On Windows, O_NOFOLLOW is not available; the is_symlink() pre-check
|
||||
# above is the only symlink defense and remains vulnerable to a narrow
|
||||
# race. The fstat()/S_ISREG guard below still rejects special files
|
||||
# (FIFOs, devices) on both platforms.
|
||||
flags |= os.O_NOFOLLOW
|
||||
nofollow_flag = getattr(os, "O_NOFOLLOW", 0)
|
||||
flags |= nofollow_flag
|
||||
|
||||
try:
|
||||
fd = os.open(str(file_path), flags)
|
||||
except OSError as exc:
|
||||
if exc.errno == getattr(errno, "ELOOP", None):
|
||||
if (
|
||||
(nofollow_flag and exc.errno == errno.ELOOP)
|
||||
or find_filesystem_link(file_path) is not None
|
||||
):
|
||||
raise ValueError(
|
||||
f"Symlink Markdown import paths are rejected: {file_path}"
|
||||
"Symlink Markdown import paths are rejected; symbolic links "
|
||||
f"and junctions are unsafe: {file_path}"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
try:
|
||||
stat_res = os.fstat(fd)
|
||||
if not stat.S_ISREG(stat_res.st_mode):
|
||||
if find_filesystem_link(file_path) is not None:
|
||||
raise ValueError(
|
||||
"Symlink Markdown import paths are rejected; symbolic links "
|
||||
f"and junctions are unsafe: {file_path}"
|
||||
)
|
||||
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
||||
raise ValueError(
|
||||
f"Markdown import path is not a regular file: {file_path}"
|
||||
)
|
||||
with open(fd, "r", encoding="utf-8", closefd=True) as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
try:
|
||||
with os.fdopen(fd, mode="r", encoding="utf-8") as source:
|
||||
fd = -1
|
||||
return source.read()
|
||||
finally:
|
||||
if fd >= 0:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
def _read_markdown_path(self, path: Path) -> List[Tuple[str, str]]:
|
||||
if path.is_symlink():
|
||||
raise ValueError(f"Symlink Markdown import paths are rejected: {path}")
|
||||
if find_filesystem_link(path) is not None:
|
||||
raise ValueError(
|
||||
"Symlink Markdown import paths are rejected; symbolic links and "
|
||||
f"junctions are unsafe: {path}"
|
||||
)
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Markdown import path does not exist: {path}")
|
||||
|
||||
if path.is_dir():
|
||||
file_paths = sorted(
|
||||
(
|
||||
file_path
|
||||
for file_path in path.iterdir()
|
||||
if file_path.is_file()
|
||||
and not file_path.is_symlink()
|
||||
and file_path.suffix.lower() in self._MARKDOWN_EXTENSIONS
|
||||
),
|
||||
key=lambda file_path: (file_path.name.casefold(), file_path.name),
|
||||
)
|
||||
file_paths = []
|
||||
for file_path in path.iterdir():
|
||||
if file_path.suffix.lower() not in self._MARKDOWN_EXTENSIONS:
|
||||
continue
|
||||
if find_filesystem_link(file_path) is not None:
|
||||
continue
|
||||
if file_path.is_file():
|
||||
file_paths.append(file_path)
|
||||
if find_filesystem_link(path) is not None:
|
||||
raise ValueError(
|
||||
"Symlink Markdown import paths are rejected; symbolic links "
|
||||
f"and junctions are unsafe: {path}"
|
||||
)
|
||||
file_paths.sort(key=lambda item: (item.name.casefold(), item.name))
|
||||
elif path.is_file():
|
||||
file_paths = [path]
|
||||
else:
|
||||
|
||||
+1039
-35
File diff suppressed because it is too large
Load Diff
@@ -278,8 +278,12 @@ class DuplicateDetector:
|
||||
for i, (entity1, entity2, score) in enumerate(similarities):
|
||||
candidate = self._create_duplicate_candidate(entity1, entity2, score)
|
||||
|
||||
# Filter by confidence threshold
|
||||
if candidate.confidence >= self.confidence_threshold:
|
||||
# Filter by confidence threshold; type mismatches are excluded
|
||||
# structurally so no threshold value can admit them.
|
||||
if (
|
||||
candidate.confidence >= self.confidence_threshold
|
||||
and "type_mismatch" not in candidate.reasons
|
||||
):
|
||||
candidates.append(candidate)
|
||||
|
||||
remaining = total_similarities - (i + 1)
|
||||
@@ -624,8 +628,12 @@ class DuplicateDetector:
|
||||
new_entity, existing_entity, similarity.score
|
||||
)
|
||||
|
||||
# Filter by confidence threshold
|
||||
if candidate.confidence >= self.confidence_threshold:
|
||||
# Filter by confidence threshold; type mismatches are
|
||||
# excluded structurally regardless of the threshold.
|
||||
if (
|
||||
candidate.confidence >= self.confidence_threshold
|
||||
and "type_mismatch" not in candidate.reasons
|
||||
):
|
||||
candidates.append(candidate)
|
||||
|
||||
processed += 1
|
||||
@@ -723,7 +731,9 @@ class DuplicateDetector:
|
||||
if key == "name":
|
||||
return getattr(entity, "text", default)
|
||||
if key == "type":
|
||||
return getattr(entity, "label", default)
|
||||
# Entity objects store the type on .type; extraction entities
|
||||
# may expose .label. Missing .label never means "no type".
|
||||
return getattr(entity, "type", default) or getattr(entity, "label", default)
|
||||
if key == "properties":
|
||||
# Check metadata for properties
|
||||
metadata = getattr(entity, "metadata", {})
|
||||
@@ -757,6 +767,25 @@ class DuplicateDetector:
|
||||
reasons = []
|
||||
confidence = similarity_score
|
||||
|
||||
# Check entity type mismatch first: two entities with different
|
||||
# explicit types are not duplicates, whatever their similarity.
|
||||
entity_type1 = self._get_entity_value(entity1, "type")
|
||||
entity_type2 = self._get_entity_value(entity2, "type")
|
||||
if entity_type1 and entity_type2 and entity_type1 != entity_type2:
|
||||
return DuplicateCandidate(
|
||||
entity1=entity1,
|
||||
entity2=entity2,
|
||||
similarity_score=similarity_score,
|
||||
confidence=0.0,
|
||||
reasons=["type_mismatch"],
|
||||
metadata={
|
||||
"name_match": False,
|
||||
"common_properties": 0,
|
||||
"type_match": False,
|
||||
"type_mismatch": True,
|
||||
},
|
||||
)
|
||||
|
||||
# Check for exact name match (strong indicator)
|
||||
name1 = str(self._get_entity_value(entity1, "name", "")).lower().strip()
|
||||
name2 = str(self._get_entity_value(entity2, "name", "")).lower().strip()
|
||||
@@ -779,9 +808,8 @@ class DuplicateDetector:
|
||||
# Boost confidence for each matching property
|
||||
confidence += 0.05 * prop_matches
|
||||
|
||||
# Check entity type match
|
||||
entity_type1 = self._get_entity_value(entity1, "type")
|
||||
entity_type2 = self._get_entity_value(entity2, "type")
|
||||
# Check entity type match (only boosts when types are equal; mismatch
|
||||
# is handled above)
|
||||
if entity_type1 and entity_type2 and entity_type1 == entity_type2:
|
||||
reasons.append("same_type")
|
||||
confidence += 0.05
|
||||
|
||||
@@ -45,6 +45,7 @@ License: MIT
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.entity_ids import get_entity_id
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
@@ -504,7 +505,7 @@ class EntityMerger:
|
||||
# Record source entities
|
||||
provenance["merged_from"] = [
|
||||
{
|
||||
"id": self._get_entity_value(e, "id"),
|
||||
"id": get_entity_id(e),
|
||||
"name": self._get_entity_value(e, "name"),
|
||||
"source": self._get_entity_value(e, "metadata", {}).get("source") if hasattr(e, "metadata") or isinstance(e, dict) else None,
|
||||
}
|
||||
|
||||
@@ -45,6 +45,7 @@ from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from ..utils.entity_ids import get_entity_id
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
@@ -317,14 +318,20 @@ class MergeStrategyManager:
|
||||
message=f"Building merged entity... ({current_step}/{total_steps}, remaining: {remaining_steps} steps)"
|
||||
)
|
||||
# Build merged entity
|
||||
merged_from = []
|
||||
for entity in entities:
|
||||
entity_id = get_entity_id(entity)
|
||||
if entity_id is not None:
|
||||
merged_from.append(entity_id)
|
||||
|
||||
merged_entity = {
|
||||
"id": base_entity.get("id"),
|
||||
"id": get_entity_id(base_entity),
|
||||
"name": self._merge_top_level_field("name", entities, base_entity),
|
||||
"type": self._merge_top_level_field("type", entities, base_entity),
|
||||
"properties": merged_properties,
|
||||
"relationships": merged_relationships,
|
||||
"metadata": self._merge_metadata(entities, base_entity),
|
||||
"merged_from": [e.get("id") for e in entities if e.get("id")],
|
||||
"merged_from": merged_from,
|
||||
"merge_strategy": strategy.value,
|
||||
}
|
||||
|
||||
|
||||
@@ -69,6 +69,15 @@ class EmbeddingGeneratorWithProvenance:
|
||||
return embeddings
|
||||
|
||||
def __getattr__(self, name):
|
||||
# __getattr__ only runs when normal lookup fails. Accessing
|
||||
# self._generator by attribute syntax HERE would re-enter
|
||||
# __getattr__ for ever when _generator itself is missing — the shape
|
||||
# pickle/copy protocol probes hit when __init__ never completed
|
||||
# (#994's RecursionError family). Fail fast on private probes.
|
||||
if name.startswith("_"):
|
||||
raise AttributeError(
|
||||
f"{type(self).__name__!r} object has no attribute {name!r}"
|
||||
)
|
||||
return getattr(self._generator, name)
|
||||
|
||||
|
||||
|
||||
@@ -80,6 +80,7 @@ import numpy as np
|
||||
|
||||
from ..utils.exceptions import ConfigurationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
|
||||
from .config import embeddings_config
|
||||
from .embedding_generator import EmbeddingGenerator
|
||||
from .pooling_strategies import PoolingStrategyFactory
|
||||
@@ -116,15 +117,15 @@ def generate_embeddings(
|
||||
>>> emb = generate_embeddings("Hello world", method="default")
|
||||
>>> embs = generate_embeddings(["text1", "text2"], method="text")
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
# Check for custom method in registry, skip self-reference
|
||||
custom_method = method_registry.get("generation", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(data, data_type=data_type, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
if custom_method and custom_method is not generate_embeddings:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, data_type=data_type, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
if method == "default":
|
||||
@@ -164,15 +165,15 @@ def embed_text(
|
||||
>>> emb = embed_text("Hello world", method="sentence_transformers")
|
||||
>>> embs = embed_text(["text1", "text2"], method="sentence_transformers")
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
# Check for custom method in registry, skip self-reference
|
||||
custom_method = method_registry.get("text", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(text, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
if custom_method and custom_method is not embed_text:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, text, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -224,15 +225,15 @@ def calculate_similarity(
|
||||
>>> similarity = calculate_similarity(emb1, emb2, method="cosine")
|
||||
>>> print(f"Similarity: {similarity:.3f}")
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
# Check for custom method in registry, skip self-reference
|
||||
custom_method = method_registry.get("similarity", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(embedding1, embedding2, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
if custom_method and custom_method is not calculate_similarity:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, embedding1, embedding2, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
generator = EmbeddingGenerator(**kwargs)
|
||||
@@ -271,15 +272,15 @@ def pool_embeddings(
|
||||
>>> pooled = pool_embeddings(embeddings, method="mean")
|
||||
>>> attention_pooled = pool_embeddings(embeddings, method="attention")
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
# Check for custom method in registry, skip self-reference
|
||||
custom_method = method_registry.get("pooling", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(embeddings, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
if custom_method and custom_method is not pool_embeddings:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, embeddings, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
strategy = PoolingStrategyFactory.create(method, **kwargs)
|
||||
|
||||
@@ -14,9 +14,10 @@ License: MIT
|
||||
"""
|
||||
|
||||
from typing import Any, Optional
|
||||
from datetime import datetime
|
||||
import uuid
|
||||
|
||||
from ..utils.helpers import utc_now_iso
|
||||
|
||||
|
||||
class ExporterWithProvenance:
|
||||
"""Base exporter with provenance tracking."""
|
||||
@@ -45,9 +46,9 @@ class ExporterWithProvenance:
|
||||
|
||||
def export(self, data: Any, destination: str, **kwargs):
|
||||
"""Export data with provenance tracking."""
|
||||
activity_started_at_time = datetime.utcnow().isoformat()
|
||||
activity_started_at_time = utc_now_iso()
|
||||
result = self._exporter.export(data, destination, **kwargs)
|
||||
activity_ended_at_time = datetime.utcnow().isoformat()
|
||||
activity_ended_at_time = utc_now_iso()
|
||||
|
||||
if self.provenance and self._prov_manager:
|
||||
self._prov_manager.track_entity(
|
||||
|
||||
@@ -24,14 +24,56 @@ License: MIT
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.helpers import ensure_directory, write_json_file
|
||||
from ..utils.helpers import ensure_directory, hash_data, utc_now_iso, write_json_file
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .rdf_exporter import SEMANTICA_NS, mint_entity_iri, mint_relationship_iri
|
||||
|
||||
|
||||
def _content_iri(prefix: str, payload: Any) -> str:
|
||||
"""Mint a document IRI from what was exported, not when.
|
||||
|
||||
Minting from ``utc_now_iso()`` gave every export of the same graph a new
|
||||
identity a few microseconds apart, so re-exporting an unchanged graph was
|
||||
never idempotent and merging exports duplicated every node (#1147). This
|
||||
mirrors ``mint_entity_iri`` (#1109): identical content hashes to the same
|
||||
IRI, and any change to the content changes it too. ``default=str`` keeps
|
||||
the hash defined for values ``json.dumps`` would otherwise reject, such as
|
||||
``datetime`` objects a caller may have left in the graph.
|
||||
|
||||
Args:
|
||||
prefix: IRI prefix the digest is appended to
|
||||
payload: JSON-serializable value whose content determines the digest
|
||||
|
||||
Returns:
|
||||
A stable IRI of the form ``{prefix}{16-hex-char digest}``
|
||||
"""
|
||||
canonical = json.dumps(payload, sort_keys=True, default=str)
|
||||
digest = hash_data(canonical)[:16]
|
||||
return f"{prefix}{digest}"
|
||||
|
||||
|
||||
def _is_jsonld_document(data: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
Report whether a dictionary is already a JSON-LD document.
|
||||
|
||||
``export_knowledge_graph`` converts a knowledge graph to JSON-LD and then
|
||||
hands the finished document to ``export()``, which converted it a second
|
||||
time. The converted document no longer carries ``entities``/
|
||||
``relationships`` keys, so the second pass treated it as an opaque value and
|
||||
buried it inside ``@graph``.
|
||||
|
||||
Args:
|
||||
data: Dictionary to test
|
||||
|
||||
Returns:
|
||||
True when the dictionary declares a JSON-LD context
|
||||
"""
|
||||
return "@context" in data
|
||||
|
||||
|
||||
class JSONExporter:
|
||||
@@ -211,7 +253,10 @@ class JSONExporter:
|
||||
- statistics: Statistics dictionary (optional)
|
||||
file_path: Output JSON file path
|
||||
format: Export format - 'json' or 'json-ld' (default: self.format)
|
||||
**options: Additional options passed to conversion methods
|
||||
**options: Additional options passed to conversion methods:
|
||||
- graph_uri: Caller-supplied IRI for the graph node when
|
||||
format='json-ld', overriding the default content-derived
|
||||
IRI (see #1147)
|
||||
|
||||
Example:
|
||||
>>> kg = {
|
||||
@@ -265,11 +310,12 @@ class JSONExporter:
|
||||
json_data = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"entities": {"@id": "semantica:entities", "@container": "@list"},
|
||||
},
|
||||
"entities": entities,
|
||||
"metadata": {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
"entity_count": len(entities),
|
||||
**options.get("metadata", {}),
|
||||
},
|
||||
@@ -294,6 +340,7 @@ class JSONExporter:
|
||||
json_data = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"relationships": {
|
||||
"@id": "semantica:relationships",
|
||||
"@container": "@list",
|
||||
@@ -301,7 +348,7 @@ class JSONExporter:
|
||||
},
|
||||
"relationships": relationships,
|
||||
"metadata": {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
"relationship_count": len(relationships),
|
||||
**options.get("metadata", {}),
|
||||
},
|
||||
@@ -339,7 +386,7 @@ class JSONExporter:
|
||||
if include_metadata:
|
||||
if "metadata" not in result:
|
||||
result["metadata"] = {}
|
||||
result["metadata"]["exported_at"] = datetime.now().isoformat()
|
||||
result["metadata"]["exported_at"] = utc_now_iso()
|
||||
if include_provenance:
|
||||
result["metadata"]["format"] = "json"
|
||||
|
||||
@@ -349,7 +396,7 @@ class JSONExporter:
|
||||
"data": data,
|
||||
"count": len(data),
|
||||
"metadata": {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
"format": "json" if include_provenance else None,
|
||||
**options.get("metadata", {}),
|
||||
},
|
||||
@@ -358,7 +405,7 @@ class JSONExporter:
|
||||
# Single value
|
||||
return {
|
||||
"value": data,
|
||||
"metadata": {"exported_at": datetime.now().isoformat()}
|
||||
"metadata": {"exported_at": utc_now_iso()}
|
||||
if include_metadata
|
||||
else {},
|
||||
}
|
||||
@@ -380,7 +427,9 @@ class JSONExporter:
|
||||
data: Data to convert (dict, list, or any value)
|
||||
include_metadata: Whether to include metadata (default: True)
|
||||
include_provenance: Whether to include provenance (default: True)
|
||||
**options: Additional options passed to knowledge graph conversion
|
||||
**options: Additional options passed to knowledge graph conversion:
|
||||
- document_uri: Caller-supplied IRI for the document node,
|
||||
overriding the default content-derived IRI (see #1147)
|
||||
|
||||
Returns:
|
||||
Dictionary in JSON-LD format with @context, @graph/@value, and metadata
|
||||
@@ -395,9 +444,29 @@ class JSONExporter:
|
||||
|
||||
# Convert data based on type
|
||||
if isinstance(data, dict):
|
||||
# A knowledge graph is converted even when it carries a context of
|
||||
# its own: the specialized conversion is what mints entity ids and
|
||||
# relationship endpoints, and skipping it leaves them raw keys.
|
||||
if "entities" in data or "relationships" in data:
|
||||
# Knowledge graph structure - use specialized conversion
|
||||
jsonld.update(self._convert_kg_to_jsonld(data, **options))
|
||||
elif _is_jsonld_document(data):
|
||||
# Already JSON-LD: merge it rather than nesting it. Wrapping a
|
||||
# converted document in @graph re-typed the payload as a named
|
||||
# graph and doubled the @context, which is what happened when
|
||||
# export_knowledge_graph handed its own output back to export().
|
||||
context = data.get("@context")
|
||||
if isinstance(context, dict):
|
||||
jsonld["@context"].update(context)
|
||||
elif context is not None:
|
||||
# A context may also be a URL or an array of them, which
|
||||
# cannot be merged key by key. Keeping both as an array
|
||||
# preserves the caller's term expansion, which wins over
|
||||
# ours, while still defining the semantica prefix. An
|
||||
# explicit null is left alone: in an array it would reset
|
||||
# the active context and take our own terms with it.
|
||||
jsonld["@context"] = [jsonld["@context"], context]
|
||||
jsonld.update({k: v for k, v in data.items() if k != "@context"})
|
||||
else:
|
||||
# Generic dictionary - wrap in @graph
|
||||
jsonld["@graph"] = [data]
|
||||
@@ -410,13 +479,67 @@ class JSONExporter:
|
||||
|
||||
# Add metadata and provenance if requested
|
||||
if include_metadata:
|
||||
jsonld["@id"] = f"https://semantica.dev/data/{datetime.now().isoformat()}"
|
||||
if include_provenance:
|
||||
jsonld["semantica:exportedAt"] = datetime.now().isoformat()
|
||||
jsonld["semantica:format"] = "json-ld"
|
||||
self._attach_document_metadata(
|
||||
jsonld, include_provenance, options.get("document_uri")
|
||||
)
|
||||
|
||||
return jsonld
|
||||
|
||||
@staticmethod
|
||||
def _attach_document_metadata(
|
||||
jsonld: Dict[str, Any],
|
||||
include_provenance: bool,
|
||||
document_uri: Optional[str] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Attach the export's own metadata without naming the graph.
|
||||
|
||||
A top-level ``@id`` alongside a top-level ``@graph`` is a *named graph*:
|
||||
the members of ``@graph`` become quads named by that ``@id`` and leave
|
||||
the default graph empty. ``rdflib.Graph.parse()`` keeps only the default
|
||||
graph, so every statement in the export was discarded without an error
|
||||
(2 of 21 statements survived a two-entity knowledge graph). When the
|
||||
payload lives in ``@graph``, the document node goes in beside it as one
|
||||
more node; otherwise it is the document itself.
|
||||
|
||||
Args:
|
||||
jsonld: Document being built, modified in place
|
||||
include_provenance: Whether to record how and when it was exported
|
||||
document_uri: Caller-supplied IRI for the document node. Falls back
|
||||
to a content-derived IRI (#1147) so re-exporting unchanged data
|
||||
is idempotent instead of minting a new identity every time.
|
||||
"""
|
||||
# A caller may hand us a document that is deliberately a named graph.
|
||||
# That name is theirs to keep, but our own statements must not end up
|
||||
# inside it, where a default-graph reader would never see them.
|
||||
payload_is_named_graph = "@id" in jsonld and "@graph" in jsonld
|
||||
|
||||
document: Dict[str, Any] = {}
|
||||
# Do not overwrite an identifier the payload already carries: the
|
||||
# knowledge-graph conversion names its own document node.
|
||||
if "@id" not in jsonld or payload_is_named_graph:
|
||||
content = {key: value for key, value in jsonld.items() if key != "@context"}
|
||||
document["@id"] = document_uri or _content_iri(
|
||||
"https://semantica.dev/data/", content
|
||||
)
|
||||
if include_provenance:
|
||||
document["semantica:exportedAt"] = utc_now_iso()
|
||||
document["semantica:format"] = "json-ld"
|
||||
|
||||
if payload_is_named_graph:
|
||||
named = {key: value for key, value in jsonld.items() if key != "@context"}
|
||||
for key in [key for key in jsonld if key != "@context"]:
|
||||
del jsonld[key]
|
||||
jsonld["@graph"] = [named, document]
|
||||
elif "@graph" in jsonld:
|
||||
# @graph may be a single node object as well as an array. list() on
|
||||
# a dictionary yields its keys, which would discard the node.
|
||||
members = jsonld["@graph"]
|
||||
members = list(members) if isinstance(members, list) else [members]
|
||||
jsonld["@graph"] = members + [document]
|
||||
else:
|
||||
jsonld.update(document)
|
||||
|
||||
def _convert_kg_to_json(self, kg: Dict[str, Any], **options) -> Dict[str, Any]:
|
||||
"""
|
||||
Convert knowledge graph to JSON format.
|
||||
@@ -444,7 +567,7 @@ class JSONExporter:
|
||||
"nodes": kg.get("nodes", []),
|
||||
"edges": kg.get("edges", []),
|
||||
"metadata": {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
**kg.get("metadata", {}),
|
||||
**options.get("metadata", {}),
|
||||
},
|
||||
@@ -468,7 +591,9 @@ class JSONExporter:
|
||||
- entities: List of entity dictionaries
|
||||
- relationships: List of relationship dictionaries
|
||||
- metadata: Metadata dictionary (optional)
|
||||
**options: Additional options (unused)
|
||||
**options: Additional options:
|
||||
- graph_uri: Caller-supplied IRI for the graph node,
|
||||
overriding the default content-derived IRI (see #1147)
|
||||
|
||||
Returns:
|
||||
Dictionary in JSON-LD format with @context, @id, @type, and graph data
|
||||
@@ -481,7 +606,11 @@ class JSONExporter:
|
||||
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
||||
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
||||
},
|
||||
"@id": f"https://semantica.dev/graph/{datetime.now().isoformat()}",
|
||||
# Minted from the graph's own content rather than the wall clock
|
||||
# (#1147): re-exporting an unchanged graph must produce the same
|
||||
# subject, or merging repeated exports duplicates every node.
|
||||
"@id": options.get("graph_uri")
|
||||
or _content_iri("https://semantica.dev/graph/", kg),
|
||||
"@type": "semantica:KnowledgeGraph",
|
||||
}
|
||||
|
||||
@@ -495,14 +624,15 @@ class JSONExporter:
|
||||
relationships = kg.get("relationships", [])
|
||||
if relationships:
|
||||
jsonld["semantica:relationships"] = [
|
||||
self._relationship_to_jsonld(r) for r in relationships
|
||||
self._relationship_to_jsonld(r, index)
|
||||
for index, r in enumerate(relationships)
|
||||
]
|
||||
self.logger.debug(
|
||||
f"Converted {len(relationships)} relationship(s) to JSON-LD"
|
||||
)
|
||||
|
||||
# Add metadata
|
||||
jsonld["semantica:exportedAt"] = datetime.now().isoformat()
|
||||
jsonld["semantica:exportedAt"] = utc_now_iso()
|
||||
if "metadata" in kg:
|
||||
jsonld["semantica:metadata"] = kg["metadata"]
|
||||
|
||||
@@ -526,11 +656,13 @@ class JSONExporter:
|
||||
Returns:
|
||||
Dictionary in JSON-LD format representing the entity
|
||||
"""
|
||||
# Generate @id if not provided
|
||||
entity_id = entity.get("id")
|
||||
if not entity_id:
|
||||
entity_text = entity.get("text") or entity.get("label", "unknown")
|
||||
entity_id = f"semantica:entity/{entity_text}"
|
||||
# Generate @id if not provided. Minted exactly as the RDF serializers
|
||||
# mint it (#1101), so the JSON-LD and Turtle exports of one knowledge
|
||||
# graph name the same entity with the same IRI. Interpolating the raw
|
||||
# text into f"semantica:entity/{text}" produced an invalid IRI for any
|
||||
# text containing a space, and a JSON-LD parser dropped the whole node.
|
||||
entity_text = entity.get("text") or entity.get("label", "unknown")
|
||||
entity_id = entity.get("id") or mint_entity_iri(entity_text)
|
||||
|
||||
jsonld = {
|
||||
"@id": entity_id,
|
||||
@@ -545,7 +677,9 @@ class JSONExporter:
|
||||
|
||||
return jsonld
|
||||
|
||||
def _relationship_to_jsonld(self, rel: Dict[str, Any]) -> Dict[str, Any]:
|
||||
def _relationship_to_jsonld(
|
||||
self, rel: Dict[str, Any], index: int = 0
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Convert relationship to JSON-LD format.
|
||||
|
||||
@@ -560,16 +694,18 @@ class JSONExporter:
|
||||
- type: Relationship type (optional)
|
||||
- confidence: Confidence score (optional)
|
||||
- metadata: Metadata dictionary (optional)
|
||||
index: Position of the relationship in the exported list, used when
|
||||
minting an IRI for a relationship that arrived without an id
|
||||
|
||||
Returns:
|
||||
Dictionary in JSON-LD format representing the relationship
|
||||
"""
|
||||
# Generate @id if not provided
|
||||
rel_id = rel.get("id")
|
||||
if not rel_id:
|
||||
source_id = rel.get("source_id") or rel.get("source", "")
|
||||
target_id = rel.get("target_id") or rel.get("target", "")
|
||||
rel_id = f"semantica:rel/{source_id}_{target_id}"
|
||||
# Generate @id if not provided, from the same mint the RDF serializers
|
||||
# use, including the list index that separates two relationships
|
||||
# sharing a pair of endpoints (#1101).
|
||||
source_id = rel.get("source_id") or rel.get("source", "")
|
||||
target_id = rel.get("target_id") or rel.get("target", "")
|
||||
rel_id = rel.get("id") or mint_relationship_iri(index, source_id, target_id)
|
||||
|
||||
jsonld = {
|
||||
"@id": rel_id,
|
||||
|
||||
+79
-78
@@ -164,6 +164,7 @@ from typing import Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
|
||||
from .arango_aql_exporter import ArangoAQLExporter
|
||||
from .arrow_exporter import ArrowExporter
|
||||
from .config import export_config
|
||||
@@ -221,12 +222,12 @@ def export_rdf(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("rdf", method)
|
||||
if custom_method and custom_method is not export_rdf:
|
||||
try:
|
||||
return custom_method(data, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -270,12 +271,12 @@ def export_json(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("json", method)
|
||||
if custom_method and custom_method is not export_json:
|
||||
try:
|
||||
return custom_method(data, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -316,12 +317,12 @@ def export_csv(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("csv", method)
|
||||
if custom_method and custom_method is not export_csv:
|
||||
try:
|
||||
return custom_method(data, file_path, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -361,12 +362,12 @@ def export_arrow(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("arrow", method)
|
||||
if custom_method and custom_method is not export_arrow:
|
||||
try:
|
||||
return custom_method(data, file_path, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -421,12 +422,12 @@ def export_parquet(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("parquet", method)
|
||||
if custom_method and custom_method is not export_parquet:
|
||||
try:
|
||||
return custom_method(data, file_path, compression=compression, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, file_path, compression=compression, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -472,12 +473,12 @@ def export_graph(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("graph", method)
|
||||
if custom_method and custom_method is not export_graph:
|
||||
try:
|
||||
return custom_method(graph_data, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, graph_data, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -538,12 +539,12 @@ def export_yaml(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("yaml", method)
|
||||
if custom_method and custom_method is not export_yaml:
|
||||
try:
|
||||
return custom_method(data, file_path, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -595,12 +596,12 @@ def export_owl(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("owl", method)
|
||||
if custom_method and custom_method is not export_owl:
|
||||
try:
|
||||
return custom_method(ontology, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, ontology, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -647,12 +648,12 @@ def export_vector(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("vector", method)
|
||||
if custom_method and custom_method is not export_vector:
|
||||
try:
|
||||
return custom_method(vectors, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, vectors, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -695,12 +696,12 @@ def export_lpg(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("lpg", method)
|
||||
if custom_method and custom_method is not export_lpg:
|
||||
try:
|
||||
return custom_method(knowledge_graph, file_path, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, knowledge_graph, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -738,12 +739,12 @@ def export_neo4j_csv(
|
||||
"""
|
||||
custom_method = method_registry.get("neo4j_csv", method)
|
||||
if custom_method and custom_method is not export_neo4j_csv:
|
||||
try:
|
||||
return custom_method(knowledge_graph, output_dir, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, knowledge_graph, output_dir, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = export_config.get_method_config("neo4j_csv")
|
||||
@@ -808,12 +809,12 @@ def export_arango(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("arango", method)
|
||||
if custom_method and custom_method is not export_arango:
|
||||
try:
|
||||
return custom_method(knowledge_graph, file_path, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, knowledge_graph, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -859,12 +860,12 @@ def generate_report(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("report", method)
|
||||
if custom_method and custom_method is not generate_report:
|
||||
try:
|
||||
return custom_method(data, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
|
||||
@@ -22,8 +22,10 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
@@ -37,6 +39,9 @@ from ..utils.progress_tracker import get_progress_tracker
|
||||
# PROV-exported URIs co-resolve under one shared namespace by default.
|
||||
from ..provenance.manager import DEFAULT_BASE_URI
|
||||
|
||||
#: Module-level logger, for the classmethod helpers that have no instance.
|
||||
logger = get_logger("owl_exporter")
|
||||
|
||||
|
||||
class OWLExporter:
|
||||
"""
|
||||
@@ -225,6 +230,7 @@ class OWLExporter:
|
||||
Returns:
|
||||
String containing OWL-XML serialization
|
||||
"""
|
||||
esc_xml = self._escape_xml
|
||||
ontology_uri = ontology.get("uri") or self.ontology_uri
|
||||
ontology_name = ontology.get("name", "SemanticaOntology")
|
||||
version = ontology.get("version") or self.version
|
||||
@@ -237,97 +243,125 @@ class OWLExporter:
|
||||
lines.append("")
|
||||
|
||||
# Ontology declaration
|
||||
lines.append(f' <owl:Ontology rdf:about="{ontology_uri}">')
|
||||
lines.append(f" <rdfs:label>{ontology_name}</rdfs:label>")
|
||||
lines.append(f" <owl:versionInfo>{version}</owl:versionInfo>")
|
||||
lines.append(f' <owl:Ontology rdf:about="{esc_xml(ontology_uri)}">')
|
||||
lines.append(f" <rdfs:label>{esc_xml(ontology_name)}</rdfs:label>")
|
||||
lines.append(f" <owl:versionInfo>{esc_xml(version)}</owl:versionInfo>")
|
||||
if ontology.get("description"):
|
||||
lines.append(
|
||||
f' <rdfs:comment>{ontology.get("description")}</rdfs:comment>'
|
||||
f' <rdfs:comment>{esc_xml(ontology.get("description"))}</rdfs:comment>'
|
||||
)
|
||||
lines.append(" </owl:Ontology>")
|
||||
lines.append("")
|
||||
|
||||
class_index = self._class_iri_index(ontology, ontology_uri)
|
||||
object_properties, data_properties = self._split_properties(ontology)
|
||||
|
||||
def _as_list(value):
|
||||
if value is None:
|
||||
return []
|
||||
return value if isinstance(value, list) else [value]
|
||||
|
||||
# Classes
|
||||
classes = ontology.get("classes", [])
|
||||
for cls in classes:
|
||||
class_uri = cls.get("uri") or cls.get("id", "")
|
||||
for cls in ontology.get("classes", []) or []:
|
||||
if not isinstance(cls, dict):
|
||||
continue
|
||||
class_uri = self._term_iri(cls, ontology_uri)
|
||||
if not class_uri:
|
||||
self.logger.warning(
|
||||
"Skipping a class with no name, uri or id: it would serialise "
|
||||
"as an empty rdf:about"
|
||||
)
|
||||
continue
|
||||
class_name = cls.get("name") or cls.get("label", "")
|
||||
|
||||
lines.append(f' <owl:Class rdf:about="{class_uri}">')
|
||||
lines.append(f" <rdfs:label>{class_name}</rdfs:label>")
|
||||
lines.append(f' <owl:Class rdf:about="{esc_xml(class_uri)}">')
|
||||
lines.append(f" <rdfs:label>{esc_xml(class_name)}</rdfs:label>")
|
||||
|
||||
if cls.get("comment"):
|
||||
lines.append(f' <rdfs:comment>{cls.get("comment")}</rdfs:comment>')
|
||||
comment = cls.get("comment") or cls.get("description")
|
||||
if comment:
|
||||
lines.append(f" <rdfs:comment>{esc_xml(comment)}</rdfs:comment>")
|
||||
|
||||
# Subclass relationships
|
||||
if cls.get("subClassOf"):
|
||||
parent = cls.get("subClassOf")
|
||||
lines.append(f' <rdfs:subClassOf rdf:resource="{parent}"/>')
|
||||
for parent in _as_list(cls.get("subClassOf") or cls.get("parent")):
|
||||
parent_iri = self._resolve_class_ref(parent, ontology_uri, class_index)
|
||||
if parent_iri:
|
||||
lines.append(
|
||||
f' <rdfs:subClassOf rdf:resource="{esc_xml(parent_iri)}"/>'
|
||||
)
|
||||
|
||||
# Equivalent classes
|
||||
if cls.get("equivalentClass"):
|
||||
equiv = cls.get("equivalentClass")
|
||||
lines.append(f' <owl:equivalentClass rdf:resource="{equiv}"/>')
|
||||
for equiv in _as_list(cls.get("equivalentClass")):
|
||||
equiv_iri = self._resolve_class_ref(equiv, ontology_uri, class_index)
|
||||
if equiv_iri:
|
||||
lines.append(
|
||||
f' <owl:equivalentClass rdf:resource="{esc_xml(equiv_iri)}"/>'
|
||||
)
|
||||
|
||||
lines.append(" </owl:Class>")
|
||||
lines.append("")
|
||||
|
||||
# Object properties
|
||||
object_properties = ontology.get("object_properties", [])
|
||||
for prop in object_properties:
|
||||
prop_uri = prop.get("uri") or prop.get("id", "")
|
||||
prop_uri = self._term_iri(prop, ontology_uri)
|
||||
if not prop_uri:
|
||||
self.logger.warning(
|
||||
"Skipping an object property with no name, uri or id"
|
||||
)
|
||||
continue
|
||||
prop_name = prop.get("name") or prop.get("label", "")
|
||||
|
||||
lines.append(f' <owl:ObjectProperty rdf:about="{prop_uri}">')
|
||||
lines.append(f" <rdfs:label>{prop_name}</rdfs:label>")
|
||||
lines.append(f' <owl:ObjectProperty rdf:about="{esc_xml(prop_uri)}">')
|
||||
lines.append(f" <rdfs:label>{esc_xml(prop_name)}</rdfs:label>")
|
||||
|
||||
if prop.get("comment"):
|
||||
lines.append(f' <rdfs:comment>{prop.get("comment")}</rdfs:comment>')
|
||||
comment = prop.get("comment") or prop.get("description")
|
||||
if comment:
|
||||
lines.append(f" <rdfs:comment>{esc_xml(comment)}</rdfs:comment>")
|
||||
|
||||
# Domain
|
||||
if prop.get("domain"):
|
||||
domain = prop.get("domain")
|
||||
if isinstance(domain, list):
|
||||
for d in domain:
|
||||
lines.append(f' <rdfs:domain rdf:resource="{d}"/>')
|
||||
else:
|
||||
lines.append(f' <rdfs:domain rdf:resource="{domain}"/>')
|
||||
for domain in _as_list(prop.get("domain")):
|
||||
domain_iri = self._resolve_class_ref(domain, ontology_uri, class_index)
|
||||
if domain_iri:
|
||||
lines.append(
|
||||
f' <rdfs:domain rdf:resource="{esc_xml(domain_iri)}"/>'
|
||||
)
|
||||
|
||||
# Range
|
||||
if prop.get("range"):
|
||||
range_val = prop.get("range")
|
||||
if isinstance(range_val, list):
|
||||
for r in range_val:
|
||||
lines.append(f' <rdfs:range rdf:resource="{r}"/>')
|
||||
else:
|
||||
lines.append(f' <rdfs:range rdf:resource="{range_val}"/>')
|
||||
for range_val in _as_list(prop.get("range")):
|
||||
range_iri = self._resolve_class_ref(range_val, ontology_uri, class_index)
|
||||
if range_iri:
|
||||
lines.append(
|
||||
f' <rdfs:range rdf:resource="{esc_xml(range_iri)}"/>'
|
||||
)
|
||||
|
||||
lines.append(" </owl:ObjectProperty>")
|
||||
lines.append("")
|
||||
|
||||
# Data properties
|
||||
data_properties = ontology.get("data_properties", [])
|
||||
for prop in data_properties:
|
||||
prop_uri = prop.get("uri") or prop.get("id", "")
|
||||
prop_uri = self._term_iri(prop, ontology_uri)
|
||||
if not prop_uri:
|
||||
self.logger.warning("Skipping a data property with no name, uri or id")
|
||||
continue
|
||||
prop_name = prop.get("name") or prop.get("label", "")
|
||||
|
||||
lines.append(f' <owl:DatatypeProperty rdf:about="{prop_uri}">')
|
||||
lines.append(f" <rdfs:label>{prop_name}</rdfs:label>")
|
||||
lines.append(f' <owl:DatatypeProperty rdf:about="{esc_xml(prop_uri)}">')
|
||||
lines.append(f" <rdfs:label>{esc_xml(prop_name)}</rdfs:label>")
|
||||
|
||||
if prop.get("comment"):
|
||||
lines.append(f' <rdfs:comment>{prop.get("comment")}</rdfs:comment>')
|
||||
comment = prop.get("comment") or prop.get("description")
|
||||
if comment:
|
||||
lines.append(f" <rdfs:comment>{esc_xml(comment)}</rdfs:comment>")
|
||||
|
||||
# Domain
|
||||
if prop.get("domain"):
|
||||
domain = prop.get("domain")
|
||||
lines.append(f' <rdfs:domain rdf:resource="{domain}"/>')
|
||||
for domain in _as_list(prop.get("domain")):
|
||||
domain_iri = self._resolve_class_ref(domain, ontology_uri, class_index)
|
||||
if domain_iri:
|
||||
lines.append(
|
||||
f' <rdfs:domain rdf:resource="{esc_xml(domain_iri)}"/>'
|
||||
)
|
||||
|
||||
# Range
|
||||
if prop.get("range"):
|
||||
range_type = prop.get("range", "xsd:string")
|
||||
lines.append(
|
||||
f' <rdfs:range rdf:resource="http://www.w3.org/2001/XMLSchema#{range_type}"/>'
|
||||
)
|
||||
for range_val in _as_list(prop.get("range")):
|
||||
range_iri = self._resolve_datatype_iri(range_val)
|
||||
if range_iri:
|
||||
lines.append(
|
||||
f' <rdfs:range rdf:resource="{esc_xml(range_iri)}"/>'
|
||||
)
|
||||
|
||||
lines.append(" </owl:DatatypeProperty>")
|
||||
lines.append("")
|
||||
@@ -335,6 +369,223 @@ class OWLExporter:
|
||||
lines.append("</rdf:RDF>")
|
||||
return "\n".join(lines)
|
||||
|
||||
# ── Ontology-dict normalisation ───────────────────────────────────────────
|
||||
#
|
||||
# OntologyGenerator emits a single `properties` list tagged with
|
||||
# type/@type, while hand-authored ontologies use `object_properties` and
|
||||
# `data_properties`. Both shapes are accepted; everything below works from
|
||||
# the normalised view so the two cannot drift apart again (#1103).
|
||||
|
||||
_XSD_NS = "http://www.w3.org/2001/XMLSchema#"
|
||||
|
||||
#: Prefixes the generator and hand-authored ontologies actually use. A
|
||||
#: prefixed name is not an absolute IRI: `owl:Thing` matches the generic
|
||||
#: scheme grammar, so treating it as one produced <owl:Thing> as a domain,
|
||||
#: which is a different term from http://www.w3.org/2002/07/owl#Thing.
|
||||
_KNOWN_PREFIXES = {
|
||||
"owl": "http://www.w3.org/2002/07/owl#",
|
||||
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
||||
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
||||
"xsd": _XSD_NS,
|
||||
"skos": "http://www.w3.org/2004/02/skos/core#",
|
||||
"dc": "http://purl.org/dc/elements/1.1/",
|
||||
"dcterms": "http://purl.org/dc/terms/",
|
||||
"foaf": "http://xmlns.com/foaf/0.1/",
|
||||
"sem": "https://semantica.dev/ns#",
|
||||
"semantica": "https://semantica.dev/ns#",
|
||||
}
|
||||
|
||||
#: Schemes that really do introduce an absolute IRI without `//`.
|
||||
_ABSOLUTE_SCHEMES = ("urn:", "doi:", "mailto:", "tag:", "uuid:")
|
||||
|
||||
@classmethod
|
||||
def _is_absolute_iri(cls, value: str) -> bool:
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
value = value.strip()
|
||||
if "://" in value:
|
||||
return bool(re.match(r"^[A-Za-z][A-Za-z0-9+.\-]*://", value))
|
||||
return value.lower().startswith(cls._ABSOLUTE_SCHEMES)
|
||||
|
||||
@classmethod
|
||||
def _expand_prefixed_name(cls, value: str) -> str:
|
||||
"""Expand a known prefixed name, or return "" when it cannot be expanded."""
|
||||
prefix, _, local = value.partition(":")
|
||||
namespace = cls._KNOWN_PREFIXES.get(prefix)
|
||||
return f"{namespace}{local}" if namespace and local else ""
|
||||
|
||||
@staticmethod
|
||||
def _iri_safe(local: str) -> str:
|
||||
"""
|
||||
Percent-encode a local name so it can sit inside <>.
|
||||
|
||||
A name is free text. "Customer Account" pasted onto a base gives an IRI
|
||||
with a space in it, which rdflib only warns about and Oxigraph rejects
|
||||
with "Invalid IRI code point".
|
||||
"""
|
||||
return quote(local.strip(), safe="~._-!$&'()*+,;=:@/?")
|
||||
|
||||
@classmethod
|
||||
def _join_iri(cls, base: str, local: str) -> str:
|
||||
"""Append a local name to a base IRI, respecting hash and slash bases."""
|
||||
if not base:
|
||||
return ""
|
||||
local = cls._iri_safe(local)
|
||||
if not local:
|
||||
return ""
|
||||
separator = "" if base.endswith(("#", "/", ":")) else "#"
|
||||
return f"{base}{separator}{local}"
|
||||
|
||||
@classmethod
|
||||
def _term_iri(cls, term: Dict[str, Any], base: str) -> str:
|
||||
"""
|
||||
Resolve the IRI of a class or property.
|
||||
|
||||
Returns "" when the term carries nothing usable, so the caller can skip
|
||||
it. Interpolating an empty string into <> silently resolves against the
|
||||
parser's base — under rdflib that is the current working directory — and
|
||||
collapses every such term onto one subject.
|
||||
"""
|
||||
for key in ("uri", "iri", "id"):
|
||||
value = term.get(key)
|
||||
if isinstance(value, str) and value.strip():
|
||||
value = value.strip()
|
||||
return value if cls._is_absolute_iri(value) else cls._join_iri(base, value)
|
||||
|
||||
name = term.get("name") or term.get("label")
|
||||
if isinstance(name, str) and name.strip():
|
||||
return cls._join_iri(base, name.strip())
|
||||
return ""
|
||||
|
||||
@classmethod
|
||||
def _class_iri_index(cls, ontology: Dict[str, Any], base: str) -> Dict[str, str]:
|
||||
"""Map class name and label to the IRI that class is actually exported under."""
|
||||
index: Dict[str, str] = {}
|
||||
for class_def in ontology.get("classes", []) or []:
|
||||
if not isinstance(class_def, dict):
|
||||
continue
|
||||
iri = cls._term_iri(class_def, base)
|
||||
if not iri:
|
||||
continue
|
||||
for key in (class_def.get("name"), class_def.get("label")):
|
||||
if isinstance(key, str) and key.strip():
|
||||
index.setdefault(key.strip(), iri)
|
||||
return index
|
||||
|
||||
@classmethod
|
||||
def _resolve_class_ref(cls, value: Any, base: str, index: Dict[str, str]) -> str:
|
||||
"""
|
||||
Resolve a domain/range reference to an absolute IRI.
|
||||
|
||||
The generator writes bare class names here. Looking the name up in the
|
||||
class index first means a reference always lands on the IRI that class
|
||||
was exported under, rather than on a re-derived guess.
|
||||
"""
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return ""
|
||||
value = value.strip()
|
||||
if cls._is_absolute_iri(value):
|
||||
return value
|
||||
if value in index:
|
||||
return index[value]
|
||||
if ":" in value:
|
||||
return cls._expand_prefixed_name(value)
|
||||
return cls._join_iri(base, value)
|
||||
|
||||
@classmethod
|
||||
def _resolve_datatype_iri(cls, value: Any) -> str:
|
||||
"""
|
||||
Resolve a data property range to an absolute datatype IRI.
|
||||
|
||||
Accepts "string", "xsd:string" and a full IRI alike. The previous
|
||||
`xsd:{range}` interpolation doubled the prefix whenever the generator
|
||||
had already written "xsd:string".
|
||||
"""
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return ""
|
||||
value = value.strip()
|
||||
if value.startswith(("xsd:", "XSD:")):
|
||||
return cls._XSD_NS + value.split(":", 1)[1]
|
||||
if cls._is_absolute_iri(value):
|
||||
return value
|
||||
return cls._XSD_NS + value
|
||||
|
||||
@classmethod
|
||||
def _ttl_datatype_ref(cls, value: Any) -> str:
|
||||
"""
|
||||
Render a data property range for Turtle.
|
||||
|
||||
XSD datatypes are written with the xsd: prefix the header already
|
||||
declares; anything else is written as a full IRI. Both are the same
|
||||
term, this only keeps the compact style the module was written in.
|
||||
"""
|
||||
iri = cls._resolve_datatype_iri(value)
|
||||
if not iri:
|
||||
return ""
|
||||
if iri.startswith(cls._XSD_NS):
|
||||
return f"xsd:{iri[len(cls._XSD_NS):]}"
|
||||
return f"<{iri}>"
|
||||
|
||||
@classmethod
|
||||
def _split_properties(
|
||||
cls, ontology: Dict[str, Any]
|
||||
) -> "tuple[List[Dict[str, Any]], List[Dict[str, Any]]]":
|
||||
"""
|
||||
Return (object_properties, data_properties) across both dict shapes.
|
||||
|
||||
A property listed under an explicit key keeps that key's kind. A
|
||||
property from the generator's combined `properties` list is classified
|
||||
by its own type/@type, defaulting to a data property.
|
||||
"""
|
||||
object_props: List[Dict[str, Any]] = []
|
||||
data_props: List[Dict[str, Any]] = []
|
||||
|
||||
for prop in ontology.get("object_properties", []) or []:
|
||||
if isinstance(prop, dict):
|
||||
object_props.append(prop)
|
||||
for prop in ontology.get("data_properties", []) or []:
|
||||
if isinstance(prop, dict):
|
||||
data_props.append(prop)
|
||||
|
||||
skipped = 0
|
||||
untyped = []
|
||||
for prop in ontology.get("properties", []) or []:
|
||||
if not isinstance(prop, dict):
|
||||
skipped += 1
|
||||
continue
|
||||
kind = str(prop.get("type") or "").strip().lower()
|
||||
owl_type = str(prop.get("@type") or "").strip().lower()
|
||||
if kind in ("object", "objectproperty") or owl_type.endswith("objectproperty"):
|
||||
object_props.append(prop)
|
||||
else:
|
||||
if not kind and not owl_type:
|
||||
untyped.append(prop.get("name") or prop.get("uri") or "<unnamed>")
|
||||
data_props.append(prop)
|
||||
|
||||
if skipped:
|
||||
logger.warning(
|
||||
f"Skipped {skipped} entr(y/ies) in 'properties' that are not "
|
||||
"dictionaries and cannot be exported"
|
||||
)
|
||||
if untyped:
|
||||
logger.warning(
|
||||
"Exported as data properties because they declare no type or "
|
||||
f"@type: {', '.join(str(name) for name in untyped)}"
|
||||
)
|
||||
|
||||
return object_props, data_props
|
||||
|
||||
@staticmethod
|
||||
def _escape_xml(value: Any) -> str:
|
||||
"""Escape a value for safe embedding in XML text or an attribute value."""
|
||||
return (
|
||||
str(value)
|
||||
.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace('"', """)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _escape_ttl_str(value: str) -> str:
|
||||
"""Escape a string value for safe embedding in a Turtle string literal."""
|
||||
@@ -387,62 +638,83 @@ class OWLExporter:
|
||||
lines.append(self._ttl_block(ontology_uri, "owl:Ontology", onto_predicates))
|
||||
lines.append("")
|
||||
|
||||
class_index = self._class_iri_index(ontology, ontology_uri)
|
||||
object_properties, data_properties = self._split_properties(ontology)
|
||||
|
||||
def _as_list(value):
|
||||
if value is None:
|
||||
return []
|
||||
return value if isinstance(value, list) else [value]
|
||||
|
||||
# Classes
|
||||
for cls in ontology.get("classes", []):
|
||||
class_uri = cls.get("uri") or cls.get("id", "")
|
||||
for cls in ontology.get("classes", []) or []:
|
||||
if not isinstance(cls, dict):
|
||||
continue
|
||||
class_uri = self._term_iri(cls, ontology_uri)
|
||||
if not class_uri:
|
||||
self.logger.warning(
|
||||
"Skipping a class with no name, uri or id: it would serialise as <>"
|
||||
)
|
||||
continue
|
||||
class_name = cls.get("name") or cls.get("label", "")
|
||||
predicates = [f'rdfs:label "{esc(class_name)}"']
|
||||
comment = cls.get("comment")
|
||||
comment = cls.get("comment") or cls.get("description")
|
||||
if comment:
|
||||
predicates.append(f'rdfs:comment "{esc(comment)}"')
|
||||
sub_class = cls.get("subClassOf")
|
||||
if sub_class:
|
||||
predicates.append(f"rdfs:subClassOf <{sub_class}>")
|
||||
equiv = cls.get("equivalentClass")
|
||||
if equiv:
|
||||
predicates.append(f"owl:equivalentClass <{equiv}>")
|
||||
for parent in _as_list(cls.get("subClassOf") or cls.get("parent")):
|
||||
parent_iri = self._resolve_class_ref(parent, ontology_uri, class_index)
|
||||
if parent_iri:
|
||||
predicates.append(f"rdfs:subClassOf <{parent_iri}>")
|
||||
for equiv in _as_list(cls.get("equivalentClass")):
|
||||
equiv_iri = self._resolve_class_ref(equiv, ontology_uri, class_index)
|
||||
if equiv_iri:
|
||||
predicates.append(f"owl:equivalentClass <{equiv_iri}>")
|
||||
lines.append(self._ttl_block(class_uri, "owl:Class", predicates))
|
||||
lines.append("")
|
||||
|
||||
# Object properties
|
||||
for prop in ontology.get("object_properties", []):
|
||||
prop_uri = prop.get("uri") or prop.get("id", "")
|
||||
for prop in object_properties:
|
||||
prop_uri = self._term_iri(prop, ontology_uri)
|
||||
if not prop_uri:
|
||||
self.logger.warning(
|
||||
"Skipping an object property with no name, uri or id"
|
||||
)
|
||||
continue
|
||||
prop_name = prop.get("name") or prop.get("label", "")
|
||||
predicates = [f'rdfs:label "{esc(prop_name)}"']
|
||||
comment = prop.get("comment")
|
||||
comment = prop.get("comment") or prop.get("description")
|
||||
if comment:
|
||||
predicates.append(f'rdfs:comment "{esc(comment)}"')
|
||||
domain = prop.get("domain")
|
||||
if domain:
|
||||
if isinstance(domain, list):
|
||||
for d in domain:
|
||||
predicates.append(f"rdfs:domain <{d}>")
|
||||
else:
|
||||
predicates.append(f"rdfs:domain <{domain}>")
|
||||
range_val = prop.get("range")
|
||||
if range_val:
|
||||
if isinstance(range_val, list):
|
||||
for r in range_val:
|
||||
predicates.append(f"rdfs:range <{r}>")
|
||||
else:
|
||||
predicates.append(f"rdfs:range <{range_val}>")
|
||||
for domain in _as_list(prop.get("domain")):
|
||||
domain_iri = self._resolve_class_ref(domain, ontology_uri, class_index)
|
||||
if domain_iri:
|
||||
predicates.append(f"rdfs:domain <{domain_iri}>")
|
||||
for range_val in _as_list(prop.get("range")):
|
||||
range_iri = self._resolve_class_ref(range_val, ontology_uri, class_index)
|
||||
if range_iri:
|
||||
predicates.append(f"rdfs:range <{range_iri}>")
|
||||
lines.append(self._ttl_block(prop_uri, "owl:ObjectProperty", predicates))
|
||||
lines.append("")
|
||||
|
||||
# Data properties
|
||||
for prop in ontology.get("data_properties", []):
|
||||
prop_uri = prop.get("uri") or prop.get("id", "")
|
||||
for prop in data_properties:
|
||||
prop_uri = self._term_iri(prop, ontology_uri)
|
||||
if not prop_uri:
|
||||
self.logger.warning("Skipping a data property with no name, uri or id")
|
||||
continue
|
||||
prop_name = prop.get("name") or prop.get("label", "")
|
||||
predicates = [f'rdfs:label "{esc(prop_name)}"']
|
||||
comment = prop.get("comment")
|
||||
comment = prop.get("comment") or prop.get("description")
|
||||
if comment:
|
||||
predicates.append(f'rdfs:comment "{esc(comment)}"')
|
||||
domain = prop.get("domain")
|
||||
if domain:
|
||||
predicates.append(f"rdfs:domain <{domain}>")
|
||||
range_type = prop.get("range")
|
||||
if range_type:
|
||||
predicates.append(f"rdfs:range xsd:{range_type}")
|
||||
for domain in _as_list(prop.get("domain")):
|
||||
domain_iri = self._resolve_class_ref(domain, ontology_uri, class_index)
|
||||
if domain_iri:
|
||||
predicates.append(f"rdfs:domain <{domain_iri}>")
|
||||
for range_val in _as_list(prop.get("range")):
|
||||
range_ref = self._ttl_datatype_ref(range_val)
|
||||
if range_ref:
|
||||
predicates.append(f"rdfs:range {range_ref}")
|
||||
lines.append(self._ttl_block(prop_uri, "owl:DatatypeProperty", predicates))
|
||||
lines.append("")
|
||||
|
||||
|
||||
@@ -29,15 +29,123 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from html import escape as xml_escape
|
||||
from typing import Any, Dict, List, Optional, Set, Union
|
||||
from urllib.parse import quote, urlsplit
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.helpers import ensure_directory
|
||||
from ..utils.helpers import ensure_directory, hash_data
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
SEMANTICA_NS = "https://semantica.dev/ns#"
|
||||
|
||||
#: Written when an entity carries no type of its own. A full IRI rather than the
|
||||
#: prefixed form, because the Turtle serializer writes it inside angle brackets,
|
||||
#: where `semantica:Entity` would be read as an IRI in the scheme `semantica`
|
||||
#: rather than as the prefix expansion (issue #1101).
|
||||
DEFAULT_ENTITY_TYPE = f"{SEMANTICA_NS}Entity"
|
||||
|
||||
#: Written when a relationship carries no type of its own. Same reasoning.
|
||||
DEFAULT_RELATION_TYPE = f"{SEMANTICA_NS}related_to"
|
||||
|
||||
#: The one datatype every serializer writes confidence in.
|
||||
#:
|
||||
#: The four paths used to disagree: Turtle wrote the value bare, which the
|
||||
#: Turtle grammar reads as xsd:decimal, N-Triples typed it xsd:float, RDF/XML
|
||||
#: emitted a plain literal with no datatype, and JSON-LD emitted a native
|
||||
#: number, which becomes xsd:double. Those are four distinct RDF terms for one
|
||||
#: value (issue #1100).
|
||||
#:
|
||||
#: xsd:decimal is the choice because it is what the Turtle path already
|
||||
#: produced, so the most used output is unchanged, and because it is exact:
|
||||
#: xsd:float is 32 bit binary, and cannot represent 0.9 or 0.95 at all.
|
||||
CONFIDENCE_DATATYPE = "http://www.w3.org/2001/XMLSchema#decimal"
|
||||
|
||||
#: Largest power of ten a confidence may carry. xsd:decimal has no exponent
|
||||
#: notation, so a value has to be written out in full, and a compact literal
|
||||
#: such as "1e100000000" would expand to a hundred million digits.
|
||||
MAX_CONFIDENCE_EXPONENT = 100
|
||||
|
||||
|
||||
def normalize_confidence(value: Any) -> Optional[str]:
|
||||
"""
|
||||
Return the canonical xsd:decimal lexical form of a confidence value.
|
||||
|
||||
Returns None when the value cannot be a decimal, so callers omit the triple
|
||||
rather than writing something the vocabulary contradicts. The Turtle path
|
||||
used to interpolate the raw value, so a confidence of "high" produced
|
||||
`semantica:confidence high .` and made the whole document unparseable
|
||||
(issue #1102).
|
||||
|
||||
Booleans are rejected. `bool` is a subclass of `int` in Python, so True
|
||||
would otherwise silently become a confidence of 1.
|
||||
"""
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
value = value.strip()
|
||||
if not value:
|
||||
return None
|
||||
if not isinstance(value, (int, float, str, Decimal)):
|
||||
return None
|
||||
|
||||
try:
|
||||
decimal_value = Decimal(str(value))
|
||||
except (InvalidOperation, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
# NaN and the infinities are Decimal values with no xsd:decimal form.
|
||||
if not decimal_value.is_finite():
|
||||
return None
|
||||
|
||||
# xsd:decimal has no exponent notation, so writing one means expanding it.
|
||||
# "1e100000000" is eleven characters that expand to a hundred million, and
|
||||
# the export path continues past validation errors, so a single malformed
|
||||
# field could exhaust memory. Nothing near this magnitude is a confidence.
|
||||
if (
|
||||
not -MAX_CONFIDENCE_EXPONENT
|
||||
<= decimal_value.adjusted()
|
||||
<= MAX_CONFIDENCE_EXPONENT
|
||||
):
|
||||
return None
|
||||
|
||||
# `str(Decimal("0.00001"))` gives "0.00001", but a float that has already
|
||||
# been through repr can arrive as "1e-05", which xsd:decimal does not allow.
|
||||
formatted = format(decimal_value, "f")
|
||||
if "." in formatted:
|
||||
formatted = formatted.rstrip("0").rstrip(".") or "0"
|
||||
|
||||
# Decimal keeps the sign of zero, so 0.0 and -0.0 would serialise as two
|
||||
# distinct RDF terms and defeat the point of a canonical form.
|
||||
if formatted.lstrip("-").strip("0.") == "":
|
||||
formatted = "0"
|
||||
return formatted
|
||||
|
||||
|
||||
def mint_entity_iri(text: str) -> str:
|
||||
"""Mint a stable IRI for an entity that arrived without an id.
|
||||
|
||||
Python's builtin ``hash()`` is randomised per process (PYTHONHASHSEED), so
|
||||
minting from it gave the same entity a different IRI on every run: exports
|
||||
could not be diffed, deduplicated against an earlier load, or joined to a
|
||||
provenance record written by an earlier process. SHA-256 is stable across
|
||||
runs and machines, which is what an identifier has to be.
|
||||
"""
|
||||
digest = hash_data(str(text))[:16]
|
||||
return f"{SEMANTICA_NS}entity_{digest}"
|
||||
|
||||
|
||||
def mint_relationship_iri(index: int, source: Any, target: Any) -> str:
|
||||
"""Mint a stable IRI for a relationship that arrived without an id."""
|
||||
digest = hash_data(f"{source}\x00{target}")[:16]
|
||||
return f"{SEMANTICA_NS}rel_{index}_{digest}"
|
||||
|
||||
|
||||
class NamespaceManager:
|
||||
"""
|
||||
RDF namespace management engine.
|
||||
@@ -294,6 +402,66 @@ class RDFSerializer:
|
||||
|
||||
# OWL-Time namespace URI
|
||||
_OWL_TIME_NS = "http://www.w3.org/2006/time#"
|
||||
_SEMANTICA_NS = "https://semantica.dev/ns#"
|
||||
|
||||
# Matches an already-valid percent-escape so it can be passed through
|
||||
# unchanged instead of being re-encoded into e.g. %2520.
|
||||
_PERCENT_ESCAPE_RE = re.compile(r"%[0-9A-Fa-f]{2}")
|
||||
|
||||
@classmethod
|
||||
def _quote_preserving_escapes(cls, value: str, safe: str) -> str:
|
||||
"""quote() that leaves existing valid %XX escapes untouched.
|
||||
|
||||
Blanket-quoting an absolute IRI double-encodes any percent-escape it
|
||||
already carries (%20 -> %2520), which changes the identity of every
|
||||
previously-valid IRI containing one. Only the spans between existing
|
||||
valid escapes are quoted; a bare '%' that isn't part of a valid
|
||||
escape (e.g. "%zz") still gets encoded to %25, keeping the malformed
|
||||
case handled.
|
||||
"""
|
||||
parts = []
|
||||
pos = 0
|
||||
for match in cls._PERCENT_ESCAPE_RE.finditer(value):
|
||||
parts.append(quote(value[pos : match.start()], safe=safe))
|
||||
parts.append(match.group(0))
|
||||
pos = match.end()
|
||||
parts.append(quote(value[pos:], safe=safe))
|
||||
return "".join(parts)
|
||||
|
||||
def _as_turtle_iri(
|
||||
self, value: Any, namespaces: Optional[Dict[str, str]] = None
|
||||
) -> str:
|
||||
"""Return an absolute, safely encoded IRI for a Turtle resource."""
|
||||
value = str(value)
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
except ValueError:
|
||||
parsed = urlsplit("")
|
||||
if parsed.scheme:
|
||||
prefix, separator, local_name = value.partition(":")
|
||||
# Built-in namespaces (semantica:, rdf:, rdfs:, owl:, ...) must
|
||||
# always be resolvable, not only when the caller passes no
|
||||
# namespaces of its own — otherwise a value like "semantica:Foo"
|
||||
# resolves fine with no @context but stops resolving the moment
|
||||
# any @context is present, since callers pass extract_namespaces()
|
||||
# (context-only) here without merging in the built-ins.
|
||||
effective_namespaces = {
|
||||
**self.namespace_manager.namespaces,
|
||||
**(namespaces or {}),
|
||||
}
|
||||
namespace = effective_namespaces.get(prefix)
|
||||
if namespace and separator:
|
||||
return self._quote_preserving_escapes(
|
||||
namespace + local_name, safe=":/?#[]@!$&'()*+,;="
|
||||
)
|
||||
# A scheme with at least two characters is an absolute IRI,
|
||||
# including opaque forms such as mailto:foo and isbn:0451450523.
|
||||
# Keep one-character schemes as the existing Windows drive-path case.
|
||||
if len(prefix) >= 2:
|
||||
return self._quote_preserving_escapes(
|
||||
value, safe=":/?#[]@!$&'()*+,;="
|
||||
)
|
||||
return self._SEMANTICA_NS + quote(value, safe="")
|
||||
|
||||
# Design decision — TemporalBound.OPEN in RDF:
|
||||
# OWL-Time has no standard predicate for "no known end date." We use
|
||||
@@ -360,15 +528,27 @@ class RDFSerializer:
|
||||
entity_id = entity.get("id")
|
||||
if not entity_id:
|
||||
entity_text = entity.get("text", "")
|
||||
entity_id = f"semantica:entity_{hash(entity_text)}"
|
||||
entity_id = mint_entity_iri(entity_text)
|
||||
|
||||
entity_type = entity.get("type", "semantica:Entity")
|
||||
entity_type = entity.get("type", DEFAULT_ENTITY_TYPE)
|
||||
text = entity.get("text") or entity.get("label", "")
|
||||
confidence = entity.get("confidence", 1.0)
|
||||
confidence = normalize_confidence(entity.get("confidence", 1.0))
|
||||
|
||||
lines.append(f"<{entity_id}> a <{entity_type}> ;")
|
||||
lines.append(f' semantica:text "{text}" ;')
|
||||
lines.append(f" semantica:confidence {confidence} .")
|
||||
lines.append(
|
||||
f"<{self._as_turtle_iri(entity_id, merged_namespaces)}> a "
|
||||
f"<{self._as_turtle_iri(entity_type, merged_namespaces)}> ;"
|
||||
)
|
||||
if confidence is None:
|
||||
self.logger.warning(
|
||||
f"Entity {entity_id} has a confidence that is not a number "
|
||||
f"({entity.get('confidence')!r}), so no confidence is written"
|
||||
)
|
||||
lines.append(f' semantica:text "{text}" .')
|
||||
else:
|
||||
lines.append(f' semantica:text "{text}" ;')
|
||||
lines.append(
|
||||
f' semantica:confidence "{confidence}"^^<{CONFIDENCE_DATATYPE}> .'
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
# Convert relationships to RDF triplets
|
||||
@@ -376,19 +556,87 @@ class RDFSerializer:
|
||||
for idx, rel in enumerate(relationships):
|
||||
source_id = rel.get("source_id") or rel.get("source")
|
||||
target_id = rel.get("target_id") or rel.get("target")
|
||||
rel_type = rel.get("type", "semantica:related_to")
|
||||
rel_type = rel.get("type", DEFAULT_RELATION_TYPE)
|
||||
|
||||
lines.append(f"<{source_id}> <{rel_type}> <{target_id}> .")
|
||||
lines.append(
|
||||
f"<{self._as_turtle_iri(source_id, merged_namespaces)}> "
|
||||
f"<{self._as_turtle_iri(rel_type, merged_namespaces)}> "
|
||||
f"<{self._as_turtle_iri(target_id, merged_namespaces)}> ."
|
||||
)
|
||||
|
||||
if include_temporal:
|
||||
owl_lines = self._owl_time_triples_for_rel(rel, idx, time_axis)
|
||||
owl_lines = self._owl_time_triples_for_rel(
|
||||
rel, idx, time_axis, merged_namespaces
|
||||
)
|
||||
if owl_lines:
|
||||
# The interval hangs off the relationship's own IRI, and a
|
||||
# relationship written as a single triple has no such node
|
||||
# in the graph. Without this the timestamps are well formed
|
||||
# and unreachable: no query can get from the edge to its
|
||||
# validity interval (#1106). The shape matches the JSON-LD
|
||||
# export, and every term is declared in the vocabulary.
|
||||
lines.extend(
|
||||
self._reified_relationship_triples(
|
||||
rel, idx, source_id, target_id, rel_type, merged_namespaces
|
||||
)
|
||||
)
|
||||
lines.extend(owl_lines)
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def _reified_relationship_triples(
|
||||
self,
|
||||
rel: Dict[str, Any],
|
||||
idx: int,
|
||||
source_id: str,
|
||||
target_id: str,
|
||||
rel_type: str,
|
||||
namespaces: Optional[Dict[str, str]] = None,
|
||||
) -> List[str]:
|
||||
"""
|
||||
Emit the reified relationship node that OWL-Time triples hang off.
|
||||
|
||||
The direct triple stays. This adds a subject the interval can attach
|
||||
to, using the same sem:Relationship shape the JSON-LD export already
|
||||
writes, so the two serializations describe relationships the same way.
|
||||
"""
|
||||
rel_id = self._as_turtle_iri(
|
||||
rel.get("id")
|
||||
or mint_relationship_iri(idx, source_id or "", target_id or ""),
|
||||
namespaces,
|
||||
)
|
||||
|
||||
# The full predicate, not its local name. Truncating to the fragment
|
||||
# made https://a.example/ns#employs and https://b.example/ns#employs the
|
||||
# same literal, so the temporal node no longer said which predicate it
|
||||
# described, and it disagreed with the direct triple beside it.
|
||||
escaped = (
|
||||
str(rel_type)
|
||||
.replace("\\", "\\\\")
|
||||
.replace('"', '\\"')
|
||||
.replace("\n", "\\n")
|
||||
.replace("\r", "\\r")
|
||||
)
|
||||
|
||||
predicates = ["a semantica:Relationship"]
|
||||
if source_id:
|
||||
predicates.append(
|
||||
f"semantica:source <{self._as_turtle_iri(source_id, namespaces)}>"
|
||||
)
|
||||
if target_id:
|
||||
predicates.append(
|
||||
f"semantica:target <{self._as_turtle_iri(target_id, namespaces)}>"
|
||||
)
|
||||
predicates.append(f'semantica:type "{escaped}"')
|
||||
|
||||
return ["", f"<{rel_id}> " + " ;\n ".join(predicates) + " ."]
|
||||
|
||||
def _owl_time_triples_for_rel(
|
||||
self, rel: Dict[str, Any], idx: int, time_axis: str
|
||||
self,
|
||||
rel: Dict[str, Any],
|
||||
idx: int,
|
||||
time_axis: str,
|
||||
namespaces: Optional[Dict[str, str]] = None,
|
||||
) -> List[str]:
|
||||
"""
|
||||
Emit OWL-Time Turtle triples for a relationship that carries temporal metadata.
|
||||
@@ -403,7 +651,7 @@ class RDFSerializer:
|
||||
def _is_open(v: Any) -> bool:
|
||||
if v is None:
|
||||
return False
|
||||
if hasattr(v, "value"): # TemporalBound enum
|
||||
if hasattr(v, "value"): # TemporalBound enum
|
||||
return v.value == _OPEN_SENTINEL
|
||||
return str(v).strip().upper() == _OPEN_SENTINEL
|
||||
|
||||
@@ -413,9 +661,16 @@ class RDFSerializer:
|
||||
if time_axis in ("transaction", "both"):
|
||||
axes.append(("tx", rel.get("recorded_at"), rel.get("superseded_at")))
|
||||
|
||||
rel_base_id = (
|
||||
rel.get("id")
|
||||
or f"semantica:rel_{idx}_{hash(str(rel.get('source_id', '')) + str(rel.get('target_id', '')))}"
|
||||
# Resolve endpoints the same way serialize_to_turtle does: both
|
||||
# representations are accepted upstream, and minting from source_id
|
||||
# alone hashes empty strings for every relationship that uses source,
|
||||
# so unrelated relationships at the same index would collide on a
|
||||
# deterministic IRI.
|
||||
source_id = rel.get("source_id") or rel.get("source") or ""
|
||||
target_id = rel.get("target_id") or rel.get("target") or ""
|
||||
rel_base_id = self._as_turtle_iri(
|
||||
rel.get("id") or mint_relationship_iri(idx, source_id, target_id),
|
||||
namespaces,
|
||||
)
|
||||
|
||||
lines = [""] # blank separator
|
||||
@@ -431,9 +686,7 @@ class RDFSerializer:
|
||||
lines.append(f" time:hasBeginning <{begin_id}> ;")
|
||||
|
||||
if _is_open(until_val):
|
||||
lines.append(
|
||||
' semantica:openEndedInterval "true"^^xsd:boolean .'
|
||||
)
|
||||
lines.append(' semantica:openEndedInterval "true"^^xsd:boolean .')
|
||||
elif until_val is not None:
|
||||
end_id = f"{rel_base_id}__{axis_name}_end"
|
||||
lines.append(f" time:hasEnd <{end_id}> .")
|
||||
@@ -442,7 +695,9 @@ class RDFSerializer:
|
||||
f' time:inXSDDateTimeStamp "{until_val}"^^xsd:dateTimeStamp .'
|
||||
)
|
||||
else:
|
||||
lines[-1] = lines[-1].rstrip(" ;") + " ." # close interval without hasEnd
|
||||
lines[-1] = (
|
||||
lines[-1].rstrip(" ;") + " ."
|
||||
) # close interval without hasEnd
|
||||
|
||||
lines.append(f"<{begin_id}> a time:Instant ;")
|
||||
lines.append(
|
||||
@@ -481,6 +736,8 @@ class RDFSerializer:
|
||||
lines.append(' xmlns:semantica="https://semantica.dev/ns#">')
|
||||
lines.append("")
|
||||
|
||||
namespaces = self.namespace_manager.extract_namespaces(rdf_data)
|
||||
|
||||
# Convert entities to RDF/XML
|
||||
entities = rdf_data.get("entities", [])
|
||||
for entity in entities:
|
||||
@@ -488,19 +745,32 @@ class RDFSerializer:
|
||||
entity_id = entity.get("id")
|
||||
if not entity_id:
|
||||
entity_text = entity.get("text", "")
|
||||
entity_id = f"semantica:entity_{hash(entity_text)}"
|
||||
entity_id = mint_entity_iri(entity_text)
|
||||
|
||||
entity_type = entity.get("type", "semantica:Entity")
|
||||
entity_type = entity.get("type") or DEFAULT_ENTITY_TYPE
|
||||
text = entity.get("text") or entity.get("label", "")
|
||||
confidence = entity.get("confidence", 1.0)
|
||||
confidence = normalize_confidence(entity.get("confidence", 1.0))
|
||||
|
||||
# RDF/XML syntax: rdf:Description with rdf:about
|
||||
lines.append(f' <rdf:Description rdf:about="{entity_id}">')
|
||||
lines.append(f' <rdf:type rdf:resource="{entity_type}"/>')
|
||||
lines.append(f" <semantica:text>{text}</semantica:text>")
|
||||
lines.append(
|
||||
f" <semantica:confidence>{confidence}</semantica:confidence>"
|
||||
entity_iri = xml_escape(
|
||||
self._as_turtle_iri(entity_id, namespaces), quote=True
|
||||
)
|
||||
entity_type_iri = xml_escape(
|
||||
self._as_turtle_iri(entity_type, namespaces), quote=True
|
||||
)
|
||||
lines.append(f' <rdf:Description rdf:about="{entity_iri}">')
|
||||
lines.append(f' <rdf:type rdf:resource="{entity_type_iri}"/>')
|
||||
lines.append(f" <semantica:text>{text}</semantica:text>")
|
||||
if confidence is None:
|
||||
self.logger.warning(
|
||||
f"Entity {entity_id} has a confidence that is not a number "
|
||||
f"({entity.get('confidence')!r}), so no confidence is written"
|
||||
)
|
||||
else:
|
||||
lines.append(
|
||||
f' <semantica:confidence rdf:datatype="{CONFIDENCE_DATATYPE}">'
|
||||
f"{confidence}</semantica:confidence>"
|
||||
)
|
||||
lines.append(" </rdf:Description>")
|
||||
lines.append("")
|
||||
|
||||
@@ -509,11 +779,19 @@ class RDFSerializer:
|
||||
for rel in relationships:
|
||||
source_id = rel.get("source_id") or rel.get("source")
|
||||
target_id = rel.get("target_id") or rel.get("target")
|
||||
rel_type = rel.get("type", "semantica:related_to")
|
||||
# RDF/XML predicates are emitted as QNames, unlike resource
|
||||
# attributes which use the shared absolute-IRI normalizer.
|
||||
rel_type = rel.get("type") or "semantica:related_to"
|
||||
|
||||
# Relationship as property on source entity
|
||||
lines.append(f' <rdf:Description rdf:about="{source_id}">')
|
||||
lines.append(f' <{rel_type} rdf:resource="{target_id}"/>')
|
||||
source_iri = xml_escape(
|
||||
self._as_turtle_iri(source_id, namespaces), quote=True
|
||||
)
|
||||
target_iri = xml_escape(
|
||||
self._as_turtle_iri(target_id, namespaces), quote=True
|
||||
)
|
||||
lines.append(f' <rdf:Description rdf:about="{source_iri}">')
|
||||
lines.append(f' <{rel_type} rdf:resource="{target_iri}"/>')
|
||||
lines.append(" </rdf:Description>")
|
||||
lines.append("")
|
||||
|
||||
@@ -565,41 +843,52 @@ class RDFSerializer:
|
||||
# Convert entities to JSON-LD
|
||||
entities = rdf_data.get("entities", [])
|
||||
for entity in entities:
|
||||
# Generate @id if not provided
|
||||
entity_id = entity.get("id")
|
||||
if not entity_id:
|
||||
entity_text = entity.get("text", "")
|
||||
entity_id = f"semantica:entity/{entity_text}"
|
||||
# Generate @id if not provided. Minted the same way the Turtle and
|
||||
# N-Triples paths mint it (#1101), so one knowledge graph carries
|
||||
# the same node identity whichever serializer wrote it. The former
|
||||
# f"semantica:entity/{text}" interpolated the raw text into an IRI:
|
||||
# any entity whose text contained a space produced an invalid IRI
|
||||
# and was dropped in full by a JSON-LD parser, silently.
|
||||
entity_id = entity.get("id") or mint_entity_iri(entity.get("text", ""))
|
||||
|
||||
jsonld["@graph"].append(
|
||||
{
|
||||
"@id": entity_id,
|
||||
"@type": entity.get("type", "semantica:Entity"),
|
||||
"semantica:text": entity.get("text") or entity.get("label", ""),
|
||||
"semantica:confidence": entity.get("confidence", 1.0),
|
||||
node = {
|
||||
"@id": entity_id,
|
||||
"@type": entity.get("type", "semantica:Entity"),
|
||||
"semantica:text": entity.get("text") or entity.get("label", ""),
|
||||
}
|
||||
confidence = normalize_confidence(entity.get("confidence", 1.0))
|
||||
if confidence is None:
|
||||
self.logger.warning(
|
||||
f"Entity {entity_id} has a confidence that is not a number "
|
||||
f"({entity.get('confidence')!r}), so no confidence is written"
|
||||
)
|
||||
else:
|
||||
# A native JSON number becomes xsd:double once expanded, so the
|
||||
# value is written as a typed literal instead.
|
||||
node["semantica:confidence"] = {
|
||||
"@value": confidence,
|
||||
"@type": CONFIDENCE_DATATYPE,
|
||||
}
|
||||
)
|
||||
jsonld["@graph"].append(node)
|
||||
|
||||
# Convert relationships to JSON-LD
|
||||
relationships = rdf_data.get("relationships", [])
|
||||
for rel in relationships:
|
||||
# Generate @id if not provided
|
||||
rel_id = rel.get("id")
|
||||
if not rel_id:
|
||||
source_id = rel.get("source_id", "")
|
||||
target_id = rel.get("target_id", "")
|
||||
rel_id = f"semantica:rel/{source_id}_{target_id}"
|
||||
for index, rel in enumerate(relationships):
|
||||
# Endpoints are resolved both ways, as serialize_to_turtle resolves
|
||||
# them: a relationship carrying source/target rather than
|
||||
# source_id/target_id used to hash into f"semantica:rel/_", so every
|
||||
# such relationship in an export collapsed onto one node and their
|
||||
# types and endpoints merged.
|
||||
source = rel.get("source_id") or rel.get("source", "")
|
||||
target = rel.get("target_id") or rel.get("target", "")
|
||||
rel_id = rel.get("id") or mint_relationship_iri(index, source, target)
|
||||
|
||||
jsonld["@graph"].append(
|
||||
{
|
||||
"@id": rel_id,
|
||||
"@type": "semantica:Relationship",
|
||||
"semantica:source": {
|
||||
"@id": rel.get("source_id") or rel.get("source")
|
||||
},
|
||||
"semantica:target": {
|
||||
"@id": rel.get("target_id") or rel.get("target")
|
||||
},
|
||||
"semantica:source": {"@id": source},
|
||||
"semantica:target": {"@id": target},
|
||||
"semantica:type": rel.get("type", "related_to"),
|
||||
}
|
||||
)
|
||||
@@ -622,20 +911,12 @@ class RDFSerializer:
|
||||
"""
|
||||
lines = []
|
||||
|
||||
namespaces = self.namespace_manager.extract_namespaces(rdf_data)
|
||||
|
||||
def expand_uri(uri: str) -> str:
|
||||
if not uri:
|
||||
return ""
|
||||
if uri.startswith("http"):
|
||||
return f"<{uri}>"
|
||||
if uri.startswith("semantica:"):
|
||||
return f"<https://semantica.dev/ns#{uri.split(':', 1)[1]}>"
|
||||
if uri.startswith("rdf:"):
|
||||
return f"<http://www.w3.org/1999/02/22-rdf-syntax-ns#{uri.split(':', 1)[1]}>"
|
||||
if uri.startswith("rdfs:"):
|
||||
return f"<http://www.w3.org/2000/01/rdf-schema#{uri.split(':', 1)[1]}>"
|
||||
if ":" in uri:
|
||||
return f"<{uri}>"
|
||||
return f"<https://semantica.dev/ns#{uri}>"
|
||||
return f"<{self._as_turtle_iri(uri, namespaces)}>"
|
||||
|
||||
# Convert entities
|
||||
entities = rdf_data.get("entities", [])
|
||||
@@ -644,12 +925,12 @@ class RDFSerializer:
|
||||
entity_id = entity.get("id")
|
||||
if not entity_id:
|
||||
entity_text = entity.get("text", "")
|
||||
entity_id = f"semantica:entity_{hash(entity_text)}"
|
||||
entity_id = mint_entity_iri(entity_text)
|
||||
|
||||
subject = expand_uri(entity_id)
|
||||
|
||||
# Type triple
|
||||
entity_type = entity.get("type", "semantica:Entity")
|
||||
entity_type = entity.get("type") or DEFAULT_ENTITY_TYPE
|
||||
lines.append(
|
||||
f"{subject} <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> {expand_uri(entity_type)} ."
|
||||
)
|
||||
@@ -662,11 +943,20 @@ class RDFSerializer:
|
||||
f'{subject} {expand_uri("semantica:text")} "{safe_text}" .'
|
||||
)
|
||||
|
||||
# Confidence property
|
||||
confidence = entity.get("confidence")
|
||||
if confidence is not None:
|
||||
# Confidence property. The default matches the other serializers,
|
||||
# which have always written one; omitting it here was half of why
|
||||
# Turtle and N-Triples of one KG were different graphs (#1100).
|
||||
raw_confidence = entity.get("confidence", 1.0)
|
||||
confidence = normalize_confidence(raw_confidence)
|
||||
if confidence is None:
|
||||
self.logger.warning(
|
||||
f"Entity {entity.get('id')} has a confidence that is not a "
|
||||
f"number ({raw_confidence!r}), so no confidence is written"
|
||||
)
|
||||
else:
|
||||
lines.append(
|
||||
f'{subject} {expand_uri("semantica:confidence")} "{confidence}"^^<http://www.w3.org/2001/XMLSchema#float> .'
|
||||
f'{subject} {expand_uri("semantica:confidence")} '
|
||||
f'"{confidence}"^^<{CONFIDENCE_DATATYPE}> .'
|
||||
)
|
||||
|
||||
# Convert relationships
|
||||
@@ -674,7 +964,7 @@ class RDFSerializer:
|
||||
for rel in relationships:
|
||||
source_id = rel.get("source_id") or rel.get("source")
|
||||
target_id = rel.get("target_id") or rel.get("target")
|
||||
rel_type = rel.get("type", "semantica:related_to")
|
||||
rel_type = rel.get("type") or DEFAULT_RELATION_TYPE
|
||||
|
||||
if source_id and target_id:
|
||||
lines.append(
|
||||
|
||||
@@ -23,13 +23,13 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
import html
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.helpers import ensure_directory
|
||||
from ..utils.helpers import ensure_directory, utc_now_iso
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
@@ -251,7 +251,7 @@ class ReportGenerator:
|
||||
# Build report data with summary
|
||||
report_data = {
|
||||
"title": "Quality Assurance Report",
|
||||
"generated_at": datetime.now().isoformat(),
|
||||
"generated_at": utc_now_iso(),
|
||||
"metrics": quality_metrics,
|
||||
"summary": self._generate_quality_summary(quality_metrics),
|
||||
}
|
||||
@@ -277,7 +277,7 @@ class ReportGenerator:
|
||||
"""
|
||||
report_data = {
|
||||
"title": "Analysis Report",
|
||||
"generated_at": datetime.now().isoformat(),
|
||||
"generated_at": utc_now_iso(),
|
||||
"analysis": analysis_results,
|
||||
"summary": self._generate_analysis_summary(analysis_results),
|
||||
}
|
||||
@@ -303,7 +303,7 @@ class ReportGenerator:
|
||||
"""
|
||||
report_data = {
|
||||
"title": "Framework Metrics Report",
|
||||
"generated_at": datetime.now().isoformat(),
|
||||
"generated_at": utc_now_iso(),
|
||||
"metrics": metrics,
|
||||
"summary": self._generate_metrics_summary(metrics),
|
||||
}
|
||||
@@ -362,7 +362,7 @@ class ReportGenerator:
|
||||
' <meta name="viewport" content="width=device-width, initial-scale=1.0">'
|
||||
)
|
||||
title = data.get("title", "Report")
|
||||
lines.append(f" <title>{title}</title>")
|
||||
lines.append(f" <title>{html.escape(str(title))}</title>")
|
||||
lines.append(" <style>")
|
||||
lines.append(" body { font-family: Arial, sans-serif; margin: 20px; }")
|
||||
lines.append(" h1 { color: #333; }")
|
||||
@@ -380,11 +380,14 @@ class ReportGenerator:
|
||||
|
||||
# Title
|
||||
title = data.get("title", "Report")
|
||||
lines.append(f" <h1>{title}</h1>")
|
||||
lines.append(f" <h1>{html.escape(str(title))}</h1>")
|
||||
|
||||
# Generated at
|
||||
if "generated_at" in data:
|
||||
lines.append(f' <p><strong>Generated:</strong> {data["generated_at"]}</p>')
|
||||
lines.append(
|
||||
f' <p><strong>Generated:</strong> '
|
||||
f'{html.escape(str(data["generated_at"]))}</p>'
|
||||
)
|
||||
|
||||
# Summary
|
||||
if "summary" in data:
|
||||
@@ -393,10 +396,13 @@ class ReportGenerator:
|
||||
if isinstance(summary, dict):
|
||||
lines.append(" <ul>")
|
||||
for key, value in summary.items():
|
||||
lines.append(f" <li><strong>{key}:</strong> {value}</li>")
|
||||
lines.append(
|
||||
f" <li><strong>{html.escape(str(key))}:</strong> "
|
||||
f"{html.escape(str(value))}</li>"
|
||||
)
|
||||
lines.append(" </ul>")
|
||||
else:
|
||||
lines.append(f" <p>{summary}</p>")
|
||||
lines.append(f" <p>{html.escape(str(summary))}</p>")
|
||||
|
||||
# Metrics
|
||||
if "metrics" in data:
|
||||
@@ -483,7 +489,10 @@ class ReportGenerator:
|
||||
else:
|
||||
value_str = str(value)
|
||||
|
||||
lines.append(f" <tr><td>{key}</td><td>{value_str}</td></tr>")
|
||||
lines.append(
|
||||
f" <tr><td>{html.escape(str(key))}</td>"
|
||||
f"<td>{html.escape(value_str)}</td></tr>"
|
||||
)
|
||||
|
||||
lines.append(" </table>")
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@ License: MIT
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
@@ -33,6 +32,7 @@ from ..utils.helpers import (
|
||||
_require_recognized_keys,
|
||||
ensure_directory,
|
||||
normalize_graph_payload,
|
||||
utc_now_iso,
|
||||
)
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
@@ -216,7 +216,7 @@ class SemanticNetworkYAMLExporter:
|
||||
records = normalize_graph_payload(semantic_network)
|
||||
yaml_data = {
|
||||
"metadata": {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
"version": "1.0",
|
||||
**semantic_network.get("metadata", {}),
|
||||
},
|
||||
@@ -309,7 +309,7 @@ class SemanticNetworkYAMLExporter:
|
||||
|
||||
if include_metadata:
|
||||
yaml_data["metadata"] = {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
"entity_count": len(entities),
|
||||
}
|
||||
|
||||
@@ -333,7 +333,7 @@ class SemanticNetworkYAMLExporter:
|
||||
|
||||
if include_properties:
|
||||
yaml_data["metadata"] = {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
"relationship_count": len(relationships),
|
||||
}
|
||||
|
||||
@@ -370,7 +370,7 @@ class SemanticNetworkYAMLExporter:
|
||||
}
|
||||
|
||||
yaml_data["metadata"] = {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
"triplet_count": len(triplets),
|
||||
}
|
||||
|
||||
@@ -410,7 +410,7 @@ class SemanticNetworkYAMLExporter:
|
||||
yaml_data = {
|
||||
"pipeline_stage": pipeline_stage,
|
||||
"metadata": {
|
||||
"extracted_at": datetime.now().isoformat(),
|
||||
"extracted_at": utc_now_iso(),
|
||||
**extracted_data.get("metadata", {}),
|
||||
},
|
||||
"semantic_network": semantic_network,
|
||||
|
||||
@@ -66,6 +66,12 @@ except (ImportError, OSError):
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# create_index's index_type reaches a raw SQL keyword position (`USING
|
||||
# {index_type}`) that can't be bound as a query parameter; only the
|
||||
# documented, PostgreSQL-recognized types are allowed through.
|
||||
_ALLOWED_INDEX_TYPES = frozenset({"btree", "gin", "hash", "gist", "brin"})
|
||||
|
||||
|
||||
def _sanitize_label(label: str) -> str:
|
||||
"""
|
||||
Sanitize a Cypher label to prevent injection.
|
||||
@@ -1214,6 +1220,11 @@ class ApacheAgeStore:
|
||||
safe_label = _sanitize_label(label)
|
||||
if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", property_name):
|
||||
raise ValidationError(f"Invalid property name: '{property_name}'")
|
||||
if index_type not in _ALLOWED_INDEX_TYPES:
|
||||
raise ValidationError(
|
||||
f"Invalid index_type: {index_type!r}. "
|
||||
f"Allowed: {sorted(_ALLOWED_INDEX_TYPES)}"
|
||||
)
|
||||
|
||||
index_name = options.get(
|
||||
"index_name", f"idx_{self.graph_name}_{safe_label}_{property_name}"
|
||||
|
||||
@@ -503,6 +503,16 @@ class Neo4jStore:
|
||||
Returns:
|
||||
List of matching nodes
|
||||
"""
|
||||
# LIMIT can't be bound as a query parameter in a way Neo4j accepts
|
||||
# here, so it's interpolated directly; validate explicitly rather
|
||||
# than trust the `limit: int` type hint, which Python doesn't
|
||||
# enforce at runtime. Done outside the try/except below so a bad
|
||||
# limit raises ValidationError, not a generic ProcessingError.
|
||||
try:
|
||||
limit = int(limit)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValidationError(f"Invalid limit: {limit!r}") from exc
|
||||
|
||||
try:
|
||||
# Build query
|
||||
if labels:
|
||||
@@ -698,6 +708,15 @@ class Neo4jStore:
|
||||
Returns:
|
||||
List of matching relationships
|
||||
"""
|
||||
# See get_nodes: LIMIT is interpolated directly, so validate
|
||||
# explicitly rather than trust the unenforced `limit: int` hint,
|
||||
# outside the try/except below so a bad limit raises
|
||||
# ValidationError, not a generic ProcessingError.
|
||||
try:
|
||||
limit = int(limit)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValidationError(f"Invalid limit: {limit!r}") from exc
|
||||
|
||||
try:
|
||||
type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else ""
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ License: MIT
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
@@ -38,6 +39,96 @@ from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
|
||||
# Fragments that turn a filter/order clause into a second statement, a
|
||||
# data-exfiltration UNION, a time-based blind-injection oracle, or schema
|
||||
# enumeration, rather than a boolean/ordering expression.
|
||||
_SQL_FRAGMENT_BLOCKLIST_RE = re.compile(
|
||||
r";|--|/\*|\*/|\bunion\b|\binsert\b|\bupdate\b|\bdelete\b|\bdrop\b|"
|
||||
r"\balter\b|\bcreate\b|\bexec\b|\bexecute\b|\bgrant\b|\brevoke\b|"
|
||||
r"\battach\b|\bpragma\b|\bxp_\w+|\bsp_\w+|\binto\s+outfile\b|\bload_file\b|"
|
||||
r"\bsleep\s*\(|\bbenchmark\s*\(|\bpg_sleep\s*\(|\bwaitfor\b|"
|
||||
r"\bdbms_\w+|\butl_\w+|\binformation_schema\b|\bpg_catalog\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# SQL single-quoted string literals ('' is the standard escaped-quote) and
|
||||
# double-quoted identifiers ("" likewise) — matched only when properly
|
||||
# closed, so a malformed/unterminated quote sequence is left alone and
|
||||
# still hits the blocklist above rather than being treated as "inside a
|
||||
# literal" and skipped.
|
||||
_SQL_STRING_LITERAL_RE = re.compile(r"'(?:[^']|'')*'")
|
||||
_SQL_QUOTED_IDENTIFIER_RE = re.compile(r'"(?:[^"]|"")*"')
|
||||
|
||||
|
||||
def _mask_sql_literals(fragment: str) -> str:
|
||||
"""Blank the contents of quoted literals so they can't trip the blocklist.
|
||||
|
||||
A legitimate value or quoted identifier that happens to contain a
|
||||
blocked word or character as *data* — e.g. ``status = 'union'`` or
|
||||
``"my--column" = 1`` — is not SQL syntax and shouldn't be rejected as
|
||||
if it were. Only the quoted span's interior is replaced (with `?`,
|
||||
keeping the surrounding quotes and the fragment's length/positions
|
||||
intact for the error message); text outside any properly closed quote
|
||||
is passed through unchanged and still fully scrutinized.
|
||||
"""
|
||||
fragment = _SQL_STRING_LITERAL_RE.sub(
|
||||
lambda m: "'" + "?" * (len(m.group(0)) - 2) + "'", fragment
|
||||
)
|
||||
fragment = _SQL_QUOTED_IDENTIFIER_RE.sub(
|
||||
lambda m: '"' + "?" * (len(m.group(0)) - 2) + '"', fragment
|
||||
)
|
||||
return fragment
|
||||
|
||||
|
||||
def _validate_sql_identifier(name: str, kind: str) -> str:
|
||||
"""Validate a table/schema name used as a raw SQL identifier.
|
||||
|
||||
``export_table_data`` interpolates *name* directly into the query text
|
||||
(SQLAlchemy has no bind-parameter syntax for identifiers), so anything
|
||||
outside a plain alphanumeric/underscore identifier is a potential
|
||||
breakout of the surrounding ``"..."`` quoting.
|
||||
"""
|
||||
if not isinstance(name, str) or not _IDENTIFIER_RE.match(name):
|
||||
raise ValidationError(
|
||||
f"Invalid {kind}: {name!r}. Must start with a letter or "
|
||||
"underscore and contain only alphanumeric characters and "
|
||||
"underscores."
|
||||
)
|
||||
return name
|
||||
|
||||
|
||||
def _validate_sql_fragment(fragment: str, kind: str) -> str:
|
||||
"""Reject WHERE/ORDER BY fragments that smuggle a second statement.
|
||||
|
||||
These clauses can't be bound as query parameters (they're arbitrary
|
||||
boolean/ordering expressions, not values), so this blocks the concrete
|
||||
injection primitives (statement separators, comments, UNION, DML/DDL
|
||||
keywords, time-based blind oracles, schema enumeration) rather than
|
||||
parameterizing.
|
||||
|
||||
This is a blocklist, not a grammar: it cannot exhaustively prove
|
||||
*fragment* is safe, only reject known-dangerous constructs, so a
|
||||
boolean-blind subquery expressed with none of the blocked keywords
|
||||
(e.g. ``id = (SELECT 1 FROM t WHERE ...)``) still passes. ``where``/
|
||||
``order_by`` are a raw-SQL-fragment API by design (see
|
||||
``export_table_data``'s docstring); treat them as trusted/operator
|
||||
input, not something to expose directly to untrusted end users.
|
||||
"""
|
||||
if not isinstance(fragment, str):
|
||||
raise ValidationError(f"Invalid {kind}: must be a string")
|
||||
# Check the blocklist against literal-masked text so a blocked word
|
||||
# appearing only as quoted data (not as SQL syntax) doesn't false-
|
||||
# positive; the original, unmodified fragment is still what's returned
|
||||
# and used in the query.
|
||||
if _SQL_FRAGMENT_BLOCKLIST_RE.search(_mask_sql_literals(fragment)):
|
||||
raise ValidationError(
|
||||
f"Invalid {kind}: {fragment!r} contains disallowed SQL "
|
||||
"keywords or statement-boundary characters"
|
||||
)
|
||||
return fragment
|
||||
|
||||
|
||||
@dataclass
|
||||
class TableData:
|
||||
@@ -253,8 +344,13 @@ class DataExporter:
|
||||
schema: Schema name (for databases with schema support, optional)
|
||||
limit: Maximum number of rows to export (optional)
|
||||
offset: Row offset for pagination (optional)
|
||||
where: WHERE clause for filtering (optional, e.g., "age > 18")
|
||||
order_by: ORDER BY clause for sorting (optional, e.g., "name ASC")
|
||||
where: WHERE clause for filtering (optional, e.g., "age > 18").
|
||||
Raw SQL, checked against a keyword/character blocklist (see
|
||||
``_validate_sql_fragment``) but not fully sanitized — treat
|
||||
as trusted/operator input, never pass untrusted end-user
|
||||
text here directly.
|
||||
order_by: ORDER BY clause for sorting (optional, e.g., "name ASC").
|
||||
Same trust requirement as ``where``.
|
||||
**options: Additional export options (unused)
|
||||
|
||||
Returns:
|
||||
@@ -269,7 +365,16 @@ class DataExporter:
|
||||
ProcessingError: If table export fails
|
||||
"""
|
||||
try:
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy import inspect, text
|
||||
|
||||
_validate_sql_identifier(table_name, "table_name")
|
||||
if schema:
|
||||
_validate_sql_identifier(schema, "schema")
|
||||
if where:
|
||||
_validate_sql_fragment(where, "where")
|
||||
if order_by:
|
||||
_validate_sql_fragment(order_by, "order_by")
|
||||
|
||||
inspector = inspect(connection)
|
||||
|
||||
# Get column information
|
||||
@@ -344,6 +449,8 @@ class DataExporter:
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
except ValidationError:
|
||||
raise
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to export table {table_name}: {e}")
|
||||
raise ProcessingError(f"Failed to export table: {e}") from e
|
||||
@@ -760,8 +867,11 @@ class DBIngestor:
|
||||
schema: Schema name (for databases with schema support, optional)
|
||||
limit: Maximum number of rows to export (optional)
|
||||
offset: Row offset for pagination (optional)
|
||||
where: WHERE clause for filtering (optional, e.g., "status = 'active'")
|
||||
order_by: ORDER BY clause for sorting (optional, e.g., "created_at DESC")
|
||||
where: WHERE clause for filtering (optional, e.g., "status = 'active'").
|
||||
Raw SQL passed through to ``export_table_data`` — same trust
|
||||
requirement documented there: not for untrusted end-user text.
|
||||
order_by: ORDER BY clause for sorting (optional, e.g., "created_at DESC").
|
||||
Same trust requirement as ``where``.
|
||||
transform: Whether to apply data transformations (default: False)
|
||||
**filters: Additional filtering options (merged with above parameters)
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user