mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-30 04:40:16 +00:00
Compare commits
201
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
943be0c10f | ||
|
|
3c00ffb019 | ||
|
|
7a6f1d0417 | ||
|
|
6c2ccfd3af | ||
|
|
cf6c9b7b9c | ||
|
|
52ba7b6890 | ||
|
|
1f868b9779 | ||
|
|
8d5479d22f | ||
|
|
14107e51c3 | ||
|
|
ea9b1f5d4a | ||
|
|
e63bad310e | ||
|
|
d79f2cfb8f | ||
|
|
48c2d2a7ed | ||
|
|
fdafffa980 | ||
|
|
abe1bc8f3e | ||
|
|
a4bcfade7f | ||
|
|
2b077c6d0e | ||
|
|
f6b31925c6 | ||
|
|
4820185924 | ||
|
|
7e1d2550b9 | ||
|
|
f124df4229 | ||
|
|
a47954c19a | ||
|
|
1ee2ae88a7 | ||
|
|
93e4b97517 | ||
|
|
48f219cd5c | ||
|
|
331c857672 | ||
|
|
727b0383cc | ||
|
|
248ae57694 | ||
|
|
ba3737c878 | ||
|
|
e3b24ef872 | ||
|
|
b891902d6d | ||
|
|
9123dcc0bd | ||
|
|
6cbe0ae438 | ||
|
|
fe3baad67c | ||
|
|
7efc66d0e3 | ||
|
|
50f2f82b95 | ||
|
|
d3f37f798e | ||
|
|
5cd79b6436 | ||
|
|
3c99f447e6 | ||
|
|
8dcbee386d | ||
|
|
e2f850e9e4 | ||
|
|
2d976963ab | ||
|
|
283b7ada0c | ||
|
|
483f53aaa6 | ||
|
|
14091d21fb | ||
|
|
58125a0a93 | ||
|
|
394ce5fe61 | ||
|
|
8e9f7c5526 | ||
|
|
d4fdc1f0d3 | ||
|
|
729f4fe932 | ||
|
|
5c6b40f36c | ||
|
|
6390652303 | ||
|
|
1b21fc4cbb | ||
|
|
719efa4794 | ||
|
|
b64e4b6600 | ||
|
|
db0a9e8bfd | ||
|
|
46451ae2e1 | ||
|
|
54bae5dffe | ||
|
|
5b01949dd8 | ||
|
|
3b710c79d2 | ||
|
|
4b312ca2fa | ||
|
|
363a9ad641 | ||
|
|
b7af18a70a | ||
|
|
560ffef59f | ||
|
|
a279e74468 | ||
|
|
6653cbe879 | ||
|
|
0dc26350f9 | ||
|
|
3063bf8096 | ||
|
|
cbb0a6dd8e | ||
|
|
a46be971e6 | ||
|
|
e3405ebc23 | ||
|
|
0ee38c2d99 | ||
|
|
a3074ec454 | ||
|
|
5e40d6e4ce | ||
|
|
96f60e6114 | ||
|
|
a2a8d776a3 | ||
|
|
4801ff3492 | ||
|
|
cfccdab8ed | ||
|
|
c5d382ee81 | ||
|
|
54c274e02c | ||
|
|
1273c4fb1e | ||
|
|
898a92062a | ||
|
|
556e786fd5 | ||
|
|
d83b21ba77 | ||
|
|
8f6948f85d | ||
|
|
60eb595d62 | ||
|
|
861b2bf757 | ||
|
|
78fc9028a8 | ||
|
|
6b7625ef9b | ||
|
|
48a05b00a6 | ||
|
|
58b77ddcf5 | ||
|
|
5d554ec586 | ||
|
|
a7ebec8fe5 | ||
|
|
71cffb15e9 | ||
|
|
d7ee22cf1f | ||
|
|
efdfa39c15 | ||
|
|
66e3333e41 | ||
|
|
9ca83d397f | ||
|
|
d5dc4eabac | ||
|
|
f60ca6a529 | ||
|
|
05c21af117 | ||
|
|
981c9d9208 | ||
|
|
c30ec14858 | ||
|
|
e5c5cf0efa | ||
|
|
e03212cd66 | ||
|
|
83c04a57d6 | ||
|
|
75b026c6dd | ||
|
|
2a303cf4da | ||
|
|
7595bad28f | ||
|
|
2d75952476 | ||
|
|
2ac3eaffd7 | ||
|
|
51c12d5c8f | ||
|
|
e55c03bd39 | ||
|
|
4d3259df31 | ||
|
|
4a886d970e | ||
|
|
e1092ac507 | ||
|
|
b77e3e8c3c | ||
|
|
68f7ae3807 | ||
|
|
56609ab3fc | ||
|
|
7b2b2efe6b | ||
|
|
08a6e7c053 | ||
|
|
5a3bdc393d | ||
|
|
e6b159e5c5 | ||
|
|
7db2e2f46b | ||
|
|
3fbe3cfd2d | ||
|
|
75bc6255d4 | ||
|
|
a96f1590f1 | ||
|
|
063f447202 | ||
|
|
a1194a155d | ||
|
|
17d878cbf3 | ||
|
|
488e381247 | ||
|
|
4acd2f9c33 | ||
|
|
7cac8a6bc3 | ||
|
|
6c77594ea6 | ||
|
|
5109c6fab2 | ||
|
|
12b9694a8e | ||
|
|
49707729ad | ||
|
|
430020c7c4 | ||
|
|
43b207c1c5 | ||
|
|
55bde673c9 | ||
|
|
0f308b2078 | ||
|
|
5c2901ae27 | ||
|
|
dae21166a1 | ||
|
|
67be421533 | ||
|
|
baf8f01f85 | ||
|
|
c58686b4ec | ||
|
|
c3a0078bfd | ||
|
|
04602a0e0e | ||
|
|
eedf1425ca | ||
|
|
d4cb14c1fb | ||
|
|
a8194dfc60 | ||
|
|
c7415f2e92 | ||
|
|
3331df28ad | ||
|
|
de5e20dc55 | ||
|
|
0f252ab355 | ||
|
|
0b77e5fe94 | ||
|
|
893b6db3c3 | ||
|
|
b8297b8077 | ||
|
|
4d37920007 | ||
|
|
6416fbb669 | ||
|
|
4b6cc09585 | ||
|
|
70aa9d01bf | ||
|
|
c53ca4e84b | ||
|
|
15171fd31a | ||
|
|
8177d88753 | ||
|
|
d94d8f6ab8 | ||
|
|
83649f6821 | ||
|
|
2f04bc01a3 | ||
|
|
5579851208 | ||
|
|
eaf51b3383 | ||
|
|
115e7965cd | ||
|
|
8639cb9f16 | ||
|
|
f1e7e64ad1 | ||
|
|
6df97cf0a0 | ||
|
|
84ce3c5155 | ||
|
|
b8175ea801 | ||
|
|
616f5ca9b9 | ||
|
|
557e29ee14 | ||
|
|
d42af280e8 | ||
|
|
c1be6dd7dc | ||
|
|
42afc06003 | ||
|
|
13915297d2 | ||
|
|
1cee3e7cb9 | ||
|
|
e7ce092ccf | ||
|
|
92dc3304f8 | ||
|
|
f737f72675 | ||
|
|
828179e115 | ||
|
|
b3e107de8f | ||
|
|
55f7eba389 | ||
|
|
a3d8064f3d | ||
|
|
1b9bb4c345 | ||
|
|
20781e8a9e | ||
|
|
6148975e83 | ||
|
|
0ca7b8d489 | ||
|
|
fb7845240b | ||
|
|
d769bf1c39 | ||
|
|
310ac7bd85 | ||
|
|
aeb1752c83 | ||
|
|
dec05b907d | ||
|
|
c77ce9394a | ||
|
|
c7174e9852 |
@@ -69,5 +69,5 @@ If you have ideas on how this could be implemented, please share.
|
||||
|
||||
---
|
||||
|
||||
**Note**: For feature requests that are ready to be implemented, consider creating a [Feature Request issue](https://github.com/Hawksight-AI/semantica/issues/new?template=feature_request.md) instead.
|
||||
**Note**: For feature requests that are ready to be implemented, consider creating a [Feature Request issue](https://github.com/semantica-agi/semantica/issues/new?template=feature_request.md) instead.
|
||||
|
||||
|
||||
@@ -46,8 +46,8 @@ If applicable, paste any error messages or describe unexpected behavior:
|
||||
|
||||
## Checklist
|
||||
|
||||
- [ ] I have searched existing [discussions](https://github.com/Hawksight-AI/semantica/discussions) and [issues](https://github.com/Hawksight-AI/semantica/issues)
|
||||
- [ ] I have checked the [documentation](https://github.com/Hawksight-AI/semantica/tree/main/docs) and [FAQ](https://github.com/Hawksight-AI/semantica/blob/main/docs/faq.md)
|
||||
- [ ] I have searched existing [discussions](https://github.com/semantica-agi/semantica/discussions) and [issues](https://github.com/semantica-agi/semantica/issues)
|
||||
- [ ] I have checked the [documentation](https://github.com/semantica-agi/semantica/tree/main/docs) and [FAQ](https://github.com/semantica-agi/semantica/blob/main/docs/faq.md)
|
||||
- [ ] I have provided a minimal code example (if applicable)
|
||||
- [ ] I have included error messages (if applicable)
|
||||
- [ ] I have provided environment details
|
||||
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
# Funding options for Semantica
|
||||
github: Hawksight-AI
|
||||
github: semantica-agi
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
blank_issues_enabled: true
|
||||
contact_links:
|
||||
- name: 📚 Documentation
|
||||
url: https://github.com/Hawksight-AI/semantica/tree/main/docs
|
||||
url: https://github.com/semantica-agi/semantica/tree/main/docs
|
||||
about: Browse the documentation
|
||||
- name: 💬 Discussions
|
||||
url: https://github.com/Hawksight-AI/semantica/discussions
|
||||
url: https://github.com/semantica-agi/semantica/discussions
|
||||
about: Ask questions and discuss with the community
|
||||
|
||||
+9
-9
@@ -3,31 +3,31 @@
|
||||
## Getting Help
|
||||
|
||||
### 📚 Documentation
|
||||
Check the [docs folder](https://github.com/Hawksight-AI/semantica/tree/main/docs) and [README](https://github.com/Hawksight-AI/semantica/blob/main/README.md) for guides and examples.
|
||||
Check the [docs folder](https://github.com/semantica-agi/semantica/tree/main/docs) and [README](https://github.com/semantica-agi/semantica/blob/main/README.md) for guides and examples.
|
||||
|
||||
### 💬 Community Support
|
||||
- **GitHub Discussions**: [Ask questions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
- **GitHub Discussions**: [Ask questions](https://github.com/semantica-agi/semantica/discussions)
|
||||
- **Discord**: Join our [Discord server](https://discord.gg/sV34vps5hH) for real-time chat
|
||||
|
||||
### 💭 Discussions
|
||||
Join the conversation on [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions):
|
||||
Join the conversation on [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions):
|
||||
- **Q&A**: Ask questions and get help from the community
|
||||
- **Ideas**: Share feature requests and suggestions
|
||||
- **Show and Tell**: Showcase your projects and use cases
|
||||
- **General**: General discussions about Semantica
|
||||
|
||||
### 🐛 Bug Reports
|
||||
Found a bug? [Create an issue](https://github.com/Hawksight-AI/semantica/issues/new/choose)
|
||||
Found a bug? [Create an issue](https://github.com/semantica-agi/semantica/issues/new/choose)
|
||||
|
||||
### 📖 Resources
|
||||
- [Quick Start Guide](https://github.com/Hawksight-AI/semantica/blob/main/docs/quickstart.md)
|
||||
- [FAQ](https://github.com/Hawksight-AI/semantica/blob/main/docs/faq.md)
|
||||
- [Cookbook Examples](https://github.com/Hawksight-AI/semantica/tree/main/cookbook)
|
||||
- [Quick Start Guide](https://github.com/semantica-agi/semantica/blob/main/docs/quickstart.md)
|
||||
- [FAQ](https://github.com/semantica-agi/semantica/blob/main/docs/faq.md)
|
||||
- [Cookbook Examples](https://github.com/semantica-agi/semantica/tree/main/cookbook)
|
||||
|
||||
## Commercial Support
|
||||
|
||||
For enterprise support, custom development, or consulting services:
|
||||
- Contact us through [GitHub Issues](https://github.com/Hawksight-AI/semantica/issues)
|
||||
- Contact us through [GitHub Issues](https://github.com/semantica-agi/semantica/issues)
|
||||
- Include "Commercial Support" in the title
|
||||
|
||||
## Sponsorship
|
||||
@@ -35,7 +35,7 @@ For enterprise support, custom development, or consulting services:
|
||||
### Sponsor this project
|
||||
|
||||
Support Semantica development:
|
||||
- [GitHub Sponsors](https://github.com/sponsors/Hawksight-AI)
|
||||
- [GitHub Sponsors](https://github.com/sponsors/semantica-agi)
|
||||
|
||||
Your sponsorship helps us:
|
||||
- Maintain and improve the framework
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
> **Before you submit:** make sure you followed the [issue workflow in CONTRIBUTING.md](https://github.com/semantica-agi/semantica/blob/main/CONTRIBUTING.md#-working-on-an-existing-issue) — comment on the issue and wait for assignment before opening a PR, to avoid duplicate work.
|
||||
> **Before you submit:** make sure you followed the [issue workflow in CONTRIBUTING.md](https://github.com/semantica-agi/semantica/blob/main/CONTRIBUTING.md#-working-on-an-existing-issue) — wait for the issue to be assigned to you before opening a PR, to avoid duplicate work.
|
||||
|
||||
## Description
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
# meaningful state carried over from a failed attempt.
|
||||
- name: Initialize CodeQL (attempt 1)
|
||||
id: codeql-init-1
|
||||
uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 2)
|
||||
id: codeql-init-2
|
||||
if: steps.codeql-init-1.outcome == 'failure'
|
||||
uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -52,17 +52,17 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 3)
|
||||
id: codeql-init-3
|
||||
if: steps.codeql-init-2.outcome == 'failure'
|
||||
uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
languages: python
|
||||
queries: security-and-quality
|
||||
config-file: .github/codeql/codeql-config.yml
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
category: "/language:python"
|
||||
upload: false
|
||||
@@ -72,7 +72,7 @@ jobs:
|
||||
# Uploads results only when Default Setup is not active.
|
||||
# If Default Setup is still enabled, this step skips gracefully
|
||||
# instead of failing the workflow with HTTP 409.
|
||||
uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.codeql.outputs.sarif-output }}
|
||||
category: "/language:python"
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
# avoiding the guardian.cmd/checkov exit-code bug in the MSDO wrapper.
|
||||
tools: eslint,templateanalyzer,terrascan
|
||||
- name: Upload results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.msdo.outputs.sarifFile }}
|
||||
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
}
|
||||
|
||||
- name: Upload Checkov results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: reports/checkov.sarif
|
||||
|
||||
+208
-2
@@ -9,8 +9,42 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.6.6] - 2026-08-20
|
||||
|
||||
### Added
|
||||
|
||||
- **Semantica RDF vocabulary, and deterministic entity/relationship IRIs** (#1109, closes #1107, closes #1101) by @fabio-rovai, reviewed by @KaifAhmad1
|
||||
- Every RDF/JSON-LD export mints terms in `https://semantica.dev/ns#`, and until now nothing declared what those terms meant — the namespace 404s and no vocabulary shipped with the package, so a consumer receiving an export had no way to tell `sem:text` from a typo of it, and no closed-world checker could validate an export at all
|
||||
- `semantica/ontology/vocabulary/semantica-ns.ttl` declares the terms the exporters actually emit — drawn from the emitting call sites in `export/rdf_exporter.py`, `export/json_exporter.py` and `provenance/manager.py`, not from what a vocabulary "ought" to contain. Ships inside the package (`from semantica.ontology.vocabulary import vocabulary_turtle`) so it loads without a network round trip, and is the same document intended to be served at the namespace IRI once hosting/content-negotiation is sorted
|
||||
- `tests/ontology/test_vocabulary.py` ties the document to the code: every term a serializer can write must be declared, so adding a term to an exporter without declaring it fails the build
|
||||
- The missing-id fallback minted entity/relationship IRIs from Python's builtin `hash()`, randomised per process (`PYTHONHASHSEED`), so the same entity got a different IRI on every run and exports couldn't be diffed, deduplicated, or joined to an earlier provenance record. It also wrote `<semantica:entity_N>`, an IRI in the scheme `semantica` rather than the expansion of the declared prefix, so those nodes never joined with anything written through it. Minting now uses SHA-256 and writes a full IRI in the declared namespace; the same fix applies to the default entity/relationship types in the Turtle path
|
||||
- **Fixed during review** (Qodo): the temporal fallback minted from `source_id` only, while the main serializer accepts `source_id` or `source` — relationships using the second form hashed two empty strings, which the previous randomised `hash()` masked by making the IRI unstable anyway; once deterministic, unrelated relationships at the same list index collided on one IRI across exports. Endpoints are now resolved the same way `serialize_to_turtle` resolves them, before minting. `sem:confidence` also lost its declared `xsd:decimal` range: the N-Triples serializer types the same value `xsd:float`, and the two are disjoint, so declaring either contradicted one of the exporters (tracked in #1100) — a new `test_declared_ranges_do_not_contradict_what_the_exporters_emit` guards the whole class of that mistake
|
||||
- **Fixed in follow-up**: `serialize_to_rdfxml`'s default entity type still wrote the bare string `"semantica:Entity"` into an `rdf:resource` attribute, which (unlike a Turtle angle-bracket or an XML element name) is not namespace-expanded — the exact #1101 failure mode, just on the untested RDF/XML path. `json_exporter.py`'s `semantica:format` and `@type: "semantica:KnowledgeGraph"` were emitted but absent from both the vocabulary and the test's `EMITTED_TERMS` guard set, so the "undeclared terms fail the build" claim didn't actually cover them — both are now declared and guarded. `MANIFEST.in` didn't mirror the `pyproject.toml` package-data addition, so a source-distribution install could omit the vocabulary file. The cross-process minting-stability test replaced the subprocess's entire environment with a POSIX-only `PATH`, breaking it on Windows; now overrides only `PYTHONHASHSEED` on top of the inherited environment
|
||||
- **Also fixed, on the JSON-LD paths**: the first fix covered the Turtle, N-Triples and RDF/XML serializers, and left both JSON-LD writers interpolating the entity's own text into `f"semantica:entity/{text}"` and the endpoints into `f"semantica:rel/{source}_{target}"`. Three consequences, all live in 0.6.5: an entity whose text contained a space produced an invalid IRI, and a JSON-LD parser dropped that node in full rather than reporting it, so the entity disappeared from the export; every relationship carrying `source`/`target` rather than `source_id`/`target_id` minted the identical `semantica:rel/_`, collapsing all of them onto one node whose types and endpoints merged; and the JSON-LD `@id` disagreed with the Turtle IRI for the same entity, so the two serializations of one knowledge graph were two different graphs. Both JSON-LD writers now use `mint_entity_iri`/`mint_relationship_iri`, and `JSONExporter.export_entities`/`export_relationships` declare the `semantica` prefix their `@context` was already writing `semantica:entities` against — without it a processor reads that as an IRI in the scheme `semantica`, which is the original #1101 defect on a third path
|
||||
- `tests/export/test_jsonld_iri_minting.py` parses each export with a real JSON-LD processor and asserts the entity survives, the relationships stay distinct, no term expands into the `semantica` scheme, and the JSON-LD `@id` equals the Turtle IRI
|
||||
- 236 export and ontology tests pass
|
||||
|
||||
- **First-class CrewAI integration** (#988, closes #962) by @Shindevrp
|
||||
- New `pip install semantica[crewai]` extra (`crewai>=0.80.0`) — crewai core provides `BaseTool`/`BaseKnowledgeSource`, so `crewai-tools` is intentionally not included, and the extra is intentionally **not** part of the `all` bundle: crewai hard-requires `chromadb~=1.1.0`, which is affected by the unpatched pre-auth code-injection CVE-2026-45829 (see `integrations/crewai/README.md`)
|
||||
- `integrations/crewai/SemanticaKGTool` — a CrewAI `BaseTool` exposing 5 KG actions (`extract_entities`, `extract_relations`, `add_to_graph`, `query_graph`, `find_related`) backed by `NERExtractor` / `RelationExtractor` / `ContextGraph`; supports both sync `run()` and async `arun()`
|
||||
- `integrations/crewai/SemanticaDecisionTool` — a CrewAI `BaseTool` wrapping `AgentContext` with 5 decision-intelligence actions (`record_decision`, `find_precedents`, `trace_causal_chain`, `analyze_impact`, `check_policy`)
|
||||
- `integrations/crewai/SemanticaKnowledgeSource` — a CrewAI `BaseKnowledgeSource` that serializes a `ContextGraph` into crew knowledge storage; implements both the legacy `load_content()` and current `validate_content()`/`aadd()` contracts so it works across `crewai>=0.80.0`
|
||||
- All three classes degrade gracefully when `crewai` is not installed (still importable, full Semantica API available)
|
||||
- New `tests/integrations/crewai/`: 70 tests covering stub-based present-case behavior (Pydantic/BaseTool subclassing, every action, knowledge-source chunking/storage) plus a subprocess isolation test for the crewai-absent degradation path
|
||||
- Docs: `docs/integrations/crewai.md` page, `docs.json` Integrations nav entry, and README integration-matrix/install updates
|
||||
- **Hardened during code review**: live `graph`/`context`/extractor state is excluded from CrewAI JSON serialization (`model_dump(mode="json")`) with `model_post_init` self-healing defaults, so checkpoint/resume no longer raises `PydanticSerializationError`; `query_graph` now searches node content (not just ids/types); `trace_causal_chain` returns an explicit error instead of substituting similarity precedents when causal tracing is unavailable, and calls `trace_decision_causality(..., max_depth=...)` with the correct argument name; `find_precedents` propagates `max_precedents` as the backend `limit`; `add_to_graph` writes are serialized under a module lock so concurrent agents can't double-count duplicate adds; nameless entities are skipped instead of creating `repr()`-junk nodes
|
||||
- **Hardened during second code review**: `check_policy` rules are now coerced type-aware — `bool("false")` was truthy, so `enabled == false` reported a violation for `enabled: false`, and string datums like `"0.90"` were compared lexicographically instead of numerically; `trace_causal_chain` no longer raises `AttributeError` (which escaped the tool) when the decision context has no `knowledge_graph`, returning honest error JSON instead; knowledge-source storage failures log an actionable ERROR (a missing crew embedder otherwise silently left agents with empty retrieval); `add_to_graph` uses a per-graph re-entrant lock instead of a process-global one (independent graphs no longer serialize each other, and re-entrant extractors can't deadlock); entity/relation `confidence=None` normalizes to `1.0` instead of failing the whole extraction; added a subprocess integration test against the real `crewai` package covering `Crew`-level serialization round-trip and restore
|
||||
|
||||
- **`ContextGraph` gains retraction and purge — the graph previously had no way to remove a node or edge without discarding everything via `clear()`** (#957, closes #955) by @pravit-amp, reviewed by @KaifAhmad1
|
||||
- `retract_node()`/`retract_edge()` close an entity's validity window rather than deleting it, reusing the existing `valid_from`/`valid_until`/`state_at()` machinery: the entity drops out of `find_active_nodes()` and future `state_at()` queries going forward, but `state_at()` calls before the retraction time still return it, so decisions recorded against it stay explainable. A `("kind", id)`-keyed retraction record captures who/why/when, retrievable via `get_retraction()`/`list_retractions()`
|
||||
- `purge_node()`/`purge_edge()` are the destructive counterpart: the entity is removed outright, from history as well as the active view, for erasure obligations retraction alone cannot satisfy (e.g. GDPR Article 17). Only a tombstone remains — that a purge happened, when, and why — deliberately never the purged content, via `get_tombstone()`/`list_tombstones()`. Purge is graph-scope only: `AgentMemory` and any bound vector store are not reached, so it is one step of an erasure workflow rather than the whole of it
|
||||
- Both operations default to `cascade=True` (also touching every incident edge, and for `purge_node`, the marker node of any cross-graph link the node exits through) since leaving edges active around an inactive/removed node produces an inconsistent active view or dangling endpoints; both accept `cascade=False` for callers that want to handle edges themselves
|
||||
- Both are idempotent: retracting/purging an already-retracted/purged entity returns `False` rather than raising, and a repeat retraction preserves the original record's reason rather than overwriting it
|
||||
- Retraction/purge closing a validity window never widens an existing one — a node or edge added with `valid_until` already in the past keeps that earlier bound rather than being pushed later by a subsequent retraction time
|
||||
- Reuses the existing audit-trail path with no changes to `change_management`: `MutationRecord` already documented `REMOVE_NODE`/`REMOVE_EDGE` in its operation vocabulary; retraction now emits `UPDATE_NODE`/`UPDATE_EDGE`, purge emits `REMOVE_NODE`/`REMOVE_EDGE`, matching the documented contract. Mutation payloads are snapshotted inside the lock and the callback fires after it is released, so a callback that itself mutates the graph (e.g. `clear()`) can't observe or lose in-flight records
|
||||
- **Fixed during review** (@KaifAhmad1): `retract_edge()`/`purge_edge()` resolved "the edge" for a given `edge_id` via the first matching object only. `edge_id` is content-derived and, prior to #926, was not guaranteed unique — a graph holding two identical `add_edge()` calls had two edge objects sharing one id. A direct `retract_edge()`/`purge_edge()` call would silently leave the second duplicate untouched (still live, still active) while returning `True` and recording a tombstone/retraction that claimed the edge was fully handled; repeat `purge_edge()` calls also silently overwrote the tombstone's `reason`/`purged_at` on each partial attempt instead of no-op'ing. The same gap let `retract_node()`'s cascade skip a duplicate outright, since it checked the live `_retractions` dict mid-loop and treated the first duplicate's just-written record as proof the second was already handled. `#926` (merged) stops *new* duplicates from being created, but any graph already holding one — loaded from a save made before that fix, or built during the window before it landed — could still trigger this. Now `retract_edge()`/`purge_edge()` act on every edge matching the id under one record, and the cascade's dedup check is snapshotted before the loop starts so within-call duplicates are still closed rather than skipped. 5 new regression tests in `TestDuplicateEdgeId`
|
||||
- New `tests/context/test_context_graph_retraction.py`: 49 tests, covering retraction/purge semantics, cascade, idempotency, validity-window narrowing, id-keyspace collisions between node and edge ids, cross-graph link teardown, `clear()`/`load_from_file()` resetting retraction/tombstone state, audit-trail integration against a real `TemporalVersionManager`, mutation-emission ordering under a concurrent `clear()`, and concurrent purges
|
||||
- Full `tests/context/` suite: 533 passed
|
||||
- **`DistanceExporter.compute_pairs()` gains an opt-in `metric_errors` column to distinguish legitimate `None` results from computation failures** (#960, follow-up to #879) by @Karunasagar12
|
||||
- Previously, a `None` in `hop_count`/`weighted_distance`/`semantic_similarity`/betweenness could mean either "no path exists" or "the underlying computation raised" — logged as a warning per #879, but not otherwise surfaced, so the two cases were indistinguishable in exported CSV/JSONL/DataFrame data. `include=["metric_errors"]` now adds a `metric_errors` field per row: `""` when all requested metrics succeeded, or a comma-separated list of metric names that raised (e.g. `"hop_count,weighted_distance"`)
|
||||
- Opt-in only — default `compute_pairs()`/`to_csv()`/`to_dataframe()`/`to_jsonl()` schema is unchanged unless `"metric_errors"` is explicitly requested
|
||||
@@ -20,6 +54,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- New `tests/export/test_distance_exporter_metric_errors.py`: 6 tests covering success, single/multiple failures, opt-out, the no-path-vs-error distinction, and default-schema stability; existing `tests/export/test_distance_exporter.py` updated for the new tuple return type
|
||||
- Full `tests/export/` suite: 77 passed
|
||||
|
||||
- **`ContextGraph.to_kg_dict()`: an adapter converting a `ContextGraph`'s internal `nodes`/`edges`/`source` shape into the canonical `entities`/`relationships`/`source_id` shape `RDFExporter` and `TemporalGraphQuery` consume** (#1081) by @cxzg007
|
||||
- Previously there was no supported way to feed a `ContextGraph` into those consumers without hand-rolling the field remapping; `to_kg_dict()` does it once, with an `entities_only` option that drops relationships left dangling by the filter
|
||||
- **Fixed during review** (Qodo): null `properties`/`metadata` on a node loaded from JSON raised `TypeError` when copied — both are now guarded with `or {}`; entity ids are coerced to `str(node_id)` to match `ContextEdge`'s already-str-coerced endpoints, so valid relationships were no longer dropped by `entities_only` filtering
|
||||
- `RDFExporter`'s validator and `TemporalGraphQuery` now also accept `source_id`/`target_id` endpoints, the shape `to_kg_dict()` emits
|
||||
|
||||
### Changed
|
||||
|
||||
- **`GraphBuilder`'s 6 public methods now have Google-style docstrings** (#878, closes #876) by @cakeni
|
||||
@@ -29,7 +68,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- **Corrected during review**: `add_temporal_edge`/`create_temporal_snapshot` docstrings overclaimed numeric-timestamp support; `_parse_time()` only special-cases `str` and `datetime`, falling back to a bare `str()` cast for anything else (not true numeric parsing). Narrowed to "datetime or ISO-formatted string"
|
||||
- **Fixed along the way**: `build()`'s `**options` documented a default only for `extract`; `extract_relations`, `extract_triplets`, `ner_method`, `relation_method`, and `triplet_method` all have concrete defaults in `_extract_from_text()` (`True`, `True`, `"llm"`, `"llm"`, `"llm"`) that were left unstated, inconsistent with CONTRIBUTING.md's own docstring example of noting defaults inline
|
||||
- `python -m pytest tests/kg/test_kg.py tests/kg/test_graph_builder_external.py -q`: 45 passed
|
||||
- **`GraphBuilder` raw-text extraction now defaults to local extractors instead of LLM extraction** (closes #930) by @dex0shubham
|
||||
- **`GraphBuilder` raw-text extraction now defaults to local extractors instead of LLM extraction** (#941, closes #930) by @dex0shubham
|
||||
- `GraphBuilder._extract_from_text()` defaulted `ner_method`, `relation_method`, and `triplet_method` to `"llm"`, and ran relation extraction unconditionally (`extract_relations` defaulted to `True`) — all four contradicting the defaults documented in the `build()` docstring at the time (`"ml"` / `"pattern"` / `False`), and diverging from the standalone extractors (`NERExtractor` defaults to `method="ml"`, `RelationExtractor` and `TripletExtractor` to `method="pattern"`). The practical effect was that any raw-text `build()` call silently required a configured provider, an API key, and network access
|
||||
- Defaults are now `ner_method="ml"`, `relation_method="pattern"`, `triplet_method="pattern"`, and `extract_relations=False`, matching the docstring. LLM extraction remains fully available and is now opt-in
|
||||
- **To restore the previous behaviour**, pass the methods explicitly:
|
||||
@@ -50,8 +89,80 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- New regression coverage in `tests/kg/test_graph_builder_extraction_defaults.py` pinning all four defaults, verifying that no default resolves to `"llm"`, confirming explicit LLM opt-in still routes correctly, asserting extractors are constructed once across repeated texts, covering fallback method lists (e.g. `ner_method=["pattern", "ml"]`) for all three extractors, asserting relations are forwarded to triplet extraction (and that `None` is forwarded when relation extraction is disabled or fails), and running the real default path end to end with no provider mocked. Verified to fail against the pre-fix code
|
||||
- Full `kg` suite: 473 passed
|
||||
|
||||
- **Explorer graph canvas now renders edge labels** (#1013, closes #1009) by @yzxcj797
|
||||
- `GraphCanvas.tsx` had no edge-label rendering path at all; Sigma's edge-label renderer draws `data.label`, but the graph state stored the relationship type under `edgeType`, so simply enabling the renderer would have left every edge blank. `graphSceneState`'s edge reducer now maps `edgeType` onto `label` (suppressed for hidden edges)
|
||||
- Rendering is gated behind a new `edgeLabelsEnabled` entry in the Effects panel (default on), wired through the existing `GraphEffectToggle`/`GraphEffectsState` plumbing, so dense graphs can still turn labels off
|
||||
- **Fixed during review** (Qodo): two follow-up passes closed gaps the first cut left — label rendering wasn't wired through `explorationEffectsPluginPhaseC.tsx`'s Phase C variant, and toggling the effect off mid-session didn't clear already-rendered labels
|
||||
- New coverage in `explorer/tests/graphSceneState.display.test.ts`
|
||||
|
||||
- **Removed `GraphWorkspaceShell.tsx`, `GraphRuntimeStage.tsx`, and `useGraphData.ts` — a second, unused implementation of the graph-loading/error-handling logic already fixed in `GraphWorkspace.tsx`** (#984, resolves the cleanup tracked in #981 by #980's review note) by @lakshayxi
|
||||
- 1,564 lines removed; the surviving `GraphWorkspace` path is now the only implementation, so the "two copies that drifted apart" root cause #980 fixed can't recur in the copy nobody was maintaining
|
||||
|
||||
- **Explorer README and `docs/explorer-setup.md` corrected to describe the authentication 0.6.5 actually shipped**, plus a documented `/ws/graph-updates` auth note (#1040, fixes #1028) by @Kyou12138
|
||||
- Both docs still claimed the Explorer API had no built-in authentication after v0.6.5 added mandatory `SEMANTICA_API_KEY` enforcement with a `503` fail-closed default; corrected to describe the actual behavior, including that only protected routes require the key (`/api/health`/`/api/info` stay open), the non-loopback-bind CLI warning only fires in anonymous mode or when the key is unset, and `SEMANTICA_API_KEY`/`SEMANTICA_ALLOW_ANONYMOUS` are documented in the environment-variable table
|
||||
|
||||
- **CI: pinned `github/codeql-action` to current v4** (#986) by @ZohaibHassan16, and **pinned Python dependencies in `requirements-ci.txt` for reproducible CI runs** (#945) by @yunaremaia, closing the gap where an unpinned CI dependency could silently change behavior between runs
|
||||
|
||||
- **README now states up front that Semantica's explainability is system-level, not foundation-model-internal** (#1033, #1034) by @KaifAhmad1
|
||||
- Nothing in the README previously scoped what "explainable" meant, leaving readers to assume Semantica could expose or reconstruct an LLM's internal reasoning. A callout now states explicitly that Semantica explains and audits what the AI *system* did — context fed in, decisions produced, provenance, relationships, policies applied — not the model's private internal reasoning, and moved the note near the top of the README rather than leaving it implicit
|
||||
|
||||
### Fixed
|
||||
|
||||
- **The temporal-evolution `stability` metric was a hardcoded placeholder, not a duration**
|
||||
- `TemporalGraphQuery.analyze_evolution()` documents `stability` as a "relationship duration/stability measure", but the implementation appended a constant `1` for every relationship with both `valid_from` and `valid_until` set (`durations.append(1) # Placeholder`). The reported stability was therefore always `1.0` when any bounded relationship existed and `0` otherwise — it never reflected how long relationships actually stayed valid, so it could not distinguish a graph of decade-long relationships from one of one-second relationships
|
||||
- `stability` now computes the mean valid-time duration in seconds (`(valid_until - valid_from).total_seconds()`) across relationships that have both bounds set. Relationships with a missing or open `valid_from`/`valid_until` are skipped (their duration is unbounded), and non-positive intervals are clamped to `0`; an empty set still reports `0`
|
||||
- New tests in `tests/kg/test_kg.py` assert the mean-duration result, the skipping of unbounded/half-open intervals, and the empty-graph zero case
|
||||
|
||||
- **Every timestamp an export or a provenance record wrote was timezone-naive** (closes #1114) by @fabio-rovai
|
||||
- `semantica/export/` stamped with `datetime.now().isoformat()`, which reads the machine's **local** clock; `semantica/provenance/` stamped with `datetime.utcnow().isoformat()`, which reads **UTC**. Both produce a naive value and both serialize identically, so nothing downstream can tell which zone a given timestamp belongs to — the same string means two different instants depending on which module wrote it
|
||||
- In RDF the consequence is silent rather than loud. Under XSD 1.1 a value with no timezone compared against one with a timezone is indeterminate whenever the two fall inside the ±14 hour window; SPARQL turns an indeterminate comparison into an error, and `FILTER` discards errors as non-matches. A timezone-qualified query over an Oxigraph store returns an answer with every Semantica-written record quietly absent from it, which is a poor property for `prov:generatedAtTime`, `prov:startedAtTime`, `prov:endedAtTime` and `prov:atTime` to have
|
||||
- New `utc_now()`/`utc_now_iso()` in `semantica/utils/helpers.py`, exported from `semantica.utils`, and used at all 29 call sites in `export/` (`json_exporter`, `yaml_exporter`, `report_generator`, `export_provenance`) and `provenance/` (`manager`, `schemas`, `bridge_axiom`). Values now read `2026-08-19T14:19:04.229937+00:00`: one unambiguous instant, comparable against any correctly stamped value, and valid `xsd:dateTimeStamp`. `sem:exportedAt`'s range in `semantica/ontology/vocabulary/semantica-ns.ttl` is tightened from `xsd:dateTime` accordingly, and its comment no longer has to explain why the weaker range was necessary
|
||||
- `datetime.utcnow()` is deprecated as of Python 3.12 and scheduled for removal; constructing a `ProvenanceEntry` under `-W error::DeprecationWarning` on 3.13 raised, and no longer does
|
||||
- New `tests/export/test_timestamp_timezones.py` and `tests/provenance/test_timestamp_timezones.py`: offset presence on every export and provenance path, PROV-O literals valid as `xsd:dateTimeStamp`, comparison against a timezone-aware instant without `TypeError`, the Oxigraph filter that dropped the naive value (with a bound inside the indeterminate window, so the test cannot pass by accident), and the document `@id` remaining a valid IRI with `+00:00` in it. 11 of the 13 fail on the parent commit
|
||||
- **Fixed during review** (Qodo): once new entries carry `+00:00` and stored ones do not, `ProvenanceManager.query_recorded_between` and `audit_log` compared ISO timestamps as raw strings, so they ordered by spelling rather than by instant — an inclusive naive bound naming a stored offset-bearing timestamp sorted *below* it and dropped the record, and a bound written in another offset landed wherever its digits fell (`19:45+05:30` is 14:15Z, but sorted after 14:19Z). Both now compare instants through a new `to_utc_datetime()` helper that reads a missing offset as UTC, which is what the values written before this change actually were; a bound that cannot be read as a timestamp keeps the historical string comparison rather than raising on a call that used to work
|
||||
- The remaining 147 naive call sites are in `context/`, `vector_store/`, `seed/` and elsewhere, where timestamps are compared against values parsed from previously stored naive strings. Converting those without a read-side migration would raise `TypeError: can't compare offset-naive and offset-aware datetimes` on existing data, so they are deliberately left for a separate change
|
||||
|
||||
- **`split`/chunking paths bypassed the centralized spaCy model cache, reloading the model on every call** (#1042, closes #998) by @Accute9, reviewed by @Sameer6305
|
||||
- `semantica/split/methods.py`'s `split_by_sentences()` and `semantica/split/semantic_chunker.py`'s `SemanticChunker.__init__` each called `spacy.load()` directly instead of reusing the process-level cache added in #889/`semantic_extract/methods.py`'s `load_spacy_model()` — every call/construction re-paid the ~120ms model-load cost independently of `NERExtractor`, which already used the cache
|
||||
- Both now route through `load_spacy_model()`, sharing one cached `Language` instance per model name across `split_by_sentences()`, `SemanticChunker`, and `NERExtractor`; a missing model still falls back to regex/paragraph chunking without poisoning the cache for a later successful load
|
||||
- **Fixed during review** (@Sameer6305): `NERExtractor.__init__()` still had a direct `spacy.load()` call site with the same cache-bypass issue, outside the two files named in #998 but sharing the same root cause; routed through the cache alongside stale test patch targets and a strengthened cache-configuration assertion
|
||||
- **Fixed during review** (@KaifAhmad1): `SemanticChunker.__init__` only caught `OSError` around `load_spacy_model()`, while the sibling fix to `NERExtractor` in this same PR added a broader `except Exception` for a model that is installed but fails at runtime (e.g. a config incompatible with the installed spaCy version). A broken-but-present model crashed `SemanticChunker()` outright instead of degrading to fallback chunking like every other path in this PR. Added the matching `except Exception` branch, leaving `self.nlp` as `None`; new `test_semantic_chunker_falls_back_when_spacy_runtime_is_broken` mirrors the existing `NERExtractor` regression test for the same scenario
|
||||
- New `tests/split/test_spacy_model_cache.py`: cache reuse across repeated calls/instances, shared cache between `split_by_sentences()`/`SemanticChunker`/`NERExtractor`, distinct model names loading separately, missing-model fallback without poisoning the cache, and the broken-runtime fallback added above
|
||||
- `pytest tests/split/test_spacy_model_cache.py tests/split/test_splitter.py tests/split/test_chunkers.py`: all passing (3 pre-existing, unrelated `tests/test_ner_configurations.py` failures confirmed present on `main` before this PR)
|
||||
|
||||
- **`export_yaml` raised a raw `AttributeError` on list input, silently wrote empty exports for unrecognized dict keys, and graph payloads were reconciled differently by every exporter** (#958, closes #956, #952, #953) by @pravit-amp, reviewed by @Sameer6305
|
||||
- Graph payloads circulate under two vocabularies, `entities`/`relationships` and `nodes`/`edges`, and each exporter reconciled them locally with a different idiom — `LPGExporter` in particular dropped every entity whenever `nodes` was present but empty, the exact shape `JSONExporter` emits. A new `normalize_graph_payload()` in `utils/helpers.py` centralizes that decision once, adopted by `LPGExporter`, `ArangoAQLExporter`, `Neo4jCSVExporter`, and both YAML exporters; `ContextGraph.to_dict()` now round-trips through YAML correctly as a result
|
||||
- `export_yaml(records, path)` on a bare list previously failed with `AttributeError` from inside the exporter; it and the other YAML methods now reject non-mapping input with an actionable `ProcessingError` naming the expected keys, since these formats distinguish entities/relationships/triplets and guessing which one a list represents would mislabel the records
|
||||
- `export_yaml({"data": [...]}, path)` previously wrote a structurally valid file with every collection empty, no exception, no warning, and the progress log reporting a completed export. `export_semantic_network`, `export_for_pipeline`, and `export_ontology_schema` now raise `ValidationError` when the payload shares no recognized key with what the method reads, or resolves to nothing while an unread key still holds records — an empty mapping is still accepted, since a genuinely empty graph has no records to lose
|
||||
- **Breaking**: the two cases above, plus a bare list, now raise instead of returning cleanly with data silently dropped or a raw `AttributeError` from exporter internals. Migration: pass records under a recognized key (`{"entities": [...]}` / `{"nodes": [...]}` for `semantic_network`, `{"classes": [...]}` for `schema`)
|
||||
- **Fixed during review** (Qodo): progress tracking could report a completed export before the output directory existed or the file was written; `export()` now creates the directory and serializes before starting tracking, so a rejected export leaves nothing behind
|
||||
- **Fixed during review** (@Sameer6305, round 1): `normalize_graph_payload()`'s collection resolver treated any truthy value as a collection — `{"entities": "abc"}` silently became three single-character records, `{"entities": 42}` leaked a raw `TypeError` from inside `list()`. Collection values are now validated before conversion, rejecting strings/bytes/mappings/non-iterable scalars by name. Separately, `Neo4jCSVExporter._normalize_graph` called the shared resolver with `require_recognized=False`, so it alone kept accepting an unrecognized mapping as a silent empty export; the opt-out (introduced earlier in this same PR, with no other caller) was removed
|
||||
- **Fixed during review** (@Sameer6305, round 2): `YAMLSchemaExporter`'s usable-schema check could treat scalar schema metadata (`version`, `uri`, `title`, `description`) as evidence records had been exported, letting records under an unread key drop silently; and `_is_record()` accepted modules and class/type objects through the generic `__dict__` path, which would have reached exporter internals instead of failing at the boundary. Both closed, with regression coverage
|
||||
- **Fixed during final maintainer review** (before merge): four more gaps in the shared boundary that the earlier rounds didn't reach
|
||||
- `LPGExporter`/`ArangoAQLExporter` called `normalize_graph_payload()` with no type guard, so non-mapping input raised `ValidationError` from inside the resolver — while YAML and `Neo4jCSVExporter` raised `ProcessingError` for the identical mistake, per this PR's own stated contract. The `_require_mapping()` guard that already existed in `yaml_exporter.py` is now shared from `utils/helpers.py` and used by all three
|
||||
- `Neo4jCSVExporter._normalize_graph` checked `isinstance(graph, dict)`, so a non-dict `Mapping` (`MappingProxyType`, `ChainMap`) fell through to the object-attribute branch and was rejected, even though the identical payload exported fine via `LPGExporter`/`ArangoAQLExporter`/YAML. Now checks `isinstance(graph, Mapping)`
|
||||
- `normalize_graph_payload()` accepts dataclass and attribute-bearing object records (`Neo4jCSVExporter._record_to_dict` reads them), but `LPGExporter`/`ArangoAQLExporter` call `.get(...)` directly on resolved entities — an object-shaped record passed validation only to crash with a raw `AttributeError` once used, the exact failure this PR's boundary exists to prevent. Records are now converted to plain dicts at the boundary (`_coerce_records` → new `_record_to_dict`), so every consumer gets a uniform shape regardless of which reading the caller used
|
||||
- Two non-empty spellings of the same collection (e.g. `entities` and `nodes`) holding identical records in a different order were rejected as conflicting, since the check used plain list equality; a caller round-tripping through a dict-keyed cache or a set has no reason to preserve order. Comparison is now an order-independent multiset of each record's canonical JSON form
|
||||
- New regression coverage in `tests/utils/test_normalize_graph_payload.py`: exception-type parity for non-mapping input across `export_lpg`/`export_arango`/`export_neo4j_csv`, dataclass-record conversion verified end-to-end through the same three exporters, `Neo4jCSVExporter` accepting a `MappingProxyType` payload, and reordered-alias equality (plus a duplicate-count case confirming the multiset check still catches real conflicts); 4 existing tests updated to assert the corrected dict-conversion behavior instead of the previous object passthrough
|
||||
- `pytest tests/export tests/utils tests/context tests/test_export_module.py tests/test_export_methods_wrapper.py tests/test_notebooks_simulation.py`: 718 passed, 4 skipped (up from 641 passed, 62 subtests at PR submission); `black`/`isort`/`flake8 --max-line-length=88` clean on every line this PR touches; `python -m build`: succeeds
|
||||
|
||||
- **`ContextGraph.add_edge` had no dedupe — identical edges were stored repeatedly under one shared edge ID, and re-ingest doubled the edge set** (#926, closes #922) by @pravit-amp
|
||||
- `_add_internal_edge` appended to `self.edges`, `edge_type_index`, and `_adjacency` unconditionally, with no check for an edge already present. Edge identity is content-derived (`_resolve_edge_identity` builds `edge_id` from `source_id`/`target_id`/`edge_type`/`weight`/`metadata`/`valid_from`/`valid_until`), so two identical `add_edge` calls produced two edge objects sharing one `edge_id` — the graph already considered them the same edge, it just kept both copies. `self.nodes` already deduped by ID; edges did not, so `stats()["edge_count"]` inflated, `density()` could exceed its mathematical maximum of `1.0`, and a refresh/restore job calling `build_from_entities_and_relationships()` (or reloading a saved graph) doubled the edge set on every cycle
|
||||
- Added an `edge_id -> ContextEdge` index (`_edge_index`), mirroring how `self.nodes` dedupes by node ID. `_add_internal_edge` now returns `False` when the `edge_id` already exists, checked before touching `edges`/`edge_type_index`/`_adjacency` and before firing the mutation callback, so a repeat `add_edge` is a silent no-op with no phantom `ADD_EDGE` audit event
|
||||
- Genuinely parallel edges are unaffected: differing type/weight/metadata/validity still produce distinct content-derived `edge_id`s, so multigraph semantics are preserved
|
||||
- Both state-reset paths (`load_from_file()` and `clear()`) also clear `_edge_index`
|
||||
- New tests: repeat `add_edge` is a no-op, parallel edges with distinct attributes are preserved, re-ingest via `build_from_entities_and_relationships()` stays at one edge, and `clear()` resets the dedupe index
|
||||
- `pytest tests/context/test_context.py`: 31 passed
|
||||
|
||||
- **`POST /api/enrich/extract` returned 503 on every request; the whole `/api/decisions*` family returned 500 as soon as a decision existed** (#886, closes #883, closes #884, closes #889) by @joseedson18jc, reviewed by @Sameer6305
|
||||
- `semantica/explorer/routes/enrich.py` imported `extract_entities`/`extract_relations` from `semantica.semantic_extract.methods`, names that module never defined (only per-strategy variants like `extract_entities_ml` exist) — the `except ImportError` handler reported this as `"semantic_extract module not available"`, masking a wiring bug as a missing dependency. The route now calls `NamedEntityRecognizer`/`RelationExtractor` directly and forwards extracted entities into relation extraction instead of re-deriving them
|
||||
- `ContextGraph.record_decision()` stores `timestamp` as `datetime.now().timestamp()` (a float), while `DecisionResponse.timestamp` was typed `Optional[str]`; passing the value through unconverted failed pydantic validation on every decision route (`/api/decisions`, `/{id}`, `/{id}/chain`, `/{id}/precedents`, `/{id}/compliance`). Added a `field_validator(mode="before")` on `DecisionResponse` normalizing float/int/datetime inputs to ISO-8601
|
||||
- Folds in the fix for #889: `extract_entities_ml`/`extract_relations_similarity`/`extract_relations_dependency` called `spacy.load()` on every invocation (~120ms of a ~132ms call, ~60x the actual extraction work). Added a process-level, lock-guarded `load_spacy_model()` cache in `semantic_extract/methods.py`, keyed by model name; failed loads are not cached, and the separate `get_nlp_model()` cache (different `disable=` pipeline config for similarity work) is kept independent to avoid handing one caller's spaCy pipeline to another
|
||||
- **Fixed during review** (@Sameer6305): capped previously-unbounded input text on `/api/enrich/extract`; tightened the route's exception handling
|
||||
- **Fixed during review** (@KaifAhmad1): the timestamp validator's `math.isfinite()` guard only rejected NaN/inf — a finite-but-out-of-range epoch (e.g. milliseconds mistakenly stored instead of seconds, such as `1723600000000`) still raised an uncaught `OverflowError`/`OSError` from `datetime.fromtimestamp()`, reintroducing an unhandled 500 on `/api/decisions*` for exactly the class of bug this PR closes. Now caught and re-raised as a `ValueError`. Also excluded `bool` from the numeric branch (`isinstance(True, int)` is `True` in Python, so `timestamp=True` was silently coerced to epoch 1 instead of being rejected)
|
||||
- New/updated tests: `tests/explorer/test_explorer_api.py` (`TestRecordedDecisions`, extraction coverage, 4 new `TestDecisionResponseTimestampValidator` cases for the range/bool fixes), `tests/semantic_extract/test_spacy_model_cache.py` (6 tests)
|
||||
- `pytest tests/explorer tests/semantic_extract/test_spacy_model_cache.py`: 266 passed
|
||||
|
||||
- **Explorer UI hid backend failures: graph load hung forever, landing page always showed "System Online"** (#980, closes #977) by @ZohaibHassan16, reviewed by @Sameer6305
|
||||
- `GraphWorkspace.tsx` only destructured `{ data, isLoading, isFetching }` from `useLoadGraph()`, ignoring the `isError`/`error`/`refetch` that `useQuery` (`retry: 0`) already returned. Combined with `GraphLoadingOverlay` having no error prop and `showLoadingOverlay` staying true whenever `loadingProgress` held a stale frame, a backend-down or failed fetch left the graph workspace stuck on the last progress frame indefinitely, with no error message and no way to recover short of a full page reload
|
||||
- `GraphLoadingOverlay` now accepts `error`/`onRetry` and renders an error card with the real fetch error message and a Retry button (`refetch()`) instead of the stuck progress UI
|
||||
@@ -112,8 +223,86 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- New regression coverage in `tests/export/test_distance_exporter.py`: warnings fire on exception for all four helpers, exported sentinel values/shape stay unchanged, and the legitimate "no KG backend" `None` path still logs nothing
|
||||
- Full `tests/export/` suite: 71 passed
|
||||
|
||||
- **`explain_violations` rendered hardcoded placeholders (`min_count=1`, `max_count=1`) instead of the SHACL shape's real constraint values, and misused the violation message text as the datatype/class value** (#1094) by @cxzg007
|
||||
- `_run_pyshacl` never read `sh:minCount`/`sh:maxCount`/`sh:datatype`/`sh:class` back from the violation's `sh:sourceShape`, so every plain-English explanation was wrong regardless of what the shape actually declared. `SHACLViolation` now carries those four fields (also exposed via `to_dict()`), populated by back-referencing `sh:sourceShape`; `explain_violations` renders the real values, falling back to `"?"` when a value is genuinely absent
|
||||
- **Known limitation**: `sh:qualifiedMinCount`/`sh:qualifiedMaxCount` are not handled yet and still fall back to the `"?"` placeholder
|
||||
- New regression tests cover both the rendering path and the `sh:sourceShape` back-reference (skipped when `pyshacl`/`rdflib` are absent)
|
||||
|
||||
- **Entity merging silently dropped `entity_id` aliases, and exact-match entity resolution had three correctness gaps** (#1086, #1026) by @T1mn
|
||||
- `entity_merger.py`/`merge_strategy.py`/`entity_resolver.py` used inconsistent logic for extracting an entity's id across the merge path, so a merged entity could lose the `entity_id` aliases that let later lookups find it under its old identity. A new `semantica/utils/entity_ids.py` unifies id extraction across all three call sites
|
||||
- `EntityResolver`'s exact-match path is now honored rather than silently falling through to fuzzy matching in some cases; entities with no identifier are preserved instead of being dropped, and blank exact-match names are ignored rather than matching every other blank name
|
||||
- New/expanded coverage in `tests/kg/test_entity_pipeline.py` and `tests/kg/test_entity_resolver_exact.py`
|
||||
|
||||
- **`flatten_dict()` silently collided keys when a flattened path from one branch matched a literal key already present at the target depth** (#1062) by @shahzaib-ahmadcs
|
||||
- Two differently-shaped inputs could flatten to the same output key, with the second write silently overwriting the first — no error, no warning, just a dropped value. Collisions are now detected and handled explicitly instead of overwriting
|
||||
|
||||
- **`ExcelParser.__init__` raised `NameError` on every instantiation — `get_progress_tracker()` was called but never imported** (#1016, closes #1014) by @pravit-amp
|
||||
- Same defect as the one fixed for `SimilarityCalculator` in #530, this time in `semantica/parse/excel_parser.py`; the existing test imported the class but never constructed it, so nothing caught the missing import. Added construction coverage for every parser exported from `semantica.parse`, driven off `__all__` so future additions are covered automatically, living outside `test_parse_comprehensive.py` (whose `setUp` mocks `get_progress_tracker` into each module and would mock away the exact interaction under test)
|
||||
|
||||
- **Graph analytics (`centrality_calculator.py`, `community_detector.py`, `connectivity_analyzer.py`) dropped isolated nodes and diverged on how each computed its working view of the graph** (#1011) by @T1mn
|
||||
- Each analyzer had its own ad hoc logic for building the node/edge set it operated over, and none of them included nodes with no edges — a node with zero connections simply vanished from centrality scores, community assignments, and connectivity reports instead of appearing with a zero/singleton value. A new shared `semantica/kg/_graph_view.py` centralizes graph-view construction (including node fallbacks and community payload shaping) for all three analyzers, which are now ~250 lines lighter combined
|
||||
- New `tests/kg/test_analytics_node_scope.py` covering isolated-node presence across all three analyzers
|
||||
|
||||
- **Explorer fired temporal-bounds and snapshot requests before the graph itself had loaded, tripling failed requests when the backend was down and leaving the timeline scrubber with nothing to scrub** (#1003) by @lakshayxi
|
||||
- Two new predicate functions gate the temporal effects on the graph having actually loaded (an empty graph still counts as loaded); confirmed against a downed backend that this cuts three failing requests per page load down to one
|
||||
|
||||
- **`SeedDataManager.load_from_database()` never actually reached the database, and connection failures were mislabeled as a missing optional dependency** (#995, closes #973) by @yzxcj797
|
||||
- `DBIngestor.execute_query`/`export_table` need the connection string as their first positional argument; `load_from_database()` only passed it into the constructor's config dict, which those methods never read, so every call raised `TypeError` before connecting. Also split the combined `except (ImportError, OSError)` handling apart — a genuine connection failure was reported as `"module not available"`, sending debugging in the wrong direction; `OSError` now propagates as an actual failure, chained via `from e`
|
||||
|
||||
- **SPARQL `CONSTRUCT` detection matched inside a leading `#`-comment, misclassifying `SELECT`/`ASK` queries as `CONSTRUCT` across all four SPARQL backends** (#951) by @pravit-amp
|
||||
- `CONSTRUCT_QUERY_RE` skipped comments with a bare `\#[^\n]*`, whose backtracking `*` let a `# CONSTRUCT ...` comment line "swallow" the real query-form keyword on the next line for a query like `# CONSTRUCT ...\nSELECT ...`. The mistaken `CONSTRUCT` classification sent `Accept: text/turtle` and tried to parse a SELECT/ASK response body as Turtle, failing with a misleading parse error. The regex now requires a comment to reach a line terminator (LF or CR, per the SPARQL grammar) before matching
|
||||
|
||||
- **`k_shortest_paths` mutated caller-visible graph state during traversal and ignored direction when excluding already-used edges** (#1000) by @T1mn
|
||||
- `semantica/kg/path_finder.py`'s search left side effects behind after returning, and edge exclusion during Yen's-algorithm-style path removal didn't respect the traversal direction of directed graphs, letting a later search see edges that should have been available. Both fixed; new coverage in `tests/kg/test_path_finder.py`
|
||||
|
||||
- **`trace_decision_causality()` ignored explicitly recorded causal edges, inferring causes only from shared NER entities plus timestamp ordering** (#983) by @hsd2514
|
||||
- A `CAUSED`/`INFLUENCED`/`PRECEDENT_FOR` edge added via `add_causal_relationship()` had no effect on the trace — when entity extraction found nothing in common between two decisions, `trace_decision_chain()` came back empty even with an explicit edge stored in the graph. Explicit causal edges are now traversed first as ground truth, with entity/timestamp inference kept as an additive fallback for pairs with no explicit link; edges whose source has no decision record (e.g. a graph restored via `from_dict`) are skipped so a stale edge can't abort the trace
|
||||
|
||||
- **`RepoIngestor`'s module-level DNS resolve cache had no lock, raising `RuntimeError: OrderedDict mutated during iteration` under concurrent `ingest_repository()` calls** (#979) by @manjunathbhaskar
|
||||
- `_REPO_HOST_RESOLVE_CACHE` is a shared `OrderedDict` read, written, and pruned by every thread with no synchronization — reliably reproduced with 32 threads hammering resolution under a low TTL and small cache cap. Now guarded by a lock
|
||||
|
||||
- **`GraphBuilder` didn't remap relationship endpoints after entity resolution merged nodes, leaving relationships pointing at ids that no longer existed in the resolved graph** (#978) by @T1mn
|
||||
- New coverage in `tests/kg/test_graph_builder_external.py`; a follow-up commit hardens the remapping against edge cases found during review
|
||||
|
||||
- **Explorer's dev server esbuild target didn't match the browser targets the production build declares**, occasionally producing dev-only syntax errors on older browsers (#966) by @le-czs
|
||||
- `explorer/vite.config.ts` now sets the dev esbuild target explicitly to match
|
||||
|
||||
- **`normalize`'s number normalizer accepted currency symbols without validating them against the surrounding text, and an earlier fix's currency-code matching wasn't token-bounded** (#940) by @Mr-Neutr0n, reviewed by @ZohaibHassan16
|
||||
- Symbol currencies are now validated before being accepted; currency codes are matched on token boundaries so a code embedded inside a longer token no longer false-positives
|
||||
|
||||
- **`ContextGraph.to_dict()` was the one reader on the class that didn't hold `self._lock`, raising `RuntimeError: dictionary changed size during iteration` under a concurrent writer and risking a torn snapshot otherwise** (#929) by @pravit-amp
|
||||
- Every other reader (`stats()`, `density()`, `find_nodes()`, `find_edges()`, `get_neighbors()`, `get_nodes_by_label()`, `state_at()`, `save_to_file()`) already took the lock after it was introduced; `to_dict()` predated that change and was missed. `save_to_file()` was safe only incidentally, since it builds its payload inline under its own lock rather than delegating to `to_dict()`
|
||||
|
||||
- **`PipelineWithProvenance` had a broken import and no working `run()` method** (#862) by @Karunasagar12
|
||||
- `from .pipeline import Pipeline` failed because `Pipeline` lives in `pipeline_builder.py`, not a nonexistent `pipeline.py` — fixed to `from .pipeline_builder import Pipeline`. The class also had no `run()`; it now delegates to `ExecutionEngine.execute_pipeline()`, the intended execution path for a built `Pipeline`. The constructor now accepts a built `Pipeline` instance directly
|
||||
|
||||
### Security
|
||||
|
||||
- **Tarball restore path traversal, latent SQL injection, DNS-rebinding TOCTOU in the shared SSRF guard, stored XSS in report generation, and unvalidated SPARQL object IRIs in AnzoStore** (#1079) by @KaifAhmad1
|
||||
- `semantica backup restore`'s tar extraction (`cli.py`) stripped only the literal `semantica-backup/` prefix and called `tar.extract()` with no path-containment check, no symlink/hardlink validation, and (on Python <3.12) no extraction filter — a crafted archive member (`../../<file>`, or a symlink pointing outside the restore root) could write arbitrary files above the restore directory. Every member is now validated for resolved-path containment before extraction, symlink/hardlink targets are rejected both lexically (absolute path, `..` segments) and by resolution, and `filter="data"` is applied on Python ≥3.12
|
||||
- `DataExporter.export_table_data()` (`db_ingestor.py`) was missing the `text` import from `sqlalchemy` — a `NameError` that made the method non-functional, but latently: the query it built from raw f-string interpolation of `table_name`/`schema`/`where`/`order_by` was already injectable, so fixing the import alone (without also fixing the injection) would have silently armed it. Both are fixed together: the import is restored, `table_name`/`schema` are now validated against a strict identifier allowlist, and `where`/`order_by` are checked against a blocklist (statement separators, comments, UNION, DDL/DML keywords, time-based blind-injection primitives, schema-enumeration terms). This is a blocklist, not a grammar — it closes the concrete UNION-exfiltration path and common injection primitives, but a boolean-blind subquery using none of the blocked keywords could still get through; `where`/`order_by` must be treated as trusted/operator input, not exposed to untrusted end users, and the docstrings now say so explicitly
|
||||
- `request_with_ssrf_guard()` (`ssrf.py`) validated a hostname's resolved IPs, then let the underlying HTTP client re-resolve the same hostname independently at connect time — a low-TTL or DNS-rebinding answer could differ between the two lookups, so a hostname that validated as public could still connect to a private/internal address. Ported the IP-pinning pattern already used by `explorer/routes/ontology.py`'s `_make_pinned_session` into the shared ingest guard: the one resolution that decides accept/reject is now also the one the connection is pinned to, via a custom `HTTPAdapter` that presents the real hostname over TLS SNI / Host header while connecting only to the validated IPs. Also closes the RFC 6598 Carrier-Grade NAT gap noted as a known limitation in #905/#868: `100.64.0.0/10` is now in `BLOCKED_NETWORKS`
|
||||
- `ReportGenerator._generate_html()` (`export/report_generator.py`) f-string-interpolated report title/summary/metrics into HTML with no escaping — an ingested entity or document whose content flowed into a report (e.g. `<img src=x onerror=...>`) executed as stored XSS when the report was opened. All interpolated values are now `html.escape()`d
|
||||
- `AnzoStore._format_object_for_sparql()` (`triplet_store/anzo_store.py`) validated the subject/predicate of a triplet via `sparql_escaping.validate_uri()` before interpolating them into a SPARQL `INSERT DATA` clause, but delegated the **object** position to a separate formatter that wrapped it as `<{obj}>` without the same validation — an object value containing `>`/`}`/`{`/`"` could close the intended `<...>` token early and inject additional SPARQL Update operations. The Blazegraph/RDF4J backends were hardened for the equivalent gap previously; Anzo's object position now goes through the same `validate_uri()` check
|
||||
- Also hardened in the same pass: Apache AGE's `create_index()` `index_type` parameter is now allowlisted (was interpolated raw into a `USING` clause); Neo4j's `limit` is now explicitly validated (raises `ValidationError` for non-integer input instead of falling through to a generic `ProcessingError`); the `ffprobe` metadata-extraction subprocess call is guarded against a filename starting with `-` being parsed as an option; the MCP server no longer echoes raw exception text to JSON-RPC clients, logging full details server-side and returning a generic message plus the exception class name instead
|
||||
- **Fixed during review** (@KaifAhmad1): the SSRF IP-pinning change introduced a connection-pool leak of its own — `requests.Session.mount()` silently drops whatever adapter it replaces without closing it, so a multi-hop redirect chain on a reused session leaked one pooled connection per hop. Pinned adapters are now tagged and explicitly closed before being replaced, both per-hop and on final restore
|
||||
- **Fixed during review** (@KaifAhmad1): mounting a pinned adapter and setting a Host header on a caller-supplied `Session` is not inherently thread-safe — two guarded calls sharing the same session from different threads could interleave their mount/restore cycles. Added a per-session lock (`_get_session_lock`) so concurrent guarded calls on the same session now serialize instead of racing; verified with a two-thread test showing correct serialization and zero cross-contamination of per-request Host headers
|
||||
- **Fixed during automated PR review** (Qodo): `export_table_data()`'s new identifier/fragment validation raised `ValidationError` from inside a `try` whose blanket `except Exception` re-wrapped it as `ProcessingError`, masking the distinction between "bad input" and "the export itself failed" that callers rely on elsewhere in this module. Added the `except ValidationError: raise` guard already used by its sibling methods
|
||||
- **Fixed during automated PR review** (Qodo): on a hop where IP pinning doesn't apply (`allow_private_ips=True`), `_apply_connection_pin()` unconditionally popped the session's `Host` header instead of restoring whatever it was before pinning touched it — a caller-supplied session carrying its own legitimate `Host` override (e.g. fronting a private endpoint under a different name) had that override silently dropped for the in-flight request, only reappearing afterward via the outer `finally` restore. It now restores the session's own pre-call header state (set back if present, popped only if it was truly absent) instead of always popping
|
||||
- **Fixed during automated PR review** (Qodo): the `where`/`order_by` blocklist matched keywords/punctuation inside properly quoted string literals and identifiers too, so legitimate data like `status = 'union'` or `name = 'a--b'` was rejected as if it were SQL syntax. The blocklist now runs against a copy with quoted-literal contents masked out (`_mask_sql_literals`) — a malformed/unterminated quote sequence doesn't match the masking pattern and is left fully exposed to the blocklist, so this closes false positives without opening a masking-based bypass; the fragment actually used in the query is unchanged
|
||||
- Re-ran each finding's proof-of-concept (or an equivalent adversarial test) against the fix and confirmed it is blocked: tar path/symlink traversal (both lexical and resolved-path forms), SQL UNION exfiltration and identifier breakout, DNS-rebinding TOCTOU (including under a configured `HTTP_PROXY`, which the pinning adapter also rejects outright since a proxy would resolve DNS itself), stored XSS, and the AnzoStore SPARQL injection
|
||||
- `pytest tests/ingest/`: 266 passed, 2 skipped (10 pre-existing failures unrelated to this change — identical failure set confirmed on unmodified `main`); full regression sweep across `graph_store`, `export`, `triplet_store`, `parse`, and backup/restore: 313 passed
|
||||
|
||||
- **`Authorization`/`Proxy-Authorization` credentials could leak to a different origin across HTTP redirects, and several ingest paths bypassed the shared SSRF/redirect guard entirely** (#1067, closes #947) by @Sameer6305, reviewed by @KaifAhmad1
|
||||
- `request_with_ssrf_guard()` previously only stripped sensitive headers from per-request `kwargs["headers"]` on a cross-origin redirect; session-level `Authorization`/`Proxy-Authorization` headers, `session.auth`, and `session.trust_env` (`.netrc` lookup) could all still resurrect credentials on the hop to a foreign origin. All five credential sources are now stripped case-insensitively, kept stripped for the remainder of a multi-hop redirect chain (no resurrection even if a later hop returns to the original host), and unconditionally restored via `finally` — including on exceptions and redirect-limit errors
|
||||
- `MCPClient._send_request_http()` and `PublicAPIIngestor.detect_public_api()`/`ingest_public_api()` called `httpx.post()`/`requests.post()`/`session.request()` directly, bypassing `request_with_ssrf_guard()` entirely. Both now route through the shared guard, including when `validate_no_auth=False`
|
||||
- `SeedDataManager.load_from_api()` mutated the caller-supplied `headers` dict in place when adding an API-key `Authorization` header, silently leaking the key back into a dict the caller might reuse elsewhere. Now copies before modifying
|
||||
- **Fixed during review** (@KaifAhmad1): `allow_private_ips=True` (used to let MCP servers run on localhost/internal networks) was applied to every redirect hop, not just the operator-configured host — a compromised or malicious MCP server could 302-redirect to an internal address (e.g. `169.254.169.254` cloud metadata) and the guard would follow it unchecked, defeating the SSRF protection this PR otherwise adds. Added `allow_private_ips_on_redirect` to `request_with_ssrf_guard()`: a redirect target inherits the original host's private-IP trust only when it matches that host; any other host falls back to strict validation. `MCPClient` now pins `allow_private_ips_on_redirect=False`, so only same-host redirects on a trusted MCP server keep working — a cross-host hop into private address space is blocked
|
||||
- **Fixed during review** (@KaifAhmad1): `detect_public_api()` only caught `requests.exceptions.RequestException`, but `request_with_ssrf_guard()` raises `ValidationError` (a disjoint hierarchy) for SSRF-blocked hosts, blocked redirect targets, missing `Location`, or exceeded redirect limits — unlike its sibling `ingest_public_api()`, which already caught it. Callers (including `is_public_api()`) got an undocumented raw `ValidationError` instead of `ProcessingError`, and the error-logging call was skipped. Now catches `(ValidationError, ProcessingError)` and re-raises, matching the sibling method
|
||||
- **Fixed during review** (@KaifAhmad1): `detect_public_api()`/`ingest_public_api()` forwarded `**options` into `request_with_ssrf_guard(..., session=self.session, allow_private_ips=self.allow_private_ips, **request_options)` without stripping `session`/`allow_private_ips` from `request_options` first — a caller passing either through the per-call `**options` (a plausible mistake, since `allow_private_ips` is also a documented constructor-level knob) got a raw `TypeError: got multiple values for keyword argument`. Both are now popped from `request_options` before the call
|
||||
- New regression coverage added during review: `TestAllowPrivateIpsOnRedirect` (cross-host redirect into private space blocked, same-host redirect trust preserved, default behavior unchanged for existing callers that don't pass the new kwarg) and `TestMCPClientAuthRedirect::test_redirect_to_private_ip_is_blocked`/`test_same_host_redirect_on_private_mcp_server_is_not_blocked` in `tests/ingest/test_auth_header_redirect_security.py`; `test_detect_public_api_propagates_ssrf_validation_error` and duplicate-kwarg regression tests for both methods in `tests/ingest/test_public_api_ingestor.py`
|
||||
- `pytest tests/ingest/test_auth_header_redirect_security.py tests/ingest/test_public_api_ingestor.py tests/test_seed_manager.py tests/ingest/test_submodules.py tests/ingest/test_cookbook_integration.py`: 111 passed
|
||||
|
||||
- **`FeedIngestor`/`FeedMonitor` (RSS/Atom feed ingestion) had no SSRF protection, allowing requests to internal/private network targets** (#928, closes #927) by @ZohaibHassan16
|
||||
- `FeedIngestor.ingest_feed()`, `discover_feeds()` (link-tag fetch, common-path HEAD probe, and feed-validation GET), and `FeedMonitor.check_updates()` all called `requests.get()`/`requests.head()` directly with default redirect-following and no scheme allowlist or private/loopback/link-local IP validation — despite `semantica/ingest/ssrf.py`'s `request_with_ssrf_guard()` already existing and being used by `web_ingestor.py`/`api_ingestor.py`. `ingest_feed()`'s own URL check only verified `urlparse(url).scheme`/`.netloc` were non-empty, never that the scheme was http/https or that the resolved target IP was safe. Reachable via the public `ingest_feed()`/`ingest()` entry points with any caller-supplied feed URL
|
||||
- All 5 call sites now route through `request_with_ssrf_guard()`, which validates scheme (http/https only) and resolved IP before the request, and re-validates every redirect `Location` before following it — closing both the direct-IP and redirect-chain SSRF paths. Added an `allow_private_ips` config option to both `FeedIngestor` and `FeedMonitor`, consistent with the other ingestors
|
||||
@@ -147,6 +336,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- **Caught by the new gate on its first run**: `python -m pip install -e ".[all]"` pulled in `setuptools==79.0.1`, vulnerable to CVE-2026-59890/GHSA-h35f-9h28-mq5c/PYSEC-2026-3447 (Unicode-normalization bypass of `MANIFEST.in` exclude/prune patterns on macOS APFS/HFS+, letting excluded files leak into a built sdist), fixed in `83.0.0`. `[build-system] requires` had the exact same too-permissive-floor pattern this whole entry is about (`setuptools>=61.0`), and `actions/setup-python`'s baked-in `setuptools` isn't governed by that pin at all since it's outside any isolated build. Bumped `[build-system] requires` to `setuptools>=83.0.0`, and the `Security` workflow now runs `pip install --upgrade pip setuptools` before auditing so the scanned environment can't have a stale ambient copy regardless of what governs it
|
||||
- Full `explorer` suite: 241 passed
|
||||
|
||||
- **`SeedDataManager.load_from_api()` made unguarded HTTP requests, with no SSRF protection at all** (#942) by @ZohaibHassan16
|
||||
- `load_from_api()` called `requests.get()` directly instead of going through `semantica/ingest/ssrf.py`'s `request_with_ssrf_guard()`, unlike every other ingestor in this module — a caller-supplied `api_url` could target internal/private network addresses with no validation. Now routes through the shared guard, gaining redirect validation and bounded DNS resolution for free
|
||||
- **Follow-up** (#959, closes #943) by @yunaremaia: added an `allow_private_ips` opt-in (parsed via the shared `parse_bool` helper) for trusted internal deployments that legitimately need to load from a private-network API, while keeping the guard's block-by-default behavior for everyone else
|
||||
|
||||
## [0.6.5] - 2026-08-11
|
||||
|
||||
### Added
|
||||
@@ -207,8 +400,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Entities and relationships round-trip as memory-local provenance only — Markdown import intentionally does not write into `ContextGraph`, matching the MVP scope agreed on in #765
|
||||
- Documented the file contract and workflow in `docs/reference/context.md`; 43 new tests in `tests/context/test_agent_memory_markdown.py` cover round-trip losslessness, idempotency, validation errors, rollback on failure, and vector-store sync ordering
|
||||
|
||||
- **Markdown directory round trips for `ContextGraph`** (#852) by @SaurabhScripts
|
||||
- `ContextGraph.save_to_file(..., format="markdown")` and `load_from_file(..., format="markdown")` persist a deterministic `graph.md` relationship manifest plus one human-editable Markdown file per node, preserving graph, node, edge, family, temporal, and cross-graph link identities
|
||||
- Imports validate the complete directory before replacing graph state, rebuild indexes and analytics state atomically, create JSON-compatible stub nodes for dangling edge endpoints, and emit the same granular node/edge audit events as JSON loading
|
||||
- Existing exports are replaced atomically only after their complete canonical layout is validated; untracked files, renamed node files, symlinks, Windows directory junctions, and other reparse points cause a fail-closed error instead of authorizing directory deletion
|
||||
- Added 30 focused tests covering deterministic round trips, manual edits, validation rollback, managed-directory identity, publish rollback, audit-manager compatibility, stale-cache clearing, mocked and real Windows junctions, and missing-path behavior
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Markdown import followed filesystem links even though Markdown export already refused to overwrite them** (#851, follow-up to #765, #786) by @SaurabhScripts
|
||||
- `AgentMemory._read_markdown_path()` now rejects symlink files, broken symlinks, symlinked directories, Windows directory junctions, and other Windows reparse points supplied directly; linked entries discovered inside an otherwise valid directory are safely skipped, preserving the current directory-import contract
|
||||
- `_read_markdown_file_content()` re-checks the file and parent directory immediately before and after opening, uses `O_NOFOLLOW` where available, and verifies the resulting descriptor is a regular file via `fstat`/`S_ISREG`, so link swaps are rejected rather than silently followed
|
||||
- Junction detection uses `os.path.isjunction()` where available and falls back to the Windows reparse-point file attribute on older Python versions; export applies the same link check before replacing a Markdown file
|
||||
- Documented the import restriction in `docs/reference/context.md`; added 11 tests to `tests/context/test_agent_memory_markdown.py` covering file/directory/broken-symlink rejection, simulated open races, mocked and real Windows junctions, and the reparse-point fallback
|
||||
- Any additional review follow-up commits land in this same PR/entry rather than as a separate changelog item
|
||||
|
||||
- **`PipelineWithProvenance` raised `ModuleNotFoundError` on import and `AttributeError` on `.run()`** (#858, closes #858) by @Karunasagar12
|
||||
- `from .pipeline import Pipeline` failed because `semantica/pipeline/pipeline.py` does not exist; corrected to `from .pipeline_builder import Pipeline`
|
||||
- `.run()` called `self._pipeline.run()` on the `Pipeline` dataclass, which has no such method; replaced with `self._engine.execute_pipeline(self._pipeline, ...)` delegating to `ExecutionEngine`
|
||||
@@ -1325,4 +1531,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
---
|
||||
|
||||
For detailed release notes, see [GitHub Releases](https://github.com/Hawksight-AI/semantica/releases).
|
||||
For detailed release notes, see [GitHub Releases](https://github.com/semantica-agi/semantica/releases).
|
||||
|
||||
+1
-1
@@ -58,7 +58,7 @@ representative at an online or offline event.
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported to the community leaders responsible for enforcement through
|
||||
[GitHub Issues](https://github.com/Hawksight-AI/semantica/issues) with "[CoC]" prefix.
|
||||
[GitHub Issues](https://github.com/semantica-agi/semantica/issues) with "[CoC]" prefix.
|
||||
All complaints will be reviewed and investigated promptly and fairly.
|
||||
|
||||
All community leaders are obligated to respect the privacy and security of the
|
||||
|
||||
+17
-3
@@ -25,9 +25,9 @@ If you want to work on an open GitHub issue, please follow these steps to keep t
|
||||
|
||||
1. **Check the issue.** Look at the issue's assignees and recent comments. If someone is already actively working on it, consider a different issue or ask in the comments whether help is welcome.
|
||||
|
||||
2. **Comment before you start.** Leave a comment on the issue saying you'd like to work on it — something like *"I'd like to take this on"* is enough. This gives maintainers the context they need to assign the issue appropriately.
|
||||
2. **Comment if you'd like the issue reserved.** Leaving a comment like *"I'd like to take this on"* is the fastest way to get assigned, but it isn't required — maintainers can also assign an issue directly to a contributor (e.g., based on recent activity in the repo) without waiting for a comment first.
|
||||
|
||||
3. **Wait for assignment.** A maintainer will review the request and assign the issue when appropriate. Please wait for this before investing significant time in implementation, as priorities and approaches can shift.
|
||||
3. **Wait for assignment.** A maintainer will assign the issue when appropriate, whether or not a comment was left. Please wait for this before investing significant time in implementation, as priorities and approaches can shift.
|
||||
|
||||
4. **Create a branch and implement.** Once assigned, fork the repository (if you haven't already), create a dedicated branch, and begin your work.
|
||||
|
||||
@@ -37,12 +37,26 @@ If you want to work on an open GitHub issue, please follow these steps to keep t
|
||||
|
||||
5. **Open a focused PR and link the issue.** When you're ready, open a pull request and reference the issue in the description (e.g., `Closes #123`). Keep the PR scoped to the work described in the issue.
|
||||
|
||||
> **Why this matters:** Commenting before opening a PR helps maintainers track who is working on what, assign issues correctly, and prevent two contributors from solving the same problem independently. It also gives you a chance to align on the expected approach before writing code.
|
||||
> **Why this matters:** Assignment (with or without a comment) helps maintainers track who is working on what and prevent two contributors from solving the same problem independently. It also gives you a chance to align on the expected approach before writing code.
|
||||
|
||||
Not sure where to start? Try a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue) or ask in [Discord](https://discord.gg/sV34vps5hH).
|
||||
|
||||
---
|
||||
|
||||
## 🔀 Duplicate PRs & Issue Priority
|
||||
|
||||
When more than one pull request targets the same issue, maintainers triage using this order of priority. These rules decide between PRs that are otherwise following the [assignment workflow above](#-working-on-an-existing-issue) — opening a PR before being assigned doesn't grant priority on its own, and an unassigned PR can still be closed as a duplicate once someone else is assigned to the issue.
|
||||
|
||||
1. **Contributor-raised issue with an existing PR.** If the person who opened the issue has also opened a PR for it, that PR is prioritized (they still need to be assigned before it's merged).
|
||||
2. **Maintainer-raised issue with a claim comment.** If we opened the issue and someone has commented asking to work on it, we assign it to them and check their PR before picking up any other PR for the same issue.
|
||||
3. **No prior assignment or comment.** If multiple PRs exist and no one was assigned or claimed the issue first, priority goes to whichever contributor has the most consistent activity in the repo over the last 60 days (e.g., merged PRs, substantive reviews, or issue triage participation) — not just PR volume.
|
||||
4. **Late duplicate PRs.** If a PR is opened after another contributor has already been assigned to the issue, we close the duplicate early rather than let it sit open, and point the author to another open issue (or ask them to check `main` for newly opened ones). This avoids contributors spending time updating a PR that won't be merged.
|
||||
5. **Overlapping scope.** If a PR covers multiple issues, or there's genuine overlap between competing PRs, maintainers discuss it on [Discord](https://discord.gg/sV34vps5hH) before deciding rather than resolving it unilaterally.
|
||||
|
||||
**Why this matters:** it keeps triage predictable, avoids wasted contributor effort on PRs that won't merge, and helps retain active contributors.
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Ways to Contribute
|
||||
|
||||
### 💻 Code
|
||||
|
||||
+4
-4
@@ -44,7 +44,7 @@ We recognize all types of contributions:
|
||||
All contributors are recognized in:
|
||||
|
||||
- This contributors list
|
||||
- [GitHub contributors page](https://github.com/Hawksight-AI/semantica/graphs/contributors)
|
||||
- [GitHub contributors page](https://github.com/semantica-agi/semantica/graphs/contributors)
|
||||
- Release notes for significant contributions
|
||||
- Community appreciation
|
||||
|
||||
@@ -54,7 +54,7 @@ All contributors are recognized in:
|
||||
|
||||
### Automatic Recognition
|
||||
|
||||
If you've made a commit, you'll automatically appear in [GitHub's contributors graph](https://github.com/Hawksight-AI/semantica/graphs/contributors).
|
||||
If you've made a commit, you'll automatically appear in [GitHub's contributors graph](https://github.com/semantica-agi/semantica/graphs/contributors).
|
||||
|
||||
### Using All-Contributors Bot
|
||||
|
||||
@@ -101,7 +101,7 @@ When using the all-contributors bot, use these codes:
|
||||
- `infra` - Infrastructure
|
||||
- `maintenance` - Maintenance
|
||||
|
||||
See [all-contributors specification](https://allcontributors.org/docs/en/emoji-key) for complete list.
|
||||
See [all-contributors specification](https://github.com/all-contributors/all-contributors#emoji-key) for complete list.
|
||||
|
||||
---
|
||||
|
||||
@@ -111,4 +111,4 @@ Every contribution, no matter how small, helps make Semantica better. Thank you
|
||||
|
||||
**Want to contribute?**
|
||||
|
||||
⭐ Give us a Star • 🍴 [Fork us](https://github.com/Hawksight-AI/semantica/fork) • Check out our [Contributing Guide](CONTRIBUTING.md) to get started!
|
||||
⭐ Give us a Star • 🍴 [Fork us](https://github.com/semantica-agi/semantica/fork) • Check out our [Contributing Guide](CONTRIBUTING.md) to get started!
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ RUN npm ci
|
||||
COPY explorer/ ./
|
||||
RUN mkdir -p /app/semantica && npm run build
|
||||
|
||||
FROM python:3.14-slim AS runtime
|
||||
FROM python:3.13-slim AS runtime
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Hawksight AI
|
||||
Copyright (c) 2026 Semantica
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
recursive-include semantica/static *
|
||||
recursive-include semantica/ontology/vocabulary *.ttl
|
||||
|
||||
@@ -2,7 +2,15 @@
|
||||
|
||||
<img src="Semantica Logo.png" alt="Semantica" width="420"/>
|
||||
|
||||
<a href="https://trendshift.io/repositories/18986?utm_source=repository-badge&utm_medium=badge&utm_campaign=badge-repository-18986" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/18986" alt="semantica-agi%2Fsemantica | Trendshift" width="250" height="55"/></a>
|
||||
<div style="display:flex; gap:10px; align-items:center; flex-wrap:wrap;">
|
||||
<a href="https://trendshift.io/repositories/18986?utm_source=repository-badge&utm_medium=badge&utm_campaign=badge-repository-18986" target="_blank" rel="noopener noreferrer">
|
||||
<img src="https://trendshift.io/api/badge/repositories/18986" alt="semantica-agi/semantica | Trendshift" width="250" height="55"/>
|
||||
</a>
|
||||
|
||||
<a href="https://trendshift.io/repositories/18986?utm_source=trendshift-badge&utm_medium=badge&utm_campaign=badge-trendshift-18986" target="_blank" rel="noopener noreferrer">
|
||||
<img src="https://trendshift.io/api/badge/trendshift/repositories/18986/weekly?language=Python" alt="semantica-agi/semantica | Trendshift" width="250" height="55"/>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
### Graph-Native Infrastructure for Context and Accountable AI Systems
|
||||
|
||||
@@ -52,6 +60,8 @@ Most AI agents act without a trail. They store embeddings, not meaning: context
|
||||
|
||||
Semantica sits underneath your LLM, vector store, and agent framework as a deterministic infrastructure layer: no LLM required for graph construction, reasoning, or provenance.
|
||||
|
||||
> ⚠️ **System-level explainability, not foundation-model explainability.** Semantica does not expose or reconstruct what happens *inside* the LLM — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. Semantica explains what's *outside* the model: the context and data fed in, the decision produced, its provenance, relevant relationships, applied policies, and the full execution trail.
|
||||
|
||||
**Who it's for:**
|
||||
|
||||
- **AI/ML platform teams** shipping agents that make consequential decisions and need structured, queryable context built from fragmented raw data, not just a vector index
|
||||
@@ -77,7 +87,7 @@ Semantica sits underneath your LLM, vector store, and agent framework as a deter
|
||||
- **Graph Analytics:** Centrality, community detection, link prediction, and shortest-path queries over the graph you just built
|
||||
- **Polyglot Graph Storage:** Native RDF (embedded Oxigraph, Blazegraph, Apache Jena, Eclipse RDF4J via SPARQL) and Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune via Cypher), plus vector stores, all swappable without touching your code
|
||||
- **Visualization:** Explore any graph, ontology, or timeline in an interactive browser workbench
|
||||
- **Drop-in Integrations:** Native Agno support, a full-featured MCP server, a comprehensive CLI, a REST API, and plugins across major editors
|
||||
- **Drop-in Integrations:** Native Agno and CrewAI support, a full-featured MCP server, a comprehensive CLI, a REST API, and plugins across major editors
|
||||
|
||||
---
|
||||
|
||||
@@ -132,7 +142,7 @@ compliant = graph.check_decision_rules({"category": "vendor_selection"}) # poli
|
||||
```bash
|
||||
semantica doctor
|
||||
# Python 3.11.9 pass
|
||||
# semantica 0.6.5 pass
|
||||
# semantica 0.6.6 pass
|
||||
# faiss vector store pass
|
||||
# Config file pass ~/.semantica/config.yaml
|
||||
```
|
||||
@@ -293,17 +303,10 @@ graph.add_causal_relationship(d1, d2, relationship_type="CAUSED")
|
||||
prov.track_entity("patient_P4821", source="ehr/medication_orders_2024.json",
|
||||
metadata={"extractor": "NamedEntityRecognizer"})
|
||||
|
||||
# Export W3C PROV-O for regulator submission - RDFExporter expects
|
||||
# {"entities": [...], "relationships": [...]}, so map ContextGraph.to_dict()'s
|
||||
# {"nodes": [...], "edges": [...]} shape onto it first
|
||||
graph_dict = graph.to_dict()
|
||||
kg = {
|
||||
"entities": [{"id": n["id"], "type": n["type"], "text": n["content"]} for n in graph_dict["nodes"]],
|
||||
"relationships": [
|
||||
{"source_id": e["source"], "target_id": e["target"], "type": e["type"]}
|
||||
for e in graph_dict["edges"]
|
||||
],
|
||||
}
|
||||
# Export W3C PROV-O for regulator submission - to_kg_dict() is the official
|
||||
# adapter that emits the {"entities": [...], "relationships": [...]} /
|
||||
# source_id shape RDFExporter expects, so no manual field mapping is needed
|
||||
kg = graph.to_kg_dict()
|
||||
RDFExporter().export(kg, "audit_trail.ttl", format="turtle")
|
||||
```
|
||||
|
||||
@@ -877,20 +880,14 @@ fact = BiTemporalFact(
|
||||
recorded_at=datetime(2024, 3, 5),
|
||||
)
|
||||
|
||||
# Query facts valid within a time window - query_time_range() expects
|
||||
# {"relationships": [...]} with source_id/target_id keys, which differs from
|
||||
# ContextGraph.to_dict()'s {"nodes", "edges"} shape, so map it first
|
||||
graph_dict = graph.to_dict()
|
||||
kg_relationships = {
|
||||
"relationships": [
|
||||
{**e, "source_id": e["source"], "target_id": e["target"]}
|
||||
for e in graph_dict["edges"]
|
||||
]
|
||||
}
|
||||
# Query facts valid within a time window - to_kg_dict() is the official
|
||||
# adapter that emits {"entities", "relationships"} with source_id/target_id
|
||||
# keys, the shape query_time_range() expects (no manual mapping required)
|
||||
kg = graph.to_kg_dict()
|
||||
|
||||
tq = TemporalGraphQuery()
|
||||
facts_in_window = tq.query_time_range(
|
||||
kg_relationships, query="valid_facts", start_time="2024-01-01", end_time="2024-12-31"
|
||||
kg, query="valid_facts", start_time="2024-01-01", end_time="2024-12-31"
|
||||
)
|
||||
|
||||
# Normalize natural language temporal expressions - returns a (start, end) range
|
||||
@@ -1189,7 +1186,7 @@ Start with `semantica`, verify with `doctor`, build a graph, and explore the com
|
||||
|
||||
## Integrations
|
||||
|
||||
Native plugin bundles for Claude Code, Cursor, Codex, Windsurf, Cline, Continue, VS Code, and OpenClaw; a full-featured MCP server for any MCP-compatible client; a comprehensive REST API; and first-class Agno support for multi-agent shared context. Every major LLM provider is already supported via `semantica.llms` and LiteLLM: OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Azure, Bedrock, Ollama, DeepSeek, HuggingFace, and more.
|
||||
Native plugin bundles for Claude Code, Cursor, Codex, Windsurf, Cline, Continue, VS Code, and OpenClaw; a full-featured MCP server for any MCP-compatible client; a comprehensive REST API; and first-class Agno and CrewAI support for agentic frameworks. Every major LLM provider is already supported via `semantica.llms` and LiteLLM: OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Azure, Bedrock, Ollama, DeepSeek, HuggingFace, and more.
|
||||
|
||||
MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
|
||||
@@ -1303,6 +1300,11 @@ MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
<strong>Agno</strong><br/>
|
||||
<sub>First-class · <code>pip install semantica[agno]</code></sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/crewAIInc/crewAI"><img src="https://github.com/crewAIInc.png?size=120" alt="CrewAI" width="48" height="48" /></a><br/>
|
||||
<strong>CrewAI</strong><br/>
|
||||
<sub>First-class · <code>pip install semantica[crewai]</code></sub>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th colspan="8" align="left">Already Supported via REST API & MCP</th>
|
||||
@@ -1319,11 +1321,6 @@ MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
<sub>REST API · MCP</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/crewAIInc/crewAI"><img src="https://github.com/crewAIInc.png?size=120" alt="CrewAI" width="48" height="48" /></a><br/>
|
||||
<strong>CrewAI</strong><br/>
|
||||
<sub>REST API · MCP</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/run-llama/llama_index"><img src="https://github.com/run-llama.png?size=120" alt="LlamaIndex" width="48" height="48" /></a><br/>
|
||||
<strong>LlamaIndex</strong><br/>
|
||||
<sub>REST API · MCP</sub>
|
||||
@@ -1354,11 +1351,6 @@ MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
<sub>Dedicated toolkit</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/crewAIInc/crewAI"><img src="https://github.com/crewAIInc.png?size=120" alt="CrewAI" width="48" height="48" /></a><br/>
|
||||
<strong>CrewAI</strong><br/>
|
||||
<sub>Dedicated toolkit</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/run-llama/llama_index"><img src="https://github.com/run-llama.png?size=120" alt="LlamaIndex" width="48" height="48" /></a><br/>
|
||||
<strong>LlamaIndex</strong><br/>
|
||||
<sub>Dedicated toolkit</sub>
|
||||
@@ -1474,18 +1466,18 @@ For contributor / dev-server setup: **[explorer/README.md: Local Setup Guide](ex
|
||||
|
||||
---
|
||||
|
||||
## What's New in v0.6.5
|
||||
## What's New in v0.6.6
|
||||
|
||||
**Security release — upgrading is strongly recommended.** Fixes for 5 externally-reported vulnerabilities in the Explorer API and graph/triplet store backends, plus a CodeQL-flagged ReDoS:
|
||||
**Security release — upgrading is strongly recommended.** Fixes for a privately disclosed batch of vulnerabilities spanning backup/restore, database export, outbound requests, and triplet-store backends, plus SSRF hardening across ingestion:
|
||||
|
||||
- **Missing authentication on all Explorer API routes** (GHSA-j4mq-hprp-987v, Critical): every route now requires `SEMANTICA_API_KEY`, fails closed (503) rather than open when unconfigured
|
||||
- **SSRF via redirect bypass in ontology URL fetching** (GHSA-8c7v-62gr-hj6g, High): redirect targets are now re-validated at every hop and the connection is pinned to the validated address, closing a DNS check-then-use race
|
||||
- **Cypher injection via unvalidated node labels and property keys** (GHSA-482h-hw99-h62p, Critical): Neptune, Neo4j, and FalkorDB now sanitize every label/relationship-type/property-key interpolation site
|
||||
- **SPARQL injection via unvalidated triplet IRIs** (GHSA-8vgg-8mr4-r236, Critical): Blazegraph, RDF4J, and Jena now validate subject/predicate/object IRIs before interpolation
|
||||
- **Missing Origin validation on the WebSocket handshake** (GHSA-4643-wpgq-w329, Moderate, anonymous-mode only): `/ws/graph-updates` now checks `Origin` against the same allowlist `CORSMiddleware` enforces for HTTP
|
||||
- **Polynomial ReDoS in SPARQL query validation** (CodeQL `py/polynomial-redos`): fixed a backtracking regex in the Explorer's SPARQL route
|
||||
- **Tarball restore path traversal**: `semantica backup restore` now validates every archive member for path containment and rejects symlink/hardlink escapes before extraction
|
||||
- **Latent SQL injection in `DataExporter.export_table_data()`**: table/schema names are now identifier-allowlisted and `where`/`order_by` fragments are blocklist-checked
|
||||
- **DNS-rebinding TOCTOU in the shared SSRF guard**: the resolved IP that passes validation is now the one the connection is pinned to, closing the check-then-use race (also closes the `100.64.0.0/10` CGNAT gap)
|
||||
- **Stored XSS in HTML report generation** and **unvalidated SPARQL object IRIs in AnzoStore** (SPARQL injection): both now escape/validate before interpolation
|
||||
- **`Authorization`/`Proxy-Authorization` credential leakage across redirects**, plus **SSRF gaps in `FeedIngestor`/`FeedMonitor`, `RepoIngestor`, and the MCP/public-API ingest paths**: all now route through the shared, redirect-safe SSRF guard
|
||||
- **HTTP response header injection and an unbounded-memory DoS** in the Explorer API, and a **`fastapi`/`python-multipart` ReDoS** (PYSEC-2024-38): floors raised, inputs sanitized, candidate pools capped
|
||||
|
||||
Also includes: embedded Oxigraph backend for `TripletStore`, PROV-O trust/spec completeness for `ProvenanceManager`, and the Altair Anzo triplet store backend.
|
||||
Also ships: **first-class CrewAI integration** (`semantica[crewai]`, extraction/decision tools + a knowledge source), **`ContextGraph` retraction and purge** (GDPR-style erasure without a full `clear()`), a declared **Semantica RDF vocabulary with deterministic entity/relationship IRIs** (stable, diffable exports), and **timezone-aware timestamps** across `export/` and `provenance/`.
|
||||
|
||||
→ [Full release notes](RELEASE_NOTES.md) · [Changelog](CHANGELOG.md)
|
||||
|
||||
@@ -1503,6 +1495,8 @@ Semantica is designed for environments where AI outputs must be explainable, aud
|
||||
- **Cybersecurity:** Threat attribution, incident response timelines, and IOC provenance tracking
|
||||
- **Autonomous Systems:** Decision logs, safety validation, and explainable AI for certification
|
||||
|
||||
> ⚠️ **This is system-level explainability, not foundation-model explainability.** Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. What Semantica explains is *outside* the model: the context and data fed in, the decision produced, its provenance, the relevant relationships, the policies applied, and the full execution trail. In short, Semantica explains and audits what the AI system did, not the LLM's private internal reasoning.
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
@@ -1514,6 +1508,7 @@ pip install semantica[all] # everything
|
||||
|
||||
```bash
|
||||
pip install semantica[agno] # Agno multi-agent integration
|
||||
pip install semantica[crewai] # CrewAI integration
|
||||
pip install semantica[llm-litellm] # OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Bedrock, Ollama, DeepSeek, and more
|
||||
pip install semantica[graph-neo4j] # Neo4j graph store (LPG)
|
||||
pip install semantica[graph-falkordb] # FalkorDB graph store (LPG)
|
||||
@@ -1599,6 +1594,23 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for full guidelines.
|
||||
|
||||
---
|
||||
|
||||
## Cite Us
|
||||
|
||||
If you use Semantica in your research or production systems, please cite it as:
|
||||
|
||||
```bibtex
|
||||
@software{semantica2026,
|
||||
title = {Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems},
|
||||
author = {Semantica},
|
||||
year = {2026},
|
||||
url = {https://github.com/semantica-agi/semantica}
|
||||
}
|
||||
```
|
||||
|
||||
All citation formats (APA, MLA, Chicago, IEEE) live on the [Citation](https://docs.getsemantica.ai/citation) page — every format attributes authorship to **Semantica**, not individual contributors.
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
|
||||
MIT License · Built by [Semantica](https://github.com/semantica-agi)
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/01_Advanced_Extraction.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/01_Advanced_Extraction.ipynb)\n",
|
||||
"\n",
|
||||
"# Advanced Extraction\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/03_Complete_Visualization_Suite.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/03_Complete_Visualization_Suite.ipynb)\n",
|
||||
"\n",
|
||||
"# Complete Visualization Suite\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/05_Multi_Format_Export.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/05_Multi_Format_Export.ipynb)\n",
|
||||
"\n",
|
||||
"# Advanced Multi-Format Export\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/08_Reasoning_and_Inference.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/08_Reasoning_and_Inference.ipynb)\n",
|
||||
"\n",
|
||||
"# Reasoning and Inference\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/09_Semantic_Layer_Construction.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/09_Semantic_Layer_Construction.ipynb)\n",
|
||||
"\n",
|
||||
"# Semantic Layer Construction\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb)\n",
|
||||
"\n",
|
||||
"# Deep Dive: Temporal Knowledge Graphs\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/12_Unstructured_to_Ontology.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/12_Unstructured_to_Ontology.ipynb)\n",
|
||||
"\n",
|
||||
"# Unstructured Text to Ontology\n",
|
||||
"\n",
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
"id": "cell-0",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/13_Manual_Ontology_Snowflake_Mapping.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/13_Manual_Ontology_Snowflake_Mapping.ipynb)\n",
|
||||
"\n",
|
||||
"# Manual Ontology + Snowflake Mapping\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/14_Datalog_Style_Reasoning.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/14_Datalog_Style_Reasoning.ipynb)\n",
|
||||
"\n",
|
||||
"# Datalog-Style Reasoning\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb)\n",
|
||||
"\n",
|
||||
"# Advanced Vector Store - Made Easy\n",
|
||||
"\n",
|
||||
@@ -352,7 +352,7 @@
|
||||
"- Build a multi-user application\n",
|
||||
"- Explore the [introduction notebook](../introduction/13_Vector_Store.ipynb) for more basics\n",
|
||||
"\n",
|
||||
"**Need Help?** Check our [documentation](https://semantica.readthedocs.io) or ask on [GitHub](https://github.com/Hawksight-AI/semantica)."
|
||||
"**Need Help?** Check our [documentation](https://semantica.readthedocs.io) or ask on [GitHub](https://github.com/semantica-agi/semantica)."
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)\n",
|
||||
"\n",
|
||||
"Semantica is a **semantic intelligence and knowledge engineering framework**. It helps you:\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb)\n",
|
||||
"\n",
|
||||
"# Data Ingestion - Comprehensive Guide\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/04_Document_Parsing.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/03_Document_Parsing.ipynb)\n",
|
||||
"\n",
|
||||
"# Document Parsing\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/05_Data_Normalization.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/04_Data_Normalization.ipynb)\n",
|
||||
"\n",
|
||||
"# Data Normalization\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/05_Entity_Extraction.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/05_Entity_Extraction.ipynb)\n",
|
||||
"\n",
|
||||
"# Entity Extraction - Comprehensive Guide\n",
|
||||
"\n",
|
||||
@@ -622,7 +622,7 @@
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/semantica-agi/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/06_Relation_Extraction.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/06_Relation_Extraction.ipynb)\n",
|
||||
"\n",
|
||||
"# Relation Extraction - Comprehensive Guide\n",
|
||||
"\n",
|
||||
@@ -599,7 +599,7 @@
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/semantica-agi/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Building_Knowledge_Graphs.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb)\n",
|
||||
"\n",
|
||||
"# Building Knowledge Graphs\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/09_Your_First_Knowledge_Graph.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)\n",
|
||||
"\n",
|
||||
"# 🚀 Your First Knowledge Graph\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/11_Graph_Analytics.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/10_Graph_Analytics.ipynb)\n",
|
||||
"\n",
|
||||
"# Graph Analytics\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/11_Chunking_and_Splitting.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/11_Chunking_and_Splitting.ipynb)\n",
|
||||
"\n",
|
||||
"# Chunking and Splitting - Comprehensive Guide\n",
|
||||
"\n",
|
||||
@@ -817,7 +817,7 @@
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/semantica-agi/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/13_Embedding_Generation.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/12_Embedding_Generation.ipynb)\n",
|
||||
"\n",
|
||||
"# Embedding Generation\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/13_Vector_Store.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/13_Vector_Store.ipynb)\n",
|
||||
"\n",
|
||||
"# Vector Store - Comprehensive Guide\n",
|
||||
"\n",
|
||||
@@ -492,7 +492,7 @@
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
"**Questions or Issues?** Check out our [GitHub repository](https://github.com/semantica-agi/semantica) or [documentation](https://semantica.readthedocs.io)."
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/14_Ontology.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/14_Ontology.ipynb)\n",
|
||||
"\n",
|
||||
"# Ontology Generation \n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/15_Export.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/15_Export.ipynb)\n",
|
||||
"\n",
|
||||
"# Export Module - Comprehensive Guide\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/17_Visualization.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/16_Visualization.ipynb)\n",
|
||||
"\n",
|
||||
"# Visualization\n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/18_Deduplication.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/18_Deduplication.ipynb)\n",
|
||||
"\n",
|
||||
"# Deduplication in Semantica\n",
|
||||
"\n",
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
"id": "c21e9c8d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/19_Context_Module.ipynb)\n",
|
||||
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/19_Context_Module.ipynb)\n",
|
||||
"\n",
|
||||
"# Context Module — Practical Guide\n",
|
||||
"\n",
|
||||
|
||||
+9
-10
@@ -13,26 +13,25 @@ icon: "quote-left"
|
||||
<Tab title="BibTeX">
|
||||
```bibtex
|
||||
@software{semantica2026,
|
||||
title = {Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems},
|
||||
author = {Semantica},
|
||||
year = {2026},
|
||||
url = {https://github.com/semantica-agi/semantica},
|
||||
version = {0.6.5},
|
||||
doi = {10.5281/zenodo.XXXXXXX}
|
||||
title = {Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems},
|
||||
author = {Semantica},
|
||||
year = {2026},
|
||||
url = {https://github.com/semantica-agi/semantica},
|
||||
doi = {10.5281/zenodo.XXXXXXX}
|
||||
}
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="APA">
|
||||
Semantica. (2026). *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems* (Version 0.6.5) \[Computer software\]. https://github.com/semantica-agi/semantica
|
||||
Semantica. (2026). *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems* \[Computer software\]. https://github.com/semantica-agi/semantica
|
||||
</Tab>
|
||||
<Tab title="MLA">
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.5, GitHub, 2026, https://github.com/semantica-agi/semantica.
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. GitHub, 2026, https://github.com/semantica-agi/semantica.
|
||||
</Tab>
|
||||
<Tab title="Chicago">
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.5. GitHub, 2026. https://github.com/semantica-agi/semantica.
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. GitHub, 2026. https://github.com/semantica-agi/semantica.
|
||||
</Tab>
|
||||
<Tab title="IEEE">
|
||||
Semantica, "Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems," Version 0.6.5, GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica
|
||||
Semantica, "Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems," GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
|
||||
@@ -16,6 +16,9 @@ At its core, Semantica adds a **context and accountability layer** on top of you
|
||||
- **Accountability Layer** — Provenance tracking, decision intelligence, conflict detection, and W3C PROV-O compliance make every claim in your AI stack auditable and explainable.
|
||||
- **Extension Layer** — `PluginRegistry` and `MethodRegistry` let you replace or augment any component: ingestors, extractors, reasoning engines, backends: without changing framework code.
|
||||
|
||||
<Warning>
|
||||
**This is system-level explainability, not foundation-model explainability.** Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. What Semantica explains is *outside* the model: the context and data fed in, the decision produced, its provenance, the relevant relationships, the policies applied, and the full execution trail. In short, Semantica explains and audits *what the AI system did*, not the foundation model's private internal reasoning.
|
||||
</Warning>
|
||||
|
||||
## Knowledge Graphs
|
||||
|
||||
|
||||
+1
-1
@@ -80,6 +80,6 @@ Deep dive into advanced features, customization, and complex workflows.
|
||||
You can also run the cookbook using Docker:
|
||||
|
||||
```bash
|
||||
docker run -p 8888:8888 hawksight/semantica-cookbook
|
||||
docker run -p 8888:8888 semantica/semantica-cookbook
|
||||
```
|
||||
</Tip>
|
||||
|
||||
@@ -102,6 +102,7 @@
|
||||
"group": "Integrations",
|
||||
"pages": [
|
||||
"integrations/agno",
|
||||
"integrations/crewai",
|
||||
"integrations/docling",
|
||||
"integrations/snowflake",
|
||||
"integrations/databricks"
|
||||
|
||||
@@ -162,7 +162,7 @@ semantica-explorer --graph my_graph.json --no-browser
|
||||
```
|
||||
|
||||
<Warning>
|
||||
`--host 0.0.0.0` makes Explorer reachable on every network interface. The server has no built-in authentication. Only use this on a trusted private network.
|
||||
`--host 0.0.0.0` makes Explorer reachable on every network interface. Since v0.6.5 the Explorer API requires `SEMANTICA_API_KEY` (sent as the `X-API-Key` header) and fails closed with `503` when unconfigured; unauthenticated access is only possible when `SEMANTICA_ALLOW_ANONYMOUS=true` is set explicitly. Only use this on a trusted private network.
|
||||
</Warning>
|
||||
|
||||
|
||||
|
||||
+11
-1
@@ -17,7 +17,7 @@ icon: "circle-question"
|
||||
| API key required? | Optional: pattern extraction works with no keys |
|
||||
| Works with LangChain / LlamaIndex? | Yes: Semantica is a layer on top, not a replacement |
|
||||
| Production-ready? | Yes: 1,000+ tests, v0.5.0 ships with 12 security fixes |
|
||||
| Latest version? | **v0.6.5** (August 2026) |
|
||||
| Latest version? | **v0.6.6** (August 2026) |
|
||||
| Local LLMs? | Yes: Ollama via LiteLLM, HuggingFaceLLM for air-gapped |
|
||||
|
||||
|
||||
@@ -52,6 +52,16 @@ Semantica works alongside these frameworks, not against them.
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Does Semantica explain an LLM's internal reasoning or chain-of-thought?" icon="triangle-exclamation">
|
||||
|
||||
No. This is **system-level explainability, not foundation-model explainability**. Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system.
|
||||
|
||||
What Semantica explains is *outside* the model: what context and data were used, what decision was produced, the provenance behind it, the relevant relationships, the policies applied, and the resulting decision trail.
|
||||
|
||||
In short: Semantica explains and audits *what the AI system did* — not the foundation model's private internal reasoning.
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Is Semantica free?" icon="tag">
|
||||
|
||||
Yes: MIT licensed, no vendor lock-in, no paywalled features. Some capabilities require third-party API keys (e.g., OpenAI embeddings, Groq inference), but Semantica itself is always free and open source.
|
||||
|
||||
@@ -42,7 +42,7 @@ icon: "rocket"
|
||||
Verify installation:
|
||||
```python
|
||||
import semantica
|
||||
print(semantica.__version__) # 0.6.5
|
||||
print(semantica.__version__) # 0.6.6
|
||||
```
|
||||
</Check>
|
||||
</Step>
|
||||
|
||||
+2
-2
@@ -4,12 +4,12 @@ description: "Project governance model: roles, decision process, release cadence
|
||||
icon: "scale-balanced"
|
||||
---
|
||||
|
||||
> Semantica is maintained by Hawksight AI with community contributions under an open governance model.
|
||||
> Semantica is maintained by the Semantica team with community contributions under an open governance model.
|
||||
|
||||
|
||||
## Roles
|
||||
|
||||
- **Maintainers** — Hawksight AI team: review and merge PRs, manage releases and code quality, set project direction and community standards.
|
||||
- **Maintainers** — Semantica team: review and merge PRs, manage releases and code quality, set project direction and community standards.
|
||||
- **Contributors** — Submit code, documentation, and bug reports. Help with issues and reviews. Recognized in [CONTRIBUTORS.md](https://github.com/semantica-agi/semantica/blob/main/CONTRIBUTORS.md).
|
||||
- **Community Members** — Use Semantica, provide feedback, share use cases, and participate in GitHub Discussions and Discord.
|
||||
|
||||
|
||||
@@ -436,6 +436,35 @@ d = graph.to_dict()
|
||||
# d["statistics"] → {"node_count": int, "edge_count": int}
|
||||
```
|
||||
|
||||
For a human-editable, version-control-friendly representation, save a Markdown
|
||||
directory instead:
|
||||
|
||||
```python
|
||||
graph.save_to_file("context_graph/", format="markdown")
|
||||
|
||||
restored = ContextGraph(advanced_analytics=True)
|
||||
restored.load_from_file("context_graph/", format="markdown")
|
||||
```
|
||||
|
||||
The directory contains a versioned `graph.md` manifest for graph identity,
|
||||
relationships, and cross-graph link descriptors, plus one file per node under
|
||||
`nodes/`. A node's content is its Markdown body; its ID, type, properties,
|
||||
metadata, and temporal validity are YAML frontmatter. Node, edge, family, graph,
|
||||
and cross-graph link IDs are preserved across round trips.
|
||||
|
||||
Markdown loading uses replacement semantics, like `from_dict()`: it parses and
|
||||
validates the complete directory before replacing the current graph. Invalid YAML,
|
||||
duplicate IDs, unsupported versions, and unsafe filesystem links fail without
|
||||
partially mutating the graph. As with JSON loading, an edge endpoint without a node
|
||||
file creates an `entity` stub node. Symlinks, Windows directory junctions, and other
|
||||
Windows reparse points are rejected.
|
||||
|
||||
Re-exporting to an existing managed directory atomically replaces it, removing stale
|
||||
node files. Before replacement, Semantica validates the complete canonical export
|
||||
layout, not just the manifest header. Untracked files, assets, extra directories, or
|
||||
renamed node files therefore cause the export to fail closed instead of being deleted.
|
||||
Keep attachments and hand-written indexes outside the managed export directory.
|
||||
|
||||
If the graph had cross-graph links created with `link_graph()`, call `resolve_links()` after loading to restore live navigation — object references cannot be serialized, so they must be reconnected manually:
|
||||
|
||||
```python
|
||||
|
||||
@@ -269,7 +269,7 @@ print("Loaded {} facts from graph".format(count))
|
||||
|
||||
## Step 5 — SPARQL queries over enriched working memory
|
||||
|
||||
After forward chaining has derived new facts, `SPARQLReasoner` lets you query the enriched working memory using SPARQL triple-pattern matching with optional inference expansion:
|
||||
After forward chaining has derived new facts, `SPARQLReasoner` prepares SPARQL queries over the enriched working memory with optional inference expansion:
|
||||
|
||||
```python
|
||||
from semantica.reasoning import SPARQLReasoner
|
||||
@@ -288,22 +288,13 @@ query = """
|
||||
}
|
||||
"""
|
||||
|
||||
# execute_query() runs: expansion → inference → deduplication
|
||||
result = sparql.execute_query(query)
|
||||
|
||||
for binding in result.bindings:
|
||||
print("Actor: {:15s} CVE: {}".format(
|
||||
binding.get("actor", "?"),
|
||||
binding.get("cve", "?"),
|
||||
))
|
||||
|
||||
# metadata shows how many results came from inference vs ground facts
|
||||
print("Original: {} Inferred: {}".format(
|
||||
result.metadata.get("original_count", 0),
|
||||
result.metadata.get("inferred_count", 0),
|
||||
))
|
||||
# expand_query() applies inference rules to the query text:
|
||||
expanded = sparql.expand_query(query)
|
||||
print(expanded)
|
||||
```
|
||||
|
||||
`execute_query()` is not implemented yet: no triplet-store execution path exists, so it raises `NotImplementedError` rather than returning an empty result set that callers would misread as "no matches". Until execution lands, run the expanded query against your RDF store directly (for example with `rdflib`).
|
||||
|
||||
Inspect the expanded query before running it:
|
||||
|
||||
```python
|
||||
|
||||
@@ -8,7 +8,7 @@ icon: "shield-check"
|
||||
|
||||
SHACL (Shapes Constraint Language) is a standard for validating graph-based data. While an ontology defines the conceptual *schema* (the "what" exists in your domain), SHACL defines the structural *rules and constraints* (the "how" it should be structured).
|
||||
|
||||
In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and `_run_pyshacl` evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated.
|
||||
In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and the public `run_shacl_validation` function evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated. The historical `_run_pyshacl` name remains available as a compatibility alias.
|
||||
|
||||
## Why Use SHACL Validation?
|
||||
|
||||
@@ -55,7 +55,7 @@ Let's look at a simple, universally understood example: ensuring every `Employee
|
||||
```python
|
||||
from semantica.context import ContextGraph
|
||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
# 1. Prepare your data graph
|
||||
graph = ContextGraph()
|
||||
@@ -95,7 +95,7 @@ data_ttl = """
|
||||
"""
|
||||
|
||||
# 5. Run Validation
|
||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
||||
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||
|
||||
# 6. Analyze the Report
|
||||
print(f"Graph conforms: {report.conforms}")
|
||||
@@ -265,10 +265,10 @@ cve_id_shape = NodeShape(
|
||||
|
||||
## Step 4 — Run validation and read the report
|
||||
|
||||
Serialize the graph to RDF, then run `_run_pyshacl` against the shapes.
|
||||
Serialize the graph to RDF, then run `run_shacl_validation` against the shapes.
|
||||
|
||||
```python
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
# Prepare your RDF data string (since export_rdf primarily exports structural metadata,
|
||||
# you typically serialize your custom data graph to Turtle using rdflib or similar).
|
||||
@@ -281,7 +281,7 @@ data_ttl = """
|
||||
"""
|
||||
|
||||
# Run SHACL validation
|
||||
report = _run_pyshacl(
|
||||
report = run_shacl_validation(
|
||||
data_ttl,
|
||||
shacl_ttl,
|
||||
data_graph_format="turtle",
|
||||
@@ -366,8 +366,8 @@ print(f"Malware nodes missing 'family': {len(missing_family)}")
|
||||
# e.g. graph.update_node(node_id, {"family": "UNKNOWN — requires triage"})
|
||||
|
||||
# After remediation, re-run validation to confirm the fix
|
||||
# (re-export the patched graph to Turtle first, then call _run_pyshacl again)
|
||||
report2 = _run_pyshacl(patched_data_ttl, shacl_ttl)
|
||||
# (re-export the patched graph to Turtle first, then call run_shacl_validation again)
|
||||
report2 = run_shacl_validation(patched_data_ttl, shacl_ttl)
|
||||
print(f"Violations after remediation: {report2.violation_count}")
|
||||
# Violations after remediation: 0
|
||||
```
|
||||
@@ -377,10 +377,49 @@ print(f"Violations after remediation: {report2.violation_count}")
|
||||
## Common Pitfalls
|
||||
|
||||
- **Assuming the ontology automatically enforces data quality**: `SHACLGenerator` generates shapes based on what it observes in the data. If your data is missing a field, the generator won't know it was mandatory unless you explicitly inject the constraint (as shown in Step 3).
|
||||
- **Passing `ContextGraph` directly to SHACL validators**: The `_run_pyshacl` function expects an RDF string (like Turtle format), not a raw Python dictionary or `ContextGraph` object.
|
||||
- **Passing `ContextGraph` directly to SHACL validators**: The `run_shacl_validation` function expects an RDF string (like Turtle format), not a raw Python dictionary or `ContextGraph` object.
|
||||
- **Forgetting RDF serialization**: You must serialize your graph (often via a temporary file using `export_rdf`) before validating it.
|
||||
- **Treating validation as a one-time step**: Validation should be integrated as an automated step in your CI/CD pipeline or data ingestion flow, acting as a recurring gatekeeper rather than a one-off script.
|
||||
- **Ignoring validation reports**: A graph that does not conform must be remediated. Failing to review the `violation_count` and address the issues negates the purpose of SHACL validation.
|
||||
- **Validating `sh:class`/`sh:node` range checks on a property that declares `rdfs:range` with RDFS entailment on**: RDFS is an entailment rule, not a constraint. When pyshacl runs with `inference="rdfs"`, it infers the range class onto every object of the property, so class-based constraints on that property can never fail — the report says `conforms: True` on data that does not conform:
|
||||
|
||||
```python
|
||||
from pyshacl import validate
|
||||
from rdflib import Graph
|
||||
|
||||
data = Graph()
|
||||
data.parse(
|
||||
data="""
|
||||
@prefix ex: <https://example.org/ns#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
ex:contains rdfs:domain ex:Container ; rdfs:range ex:Item .
|
||||
ex:box a ex:Container ; ex:contains ex:notAnItem .
|
||||
ex:notAnItem a ex:Fish .
|
||||
""",
|
||||
format="turtle",
|
||||
)
|
||||
|
||||
shapes = Graph()
|
||||
shapes.parse(
|
||||
data="""
|
||||
@prefix ex: <https://example.org/ns#> .
|
||||
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
||||
ex:ContainerShape a sh:NodeShape ;
|
||||
sh:targetClass ex:Container ;
|
||||
sh:property [ sh:path ex:contains ; sh:class ex:Item ] .
|
||||
""",
|
||||
format="turtle",
|
||||
)
|
||||
|
||||
for inference in ("none", "rdfs"):
|
||||
conforms, _, _ = validate(data, shacl_graph=shapes, inference=inference)
|
||||
print(inference, conforms)
|
||||
# none False <- correct: notAnItem is a Fish, not an Item
|
||||
# rdfs True <- the entailment manufactured the type
|
||||
```
|
||||
|
||||
Mitigations: prefer not to declare `rdfs:range` on properties you intend to constrain with `sh:class`; when class membership is the thing under test, run validation without RDFS entailment (`inference="none"`); or express the check as a constraint the entailment cannot satisfy (for example a literal property constraint). Note the trade-off: with entailment off, `sh:targetClass` no longer reaches subclasses, so subclass hierarchies need explicit typing or inference-aware target selection. Semantica's own `run_shacl_validation` wrapper already calls pyshacl with `inference="none"`, so this pitfall only bites when calling `pyshacl.validate` directly with entailment enabled.
|
||||
- **Trusting `conforms: True` without checking the inference mode**: an inference-enabled run can hide the exact violations the shapes were written to catch (see above). Record which inference mode validation ran under alongside the result, and re-run shape sets that contain `sh:class`/`sh:node` with entailment off before treating a pass as authoritative.
|
||||
|
||||
---
|
||||
|
||||
@@ -396,7 +435,7 @@ A DoD CTI team enforces STIX-compatible constraints on a threat graph before sha
|
||||
from semantica.context import AgentContext, ContextGraph
|
||||
from semantica.vector_store import VectorStore
|
||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
graph = ContextGraph()
|
||||
ctx = AgentContext(
|
||||
@@ -448,7 +487,7 @@ data_ttl = """
|
||||
<http://example.org/hammertoss> a ex:Malware .
|
||||
"""
|
||||
|
||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
||||
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||
print(f"CTI graph conforms : {report.conforms}")
|
||||
print(f"Violations : {report.violation_count}")
|
||||
print(f"Warnings : {report.warning_count}")
|
||||
@@ -469,7 +508,7 @@ A SOC team validates zero-trust policy nodes before publishing them to the polic
|
||||
```python
|
||||
from semantica.context import ContextGraph
|
||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
graph = ContextGraph()
|
||||
graph.add_node("policy-001", "Policy", "MFA Required for Tier-1 Resources",
|
||||
@@ -516,7 +555,7 @@ data_ttl = """
|
||||
<http://example.org/policy-002> a ex:Policy .
|
||||
"""
|
||||
|
||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
||||
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||
print(f"Policy graph conforms: {report.conforms}")
|
||||
# Policy graph conforms: False
|
||||
|
||||
@@ -534,7 +573,7 @@ A clinical informatics team validates trial ontology nodes before loading them i
|
||||
|
||||
```python
|
||||
from semantica.ontology import LLMOntologyGenerator, SHACLGenerator, PropertyShape
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
from semantica.export import export_rdf
|
||||
import tempfile, os
|
||||
|
||||
@@ -586,7 +625,7 @@ with open(tmp.name) as f:
|
||||
data_ttl = f.read()
|
||||
os.unlink(tmp.name)
|
||||
|
||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
||||
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||
print(f"Trial data conforms: {report.conforms}")
|
||||
print(f"Warnings : {report.warning_count}")
|
||||
```
|
||||
@@ -600,7 +639,7 @@ A credit risk team validates every `LoanApplication` node against Basel III CRE2
|
||||
```python
|
||||
from semantica.context import ContextGraph
|
||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
graph = ContextGraph()
|
||||
graph.add_node("loan-001", "LoanApplication", "Prime mortgage APP-2025-88421",
|
||||
@@ -645,7 +684,7 @@ data_ttl = """
|
||||
ex:ltv "0.65" .
|
||||
"""
|
||||
|
||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
||||
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||
print(f"Loan portfolio conforms: {report.conforms}")
|
||||
# Loan portfolio conforms: False
|
||||
|
||||
@@ -675,14 +714,14 @@ Call this function as a pre-publish gate; exit code 1 blocks the pipeline.
|
||||
```python
|
||||
import sys
|
||||
from semantica.ontology import OntologyGenerator, SHACLGenerator
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
def validate_before_publish(data_graph_str: str, ontology: dict) -> None:
|
||||
shacl_gen = SHACLGenerator(base_uri="https://example.org/shapes/")
|
||||
shacl_graph = shacl_gen.generate(ontology)
|
||||
shacl_ttl = shacl_gen.serialize(shacl_graph, format="turtle")
|
||||
|
||||
report = _run_pyshacl(data_graph_str, shacl_ttl)
|
||||
report = run_shacl_validation(data_graph_str, shacl_ttl)
|
||||
|
||||
if not report.conforms:
|
||||
print(f"Graph validation FAILED — {report.violation_count} violation(s)")
|
||||
@@ -700,7 +739,6 @@ def validate_before_publish(data_graph_str: str, ontology: dict) -> None:
|
||||
|
||||
- [Ontology Management](ontology) — generate the OWL ontology that SHACL shapes are derived from
|
||||
- [Reasoning & Rules](reasoning) — complement SHACL structural constraints with logical inference rules
|
||||
- [Export & Serialization](export) — serialize graph data to Turtle/RDF/XML for `_run_pyshacl` input
|
||||
- [Export & Serialization](export) — serialize graph data to Turtle/RDF/XML for `run_shacl_validation` input
|
||||
- [Conflict Resolution](conflict-resolution) — detect and resolve data conflicts before SHACL validation
|
||||
- [Change Management](change-management) — version-gate SHACL shapes alongside ontology versions
|
||||
|
||||
|
||||
+5
-1
@@ -192,7 +192,11 @@ decision_id = context.record_decision(
|
||||
|
||||
## Built for Where Mistakes Have Consequences
|
||||
|
||||
Semantica was designed for domains where every decision must be explainable and every fact must be traceable:
|
||||
Semantica was designed for domains where every decision must be explainable and every fact must be traceable.
|
||||
|
||||
<Warning>
|
||||
**This is system-level explainability, not foundation-model explainability.** Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. What Semantica explains is *outside* the model: the context and data fed in, the decision produced, its provenance, the relevant relationships, the policies applied, and the full execution trail. See [Core Concepts](concepts) for the full scope note.
|
||||
</Warning>
|
||||
|
||||
**Healthcare & Life Sciences**
|
||||
- Clinical decision support with full audit trails
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
---
|
||||
title: "CrewAI Integration"
|
||||
description: "Give CrewAI crews a shared semantic knowledge graph, decision intelligence, and graph-based retrieval via three drop-in components."
|
||||
icon: "users"
|
||||
---
|
||||
|
||||
> Three drop-in components that bring Semantica's knowledge graph and decision intelligence into any CrewAI crew.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install "semantica[crewai]"
|
||||
```
|
||||
|
||||
Requires `crewai >= 0.80.0`. If `crewai` is not installed, the integration still imports — every class carries the full Semantica API and degrades gracefully, but cannot be passed to a `Crew`.
|
||||
|
||||
## Components at a Glance
|
||||
|
||||
- **SemanticaKGTool** — `Agent(tools=[…])`: 5 KG construction/query actions: extract entities, extract relations, add to graph, query graph, find related.
|
||||
- **SemanticaDecisionTool** — `Agent(tools=[…])`: 5 decision intelligence actions: record decisions, find precedents, trace causal chains, analyze impact, check policies.
|
||||
- **SemanticaKnowledgeSource** — `Crew(knowledge_sources=[…])`: Serializes a `ContextGraph` into CrewAI knowledge storage so every agent gets retrieval access to the graph.
|
||||
|
||||
## Component Details
|
||||
|
||||
<Tabs>
|
||||
<Tab title="SemanticaKGTool">
|
||||
Lets agents actively **build and query** a shared `ContextGraph` mid-reasoning.
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKGTool
|
||||
|
||||
graph = ContextGraph()
|
||||
|
||||
analyst = Agent(
|
||||
role="Knowledge Analyst",
|
||||
goal="Build and explore a knowledge graph from documents",
|
||||
backstory="You map entities and relationships into a shared graph.",
|
||||
tools=[SemanticaKGTool(graph=graph)],
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[analyst],
|
||||
tasks=[Task(
|
||||
description="Extract and link key entities from the brief",
|
||||
expected_output="JSON",
|
||||
agent=analyst,
|
||||
)],
|
||||
)
|
||||
crew.kickoff()
|
||||
```
|
||||
|
||||
| Tool | Description |
|
||||
| :------ | :------------- |
|
||||
| `extract_entities` | Extract named entities from `text` |
|
||||
| `extract_relations` | Extract relationships between entities in `text` |
|
||||
| `add_to_graph` | Extract entities/relations from `text` and add them to the shared graph |
|
||||
| `query_graph` | Keyword-search the graph by node id, type, and content using `query` |
|
||||
| `find_related` | Find concepts related to `entity` within `hops` hops |
|
||||
|
||||
All actions return JSON so agents get parseable results.
|
||||
|
||||
**Sharing a graph:** the tool reads/writes whatever `graph` you pass in. When no `graph` is given, a fresh in-memory `ContextGraph()` is created (and a warning is logged) — two tool instances that each auto-create their own graph do **not** share knowledge. Pass the same `ContextGraph` to every agent that must share state.
|
||||
</Tab>
|
||||
<Tab title="SemanticaDecisionTool">
|
||||
Exposes Semantica's decision intelligence as a native CrewAI tool, backed by `AgentContext`.
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from integrations.crewai import SemanticaDecisionTool
|
||||
|
||||
planner = Agent(
|
||||
role="Decision Planner",
|
||||
goal="Make grounded, precedented decisions",
|
||||
backstory="You record decisions and validate them against policy.",
|
||||
tools=[SemanticaDecisionTool()],
|
||||
)
|
||||
|
||||
crew = Crew(agents=[planner], tasks=[...])
|
||||
```
|
||||
|
||||
When no `AgentContext` is passed, one is created in-memory with `decision_tracking=True` and its own `ContextGraph`, so decision actions work out of the box (a warning is logged — pass the same `AgentContext` to every agent that must share decision state). Missing optional fields in `record_decision` fall back to `category="general"`, `reasoning="agent decision"`, and `outcome="recorded"`. `find_precedents` returns up to `max_precedents` results. If a knowledge graph cannot trace causality, `trace_causal_chain` returns an explicit error rather than substituting similarity-based results.
|
||||
|
||||
| Tool | Description |
|
||||
| :------ | :------------- |
|
||||
| `record_decision` | Record a decision with reasoning, outcome, and confidence |
|
||||
| `find_precedents` | Search for similar past decisions |
|
||||
| `trace_causal_chain` | Trace the causal chain from a decision |
|
||||
| `analyze_impact` | Assess downstream influence of a decision |
|
||||
| `check_policy` | Validate a proposed decision against policy rules |
|
||||
</Tab>
|
||||
<Tab title="SemanticaKnowledgeSource">
|
||||
Gives **every agent in the crew** retrieval access to a `ContextGraph`.
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKnowledgeSource
|
||||
|
||||
graph = ContextGraph()
|
||||
graph.add_node(node_id="privacy", node_type="policy", content="...")
|
||||
|
||||
researcher = Agent(
|
||||
role="Policy Researcher",
|
||||
goal="Answer questions from the knowledge base",
|
||||
backstory="You retrieve from graph knowledge to answer accurately.",
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[researcher],
|
||||
tasks=[...],
|
||||
knowledge_sources=[SemanticaKnowledgeSource(graph=graph)],
|
||||
)
|
||||
```
|
||||
|
||||
On kickoff the graph's nodes and edges are serialized, chunked, and stored through CrewAI's knowledge pipeline.
|
||||
|
||||
> **Embedder required:** storing chunks goes through CrewAI's knowledge pipeline, which needs an embedder to be configured. Set `Crew(embedder=...)` (or provide the default credentials CrewAI falls back to, e.g. `OPENAI_API_KEY`). If no working embedder is configured, storage fails, an ERROR is logged, and agents will retrieve **nothing** — the crew still runs, but its knowledge queries return empty.
|
||||
|
||||
**Compatibility:** CrewAI's `BaseKnowledgeSource` contract changed between `0.80.x` and current releases (`load_content()` → `validate_content()`/`aadd()`). `SemanticaKnowledgeSource` implements both legacy and current methods, so it works across `crewai>=0.80.0`.
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
## Checkpoints & Serialization
|
||||
|
||||
CrewAI serializes tools and knowledge sources to JSON for checkpointing/resume. Live Semantica state (`ContextGraph`, `AgentContext`, extractors) is **excluded from that serialization** — a restored tool/source comes back with a fresh in-memory `ContextGraph` and logs a warning. Until you re-attach the live graph/context, the restored objects answer queries against an **empty** graph, so re-wire them after resuming (e.g. `restored_tool.graph = live_graph`) before agents continue.
|
||||
|
||||
## API Reference
|
||||
|
||||
```python
|
||||
from integrations.crewai import (
|
||||
SemanticaKGTool, # BaseTool: KG construction/query actions
|
||||
SemanticaDecisionTool, # BaseTool: decision intelligence actions
|
||||
SemanticaKnowledgeSource, # BaseKnowledgeSource: graph → crew knowledge
|
||||
CREWAI_AVAILABLE, # bool: True if crewai is installed
|
||||
)
|
||||
```
|
||||
|
||||
All three classes are usable without `crewai` installed: they carry the full Semantica API and degrade gracefully.
|
||||
|
||||
## See Also
|
||||
|
||||
- [Context Module](../reference/context) — AgentContext and ContextGraph backing the integration.
|
||||
- [Semantic Extraction](../reference/semantic_extract) — NERExtractor / RelationExtractor used by SemanticaKGTool.
|
||||
- [LLMs](../reference/llms) — Configure LLM providers for your crew's agents.
|
||||
- [Vector Store](../reference/vector_store) — Vector backend used by SemanticaDecisionTool.
|
||||
@@ -12,7 +12,7 @@ icon: "file-contract"
|
||||
```
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Hawksight AI
|
||||
Copyright (c) 2026 Semantica
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -435,8 +435,8 @@ print("Nodes: {}, Edges: {}".format(stats["node_count"], stats["edge_count"]))
|
||||
| `query(query, skip, limit)` | `List[Dict]` | Full-text search over node content |
|
||||
| `stats()` | `Dict` | Node/edge counts, type breakdowns, graph density |
|
||||
| `density()` | `float` | Graph density score |
|
||||
| `save_to_file(path)` | `None` | Persist graph to JSON |
|
||||
| `load_from_file(path)` | `None` | Load graph from JSON |
|
||||
| `save_to_file(path, format="json")` | `None` | Persist graph as JSON or a Markdown directory |
|
||||
| `load_from_file(path, format="json")` | `None` | Replace graph state from JSON or a Markdown directory |
|
||||
| `build_from_conversations(conversations, link_entities)` | `Dict` | Build graph from conversation data |
|
||||
| `link_graph(other_graph, source_node_id, target_node_id, link_type)` | `str` | Create cross-graph navigation link; returns `link_id` |
|
||||
| `navigate_to(link_id)` | `Tuple` | Follow a cross-graph link to `(target_graph, target_node_id)` |
|
||||
@@ -625,8 +625,10 @@ malformed or duplicate fields before changing memory, and re-importing unchanged
|
||||
files is idempotent. Memory-local `entities` and `relationships` are preserved as
|
||||
provenance but are not applied to `ContextGraph` by Markdown import. Use a dedicated
|
||||
export directory: matching files are overwritten, but unrelated or stale Markdown
|
||||
files are not deleted automatically. Export refuses to overwrite symbolic links and
|
||||
uses atomic file replacement. Timestamp offsets are preserved in Markdown and
|
||||
files are not deleted automatically. Export refuses to overwrite filesystem links and
|
||||
uses atomic file replacement; import also refuses symlinks, Windows directory
|
||||
junctions, and other Windows reparse points.
|
||||
Timestamp offsets are preserved in Markdown and
|
||||
normalized to UTC only for comparisons, so aware and local-naive records can be
|
||||
queried together safely. Vector-store writes are deferred until the in-memory import
|
||||
commits; adapter synchronization remains best-effort and logs failures.
|
||||
|
||||
@@ -203,6 +203,13 @@ export_lpg(graph, "import.cypher", method="cypher")
|
||||
exporter = SemanticNetworkYAMLExporter()
|
||||
exporter.export(graph, "graph.yaml")
|
||||
```
|
||||
|
||||
The YAML exporters read `entities`/`relationships`/`triplets` (with
|
||||
`nodes`/`edges` accepted as aliases, so `ContextGraph.to_dict()` exports
|
||||
directly). A non-empty mapping supplying none of them raises
|
||||
`ValidationError` rather than writing a file with every collection empty,
|
||||
as does one whose collection value is not a list of records
|
||||
(`{"entities": "abc"}`).
|
||||
</Tab>
|
||||
<Tab title="Graph DB Import">
|
||||
**LPGExporter** writes Cypher `CREATE` statements for Neo4j and Memgraph:
|
||||
@@ -236,6 +243,12 @@ export_lpg(graph, "import.cypher", method="cypher")
|
||||
|
||||
Both exporters write to a file and return `None`.
|
||||
|
||||
`LPGExporter`, `ArangoAQLExporter`, and `Neo4jCSVExporter` resolve mapping
|
||||
payloads on the same terms as the YAML exporters above, so an unrecognized
|
||||
or malformed mapping is rejected instead of exported as an empty graph.
|
||||
`Neo4jCSVExporter` still reads graph *objects* off their
|
||||
`nodes`/`entities` and `edges`/`relationships` attributes.
|
||||
|
||||
<Warning>
|
||||
**`ArangoAQLExporter.export()` and `LPGExporter.export()` write to a file and return `None`.** They do not return the AQL/Cypher string. Write to a file and read it back if you need the string.
|
||||
</Warning>
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
# Graph storage backends and feature matrix
|
||||
|
||||
Semantica separates graph modeling from physical storage. LPG backends are accessed through `graph_store` adapters; RDF backends are accessed through `triplet_store` adapters.
|
||||
|
||||
This page is intentionally conservative: it distinguishes between an adapter existing, a feature being generally available with that model, and a backend needing user-supplied wiring.
|
||||
|
||||
## Status labels
|
||||
|
||||
- `built-in`: adapter implementation exists in Semantica core.
|
||||
- `tested`: covered by automated integration fixtures or tests.
|
||||
- `example-only`: usable example exists, but support is not asserted by integration tests.
|
||||
- `interface/BYO`: interface or integration point exists; bring your own backend wiring.
|
||||
|
||||
## Adapter inventory
|
||||
|
||||
| Backend | Model | Adapter | Status | Reference |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Neo4j | LPG | `semantica.graph_store.Neo4jStore` | built-in | `cookbook/introduction/09_Graph_Store.ipynb` |
|
||||
| FalkorDB | LPG | `semantica.graph_store.FalkorDBStore` | built-in | `docs/reference/graph_store.md` |
|
||||
| Amazon Neptune | LPG | `semantica.graph_store.AmazonNeptuneStore` | built-in | `cookbook/introduction/21_Amazon_Neptune_Store.ipynb` |
|
||||
| Apache AGE | LPG | `semantica.graph_store.ApacheAgeStore` | built-in | `docs/graph_stores/apache_age.md` |
|
||||
| RDF4J | RDF | `semantica.triplet_store.RDF4JStore` | built-in | `cookbook/introduction/20_Triplet_Store.ipynb` |
|
||||
| Apache Jena | RDF | `semantica.triplet_store.JenaStore` | built-in | `cookbook/introduction/20_Triplet_Store.ipynb` |
|
||||
| Blazegraph | RDF | `semantica.triplet_store.BlazegraphStore` | built-in | `cookbook/introduction/20_Triplet_Store.ipynb` |
|
||||
| Anzo | RDF | `semantica.triplet_store.AnzoStore` | built-in | `cookbook/introduction/20_Triplet_Store.ipynb` |
|
||||
| Oxigraph | RDF | `semantica.triplet_store.OxigraphStore` | built-in | `docs/reference/triplet_store.md` |
|
||||
|
||||
## Feature matrix
|
||||
|
||||
`Yes` means the capability is expected to work with the adapter and graph model. `Partial` means the capability works with model-specific constraints. `BYO` means the user must supply or validate wiring for the backend.
|
||||
|
||||
| Backend | Model | Ingestion | Context graph construction | Reasoning/analytics | Provenance | Known limitations |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Neo4j | LPG | Yes | Yes | Yes | Partial | Provenance and context metadata are stored as node and edge properties; relationship properties and stable node identifiers are required. |
|
||||
| FalkorDB | LPG | Yes | Yes | Partial | Partial | Redis-based; provenance depends on node/edge properties, and multi-graph isolation depends on the selected graph name. |
|
||||
| Amazon Neptune | LPG | Yes | Yes | Partial | Partial | Use the property-graph endpoint; AWS auth, VPC, and endpoint configuration can affect local tests. Provenance depends on node/edge properties. |
|
||||
| Apache AGE | LPG | Yes | Yes | Partial | Partial | Runs through PostgreSQL/AGE; Cypher compatibility and property handling can differ from standalone LPG engines. |
|
||||
| RDF4J | RDF | Yes | Partial | Partial | Partial | Context separation relies on named graphs; triple-level provenance may require reification or graph-level metadata. `RDF4JStore(repository_id=...)` currently has no effect — the constructor always connects to the `"default"` repository regardless of the value passed; track a fix separately. |
|
||||
| Apache Jena | RDF | Yes | Partial | Partial | Partial | Named graphs are needed for context separation; backend configuration and transaction behavior matter. |
|
||||
| Blazegraph | RDF | Yes | Partial | Partial | Partial | Use quads/named graphs for context; IRI stability and graph naming matter for provenance. |
|
||||
| Anzo | RDF | Yes | Partial | Partial | Partial | Anzo deployments are environment-specific; validate `dataset_uri`/graphmart naming, named-graph support, and provenance mapping. |
|
||||
| Oxigraph | RDF | Yes | Partial | Partial | Partial | Embedded, single-process store (in-memory or on-disk); named graphs are supported, but there is no separate server process to scale independently. |
|
||||
|
||||
## RDF and LPG differences
|
||||
|
||||
- LPG backends store context and provenance as graph elements and properties. If a backend does not support relationship properties, some provenance patterns may be degraded.
|
||||
- RDF backends rely on IRIs, named graphs, and optional reification. Context graphs and provenance are easiest to preserve when the store supports named graphs/quads.
|
||||
- Ingestion works across both models, but the physical representation differs: LPG stores nodes/edges directly, while RDF stores subject-predicate-object statements.
|
||||
- Reasoning and analytics should be validated against the adapter's query capabilities, especially for path traversal, property filters, and named-graph queries.
|
||||
|
||||
## Minimal connection examples
|
||||
|
||||
Prefer the referenced notebook cells for a working setup. The examples below show the intended adapter entrypoints, not a universal connection DSL.
|
||||
|
||||
### Neo4j
|
||||
|
||||
```python
|
||||
import os
|
||||
from semantica.graph_store import Neo4jStore
|
||||
|
||||
store = Neo4jStore(
|
||||
uri='bolt://localhost:7687',
|
||||
user='neo4j',
|
||||
password=os.environ['NEO4J_PASSWORD']
|
||||
)
|
||||
```
|
||||
|
||||
### FalkorDB
|
||||
|
||||
```python
|
||||
from semantica.graph_store import FalkorDBStore
|
||||
|
||||
store = FalkorDBStore(
|
||||
host='localhost',
|
||||
port=6379,
|
||||
graph_name='semantica'
|
||||
)
|
||||
```
|
||||
|
||||
### Amazon Neptune
|
||||
|
||||
```python
|
||||
from semantica.graph_store import AmazonNeptuneStore
|
||||
|
||||
store = AmazonNeptuneStore(
|
||||
endpoint='your-neptune-cluster-endpoint',
|
||||
port=8182,
|
||||
region='us-east-1'
|
||||
)
|
||||
```
|
||||
|
||||
### Apache AGE
|
||||
|
||||
```python
|
||||
from semantica.graph_store import ApacheAgeStore
|
||||
|
||||
store = ApacheAgeStore(
|
||||
connection_string='host=localhost dbname=agedb user=postgres password=postgres',
|
||||
graph_name='semantica'
|
||||
)
|
||||
```
|
||||
|
||||
### RDF4J
|
||||
|
||||
```python
|
||||
from semantica.triplet_store import RDF4JStore
|
||||
|
||||
store = RDF4JStore(
|
||||
endpoint='http://localhost:8080/rdf4j-server',
|
||||
repository_id='semantica' # currently has no effect; connects to "default" (see Known limitations)
|
||||
)
|
||||
```
|
||||
|
||||
### Apache Jena
|
||||
|
||||
```python
|
||||
from semantica.triplet_store import JenaStore
|
||||
|
||||
store = JenaStore(
|
||||
endpoint='http://localhost:3030/ds'
|
||||
)
|
||||
```
|
||||
|
||||
### Blazegraph
|
||||
|
||||
```python
|
||||
from semantica.triplet_store import BlazegraphStore
|
||||
|
||||
store = BlazegraphStore(
|
||||
endpoint='http://localhost:9999/blazegraph/sparql'
|
||||
)
|
||||
```
|
||||
|
||||
### Anzo
|
||||
|
||||
```python
|
||||
from semantica.triplet_store import AnzoStore
|
||||
|
||||
store = AnzoStore(
|
||||
endpoint='http://anzo-host:8080',
|
||||
dataset_uri='http://cambridgesemantics.com/Graphmart/your-graphmart-id'
|
||||
)
|
||||
```
|
||||
|
||||
### Oxigraph
|
||||
|
||||
```python
|
||||
from semantica.triplet_store import OxigraphStore
|
||||
|
||||
# Omit `path` for an in-memory store; pass a directory for on-disk persistence.
|
||||
store = OxigraphStore(path='./semantica-oxigraph-data')
|
||||
```
|
||||
|
||||
Replace hostnames, ports, repositories, graphs, and credentials with values from your environment. For regulated or self-hosted deployments, keep credentials in environment variables or secret storage rather than source code.
|
||||
+10
-1
@@ -63,7 +63,9 @@ semantica-explorer --graph my_graph.json --no-browser
|
||||
python -m semantica.explorer --graph my_graph.json
|
||||
```
|
||||
|
||||
> **Security note:** The Explorer API has no built-in authentication. The default `--host 127.0.0.1` binds to localhost only, so it is not reachable from other machines on your network. If you bind to `0.0.0.0`, all graph data is readable and writable by any host that can reach the port. The CLI will print a warning in that case.
|
||||
> **Security note:** Since v0.6.5 the Explorer API requires an API key on protected routes. Set the `SEMANTICA_API_KEY` environment variable and send it as the `X-API-Key` header; without a configured key, protected routes fail closed with `503` rather than serving anonymously. To opt into unauthenticated access for local development only, set `SEMANTICA_ALLOW_ANONYMOUS=true` explicitly. (`/api/health` and `/api/info` are intentionally unauthenticated.)
|
||||
>
|
||||
> The default `--host 127.0.0.1` binds to localhost only, so it is not reachable from other machines on your network. If you bind to `0.0.0.0`, all graph data is readable and writable by any host that can reach the port (subject to API-key auth). The CLI prints a warning when binding to a non-loopback host in anonymous mode or when `SEMANTICA_API_KEY` is unset.
|
||||
|
||||
---
|
||||
|
||||
@@ -148,6 +150,8 @@ This writes the compiled assets to `../semantica/static/`. The Python server the
|
||||
| --- | --- | --- |
|
||||
| `EXPLORER_CORS_ORIGINS` | `http://localhost:5173,http://127.0.0.1:5173` | Comma-separated list of allowed CORS origins |
|
||||
| `EXPLORER_CORS_CREDENTIALS` | `false` | Set to `true` to allow credentialed cross-origin requests (only needed behind an authenticating reverse proxy) |
|
||||
| `SEMANTICA_API_KEY` | *(unset)* | API key required on protected routes since v0.6.5; send it as the `X-API-Key` header. When unset, protected routes fail closed with `503`. |
|
||||
| `SEMANTICA_ALLOW_ANONYMOUS` | `false` | Set to `true` to opt into unauthenticated access (local development only). |
|
||||
|
||||
---
|
||||
|
||||
@@ -251,6 +255,11 @@ Vite automatically tries the next available port and prints the actual URL in th
|
||||
- Confirm the backend exposes the `/ws/graph-updates` WebSocket endpoint.
|
||||
- Check DevTools → Network → WS tab for the connection status and error code.
|
||||
- Ensure the backend version matches the frontend — mixing major versions can cause protocol mismatches.
|
||||
- **Authentication:** `/ws/graph-updates` enforces the same API key as the REST routes. Browsers cannot set custom headers on a WebSocket handshake, so pass the key as a query parameter instead:
|
||||
```
|
||||
ws://127.0.0.1:8000/ws/graph-updates?api_key=<your-key>
|
||||
```
|
||||
Non-browser clients (native apps, scripts) may send it as the `X-API-Key` header. A missing or incorrect key results in close code `4401`; if `SEMANTICA_API_KEY` is unset and `SEMANTICA_ALLOW_ANONYMOUS` is not `true`, the connection is also rejected. Note that API keys in URLs appear in server logs — prefer the header for non-browser clients.
|
||||
|
||||
---
|
||||
|
||||
|
||||
Generated
+1489
-14
File diff suppressed because it is too large
Load Diff
@@ -9,7 +9,7 @@
|
||||
"lint": "eslint .",
|
||||
"preview": "vite preview",
|
||||
"test:graph-store": "node --test tests/graphStore.multi-edge.test.mjs",
|
||||
"test:graph-workspace": "node --import tsx --test tests/graphSceneState.display.test.ts",
|
||||
"test:graph-workspace": "node --import tsx --test tests/markdownContentViewer.test.ts tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts",
|
||||
"test:plugin-registry": "node --import tsx --test tests/pluginRegistry.temporal.test.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
@@ -29,6 +29,8 @@
|
||||
"react-arborist": "^3.4.3",
|
||||
"react-dom": "^19.2.4",
|
||||
"react-dropzone": "^15.0.0",
|
||||
"react-markdown": "^10.1.0",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"sigma": "^3.0.2",
|
||||
"vis-data": "^8.0.3",
|
||||
"vis-timeline": "^8.5.0"
|
||||
|
||||
@@ -162,7 +162,11 @@ const SIGMA_SETTINGS = {
|
||||
hideLabelsOnMove: true,
|
||||
hideEdgesOnMove: true,
|
||||
enableEdgeEvents: true,
|
||||
renderEdgeLabels: false,
|
||||
// #1009: edge labels (the edge `type` — "works_for", "leads", ...) were
|
||||
// hardcoded off, so edge text never rendered regardless of data. The
|
||||
// labelDensity / labelGridCellSize / labelRenderedSizeThreshold settings
|
||||
// below already throttle label density for both nodes and edges.
|
||||
renderEdgeLabels: true,
|
||||
labelDensity: 0.7,
|
||||
labelGridCellSize: 140,
|
||||
zIndex: true,
|
||||
@@ -741,6 +745,12 @@ function buildEffectAvailability(
|
||||
? { enabled: true, available: true, reason: "Panel enabled" }
|
||||
: { enabled: false, available: false, reason: "Disabled by toggle" };
|
||||
|
||||
// #1009: edge labels are immediately available once the graph is loaded —
|
||||
// they have no async analytics or zoom-tier dependency.
|
||||
const edgeLabels = effectsState.edgeLabelsEnabled
|
||||
? { enabled: true, available: true, reason: "Ready" }
|
||||
: { enabled: false, available: false, reason: "Disabled by toggle" };
|
||||
|
||||
const diagnostics = !GRAPH_THEME.effects.diagnostics.enabledInDev
|
||||
? { enabled: false, available: false, reason: "Disabled in production" }
|
||||
: effectsState.diagnosticsEnabled
|
||||
@@ -758,6 +768,7 @@ function buildEffectAvailability(
|
||||
communities,
|
||||
centrality,
|
||||
legend,
|
||||
edgeLabels,
|
||||
diagnostics,
|
||||
};
|
||||
}
|
||||
@@ -1211,6 +1222,12 @@ function applySceneState(
|
||||
size: resolvedStyle.size,
|
||||
zIndex: resolvedStyle.zIndex,
|
||||
curvature: resolvedStyle.curvature,
|
||||
// #1009: Sigma's edge label renderer draws data.label — the graph
|
||||
// stores the relationship type in edgeType, which the renderer never
|
||||
// saw, so enabling renderEdgeLabels alone left edges blank.
|
||||
// Use || rather than ?? so that an empty-string edgeType (possible
|
||||
// when the API returns type: "") does not produce a blank label.
|
||||
label: resolvedStyle.hidden ? undefined : String(attrs.edgeType || data.label || ""),
|
||||
};
|
||||
});
|
||||
|
||||
@@ -1295,6 +1312,9 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
const onEdgeClickRef = useRef(onEdgeClick);
|
||||
const onSceneRuntimeChangeRef = useRef(onSceneRuntimeChange);
|
||||
const onCameraStateChangeRef = useRef(onCameraStateChange);
|
||||
// #1009: tracked as a ref so the Sigma creation effect always reads the
|
||||
// current value without needing effectsState in its dependency array.
|
||||
const effectsStateRef = useRef(effectsState);
|
||||
const [hoveredNodeId, setHoveredNodeId] = useState<string | null>(null);
|
||||
const [zoomTier, setZoomTier] = useState<GraphZoomTier>("overview");
|
||||
const [analyticsSnapshot, setAnalyticsSnapshot] = useState<GraphAnalyticsSnapshot | null>(null);
|
||||
@@ -1323,6 +1343,7 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
onEdgeClickRef.current = onEdgeClick;
|
||||
onSceneRuntimeChangeRef.current = onSceneRuntimeChange;
|
||||
onCameraStateChangeRef.current = onCameraStateChange;
|
||||
effectsStateRef.current = effectsState;
|
||||
|
||||
const behaviors = useMemo<GraphBehavior[]>(
|
||||
() => [
|
||||
@@ -1835,7 +1856,13 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
return;
|
||||
}
|
||||
|
||||
const sigma = new Sigma(displayGraphRef.current, containerRef.current, SIGMA_SETTINGS);
|
||||
const sigma = new Sigma(displayGraphRef.current, containerRef.current, {
|
||||
...SIGMA_SETTINGS,
|
||||
// #1009: initialize with the current toggle value rather than the
|
||||
// static default so that a user who disabled Edge Labels before
|
||||
// graph/Sigma initialization sees the correct state after mount.
|
||||
renderEdgeLabels: effectsStateRef.current.edgeLabelsEnabled,
|
||||
});
|
||||
sigmaRef.current = sigma;
|
||||
appliedGraphVersionRef.current = graphVersionRef.current;
|
||||
|
||||
@@ -1937,6 +1964,17 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
});
|
||||
}, [behaviors, dispatchToBehaviors, getBehaviorContext, graphReady, syncCameraState]);
|
||||
|
||||
// #1009: renderEdgeLabels follows the Effects-panel toggle instead of
|
||||
// staying hardcoded — dense graphs get their label-free edges back.
|
||||
useEffect(() => {
|
||||
const sigma = sigmaRef.current;
|
||||
if (!sigma) {
|
||||
return;
|
||||
}
|
||||
sigma.setSetting("renderEdgeLabels", effectsState.edgeLabelsEnabled);
|
||||
sigma.scheduleRefresh();
|
||||
}, [effectsState.edgeLabelsEnabled]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
const sigma = sigmaRef.current;
|
||||
|
||||
@@ -3,6 +3,7 @@ import { Loader2 } from "lucide-react";
|
||||
import { graph } from "../../store/graphStore";
|
||||
import { GRAPH_THEME, withAlpha } from "./graphTheme";
|
||||
import type { GraphSelectedNodeKind } from "./types";
|
||||
import { MarkdownContentViewer } from "./MarkdownContentViewer";
|
||||
|
||||
export type LinkPrediction = {
|
||||
target: string;
|
||||
@@ -364,6 +365,11 @@ export function GraphInspectorPanel({
|
||||
([key]) =>
|
||||
!["x","y","valid_from","valid_until","content","source","source_url","pmid","pmids","evidence","provenance","confidence"].includes(key),
|
||||
);
|
||||
const nodeContent = (typeof attributes?.content === "string" && attributes.content)
|
||||
? attributes.content
|
||||
: (typeof properties.content === "string" && properties.content)
|
||||
? properties.content
|
||||
: "";
|
||||
|
||||
return (
|
||||
<aside style={{ padding: 24, display: "flex", flexDirection: "column", gap: 18 }}>
|
||||
@@ -408,6 +414,20 @@ export function GraphInspectorPanel({
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{/* Content Section — only rendered when the node carries actual content.
|
||||
This matches the existing inspector convention: sections that have no
|
||||
data for the current node are either hidden (temporal bounds) or closed
|
||||
by default (Source Attribution, Properties). Always showing an open
|
||||
empty panel would add noise for every relationship/predicate node. */}
|
||||
{nodeContent && (
|
||||
<details className="node-panel-collapse" open>
|
||||
<summary className="node-panel-summary">Content</summary>
|
||||
<div className="node-panel-body" style={{ marginTop: 8 }}>
|
||||
<MarkdownContentViewer content={nodeContent} />
|
||||
</div>
|
||||
</details>
|
||||
)}
|
||||
|
||||
{/* Actions */}
|
||||
<section style={sectionStyle}>
|
||||
<div style={sectionTitleStyle}>Actions</div>
|
||||
|
||||
@@ -40,6 +40,7 @@ import {
|
||||
type GraphPluginToolbarItem,
|
||||
} from "./plugins";
|
||||
import { explorationEffectsShouldLoad, neighborhoodPanelShouldLoad, temporalOverlayShouldLoad } from "./pluginRegistryPredicates";
|
||||
import { shouldFetchTemporalBounds, shouldFetchTemporalSnapshot } from "./temporalLifecyclePredicates";
|
||||
import type { LinkPrediction, PathResponse } from "./GraphInspectorPanel";
|
||||
import type { GraphSceneHandle, GraphSceneRuntime } from "./scene";
|
||||
import type {
|
||||
@@ -147,6 +148,7 @@ const DEFAULT_EFFECTS_STATE: GraphEffectsState = {
|
||||
communitiesEnabled: false,
|
||||
centralityEnabled: false,
|
||||
legendEnabled: false,
|
||||
edgeLabelsEnabled: true,
|
||||
diagnosticsEnabled: false,
|
||||
lensMode: "neighborhood",
|
||||
effectQuality: "bounded",
|
||||
@@ -1440,7 +1442,18 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
applyGraphReadySummary(summary);
|
||||
}, [applyGraphReadySummary, graphReady, summary]);
|
||||
|
||||
const canFetchTemporalBounds = shouldFetchTemporalBounds(summary);
|
||||
const canFetchTemporalSnapshot = shouldFetchTemporalSnapshot({
|
||||
debouncedTime,
|
||||
isLoading,
|
||||
summary,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (!canFetchTemporalBounds) {
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
const loadBounds = async () => {
|
||||
try {
|
||||
@@ -1460,10 +1473,21 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [summary?.nodeCount, summary?.edgeCount]);
|
||||
}, [
|
||||
canFetchTemporalBounds,
|
||||
summary?.nodeCount,
|
||||
summary?.edgeCount,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!debouncedTime || isLoading) return;
|
||||
if (!canFetchTemporalSnapshot) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!debouncedTime) {
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
const applySnapshot = async () => {
|
||||
@@ -1505,7 +1529,10 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [debouncedTime, isLoading]);
|
||||
}, [
|
||||
canFetchTemporalSnapshot,
|
||||
debouncedTime,
|
||||
]);
|
||||
|
||||
const resolveNodeIdForFocusedMode = useCallback((
|
||||
nodeId: string,
|
||||
|
||||
@@ -0,0 +1,403 @@
|
||||
import { useState, useRef, useEffect, type CSSProperties } from "react";
|
||||
import ReactMarkdown from "react-markdown";
|
||||
import remarkGfm from "remark-gfm";
|
||||
import { Check, Copy, Code2, Eye, ExternalLink, Image as ImageIcon } from "lucide-react";
|
||||
import { GRAPH_THEME } from "./graphTheme";
|
||||
|
||||
export interface MarkdownContentViewerProps {
|
||||
content?: string | null;
|
||||
className?: string;
|
||||
defaultMode?: "preview" | "source";
|
||||
}
|
||||
|
||||
export function isSafeUrl(url?: string): boolean {
|
||||
if (!url) return false;
|
||||
const trimmed = url.trim();
|
||||
// Reject whitespace-only strings — new URL("", base) would resolve to the base
|
||||
// protocol and produce a false positive. This guards direct callers of the exported
|
||||
// function; markdown parsers normalise whitespace-only destinations to "" which
|
||||
// already fails the !url check above.
|
||||
if (!trimmed) return false;
|
||||
if (trimmed.startsWith("//")) return false;
|
||||
if (trimmed.startsWith("#")) return true;
|
||||
if (trimmed.startsWith("/")) return true;
|
||||
try {
|
||||
const parsed = new URL(trimmed, "http://localhost");
|
||||
return ["http:", "https:", "mailto:"].includes(parsed.protocol);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function MarkdownContentViewer({
|
||||
content,
|
||||
className,
|
||||
defaultMode = "preview",
|
||||
}: MarkdownContentViewerProps) {
|
||||
const [activeMode, setActiveMode] = useState<"preview" | "source">(defaultMode);
|
||||
const [copied, setCopied] = useState(false);
|
||||
// Track the content value for which the copied indicator is valid.
|
||||
// When content changes (i.e. the user selects a different node), reset the
|
||||
// copied indicator inline during render rather than in a useEffect — this
|
||||
// avoids a cascading-render lint error and is the React-recommended pattern
|
||||
// for resetting derived visual state on prop changes.
|
||||
const [copiedForContent, setCopiedForContent] = useState<string | null | undefined>(content);
|
||||
if (copiedForContent !== content) {
|
||||
setCopiedForContent(content);
|
||||
if (copied) {
|
||||
// Clear the stale indicator synchronously so the new node's copy button
|
||||
// never shows "Copied" from the previous selection.
|
||||
setCopied(false);
|
||||
}
|
||||
}
|
||||
|
||||
const copyTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
// Clean up any outstanding timeout on unmount.
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (copyTimeoutRef.current) {
|
||||
clearTimeout(copyTimeoutRef.current);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const rawContent = typeof content === "string" ? content : "";
|
||||
const hasContent = rawContent.trim().length > 0;
|
||||
|
||||
const handleCopy = async () => {
|
||||
if (!hasContent) return;
|
||||
try {
|
||||
await navigator.clipboard.writeText(rawContent);
|
||||
if (copyTimeoutRef.current) {
|
||||
clearTimeout(copyTimeoutRef.current);
|
||||
}
|
||||
setCopied(true);
|
||||
copyTimeoutRef.current = setTimeout(() => setCopied(false), 1500);
|
||||
} catch {
|
||||
// Clipboard write unavailable
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className={className} style={viewerContainerStyle}>
|
||||
<div style={viewerHeaderStyle}>
|
||||
<div style={{ display: "flex", gap: 4 }} role="tablist">
|
||||
<button
|
||||
type="button"
|
||||
role="tab"
|
||||
aria-selected={activeMode === "preview"}
|
||||
onClick={() => setActiveMode("preview")}
|
||||
style={{ ...tabBtnStyle, ...(activeMode === "preview" ? activeTabBtnStyle : {}) }}
|
||||
>
|
||||
<Eye size={12} style={{ marginRight: 5 }} />
|
||||
Preview
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
role="tab"
|
||||
aria-selected={activeMode === "source"}
|
||||
onClick={() => setActiveMode("source")}
|
||||
style={{ ...tabBtnStyle, ...(activeMode === "source" ? activeTabBtnStyle : {}) }}
|
||||
>
|
||||
<Code2 size={12} style={{ marginRight: 5 }} />
|
||||
Source
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{hasContent && (
|
||||
<button type="button" onClick={() => void handleCopy()} style={copyBtnStyle} title="Copy raw content">
|
||||
{copied ? (
|
||||
<>
|
||||
<Check size={12} color="#3fb950" style={{ marginRight: 4 }} />
|
||||
<span style={{ color: "#3fb950", fontSize: 11 }}>Copied</span>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Copy size={12} style={{ marginRight: 4 }} />
|
||||
<span style={{ fontSize: 11 }}>Copy</span>
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div style={viewerBodyStyle}>
|
||||
{!hasContent ? (
|
||||
<div style={emptyTextStyle}>No content available for this node.</div>
|
||||
) : activeMode === "source" ? (
|
||||
<pre style={sourcePreStyle}>
|
||||
<code style={sourceCodeStyle}>{rawContent}</code>
|
||||
</pre>
|
||||
) : (
|
||||
<div style={previewStyle}>
|
||||
<ReactMarkdown
|
||||
remarkPlugins={[remarkGfm]}
|
||||
components={{
|
||||
// C-1: react-markdown passes a HAST `node` prop (the raw AST
|
||||
// Element) to every custom component override via passNode:true.
|
||||
// In React 19 any unknown prop spreads onto a native element are
|
||||
// serialised as HTML attributes, producing node="[object Object]"
|
||||
// on every rendered link. Fix: destructure `node` by name so it
|
||||
// is explicitly discarded, then spread `...rest` to preserve all
|
||||
// other legitimate HAST/remark-gfm attributes — e.g. the `id`,
|
||||
// `aria-describedby`, `aria-label`, `data-footnote-ref`,
|
||||
// `data-footnote-backref`, and `class` attrs that GFM footnotes
|
||||
// require for correct in-page navigation and accessibility.
|
||||
//
|
||||
// C-2: fragment links (#anchor, GFM footnote backlinks) must
|
||||
// navigate within the current document. External links continue
|
||||
// to use target="_blank" with noopener noreferrer.
|
||||
//
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
a: ({ href, children, title, node: _node, ...rest }) => {
|
||||
if (!isSafeUrl(href)) {
|
||||
return <span style={{ color: GRAPH_THEME.ui.text.muted, textDecoration: "line-through" }}>{children}</span>;
|
||||
}
|
||||
// isSafeUrl returning true guarantees href is a non-empty string.
|
||||
const safeHref = href ?? "";
|
||||
// Fragment links (#section, footnote backlinks like
|
||||
// #user-content-fnref-1) are in-document anchors. Opening them
|
||||
// in a new tab would break GFM footnote back-navigation.
|
||||
const isFragment = safeHref.startsWith("#");
|
||||
if (isFragment) {
|
||||
return (
|
||||
<a href={safeHref} title={title} style={linkStyle} {...rest}>
|
||||
{children}
|
||||
</a>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<a href={safeHref} title={title} target="_blank" rel="noopener noreferrer" style={linkStyle} {...rest}>
|
||||
{children}
|
||||
<ExternalLink size={10} style={{ marginLeft: 3, verticalAlign: "middle", display: "inline" }} />
|
||||
</a>
|
||||
);
|
||||
},
|
||||
img: ({ src, alt }) => (
|
||||
<span style={imageBadgeStyle} title={src || "Image"}>
|
||||
<ImageIcon size={12} style={{ marginRight: 5 }} />
|
||||
<span>Image: {alt || src || "unlabeled"}</span>
|
||||
</span>
|
||||
),
|
||||
h1: ({ children }) => <h1 style={h1Style}>{children}</h1>,
|
||||
h2: ({ children }) => <h2 style={h2Style}>{children}</h2>,
|
||||
h3: ({ children }) => <h3 style={h3Style}>{children}</h3>,
|
||||
h4: ({ children }) => <h4 style={h4Style}>{children}</h4>,
|
||||
p: ({ children }) => <p style={{ margin: "0 0 8px 0" }}>{children}</p>,
|
||||
ul: ({ children }) => <ul style={{ margin: "0 0 8px 0", paddingLeft: 18 }}>{children}</ul>,
|
||||
ol: ({ children }) => <ol style={{ margin: "0 0 8px 0", paddingLeft: 18 }}>{children}</ol>,
|
||||
li: ({ children }) => <li style={{ marginBottom: 3 }}>{children}</li>,
|
||||
blockquote: ({ children }) => <blockquote style={blockquoteStyle}>{children}</blockquote>,
|
||||
hr: () => <hr style={{ border: "none", borderTop: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}`, margin: "10px 0" }} />,
|
||||
table: ({ children }) => (
|
||||
<div style={{ width: "100%", overflowX: "auto", margin: "8px 0", borderRadius: 6, border: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}` }}>
|
||||
<table style={{ width: "100%", borderCollapse: "collapse", fontSize: 12 }}>{children}</table>
|
||||
</div>
|
||||
),
|
||||
thead: ({ children }) => <thead style={{ background: "rgba(255, 255, 255, 0.04)" }}>{children}</thead>,
|
||||
tbody: ({ children }) => <tbody>{children}</tbody>,
|
||||
tr: ({ children }) => <tr style={{ borderBottom: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}` }}>{children}</tr>,
|
||||
th: ({ children }) => <th style={{ padding: "6px 8px", textAlign: "left", fontWeight: 700, color: GRAPH_THEME.ui.text.strong, borderRight: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}` }}>{children}</th>,
|
||||
td: ({ children }) => <td style={{ padding: "6px 8px", color: GRAPH_THEME.ui.text.body, borderRight: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}` }}>{children}</td>,
|
||||
pre: ({ children }) => <pre style={preBlockStyle}>{children}</pre>,
|
||||
// C-1: discard `node` here too — code elements are custom components
|
||||
// and would otherwise receive node="[object Object]" in the DOM.
|
||||
code: ({ className: codeClass, children }) => {
|
||||
const isInline = !codeClass && typeof children === "string" && !children.includes("\n");
|
||||
return (
|
||||
<code style={isInline ? inlineCodeStyle : blockCodeStyle}>
|
||||
{children}
|
||||
</code>
|
||||
);
|
||||
},
|
||||
}}
|
||||
>
|
||||
{rawContent}
|
||||
</ReactMarkdown>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/* ─── Styles ──────────────────────────────────────────────────────── */
|
||||
|
||||
const viewerContainerStyle: CSSProperties = {
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
background: "rgba(255, 255, 255, 0.025)",
|
||||
border: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}`,
|
||||
borderRadius: 12,
|
||||
overflow: "hidden",
|
||||
};
|
||||
|
||||
const viewerHeaderStyle: CSSProperties = {
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "space-between",
|
||||
padding: "6px 10px",
|
||||
background: "rgba(0, 0, 0, 0.2)",
|
||||
borderBottom: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}`,
|
||||
};
|
||||
|
||||
const tabBtnStyle: CSSProperties = {
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
padding: "4px 9px",
|
||||
borderRadius: 6,
|
||||
border: "1px solid transparent",
|
||||
background: "transparent",
|
||||
color: GRAPH_THEME.ui.text.muted,
|
||||
fontSize: 12,
|
||||
fontWeight: 600,
|
||||
cursor: "pointer",
|
||||
transition: "all 150ms ease",
|
||||
};
|
||||
|
||||
const activeTabBtnStyle: CSSProperties = {
|
||||
background: GRAPH_THEME.ui.timeline.playheadSoft,
|
||||
border: `1px solid ${GRAPH_THEME.ui.control.activeBorder}`,
|
||||
color: GRAPH_THEME.ui.timeline.playhead,
|
||||
};
|
||||
|
||||
const copyBtnStyle: CSSProperties = {
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
padding: "3px 8px",
|
||||
borderRadius: 6,
|
||||
border: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}`,
|
||||
background: "rgba(255, 255, 255, 0.04)",
|
||||
color: GRAPH_THEME.ui.text.subtle,
|
||||
fontSize: 11,
|
||||
cursor: "pointer",
|
||||
};
|
||||
|
||||
const viewerBodyStyle: CSSProperties = {
|
||||
padding: 12,
|
||||
maxHeight: 380,
|
||||
overflowY: "auto",
|
||||
};
|
||||
|
||||
const emptyTextStyle: CSSProperties = {
|
||||
color: GRAPH_THEME.ui.text.muted,
|
||||
fontSize: 12,
|
||||
lineHeight: 1.5,
|
||||
fontStyle: "italic",
|
||||
};
|
||||
|
||||
const sourcePreStyle: CSSProperties = {
|
||||
margin: 0,
|
||||
padding: 10,
|
||||
borderRadius: 8,
|
||||
background: "rgba(0, 0, 0, 0.3)",
|
||||
border: "1px solid rgba(255, 255, 255, 0.05)",
|
||||
overflowX: "auto",
|
||||
};
|
||||
|
||||
const sourceCodeStyle: CSSProperties = {
|
||||
fontFamily: "'JetBrains Mono', 'Fira Code', monospace",
|
||||
fontSize: 12,
|
||||
lineHeight: 1.6,
|
||||
color: GRAPH_THEME.ui.text.strong,
|
||||
whiteSpace: "pre-wrap",
|
||||
wordBreak: "break-word",
|
||||
userSelect: "text",
|
||||
};
|
||||
|
||||
const previewStyle: CSSProperties = {
|
||||
color: GRAPH_THEME.ui.text.body,
|
||||
fontSize: 13,
|
||||
lineHeight: 1.6,
|
||||
wordBreak: "break-word",
|
||||
};
|
||||
|
||||
const h1Style: CSSProperties = {
|
||||
fontSize: 16,
|
||||
fontWeight: 700,
|
||||
color: GRAPH_THEME.ui.text.strong,
|
||||
marginTop: 8,
|
||||
marginBottom: 6,
|
||||
paddingBottom: 3,
|
||||
borderBottom: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}`,
|
||||
};
|
||||
|
||||
const h2Style: CSSProperties = {
|
||||
fontSize: 14,
|
||||
fontWeight: 700,
|
||||
color: GRAPH_THEME.ui.text.strong,
|
||||
marginTop: 8,
|
||||
marginBottom: 4,
|
||||
};
|
||||
|
||||
const h3Style: CSSProperties = {
|
||||
fontSize: 13,
|
||||
fontWeight: 600,
|
||||
color: GRAPH_THEME.ui.text.strong,
|
||||
marginTop: 6,
|
||||
marginBottom: 4,
|
||||
};
|
||||
|
||||
const h4Style: CSSProperties = {
|
||||
fontSize: 12,
|
||||
fontWeight: 600,
|
||||
color: GRAPH_THEME.ui.text.strong,
|
||||
marginTop: 4,
|
||||
marginBottom: 2,
|
||||
};
|
||||
|
||||
const blockquoteStyle: CSSProperties = {
|
||||
margin: "8px 0",
|
||||
padding: "6px 12px",
|
||||
borderLeft: `3px solid ${GRAPH_THEME.ui.timeline.playhead}`,
|
||||
background: "rgba(98, 226, 205, 0.05)",
|
||||
borderRadius: "0 6px 6px 0",
|
||||
color: GRAPH_THEME.ui.text.body,
|
||||
fontStyle: "italic",
|
||||
};
|
||||
|
||||
const linkStyle: CSSProperties = {
|
||||
color: "#79c0ff",
|
||||
textDecoration: "underline",
|
||||
textUnderlineOffset: "3px",
|
||||
wordBreak: "break-all",
|
||||
};
|
||||
|
||||
const imageBadgeStyle: CSSProperties = {
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
padding: "3px 7px",
|
||||
background: "rgba(255, 255, 255, 0.04)",
|
||||
border: `1px solid ${GRAPH_THEME.ui.surface.panelBorder}`,
|
||||
borderRadius: 6,
|
||||
color: GRAPH_THEME.ui.text.muted,
|
||||
fontSize: 11,
|
||||
margin: "3px 0",
|
||||
};
|
||||
|
||||
const inlineCodeStyle: CSSProperties = {
|
||||
fontFamily: "'JetBrains Mono', monospace",
|
||||
fontSize: 12,
|
||||
padding: "2px 5px",
|
||||
borderRadius: 4,
|
||||
background: "rgba(255, 255, 255, 0.07)",
|
||||
color: "#e6edf3",
|
||||
border: "1px solid rgba(255, 255, 255, 0.08)",
|
||||
};
|
||||
|
||||
const preBlockStyle: CSSProperties = {
|
||||
margin: "8px 0",
|
||||
padding: 10,
|
||||
borderRadius: 8,
|
||||
background: "rgba(0, 0, 0, 0.35)",
|
||||
border: "1px solid rgba(255, 255, 255, 0.08)",
|
||||
overflowX: "auto",
|
||||
};
|
||||
|
||||
const blockCodeStyle: CSSProperties = {
|
||||
fontFamily: "'JetBrains Mono', monospace",
|
||||
fontSize: 12,
|
||||
lineHeight: 1.5,
|
||||
color: "#e6edf3",
|
||||
};
|
||||
@@ -2099,6 +2099,15 @@ function createCollapsedNeighborhoodGraph(
|
||||
return collapsedGraph;
|
||||
}
|
||||
|
||||
// Normalize an edge relationship type: empty string, null, and undefined all
|
||||
// fall back to the project-wide default used consistently across every
|
||||
// aggregation path. Keep this local — it exists only to guarantee that the
|
||||
// three code paths (single-entry, multi-entry, community-grouped) produce the
|
||||
// same semantics and do not diverge again.
|
||||
function normalizeEdgeType(value: string | null | undefined): string {
|
||||
return value || "related_to";
|
||||
}
|
||||
|
||||
function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAttributes> {
|
||||
const aggregated = new Graph<NodeAttributes, EdgeAttributes>({
|
||||
type: "directed",
|
||||
@@ -2124,10 +2133,13 @@ function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAt
|
||||
const [{ edgeId, attrs }] = entries;
|
||||
aggregated.mergeDirectedEdgeWithKey(edgeId, sourceId, targetId, {
|
||||
...attrs,
|
||||
// #1009: normalize empty/null/undefined edgeType so Sigma's label
|
||||
// renderer never receives a blank string on the single-entry path.
|
||||
edgeType: normalizeEdgeType(attrs.edgeType),
|
||||
dominantEdgeType: normalizeEdgeType(attrs.dominantEdgeType ?? attrs.edgeType),
|
||||
rawEdgeIds: collectRawEdgeIds(attrs, edgeId),
|
||||
isAggregated: isAggregatedEdgeAttributes(attrs),
|
||||
aggregateCount: attrs.aggregateCount ?? collectRawEdgeIds(attrs, edgeId).length,
|
||||
dominantEdgeType: attrs.dominantEdgeType ?? attrs.edgeType,
|
||||
representativeWeight: attrs.representativeWeight ?? Number(attrs.weight ?? 1),
|
||||
});
|
||||
return;
|
||||
@@ -2150,10 +2162,11 @@ function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAt
|
||||
const rawEdgeIds = entries.flatMap(({ edgeId, attrs }) => collectRawEdgeIds(attrs, edgeId));
|
||||
const typeCounts = new Map<string, number>();
|
||||
entries.forEach(({ attrs }) => {
|
||||
const edgeType = String(attrs.edgeType ?? "related_to");
|
||||
const edgeType = normalizeEdgeType(attrs.edgeType);
|
||||
typeCounts.set(edgeType, (typeCounts.get(edgeType) ?? 0) + 1);
|
||||
});
|
||||
const dominantEdgeType = [...typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0] ?? representative.attrs.edgeType ?? "related_to";
|
||||
const dominantEdgeType = [...typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0]
|
||||
?? normalizeEdgeType(representative.attrs.edgeType);
|
||||
const reverseKey = `${targetId}→${sourceId}`;
|
||||
const isBidirectionalBundle = groupedEdges.has(reverseKey);
|
||||
const syntheticEdgeId = `${AGGREGATED_EDGE_PREFIX}${sourceId}::${targetId}`;
|
||||
@@ -2167,10 +2180,10 @@ function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAt
|
||||
rawEdgeIds,
|
||||
isAggregated: true,
|
||||
aggregateCount: rawEdgeIds.length,
|
||||
dominantEdgeType: String(dominantEdgeType),
|
||||
dominantEdgeType: dominantEdgeType,
|
||||
representativeWeight: Number(representative.attrs.weight ?? 1),
|
||||
weight: Number(representative.attrs.weight ?? 1),
|
||||
edgeType: String(representative.attrs.edgeType ?? dominantEdgeType ?? "related_to"),
|
||||
edgeType: representative.attrs.edgeType || dominantEdgeType,
|
||||
parallelCount: rawEdgeIds.length,
|
||||
familySize: rawEdgeIds.length,
|
||||
bundleKind: isBidirectionalBundle ? "bidirectional" : "parallel",
|
||||
@@ -2280,7 +2293,7 @@ function buildCommunityGroupedGraph(): GraphDisplayResult {
|
||||
};
|
||||
bucket.rawEdgeIds.push(String(edgeId));
|
||||
bucket.weight = Math.max(bucket.weight, Number((attrs as EdgeAttributes).weight ?? 1));
|
||||
const edgeType = String((attrs as EdgeAttributes).edgeType ?? "related_to");
|
||||
const edgeType = normalizeEdgeType((attrs as EdgeAttributes).edgeType);
|
||||
bucket.typeCounts.set(edgeType, (bucket.typeCounts.get(edgeType) ?? 0) + 1);
|
||||
groupedEdges.set(key, bucket);
|
||||
});
|
||||
@@ -2396,7 +2409,8 @@ function buildCommunityGroupedGraph(): GraphDisplayResult {
|
||||
if (!visibleGroupedEdgeKeys.has(key)) {
|
||||
return;
|
||||
}
|
||||
const dominantEdgeType = [...bundle.typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0] ?? "related_to";
|
||||
const dominantEdgeType = [...bundle.typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0]
|
||||
?? "related_to";
|
||||
const reverseKey = `${bundle.targetId}→${bundle.sourceId}`;
|
||||
const syntheticEdgeId = `${AGGREGATED_EDGE_PREFIX}${key}`;
|
||||
const aggregateCount = bundle.rawEdgeIds.length;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { CSSProperties } from "react";
|
||||
|
||||
import type {
|
||||
GraphDiagnosticsSnapshot,
|
||||
GraphEffectAvailability,
|
||||
GraphEffectToggle,
|
||||
} from "../types";
|
||||
@@ -30,6 +31,11 @@ const EFFECT_ROWS: EffectRowConfig[] = [
|
||||
label: "Neighborhood Lens",
|
||||
description: "Local emphasis around the hovered or selected node.",
|
||||
},
|
||||
{
|
||||
key: "edgeLabelsEnabled",
|
||||
label: "Edge Labels",
|
||||
description: "Draw the relationship type on graph edges. Off restores label-free edges on dense graphs.",
|
||||
},
|
||||
{
|
||||
key: "legendEnabled",
|
||||
label: "Semantic Legend",
|
||||
@@ -37,6 +43,17 @@ const EFFECT_ROWS: EffectRowConfig[] = [
|
||||
},
|
||||
];
|
||||
|
||||
// Maps the effect toggle keys rendered by this plugin to their corresponding
|
||||
// availability keys in GraphDiagnosticsSnapshot["effectAvailability"]. Kept
|
||||
// local because this plugin only renders a subset of all effects.
|
||||
const EFFECT_AVAILABILITY_KEYS: Partial<Record<GraphEffectToggle, keyof GraphDiagnosticsSnapshot["effectAvailability"]>> = {
|
||||
pathPulseEnabled: "pathPulse",
|
||||
pathFlowEnabled: "pathFlow",
|
||||
lensEnabled: "lens",
|
||||
edgeLabelsEnabled: "edgeLabels",
|
||||
legendEnabled: "legend",
|
||||
};
|
||||
|
||||
function renderAvailabilityText(availability: GraphEffectAvailability) {
|
||||
if (availability.available) {
|
||||
if (typeof availability.visibleSegments === "number" && typeof availability.segmentCap === "number") {
|
||||
@@ -139,15 +156,9 @@ export const explorationEffectsPlugin: GraphPlugin = {
|
||||
description={row.description}
|
||||
checked={effectsState[row.key]}
|
||||
availability={
|
||||
availability?.[
|
||||
row.key === "pathPulseEnabled"
|
||||
? "pathPulse"
|
||||
: row.key === "pathFlowEnabled"
|
||||
? "pathFlow"
|
||||
: row.key === "lensEnabled"
|
||||
? "lens"
|
||||
: "legend"
|
||||
] ?? {
|
||||
(EFFECT_AVAILABILITY_KEYS[row.key] !== undefined
|
||||
? availability?.[EFFECT_AVAILABILITY_KEYS[row.key]!]
|
||||
: undefined) ?? {
|
||||
enabled: effectsState[row.key],
|
||||
available: false,
|
||||
reason: "Waiting for graph runtime",
|
||||
|
||||
@@ -47,6 +47,11 @@ const SCENE_EFFECT_ROWS: EffectRowConfig[] = [
|
||||
label: "Contours",
|
||||
description: "Low-contrast density halos around the strongest visible anchors.",
|
||||
},
|
||||
{
|
||||
key: "edgeLabelsEnabled",
|
||||
label: "Edge Labels",
|
||||
description: "Draw the relationship type on graph edges. Off restores label-free edges on dense graphs.",
|
||||
},
|
||||
{
|
||||
key: "legendEnabled",
|
||||
label: "Regions Summary",
|
||||
@@ -83,6 +88,7 @@ const AVAILABILITY_KEYS: Record<GraphEffectToggle, keyof GraphDiagnosticsSnapsho
|
||||
communitiesEnabled: "communities",
|
||||
centralityEnabled: "centrality",
|
||||
legendEnabled: "legend",
|
||||
edgeLabelsEnabled: "edgeLabels",
|
||||
diagnosticsEnabled: "diagnostics",
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import type { GraphLoadSummary } from "./types";
|
||||
|
||||
/**
|
||||
* Predicates for gating GraphWorkspace temporal API requests.
|
||||
*
|
||||
* Temporal bounds and snapshot requests must strictly not execute until the
|
||||
* initial graph load has succeeded (summary !== undefined). An empty graph
|
||||
* (nodeCount: 0) is still a successful load and must not be rejected.
|
||||
*/
|
||||
|
||||
export function shouldFetchTemporalBounds(
|
||||
summary: GraphLoadSummary | undefined,
|
||||
): boolean {
|
||||
return summary !== undefined;
|
||||
}
|
||||
|
||||
export function shouldFetchTemporalSnapshot({
|
||||
debouncedTime,
|
||||
isLoading,
|
||||
summary,
|
||||
}: {
|
||||
debouncedTime: Date | null;
|
||||
isLoading: boolean;
|
||||
summary: GraphLoadSummary | undefined;
|
||||
}): boolean {
|
||||
return (
|
||||
summary !== undefined &&
|
||||
debouncedTime !== null &&
|
||||
!isLoading
|
||||
);
|
||||
}
|
||||
@@ -103,6 +103,7 @@ export type GraphEffectToggle =
|
||||
| "communitiesEnabled"
|
||||
| "centralityEnabled"
|
||||
| "legendEnabled"
|
||||
| "edgeLabelsEnabled"
|
||||
| "diagnosticsEnabled";
|
||||
|
||||
export interface GraphEffectsState {
|
||||
@@ -113,6 +114,7 @@ export interface GraphEffectsState {
|
||||
semanticRegionsEnabled: boolean;
|
||||
contoursEnabled: boolean;
|
||||
pathfindingEnabled: boolean;
|
||||
edgeLabelsEnabled: boolean;
|
||||
communitiesEnabled: boolean;
|
||||
centralityEnabled: boolean;
|
||||
legendEnabled: boolean;
|
||||
@@ -186,6 +188,7 @@ export interface GraphDiagnosticsSnapshot {
|
||||
communities: GraphEffectAvailability;
|
||||
centrality: GraphEffectAvailability;
|
||||
legend: GraphEffectAvailability;
|
||||
edgeLabels: GraphEffectAvailability;
|
||||
diagnostics: GraphEffectAvailability;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1061,3 +1061,198 @@ test("checkGroupedViewAvailability returns available when communities exist", ()
|
||||
assert.equal(result.reason, null);
|
||||
});
|
||||
|
||||
|
||||
// ── #1009: edge label data-path regression tests ─────────────────────────────
|
||||
|
||||
test("resolveDisplayGraph parallel-bundle preserves edgeType on aggregated edge", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
batchMergeEdges([
|
||||
{ id: "e1", source: "a", target: "b", attributes: { edgeType: "causes", weight: 1, properties: {} } },
|
||||
{ id: "e2", source: "a", target: "b", attributes: { edgeType: "causes", weight: 2, properties: {} } },
|
||||
]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string; isAggregated?: boolean };
|
||||
assert.equal(attrs.isAggregated, true);
|
||||
// The aggregated representative must carry the relationship text through to
|
||||
// the edgeReducer's label assignment.
|
||||
assert.equal(typeof attrs.edgeType, "string");
|
||||
assert.ok((attrs.edgeType ?? "").length > 0, "aggregated edge must have a non-empty edgeType");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph parallel-bundle picks dominant edgeType across mixed types", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
batchMergeEdges([
|
||||
{ id: "e1", source: "a", target: "b", attributes: { edgeType: "inhibits", weight: 1, properties: {} } },
|
||||
{ id: "e2", source: "a", target: "b", attributes: { edgeType: "inhibits", weight: 1, properties: {} } },
|
||||
{ id: "e3", source: "a", target: "b", attributes: { edgeType: "activates", weight: 1, properties: {} } },
|
||||
]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string; dominantEdgeType?: string };
|
||||
// "inhibits" appears twice so it must be the dominant type.
|
||||
assert.equal(attrs.edgeType, "inhibits");
|
||||
assert.equal(attrs.dominantEdgeType, "inhibits");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph grouped view community edges carry non-empty edgeType", () => {
|
||||
const left = ["g1", "g2", "g3", "g4"];
|
||||
const right = ["h1", "h2", "h3", "h4"];
|
||||
[...left, ...right].forEach((nodeId, index) => addNode(nodeId, index < left.length ? "left" : "right"));
|
||||
|
||||
let edgeIndex = 0;
|
||||
for (let i = 0; i < left.length; i += 1) {
|
||||
for (let j = 0; j < left.length; j += 1) {
|
||||
if (i !== j) {
|
||||
batchMergeEdges([{
|
||||
id: `lg-${edgeIndex++}`,
|
||||
source: left[i],
|
||||
target: left[j],
|
||||
attributes: { edgeType: "co_occurs", weight: 3, properties: {} },
|
||||
}]);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < right.length; i += 1) {
|
||||
for (let j = 0; j < right.length; j += 1) {
|
||||
if (i !== j) {
|
||||
batchMergeEdges([{
|
||||
id: `rg-${edgeIndex++}`,
|
||||
source: right[i],
|
||||
target: right[j],
|
||||
attributes: { edgeType: "co_occurs", weight: 3, properties: {} },
|
||||
}]);
|
||||
}
|
||||
}
|
||||
}
|
||||
batchMergeEdges([{ id: "bridge-g", source: "g1", target: "h1", attributes: { edgeType: "interacts_with", weight: 0.1, properties: {} } }]);
|
||||
|
||||
const { graph: displayGraph, state } = resolveDisplayGraph("", [], [], "grouped", { aggregationEnabled: true });
|
||||
assert.equal(state.groupedViewAvailable, true);
|
||||
|
||||
const communityEdges = displayGraph.edges().filter((edgeId) => {
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { bundleKind?: string };
|
||||
return attrs.bundleKind === "community";
|
||||
});
|
||||
assert.ok(communityEdges.length > 0, "expected at least one community bundle edge");
|
||||
|
||||
for (const edgeId of communityEdges) {
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string };
|
||||
assert.equal(typeof attrs.edgeType, "string");
|
||||
assert.ok((attrs.edgeType ?? "").length > 0, `community edge ${edgeId} must have a non-empty edgeType`);
|
||||
}
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph raw edge preserves exact edgeType string for label rendering", () => {
|
||||
addNode("src");
|
||||
addNode("tgt");
|
||||
batchMergeEdges([{
|
||||
id: "raw-1",
|
||||
source: "src",
|
||||
target: "tgt",
|
||||
attributes: { edgeType: "works_for", weight: 1, properties: {} },
|
||||
}]);
|
||||
|
||||
// In full view without aggregation the edge passes through unchanged.
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: false });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string };
|
||||
assert.equal(attrs.edgeType, "works_for");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph does not produce empty-string edgeType on aggregated edges when source has empty type", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
// Simulate an API response where type is empty string — the aggregation
|
||||
// path must not propagate a blank label.
|
||||
batchMergeEdges([
|
||||
{ id: "e-empty-1", source: "a", target: "b", attributes: { edgeType: "", weight: 1, properties: {} } },
|
||||
{ id: "e-empty-2", source: "a", target: "b", attributes: { edgeType: "", weight: 1, properties: {} } },
|
||||
]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as {
|
||||
edgeType?: string;
|
||||
isAggregated?: boolean;
|
||||
};
|
||||
assert.equal(attrs.isAggregated, true);
|
||||
// The aggregation falls back to "related_to" when all source edgeTypes are
|
||||
// empty, so the rendered label should never be an empty string.
|
||||
assert.equal(attrs.edgeType, "related_to");
|
||||
});
|
||||
|
||||
test("resolveEdgeElementStyle hidden class produces hidden:true for suppressed edges", () => {
|
||||
// Verify the data condition the edgeReducer relies on: hidden-classified
|
||||
// edges must have hidden:true so that the label assignment sets undefined.
|
||||
const style = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"overview",
|
||||
"inactive",
|
||||
{
|
||||
edgeType: "causes",
|
||||
weight: 1,
|
||||
properties: {},
|
||||
edgeVariant: "line",
|
||||
visualPriority: 0.05,
|
||||
baseSize: 0.3,
|
||||
},
|
||||
"source",
|
||||
"target",
|
||||
"full",
|
||||
"inactive-edge",
|
||||
"hidden",
|
||||
);
|
||||
assert.equal(style.hidden, true);
|
||||
});
|
||||
|
||||
// ── #1009 maintainer-blocking regression: single-edge empty edgeType ─────────
|
||||
|
||||
test("resolveDisplayGraph single-edge normalizes empty-string edgeType to related_to", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
// One edge only — exercises the entries.length === 1 path in aggregateDisplayGraph.
|
||||
batchMergeEdges([{
|
||||
id: "e-single-empty",
|
||||
source: "a",
|
||||
target: "b",
|
||||
attributes: { edgeType: "", weight: 1, properties: {} },
|
||||
}]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string; dominantEdgeType?: string };
|
||||
assert.equal(attrs.edgeType, "related_to",
|
||||
"single-edge path must normalize empty edgeType to the canonical fallback");
|
||||
assert.equal(attrs.dominantEdgeType, "related_to",
|
||||
"single-edge dominantEdgeType must also be normalized");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph single-edge preserves a valid non-empty edgeType unchanged", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
batchMergeEdges([{
|
||||
id: "e-single-valid",
|
||||
source: "a",
|
||||
target: "b",
|
||||
attributes: { edgeType: "works_for", weight: 1, properties: {} },
|
||||
}]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string };
|
||||
assert.equal(attrs.edgeType, "works_for",
|
||||
"single-edge path must not alter a valid relationship type");
|
||||
});
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import React from "react";
|
||||
import { renderToString } from "react-dom/server";
|
||||
|
||||
(globalThis as any).React = React;
|
||||
|
||||
import { isSafeUrl, MarkdownContentViewer } from "../src/workspaces/GraphWorkspace/MarkdownContentViewer.tsx";
|
||||
|
||||
test("isSafeUrl permits safe http, https, and mailto URLs and relative paths", () => {
|
||||
assert.equal(isSafeUrl("https://example.com"), true);
|
||||
assert.equal(isSafeUrl("http://localhost:8000"), true);
|
||||
assert.equal(isSafeUrl("mailto:user@example.com"), true);
|
||||
assert.equal(isSafeUrl("#section-1"), true);
|
||||
assert.equal(isSafeUrl("/relative/path"), true);
|
||||
});
|
||||
|
||||
test("isSafeUrl rejects protocol-relative URLs and dangerous schemes", () => {
|
||||
// Protocol-relative URLs (must be blocked)
|
||||
assert.equal(isSafeUrl("//evil.com"), false);
|
||||
assert.equal(isSafeUrl("//localhost:8000"), false);
|
||||
assert.equal(isSafeUrl("//"), false);
|
||||
|
||||
// Dangerous schemes
|
||||
assert.equal(isSafeUrl("javascript:alert('xss')"), false);
|
||||
assert.equal(isSafeUrl("JAVASCRIPT:alert(1)"), false);
|
||||
assert.equal(isSafeUrl("data:text/html;base64,PHNjcmlwdD4="), false);
|
||||
assert.equal(isSafeUrl("vbscript:MsgBox(1)"), false);
|
||||
assert.equal(isSafeUrl(""), false);
|
||||
assert.equal(isSafeUrl(undefined), false);
|
||||
});
|
||||
|
||||
// ─── C URL contract: whitespace-only strings ────────────────────────────────
|
||||
// The CommonMark parser normalises whitespace-only link destinations to "" so
|
||||
// these values are unreachable through normal markdown rendering. However, the
|
||||
// function is exported and its direct-call contract must be correct.
|
||||
test("isSafeUrl rejects whitespace-only strings (contract correctness)", () => {
|
||||
assert.equal(isSafeUrl(" "), false, "single space must be rejected");
|
||||
assert.equal(isSafeUrl("\t"), false, "tab must be rejected");
|
||||
assert.equal(isSafeUrl("\n"), false, "newline must be rejected");
|
||||
assert.equal(isSafeUrl(" "), false, "multiple spaces must be rejected");
|
||||
assert.equal(isSafeUrl(" \t\n "), false, "mixed whitespace must be rejected");
|
||||
});
|
||||
|
||||
test("renders Preview mode with formatted Markdown elements and tabs", () => {
|
||||
const markdown = `# Main Title\n\n**Bold Statement**\n\n* Item A\n* Item B`;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content: markdown, defaultMode: "preview" }));
|
||||
|
||||
// Tab buttons are present
|
||||
assert.equal(html.includes("Preview"), true);
|
||||
assert.equal(html.includes("Source"), true);
|
||||
assert.equal(html.includes("Copy"), true);
|
||||
|
||||
// Formatted preview elements
|
||||
assert.equal(html.includes("Main Title"), true);
|
||||
assert.equal(html.includes("Bold Statement"), true);
|
||||
assert.equal(html.includes("<strong>Bold Statement</strong>"), true);
|
||||
assert.equal(html.includes("Item A"), true);
|
||||
assert.equal(html.includes("Item B"), true);
|
||||
});
|
||||
|
||||
test("renders Source mode with exact unmodified text inside pre/code", () => {
|
||||
const markdown = `# Title 🚀\n\n * Indented item\n\n\`\`\`python\ndef test():\n return "α + β"\n\`\`\``;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content: markdown, defaultMode: "source" }));
|
||||
|
||||
assert.equal(html.includes("<pre"), true);
|
||||
assert.equal(html.includes("<code"), true);
|
||||
assert.equal(html.includes("# Title 🚀"), true);
|
||||
assert.equal(html.includes(" * Indented item"), true);
|
||||
assert.equal(html.includes('return "α + β"'), true);
|
||||
});
|
||||
|
||||
test("renders raw HTML safely as escaped text without executing elements", () => {
|
||||
const dangerousHtml = `<script>alert("XSS")</script><iframe src="https://evil.com"></iframe>`;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content: dangerousHtml, defaultMode: "preview" }));
|
||||
|
||||
// Script and iframe tags must NOT be rendered as active DOM tags
|
||||
assert.equal(html.includes("<script>"), false);
|
||||
assert.equal(html.includes("<iframe"), false);
|
||||
// Content is escaped as text
|
||||
assert.equal(html.includes("<script>"), true);
|
||||
});
|
||||
|
||||
// ─── C-1: HAST node prop must not reach the DOM ─────────────────────────────
|
||||
// react-markdown passes a HAST `node` (Element) object to custom component
|
||||
// overrides. Before this fix, ...props spread caused React 19 to serialise it
|
||||
// as node="[object Object]" on every <a> and <code> element.
|
||||
test("rendered links do not expose the HAST node object as a DOM attribute", () => {
|
||||
const content = `[Example](https://example.com)\n\nInline \`code\` here.`;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content, defaultMode: "preview" }));
|
||||
|
||||
// The rendered HTML must not contain the serialised HAST object
|
||||
assert.equal(html.includes("node="), false, "node= attribute must not appear in rendered HTML");
|
||||
assert.equal(html.includes("[object Object]"), false, "serialised HAST object must not appear in rendered HTML");
|
||||
|
||||
// The link must still render correctly with the right href
|
||||
assert.equal(html.includes('href="https://example.com"'), true, "href must be present");
|
||||
});
|
||||
|
||||
// ─── C-2: Fragment links must not open in a new tab ─────────────────────────
|
||||
// Links to in-document anchors such as #section or GFM footnote backlinks like
|
||||
// #user-content-fn-1 must stay in the current document. Only external links
|
||||
// use target="_blank".
|
||||
test("fragment links render in the current document without target blank", () => {
|
||||
const content = `[Jump to section](#introduction)\n\n[External](https://example.com)`;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content, defaultMode: "preview" }));
|
||||
|
||||
// Fragment link must have the href
|
||||
assert.equal(html.includes('href="#introduction"'), true, "fragment href must be present");
|
||||
|
||||
// Confirm no target=_blank attribute appears anywhere near the fragment link.
|
||||
// We check that the output contains a fragment href WITHOUT target="_blank"
|
||||
// by verifying the two strings are not both present (the external link has
|
||||
// target blank; the fragment link must not).
|
||||
const fragmentLinkIdx = html.indexOf('href="#introduction"');
|
||||
assert.notEqual(fragmentLinkIdx, -1, "fragment link must be rendered");
|
||||
// Inspect the 80 chars around the fragment href — should not contain target
|
||||
const fragmentContext = html.slice(Math.max(0, fragmentLinkIdx - 10), fragmentLinkIdx + 90);
|
||||
assert.equal(fragmentContext.includes('target="_blank"'), false, "fragment link must not have target=_blank");
|
||||
|
||||
// External link must still have target blank
|
||||
assert.equal(html.includes('href="https://example.com"'), true, "external href must be present");
|
||||
assert.equal(html.includes('target="_blank"'), true, "external link must have target=_blank");
|
||||
assert.equal(html.includes('rel="noopener noreferrer"'), true, "external link must have rel");
|
||||
});
|
||||
|
||||
test("GFM footnote backlinks render without target blank", () => {
|
||||
// GFM footnote syntax: footnote ref in text + definition below
|
||||
const content = `See the note[^1] for more.\n\n[^1]: This is the footnote text.`;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content, defaultMode: "preview" }));
|
||||
|
||||
// The footnote reference link (#user-content-fn-1) and backlink
|
||||
// (#user-content-fnref-1) are fragment links and must not open in a new tab.
|
||||
// We verify no fragment href is paired with target=_blank.
|
||||
// Extract all href="#..." occurrences and confirm none is adjacent to target=_blank.
|
||||
const anchorMatches = [...html.matchAll(/href="#[^"]*"/g)];
|
||||
assert.ok(anchorMatches.length > 0, "GFM footnotes must produce fragment links");
|
||||
for (const match of anchorMatches) {
|
||||
const start = match.index ?? 0;
|
||||
const context = html.slice(Math.max(0, start - 10), start + 120);
|
||||
assert.equal(
|
||||
context.includes('target="_blank"'),
|
||||
false,
|
||||
`fragment link ${match[0]} must not have target=_blank`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// ─── C-1-R: GFM footnote attributes must be preserved (regression test) ─────
|
||||
// The C-1 fix (removing the HAST `node` prop) must NOT silently drop other
|
||||
// legitimate HAST attributes. remark-gfm generates the following on footnote
|
||||
// links that are required for correct in-page navigation and accessibility:
|
||||
//
|
||||
// Footnote reference anchor:
|
||||
// id="user-content-fnref-1" ← backlink target
|
||||
// data-footnote-ref="true"
|
||||
// aria-describedby="footnote-label"
|
||||
//
|
||||
// Footnote back-link anchor:
|
||||
// data-footnote-backref=""
|
||||
// aria-label="Back to reference 1" ← screen-reader label
|
||||
// class="data-footnote-backref"
|
||||
//
|
||||
// If these are absent, clicking the ↩ back-link cannot scroll back to the
|
||||
// in-text reference, and screen readers cannot announce the backlink purpose.
|
||||
test("GFM footnote links preserve generated id, aria, and class attributes", () => {
|
||||
const content = `See the note[^1] for more.\n\n[^1]: This is the footnote text.`;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content, defaultMode: "preview" }));
|
||||
|
||||
// The HAST `node` object must not appear serialised as a DOM attribute.
|
||||
assert.equal(html.includes("node="), false, "node= attribute must not appear in HTML");
|
||||
assert.equal(html.includes("[object Object]"), false, "serialised HAST object must not appear in HTML");
|
||||
|
||||
// Footnote reference anchor must retain its id so the backlink can navigate to it.
|
||||
assert.equal(
|
||||
html.includes('id="user-content-fnref-1"'),
|
||||
true,
|
||||
"footnote reference anchor must retain id for back-navigation",
|
||||
);
|
||||
|
||||
// Footnote backlink must retain its aria-label for screen-reader accessibility.
|
||||
assert.equal(
|
||||
html.includes('aria-label="Back to reference 1"'),
|
||||
true,
|
||||
"footnote backlink must retain aria-label for accessibility",
|
||||
);
|
||||
|
||||
// Footnote backlink must retain its class attribute.
|
||||
assert.equal(
|
||||
html.includes('class="data-footnote-backref"'),
|
||||
true,
|
||||
"footnote backlink must retain class attribute",
|
||||
);
|
||||
});
|
||||
|
||||
test("renders safe links as <a> with target blank and unclickable span for unsafe links", () => {
|
||||
const content = `[Safe Link](https://getsemantica.ai)\n\n[Unsafe Scheme](javascript:alert(1))\n\n[Protocol Relative](//evil.com)`;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content, defaultMode: "preview" }));
|
||||
|
||||
// Safe link renders as <a> with security attributes
|
||||
assert.equal(html.includes('href="https://getsemantica.ai"'), true);
|
||||
assert.equal(html.includes('target="_blank"'), true);
|
||||
assert.equal(html.includes('rel="noopener noreferrer"'), true);
|
||||
|
||||
// Unsafe links do NOT render as <a> tags
|
||||
assert.equal(html.includes('href="javascript:alert(1)"'), false);
|
||||
assert.equal(html.includes('href="//evil.com"'), false);
|
||||
assert.equal(html.includes("Unsafe Scheme"), true);
|
||||
assert.equal(html.includes("Protocol Relative"), true);
|
||||
});
|
||||
|
||||
test("renders remote images as safe placeholder badges instead of <img> tags", () => {
|
||||
const content = ``;
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content, defaultMode: "preview" }));
|
||||
|
||||
// No <img> tag rendered
|
||||
assert.equal(html.includes("<img"), false);
|
||||
// Image placeholder badge rendered
|
||||
assert.equal(html.includes("Image:"), true);
|
||||
assert.equal(html.includes("System Diagram"), true);
|
||||
});
|
||||
|
||||
test("renders clear empty-state message when content is empty or null", () => {
|
||||
const emptyHtml = renderToString(React.createElement(MarkdownContentViewer, { content: "" }));
|
||||
assert.equal(emptyHtml.includes("No content available for this node."), true);
|
||||
|
||||
const nullHtml = renderToString(React.createElement(MarkdownContentViewer, { content: null }));
|
||||
assert.equal(nullHtml.includes("No content available for this node."), true);
|
||||
});
|
||||
|
||||
test("renders plain text cleanly without requiring Markdown formatting", () => {
|
||||
const plainText = "Plain entity summary text without markdown formatting.";
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content: plainText, defaultMode: "preview" }));
|
||||
|
||||
assert.equal(html.includes(plainText), true);
|
||||
});
|
||||
|
||||
test("handles very large Markdown content without failure", () => {
|
||||
const largeContent = `# Large Knowledge Node\n\n` + "Structured observation paragraph. ".repeat(400);
|
||||
assert.equal(largeContent.length > 10000, true);
|
||||
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, { content: largeContent, defaultMode: "preview" }));
|
||||
assert.equal(html.includes("Large Knowledge Node"), true);
|
||||
});
|
||||
|
||||
// ─── H-2: Stale copied state lifecycle (SSR-compatible portion) ─────────────
|
||||
// Full state-transition testing (Node A → copy → Node B) requires an interactive
|
||||
// framework. The lifecycle correctness is guaranteed by the render-phase
|
||||
// previous-prop synchronisation pattern: a `copiedForContent` state value tracks
|
||||
// the content for which the copied indicator was set; when `content` changes, the
|
||||
// mismatch is detected during render and `copied` is reset to false in the same
|
||||
// React batch, before the new node's UI is painted. What we CAN verify in SSR
|
||||
// is that the initial render for any content value shows the Copy button (not the
|
||||
// Copied indicator), which confirms the initial state is always clean.
|
||||
test("copy button always starts in un-copied state on initial render", () => {
|
||||
const html = renderToString(React.createElement(MarkdownContentViewer, {
|
||||
content: "# Some Node\n\nDescription text.",
|
||||
defaultMode: "preview",
|
||||
}));
|
||||
|
||||
// Initial render must show 'Copy', never 'Copied'
|
||||
assert.equal(html.includes("Copy"), true, "Copy button must be present on initial render");
|
||||
assert.equal(html.includes("Copied"), false, "Copied indicator must NOT be present on initial render");
|
||||
});
|
||||
@@ -0,0 +1,114 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
shouldFetchTemporalBounds,
|
||||
shouldFetchTemporalSnapshot,
|
||||
} from "../src/workspaces/GraphWorkspace/temporalLifecyclePredicates.ts";
|
||||
import type { GraphLoadSummary } from "../src/workspaces/GraphWorkspace/types.ts";
|
||||
|
||||
const sampleSummary: GraphLoadSummary = {
|
||||
nodeCount: 42,
|
||||
edgeCount: 78,
|
||||
loadTimeMs: 120,
|
||||
hasCoordinates: true,
|
||||
layoutSource: "provided",
|
||||
layoutReady: true,
|
||||
};
|
||||
|
||||
const emptyGraphSummary: GraphLoadSummary = {
|
||||
nodeCount: 0,
|
||||
edgeCount: 0,
|
||||
loadTimeMs: 15,
|
||||
hasCoordinates: false,
|
||||
layoutSource: "runtime",
|
||||
layoutReady: false,
|
||||
};
|
||||
|
||||
// ── shouldFetchTemporalBounds ────────────────────────────────────────────────
|
||||
|
||||
test("temporal bounds: false when summary is undefined (initial mount or failed load)", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalBounds(undefined),
|
||||
false,
|
||||
"bounds request must not run before graph load succeeds",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal bounds: true when non-empty summary is present", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalBounds(sampleSummary),
|
||||
true,
|
||||
"bounds request should run when successful graph summary exists",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal bounds: true when successful summary has nodeCount of 0", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalBounds(emptyGraphSummary),
|
||||
true,
|
||||
"an empty graph is still a successful load and must allow bounds fetching",
|
||||
);
|
||||
});
|
||||
|
||||
// ── shouldFetchTemporalSnapshot ──────────────────────────────────────────────
|
||||
|
||||
test("temporal snapshot: false when summary is undefined even if scrubber time is set and isLoading is false", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: false,
|
||||
summary: undefined,
|
||||
}),
|
||||
false,
|
||||
"snapshot request must not run when graph load failed",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: false when graph is currently loading", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: true,
|
||||
summary: sampleSummary,
|
||||
}),
|
||||
false,
|
||||
"snapshot request must not run while graph is loading",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: false when debouncedTime is null", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: null,
|
||||
isLoading: false,
|
||||
summary: sampleSummary,
|
||||
}),
|
||||
false,
|
||||
"snapshot request must not run without a scrubber timestamp",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: true when summary exists, isLoading is false, and time is set", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: false,
|
||||
summary: sampleSummary,
|
||||
}),
|
||||
true,
|
||||
"snapshot request should run after graph load succeeds and time is set",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: true when successful summary has 0 nodes, isLoading is false, and time is set", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: false,
|
||||
summary: emptyGraphSummary,
|
||||
}),
|
||||
true,
|
||||
"empty successful graph must allow snapshot requests once ready",
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,108 @@
|
||||
# Semantica × CrewAI
|
||||
|
||||
First-class integration between Semantica and [CrewAI](https://github.com/crewAIInc/crewAI) — give your crews a shared semantic knowledge graph, decision intelligence, and graph-based retrieval.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install semantica[crewai]
|
||||
```
|
||||
|
||||
Requires `crewai >= 0.80.0`. If `crewai` is not installed, the integration still imports (classes degrade gracefully), but you can't pass the objects to a `Crew`.
|
||||
|
||||
> **⚠️ Security note:** crewai hard-requires `chromadb~=1.1.0`, which is currently affected by the unpatched pre-authentication code-injection advisory **CVE-2026-45829** (no fixed release — even the latest chromadb 1.5.9 is affected). Installing `semantica[crewai]` pulls that dependency into your environment. The `crewai` extra is intentionally **not** part of `semantica[all]` for this reason — only install it where you actually use CrewAI, and follow chromadb for a patched release.
|
||||
|
||||
## 1. SemanticaKGTool
|
||||
|
||||
A `BaseTool` that lets agents **build and query** a shared `ContextGraph` mid-reasoning:
|
||||
|
||||
- `extract_entities` — extract named entities from `text`
|
||||
- `extract_relations` — extract relationships from `text`
|
||||
- `add_to_graph` — extract entities/relations from `text` and add them to the shared graph
|
||||
- `query_graph` — keyword-search the graph using `query`
|
||||
- `find_related` — find concepts related to `entity` within `hops`
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKGTool
|
||||
|
||||
graph = ContextGraph()
|
||||
|
||||
analyst = Agent(
|
||||
role="Knowledge Analyst",
|
||||
goal="Build and explore a knowledge graph from documents",
|
||||
backstory="You map entities and relationships into a shared graph.",
|
||||
tools=[SemanticaKGTool(graph=graph)],
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[analyst],
|
||||
tasks=[Task(description="Extract and link key entities from the brief", expected_output="JSON", agent=analyst)],
|
||||
)
|
||||
result = crew.kickoff()
|
||||
```
|
||||
|
||||
All actions return JSON, so agents get parseable results.
|
||||
|
||||
## 2. SemanticaDecisionTool
|
||||
|
||||
A `BaseTool` that wraps `AgentContext` and exposes decision intelligence:
|
||||
|
||||
- `record_decision` — record a decision with reasoning and outcome
|
||||
- `find_precedents` — retrieve past decisions similar to a scenario
|
||||
- `trace_causal_chain` — trace the causal chain from a decision
|
||||
- `analyze_impact` — assess downstream influence using graph centrality
|
||||
- `check_policy` — validate a proposed decision against rule-based policies
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from integrations.crewai import SemanticaDecisionTool
|
||||
|
||||
planner = Agent(
|
||||
role="Decision Planner",
|
||||
goal="Make grounded, precedented decisions",
|
||||
backstory="You record decisions and validate them against policy.",
|
||||
tools=[SemanticaDecisionTool()],
|
||||
)
|
||||
|
||||
crew = Crew(agents=[planner], tasks=[...])
|
||||
```
|
||||
|
||||
When no `AgentContext` is passed, one is created in-memory with `decision_tracking=True`.
|
||||
|
||||
## 3. SemanticaKnowledgeSource
|
||||
|
||||
A `BaseKnowledgeSource` that serializes the current state of a `ContextGraph` (nodes, edges, metadata) into CrewAI's knowledge storage, giving **every agent in the crew** retrieval access to the graph:
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKnowledgeSource
|
||||
|
||||
graph = ContextGraph()
|
||||
graph.add_node(node_id="privacy", node_type="policy", content="...")
|
||||
|
||||
researcher = Agent(
|
||||
role="Policy Researcher",
|
||||
goal="Answer questions from the knowledge base",
|
||||
backstory="You retrieve from graph knowledge to answer accurately.",
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[researcher],
|
||||
tasks=[...],
|
||||
knowledge_sources=[SemanticaKnowledgeSource(graph=graph)],
|
||||
)
|
||||
```
|
||||
|
||||
> **Embedder required:** storing chunks goes through CrewAI's knowledge pipeline, which needs an embedder. Set `Crew(embedder=...)` (or provide CrewAI's default credentials, e.g. `OPENAI_API_KEY`). Without a working embedder, storage fails, an ERROR is logged, and agents retrieve nothing — the crew still runs with empty knowledge queries.
|
||||
|
||||
### Compatibility note
|
||||
|
||||
CrewAI's `BaseKnowledgeSource` contract changed between `0.80.x` and current releases (`load_content()` → `validate_content()`/`aadd()`). `SemanticaKnowledgeSource` implements both the legacy and current methods, so it works across `crewai>=0.80.0`.
|
||||
|
||||
### Sharing state & checkpoints
|
||||
|
||||
- Each tool/source holds whatever `graph`/`context` you pass it. When omitted, a fresh in-memory object is created and a warning is logged — instances that auto-create their own state do **not** share knowledge, so pass the same object to every agent that must share.
|
||||
- Live state (`ContextGraph`, `AgentContext`, extractors) is excluded from CrewAI's JSON serialization. After restoring from a checkpoint, re-attach the live graph/context to the restored objects.
|
||||
@@ -0,0 +1,44 @@
|
||||
"""
|
||||
Semantica × CrewAI Integration
|
||||
==============================
|
||||
|
||||
First-class integration between the Semantica semantic intelligence stack and
|
||||
the `CrewAI <https://github.com/crewAIInc/crewAI>`_ agentic framework.
|
||||
|
||||
Public surface
|
||||
--------------
|
||||
SemanticaKGTool — CrewAI ``BaseTool`` exposing KG construction/query actions
|
||||
SemanticaDecisionTool — CrewAI ``BaseTool`` exposing decision-intelligence actions
|
||||
SemanticaKnowledgeSource— CrewAI ``BaseKnowledgeSource`` giving crews graph knowledge
|
||||
|
||||
Quick start
|
||||
-----------
|
||||
pip install semantica[crewai]
|
||||
|
||||
>>> from integrations.crewai import (
|
||||
... SemanticaKGTool,
|
||||
... SemanticaDecisionTool,
|
||||
... SemanticaKnowledgeSource,
|
||||
... )
|
||||
|
||||
Compatibility
|
||||
-------------
|
||||
Requires ``crewai >= 0.80.0``. All three classes degrade gracefully when
|
||||
``crewai`` is not installed — they are still importable and carry the full
|
||||
Semantica API, but cannot be passed to ``Crew`` / ``Agent`` constructors.
|
||||
"""
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE, CREWAI_IMPORT_ERROR
|
||||
from .decision_tool import SemanticaDecisionTool
|
||||
from .kg_tool import SemanticaKGTool
|
||||
from .knowledge_source import SemanticaKnowledgeSource
|
||||
|
||||
__all__ = [
|
||||
"SemanticaKGTool",
|
||||
"SemanticaDecisionTool",
|
||||
"SemanticaKnowledgeSource",
|
||||
"CREWAI_AVAILABLE",
|
||||
"CREWAI_IMPORT_ERROR",
|
||||
]
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,24 @@
|
||||
"""
|
||||
Shared CrewAI availability probe.
|
||||
|
||||
Every integration module needs to know whether the real ``crewai`` package is
|
||||
installed. Probing once here (instead of once per module) guarantees the
|
||||
exported ``CREWAI_AVAILABLE`` flag means the *whole* integration is ready — a
|
||||
caller gating on it will never see tools using CrewAI while a knowledge source
|
||||
silently degrades (or vice versa).
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
CREWAI_AVAILABLE = False
|
||||
CREWAI_IMPORT_ERROR: Optional[str] = None
|
||||
|
||||
try:
|
||||
from crewai.knowledge.source.base_knowledge_source import ( # noqa: F401
|
||||
BaseKnowledgeSource,
|
||||
)
|
||||
from crewai.tools import BaseTool # noqa: F401
|
||||
|
||||
CREWAI_AVAILABLE = True
|
||||
except ImportError as exc:
|
||||
CREWAI_IMPORT_ERROR = str(exc)
|
||||
@@ -0,0 +1,555 @@
|
||||
"""
|
||||
SemanticaDecisionTool — a CrewAI ``BaseTool`` exposing Semantica's decision
|
||||
intelligence (``AgentContext``) to agents.
|
||||
|
||||
Lets agents record decisions with reasoning, retrieve past precedents, trace
|
||||
causal chains, analyse downstream impact, and validate proposed decisions
|
||||
against policy rules.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[crewai]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from integrations.crewai import SemanticaDecisionTool
|
||||
>>> from crewai import Agent, Crew, Task
|
||||
>>> tool = SemanticaDecisionTool()
|
||||
>>> crew = Crew(
|
||||
... agents=[Agent(role="...", goal="...", backstory="...", tools=[tool])],
|
||||
... tasks=[...],
|
||||
... )
|
||||
|
||||
Tools exposed
|
||||
-------------
|
||||
record_decision — Record a decision with reasoning and outcome
|
||||
find_precedents — Search past decisions similar to a scenario
|
||||
trace_causal_chain— Trace the causal chain from a decision node
|
||||
analyze_impact — Assess downstream influence of a decision
|
||||
check_policy — Validate a proposed decision against policy rules
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Literal, Optional, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: CrewAI BaseTool base class
|
||||
# ---------------------------------------------------------------------------
|
||||
_BaseTool: Any = object
|
||||
|
||||
if CREWAI_AVAILABLE:
|
||||
from crewai.tools import BaseTool as _BaseTool # type: ignore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input schema
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaDecisionToolInput(BaseModel):
|
||||
"""
|
||||
Input schema for ``SemanticaDecisionTool``.
|
||||
|
||||
Exactly one action is dispatched per call; the remaining fields are only
|
||||
used by the actions that need them.
|
||||
"""
|
||||
|
||||
action: Literal[
|
||||
"record_decision",
|
||||
"find_precedents",
|
||||
"trace_causal_chain",
|
||||
"analyze_impact",
|
||||
"check_policy",
|
||||
] = Field(
|
||||
...,
|
||||
description=(
|
||||
"Which decision-intelligence operation to run. One of: "
|
||||
"'record_decision', 'find_precedents', 'trace_causal_chain', "
|
||||
"'analyze_impact', 'check_policy'."
|
||||
),
|
||||
)
|
||||
category: Optional[str] = Field(
|
||||
None,
|
||||
description="Domain category, e.g. 'loan_approval'. Used by 'record_decision'.",
|
||||
)
|
||||
scenario: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"Short description of the situation. Used by 'record_decision' and "
|
||||
"'find_precedents'."
|
||||
),
|
||||
)
|
||||
reasoning: Optional[str] = Field(
|
||||
None, description="Why this outcome was chosen. Used by 'record_decision'."
|
||||
)
|
||||
outcome: Optional[str] = Field(
|
||||
None, description="The decision result. Used by 'record_decision'."
|
||||
)
|
||||
confidence: float = Field(
|
||||
0.8,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Confidence score in [0, 1]. Used by 'record_decision'.",
|
||||
)
|
||||
entities: Optional[str] = Field(
|
||||
None,
|
||||
description="Comma-separated entity names. Used by 'record_decision'.",
|
||||
)
|
||||
decision_id: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"Identifier of a decision. Used by 'trace_causal_chain' and "
|
||||
"'analyze_impact'."
|
||||
),
|
||||
)
|
||||
depth: int = Field(
|
||||
3,
|
||||
ge=1,
|
||||
le=20,
|
||||
description="Maximum chain depth. Used by 'trace_causal_chain'.",
|
||||
)
|
||||
decision_data: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"JSON object describing a proposed decision. Used by 'check_policy'."
|
||||
),
|
||||
)
|
||||
policy_rules: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"JSON list of rule strings like 'confidence >= 0.7'. Used by "
|
||||
"'check_policy'."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SemanticaDecisionTool
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaDecisionTool(_BaseTool): # type: ignore[misc]
|
||||
"""
|
||||
CrewAI tool that surfaces Semantica's decision intelligence as agent actions.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
context:
|
||||
A ``semantica.context.AgentContext`` (or compatible object exposing
|
||||
``record_decision``, ``find_precedents_advanced``,
|
||||
``analyze_decision_influence``). A fresh in-memory context is created
|
||||
when ``None``.
|
||||
max_precedents:
|
||||
Default number of precedents returned by ``find_precedents``.
|
||||
causal_depth:
|
||||
Default chain depth used by ``trace_causal_chain``.
|
||||
"""
|
||||
|
||||
name: str = "semantica_decision"
|
||||
description: str = (
|
||||
"Decision intelligence toolkit. Actions: 'record_decision' (record a "
|
||||
"decision with category, scenario, reasoning, outcome, confidence), "
|
||||
"'find_precedents' (search past decisions similar to 'scenario'), "
|
||||
"'trace_causal_chain' (trace the causal chain from 'decision_id'), "
|
||||
"'analyze_impact' (assess downstream influence of 'decision_id'), "
|
||||
"'check_policy' (validate 'decision_data' JSON against 'policy_rules' "
|
||||
"rules like 'confidence >= 0.7'). Returns JSON."
|
||||
)
|
||||
args_schema: Type[BaseModel] = SemanticaDecisionToolInput
|
||||
context: Any = Field(default=None, exclude=True)
|
||||
max_precedents: int = 5
|
||||
causal_depth: int = 3
|
||||
had_live_state: bool = False
|
||||
reconstructed_state: bool = Field(default=False, exclude=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
context: Any = None,
|
||||
max_precedents: int = 5,
|
||||
causal_depth: int = 3,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if CREWAI_AVAILABLE:
|
||||
super().__init__(
|
||||
context=context,
|
||||
max_precedents=max_precedents,
|
||||
causal_depth=causal_depth,
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
super().__init__()
|
||||
self.context = context
|
||||
self.max_precedents = max_precedents
|
||||
self.causal_depth = causal_depth
|
||||
# Degraded mode is a plain class — no model_post_init lifecycle.
|
||||
self._ensure_defaults()
|
||||
|
||||
logger.info("SemanticaDecisionTool initialised (crewai=%s)", CREWAI_AVAILABLE)
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
"""Re-create default state after validation/deserialisation.
|
||||
|
||||
``context`` is excluded from JSON serialisation (CrewAI checkpoints
|
||||
serialise every tool via ``model_dump(mode="json")``), so a tool
|
||||
restored from a checkpoint has ``None`` state until this runs.
|
||||
"""
|
||||
self._ensure_defaults()
|
||||
super().model_post_init(__context)
|
||||
|
||||
def _ensure_defaults(self) -> None:
|
||||
"""Lazy-import and build a real AgentContext when none is wired."""
|
||||
if self.context is None:
|
||||
from semantica.context import AgentContext, ContextGraph
|
||||
from semantica.vector_store import VectorStore
|
||||
|
||||
self.context = AgentContext(
|
||||
vector_store=VectorStore(backend="faiss"),
|
||||
decision_tracking=True,
|
||||
knowledge_graph=ContextGraph(),
|
||||
)
|
||||
if self.had_live_state:
|
||||
self.reconstructed_state = True
|
||||
logger.warning(
|
||||
"SemanticaDecisionTool: the live decision context was lost "
|
||||
"during serialization/checkpoint restore — an EMPTY "
|
||||
"context was reconstructed; re-attach the original context "
|
||||
"before continuing"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"SemanticaDecisionTool created a fresh in-memory "
|
||||
"AgentContext — agents sharing decision state must be "
|
||||
"wired to the same context"
|
||||
)
|
||||
self.had_live_state = True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CrewAI entry points
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run(
|
||||
self,
|
||||
action: str,
|
||||
category: Optional[str] = None,
|
||||
scenario: Optional[str] = None,
|
||||
reasoning: Optional[str] = None,
|
||||
outcome: Optional[str] = None,
|
||||
confidence: float = 0.8,
|
||||
entities: Optional[str] = None,
|
||||
decision_id: Optional[str] = None,
|
||||
depth: int = 3,
|
||||
decision_data: Optional[str] = None,
|
||||
policy_rules: Optional[str] = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
valid = {
|
||||
"record_decision",
|
||||
"find_precedents",
|
||||
"trace_causal_chain",
|
||||
"analyze_impact",
|
||||
"check_policy",
|
||||
}
|
||||
if action not in valid:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": f"Unknown action '{action}'. Valid actions: "
|
||||
+ ", ".join(sorted(valid))
|
||||
}
|
||||
)
|
||||
|
||||
if action == "record_decision":
|
||||
return self._record_decision(
|
||||
category=category or "general",
|
||||
scenario=scenario or "decision recorded",
|
||||
reasoning=reasoning or "agent decision",
|
||||
outcome=outcome or "recorded",
|
||||
confidence=confidence,
|
||||
entities=entities,
|
||||
)
|
||||
if action == "find_precedents":
|
||||
return self._find_precedents(scenario=scenario or "", category=category)
|
||||
if action == "trace_causal_chain":
|
||||
return self._trace_causal_chain(decision_id or "", depth=depth)
|
||||
if action == "analyze_impact":
|
||||
return self._analyze_impact(decision_id or "")
|
||||
return self._check_policy(decision_data or "", policy_rules)
|
||||
|
||||
async def _arun(self, action: str, **kwargs: Any) -> str:
|
||||
"""Async variant of ``_run`` for CrewAI's async tool path."""
|
||||
return self._run(action=action, **kwargs)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Actions
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _record_decision(
|
||||
self,
|
||||
category: str,
|
||||
scenario: str,
|
||||
reasoning: str,
|
||||
outcome: str,
|
||||
confidence: float = 0.8,
|
||||
entities: Optional[str] = None,
|
||||
) -> str:
|
||||
entity_list: Optional[List[str]] = None
|
||||
if entities:
|
||||
entity_list = [e.strip() for e in entities.split(",") if e.strip()]
|
||||
|
||||
try:
|
||||
decision_id = self.context.record_decision(
|
||||
category=category,
|
||||
scenario=scenario,
|
||||
reasoning=reasoning,
|
||||
outcome=outcome,
|
||||
confidence=float(confidence),
|
||||
entities=entity_list,
|
||||
)
|
||||
result = {"decision_id": str(decision_id), "status": "recorded"}
|
||||
logger.info("record_decision → %s", decision_id)
|
||||
except Exception as exc:
|
||||
result = {"error": str(exc), "status": "failed"}
|
||||
logger.warning("record_decision failed: %s", exc)
|
||||
|
||||
return json.dumps(result)
|
||||
|
||||
def _find_precedents(
|
||||
self,
|
||||
scenario: str,
|
||||
category: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> str:
|
||||
k = limit if limit is not None else self.max_precedents
|
||||
try:
|
||||
precedents = self.context.find_precedents_advanced(
|
||||
scenario=scenario,
|
||||
category=category,
|
||||
limit=k,
|
||||
)
|
||||
out: List[Dict[str, Any]] = []
|
||||
for p in (precedents or [])[:k]:
|
||||
if isinstance(p, dict):
|
||||
out.append(p)
|
||||
else:
|
||||
out.append(
|
||||
{
|
||||
"scenario": getattr(p, "scenario", str(p)),
|
||||
"outcome": getattr(p, "outcome", ""),
|
||||
"confidence": getattr(p, "confidence", 0.0),
|
||||
"category": getattr(p, "category", ""),
|
||||
}
|
||||
)
|
||||
logger.info("find_precedents('%s') → %d results", scenario, len(out))
|
||||
return json.dumps({"precedents": out, "count": len(out)})
|
||||
except Exception as exc:
|
||||
logger.warning("find_precedents failed: %s", exc)
|
||||
return json.dumps({"precedents": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _trace_causal_chain(self, decision_id: str, depth: Optional[int] = None) -> str:
|
||||
if not decision_id:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": "decision_id is required for trace_causal_chain",
|
||||
"causal_chain": [],
|
||||
"decision_id": "",
|
||||
}
|
||||
)
|
||||
max_depth = depth or self.causal_depth
|
||||
try:
|
||||
graph = getattr(self.context, "knowledge_graph", None)
|
||||
if graph is None:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": (
|
||||
"causal tracing is not available on this knowledge "
|
||||
"graph (the decision context has no knowledge_graph)"
|
||||
),
|
||||
"causal_chain": [],
|
||||
"decision_id": decision_id,
|
||||
}
|
||||
)
|
||||
trace = getattr(graph, "trace_decision_causality", None)
|
||||
if trace is None:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": (
|
||||
"causal tracing is not available on this knowledge graph "
|
||||
"(graph.trace_decision_causality is not implemented)"
|
||||
),
|
||||
"causal_chain": [],
|
||||
"decision_id": decision_id,
|
||||
}
|
||||
)
|
||||
chain = trace(decision_id, max_depth=max_depth)
|
||||
return json.dumps({"causal_chain": chain, "decision_id": decision_id})
|
||||
except Exception as exc:
|
||||
logger.warning("trace_causal_chain failed: %s", exc)
|
||||
return json.dumps(
|
||||
{"error": str(exc), "causal_chain": [], "decision_id": decision_id}
|
||||
)
|
||||
|
||||
def _analyze_impact(self, decision_id: str) -> str:
|
||||
try:
|
||||
influence = self.context.analyze_decision_influence(decision_id)
|
||||
if not isinstance(influence, dict):
|
||||
influence = {"influence": str(influence)}
|
||||
influence["decision_id"] = decision_id
|
||||
return json.dumps(influence)
|
||||
except Exception as exc:
|
||||
logger.warning("analyze_impact failed: %s", exc)
|
||||
return json.dumps({"error": str(exc), "decision_id": decision_id})
|
||||
|
||||
def _check_policy(
|
||||
self,
|
||||
decision_data: str,
|
||||
policy_rules: Optional[str] = None,
|
||||
) -> str:
|
||||
try:
|
||||
data = (
|
||||
json.loads(decision_data)
|
||||
if isinstance(decision_data, str)
|
||||
else decision_data
|
||||
)
|
||||
except json.JSONDecodeError as exc:
|
||||
return json.dumps(
|
||||
{
|
||||
"compliant": False,
|
||||
"violations": [f"Invalid decision_data JSON: {exc}"],
|
||||
"warnings": [],
|
||||
}
|
||||
)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return json.dumps(
|
||||
{
|
||||
"compliant": False,
|
||||
"violations": [
|
||||
f"decision_data must decode to a JSON object, "
|
||||
f"got {type(data).__name__}: {data!r}"
|
||||
],
|
||||
"warnings": [],
|
||||
}
|
||||
)
|
||||
|
||||
violations: List[str] = []
|
||||
warnings: List[str] = []
|
||||
|
||||
rules: List[str] = []
|
||||
if policy_rules:
|
||||
try:
|
||||
parsed_rules = json.loads(policy_rules)
|
||||
except json.JSONDecodeError:
|
||||
rules = [r.strip() for r in policy_rules.split(",") if r.strip()]
|
||||
else:
|
||||
if isinstance(parsed_rules, str):
|
||||
rules = [parsed_rules]
|
||||
elif isinstance(parsed_rules, list):
|
||||
for item in parsed_rules:
|
||||
if isinstance(item, str):
|
||||
rules.append(item)
|
||||
else:
|
||||
warnings.append(
|
||||
f"Ignoring non-string policy rule entry: {item!r}"
|
||||
)
|
||||
else:
|
||||
warnings.append(
|
||||
f"policy_rules must decode to a JSON list of rule strings, "
|
||||
f"got {type(parsed_rules).__name__}: {parsed_rules!r}"
|
||||
)
|
||||
|
||||
for rule in rules:
|
||||
try:
|
||||
if not self._eval_rule(rule, data):
|
||||
violations.append(f"Rule violated: {rule}")
|
||||
except Exception as exc:
|
||||
warnings.append(f"Could not evaluate rule '{rule}': {exc}")
|
||||
|
||||
compliant = len(violations) == 0
|
||||
logger.debug(
|
||||
"check_policy: compliant=%s, violations=%d", compliant, len(violations)
|
||||
)
|
||||
return json.dumps(
|
||||
{
|
||||
"compliant": compliant,
|
||||
"violations": violations,
|
||||
"warnings": warnings,
|
||||
}
|
||||
)
|
||||
|
||||
def _eval_rule(self, rule: str, data: Dict[str, Any]) -> bool:
|
||||
"""Evaluate a simple comparison rule (``field op value``) against data.
|
||||
|
||||
This is a small standalone evaluator for the tool's ``check_policy``
|
||||
action — it is intentionally independent of Semantica's policy engine
|
||||
so agents get a bounded, side-effect-free rule check. Rules are
|
||||
``<field> <op> <value>`` comparisons only; there is no expression
|
||||
evaluation (no ``eval``), so untrusted rule strings are safe to pass.
|
||||
|
||||
Values are coerced type-aware: ``true``/``false`` (and ``1``/``0``)
|
||||
become booleans, numeric literals become numbers, and string values
|
||||
that parse as numbers are compared numerically, so ``score == 0.9``
|
||||
holds for ``score: "0.90"`` and ``enabled == false`` holds for
|
||||
``enabled: false``. Field names may contain hyphens, dots and spaces
|
||||
(e.g. ``risk-score >= 0.9``); they are matched against ``data`` keys
|
||||
as-is.
|
||||
"""
|
||||
m = re.match(r"(.+?)\s*(>=|<=|!=|==|>|<)\s*(.+)$", rule.strip())
|
||||
if not m:
|
||||
raise ValueError(f"unrecognised rule format: {rule!r}")
|
||||
field, op, val_str = m.group(1), m.group(2), m.group(3).strip().strip("\"'")
|
||||
if field not in data:
|
||||
raise ValueError(f"rule references undefined field {field!r}")
|
||||
actual = data[field]
|
||||
if actual is None:
|
||||
raise ValueError(f"field {field!r} is null — cannot evaluate rule")
|
||||
val = self._coerce_value(val_str)
|
||||
if isinstance(actual, str):
|
||||
actual = self._coerce_value(actual)
|
||||
ops = {
|
||||
">=": lambda a, b: a >= b,
|
||||
"<=": lambda a, b: a <= b,
|
||||
"!=": lambda a, b: a != b,
|
||||
"==": lambda a, b: a == b,
|
||||
">": lambda a, b: a > b,
|
||||
"<": lambda a, b: a < b,
|
||||
}
|
||||
return ops[op](actual, val)
|
||||
|
||||
@staticmethod
|
||||
def _coerce_value(value: str) -> Any:
|
||||
"""Parse a rule literal into its most specific Python type."""
|
||||
text = value.strip()
|
||||
lowered = text.lower()
|
||||
if lowered in ("true", "1"):
|
||||
return True
|
||||
if lowered in ("false", "0"):
|
||||
return False
|
||||
try:
|
||||
return int(text)
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
return float(text)
|
||||
except ValueError:
|
||||
pass
|
||||
return text
|
||||
|
||||
# When crewai is absent there is no BaseTool to provide the public
|
||||
# ``run``/``arun`` entry points, so expose them directly. With crewai
|
||||
# installed these are left untouched so crewai's own implementations
|
||||
# (usage tracking, ``result_as_answer``) win.
|
||||
if not CREWAI_AVAILABLE:
|
||||
|
||||
def run(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool synchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
|
||||
async def arun(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool asynchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
@@ -0,0 +1,573 @@
|
||||
"""
|
||||
SemanticaKGTool — a CrewAI ``BaseTool`` exposing Semantica's knowledge-graph
|
||||
pipeline (``NERExtractor``, ``RelationExtractor``, ``ContextGraph``) to agents.
|
||||
|
||||
Lets agents build and query a shared ``ContextGraph`` as part of their
|
||||
reasoning loop.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[crewai]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from integrations.crewai import SemanticaKGTool
|
||||
>>> from semantica.context import ContextGraph
|
||||
>>> from crewai import Agent, Crew, Task
|
||||
>>> graph = ContextGraph()
|
||||
>>> tool = SemanticaKGTool(graph=graph)
|
||||
>>> crew = Crew(
|
||||
... agents=[Agent(role="...", goal="...", backstory="...", tools=[tool])],
|
||||
... tasks=[...],
|
||||
... )
|
||||
|
||||
Tools exposed
|
||||
-------------
|
||||
extract_entities — Extract named entities from text
|
||||
extract_relations — Extract relationships between entities
|
||||
add_to_graph — Extract entities/relations from text and add them to the graph
|
||||
query_graph — Query the graph by keyword
|
||||
find_related — Find concepts related to a given entity within ``hops``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import weakref
|
||||
from typing import Any, Dict, List, Literal, Optional, Sequence, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE, CREWAI_IMPORT_ERROR # noqa: F401
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: CrewAI BaseTool base class
|
||||
# ---------------------------------------------------------------------------
|
||||
_BaseTool: Any = object
|
||||
|
||||
if CREWAI_AVAILABLE:
|
||||
from crewai.tools import BaseTool as _BaseTool # type: ignore
|
||||
|
||||
# One re-entrant lock per graph so concurrent tool invocations sharing a graph
|
||||
# cannot double-count duplicate adds (check-then-act is not atomic), while
|
||||
# independent graphs are never serialised against each other. An RLock also
|
||||
# means an extractor callback that re-enters add_to_graph on the same graph
|
||||
# cannot deadlock.
|
||||
_graph_locks_guard = threading.Lock()
|
||||
_graph_locks: "weakref.WeakKeyDictionary[Any, threading.RLock]" = (
|
||||
weakref.WeakKeyDictionary()
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input schema
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaKGToolInput(BaseModel):
|
||||
"""
|
||||
Input schema for ``SemanticaKGTool``.
|
||||
|
||||
Exactly one action is dispatched per call; the remaining fields are only
|
||||
used by the actions that need them.
|
||||
"""
|
||||
|
||||
action: Literal[
|
||||
"extract_entities",
|
||||
"extract_relations",
|
||||
"add_to_graph",
|
||||
"query_graph",
|
||||
"find_related",
|
||||
] = Field(
|
||||
...,
|
||||
description=(
|
||||
"Which graph operation to run. One of: 'extract_entities', "
|
||||
"'extract_relations', 'add_to_graph', 'query_graph', 'find_related'."
|
||||
),
|
||||
)
|
||||
text: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"Input text. Used by 'extract_entities', 'extract_relations' and "
|
||||
"'add_to_graph'."
|
||||
),
|
||||
)
|
||||
query: Optional[str] = Field(
|
||||
None, description="Search query. Used by 'query_graph'."
|
||||
)
|
||||
entity: Optional[str] = Field(
|
||||
None,
|
||||
description="Root entity name. Used by 'find_related'.",
|
||||
)
|
||||
hops: int = Field(
|
||||
1,
|
||||
ge=1,
|
||||
le=10,
|
||||
description="Maximum relationship hops. Used by 'find_related'.",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SemanticaKGTool
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaKGTool(_BaseTool): # type: ignore[misc]
|
||||
"""
|
||||
CrewAI tool that surfaces Semantica's KG pipeline as agent actions.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph:
|
||||
A ``semantica.context.ContextGraph`` to read/write. A fresh in-memory
|
||||
graph is used when ``None``.
|
||||
ner_extractor:
|
||||
A ``semantica.semantic_extract.NERExtractor`` instance; auto-created
|
||||
when ``None``.
|
||||
relation_extractor:
|
||||
A ``semantica.semantic_extract.RelationExtractor`` instance; auto-
|
||||
created when ``None``.
|
||||
"""
|
||||
|
||||
name: str = "semantica_knowledge_graph"
|
||||
description: str = (
|
||||
"Build and query a semantic knowledge graph. Actions: "
|
||||
"'extract_entities' (extract named entities from 'text'), "
|
||||
"'extract_relations' (extract relationships from 'text'), "
|
||||
"'add_to_graph' (extract entities/relations from 'text' and add them "
|
||||
"to the shared graph), 'query_graph' (keyword search using 'query'), "
|
||||
"'find_related' (find concepts related to 'entity' within 'hops' "
|
||||
"hops). Returns JSON."
|
||||
)
|
||||
args_schema: Type[BaseModel] = SemanticaKGToolInput
|
||||
graph: Any = Field(default=None, exclude=True)
|
||||
ner_extractor: Any = Field(default=None, exclude=True)
|
||||
relation_extractor: Any = Field(default=None, exclude=True)
|
||||
had_live_state: bool = False
|
||||
reconstructed_state: bool = Field(default=False, exclude=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
graph: Any = None,
|
||||
ner_extractor: Any = None,
|
||||
relation_extractor: Any = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if CREWAI_AVAILABLE:
|
||||
super().__init__(
|
||||
graph=graph,
|
||||
ner_extractor=ner_extractor,
|
||||
relation_extractor=relation_extractor,
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
super().__init__()
|
||||
self.graph = graph
|
||||
self.ner_extractor = ner_extractor
|
||||
self.relation_extractor = relation_extractor
|
||||
# Degraded mode is a plain class — no model_post_init lifecycle.
|
||||
self._ensure_defaults()
|
||||
|
||||
logger.info("SemanticaKGTool initialised (crewai=%s)", CREWAI_AVAILABLE)
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
"""Re-create default state after validation/deserialisation.
|
||||
|
||||
``graph``/extractors are excluded from JSON serialisation (CrewAI
|
||||
checkpoints serialise every tool via ``model_dump(mode="json")``), so a
|
||||
tool restored from a checkpoint has ``None`` state until this runs.
|
||||
"""
|
||||
self._ensure_defaults()
|
||||
super().model_post_init(__context)
|
||||
|
||||
def _ensure_defaults(self) -> None:
|
||||
"""Lazy-import and build defaults for any missing shared state."""
|
||||
# Lazy imports keep the module importable without heavy deps
|
||||
if self.graph is None:
|
||||
from semantica.context import ContextGraph
|
||||
|
||||
self.graph = ContextGraph()
|
||||
if self.had_live_state:
|
||||
self.reconstructed_state = True
|
||||
logger.warning(
|
||||
"SemanticaKGTool: the live graph was lost during "
|
||||
"serialization/checkpoint restore — an EMPTY graph was "
|
||||
"reconstructed; re-attach the original graph before "
|
||||
"continuing"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"SemanticaKGTool created a fresh in-memory ContextGraph — "
|
||||
"agents sharing this tool's graph must be wired explicitly"
|
||||
)
|
||||
self.had_live_state = True
|
||||
if self.ner_extractor is None:
|
||||
from semantica.semantic_extract import NERExtractor
|
||||
|
||||
self.ner_extractor = NERExtractor()
|
||||
if self.relation_extractor is None:
|
||||
from semantica.semantic_extract import RelationExtractor
|
||||
|
||||
self.relation_extractor = RelationExtractor()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CrewAI entry points
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run(
|
||||
self,
|
||||
action: str,
|
||||
text: Optional[str] = None,
|
||||
query: Optional[str] = None,
|
||||
entity: Optional[str] = None,
|
||||
hops: int = 1,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
"""
|
||||
Dispatch a graph action. Always returns a JSON string so the agent
|
||||
receives a structured, parseable result.
|
||||
"""
|
||||
valid = {
|
||||
"extract_entities",
|
||||
"extract_relations",
|
||||
"add_to_graph",
|
||||
"query_graph",
|
||||
"find_related",
|
||||
}
|
||||
if action not in valid:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": f"Unknown action '{action}'. Valid actions: "
|
||||
+ ", ".join(sorted(valid))
|
||||
}
|
||||
)
|
||||
|
||||
if action == "extract_entities":
|
||||
return self._extract_entities(text or "")
|
||||
if action == "extract_relations":
|
||||
return self._extract_relations(text or "")
|
||||
if action == "add_to_graph":
|
||||
return self._add_from_text(text or "")
|
||||
if action == "query_graph":
|
||||
return self._query_graph(query or "")
|
||||
return self._find_related(entity or "", hops=hops)
|
||||
|
||||
async def _arun(
|
||||
self,
|
||||
action: str,
|
||||
text: Optional[str] = None,
|
||||
query: Optional[str] = None,
|
||||
entity: Optional[str] = None,
|
||||
hops: int = 1,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
"""
|
||||
Async variant of ``_run`` for CrewAI's async tool path.
|
||||
"""
|
||||
return self._run(
|
||||
action=action, text=text, query=query, entity=entity, hops=hops, **kwargs
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Entity/relation field access (handles both Semantica dataclasses and
|
||||
# third-party shapes like MagicMock/plain dicts in stubs)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _first_str(obj: Any, attrs: Sequence[str]) -> str:
|
||||
"""Return the first attribute value that is a non-empty string."""
|
||||
for attr in attrs:
|
||||
value = getattr(obj, attr, None)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
if isinstance(obj, dict):
|
||||
for key in attrs:
|
||||
value = obj.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return ""
|
||||
|
||||
@classmethod
|
||||
def _entity_name(cls, e: Any) -> str:
|
||||
"""Best-effort name for an entity-like object."""
|
||||
return cls._first_str(e, ("name", "text", "label", "node_id", "id"))
|
||||
|
||||
@classmethod
|
||||
def _entity_type(cls, e: Any) -> str:
|
||||
"""Best-effort type/label for an entity-like object."""
|
||||
return cls._first_str(e, ("type", "label")) or "Entity"
|
||||
|
||||
@classmethod
|
||||
def _relation_source(cls, r: Any) -> str:
|
||||
"""Best-effort source of a relation-like object."""
|
||||
src = cls._first_str(r, ("source",))
|
||||
if not src:
|
||||
src = cls._entity_name(getattr(r, "subject", None))
|
||||
return src
|
||||
|
||||
@classmethod
|
||||
def _relation_target(cls, r: Any) -> str:
|
||||
"""Best-effort target of a relation-like object."""
|
||||
tgt = cls._first_str(r, ("target",))
|
||||
if not tgt:
|
||||
tgt = cls._entity_name(getattr(r, "object", None))
|
||||
return tgt
|
||||
|
||||
@classmethod
|
||||
def _relation_type(cls, r: Any) -> str:
|
||||
"""Best-effort relation type of a relation-like object."""
|
||||
rtype = cls._first_str(r, ("type", "relation", "predicate"))
|
||||
return rtype or "related_to"
|
||||
|
||||
@classmethod
|
||||
def _confidence(cls, e: Any) -> float:
|
||||
"""Normalise an entity/relation confidence value to a float."""
|
||||
try:
|
||||
val = getattr(e, "confidence", None)
|
||||
if val is None:
|
||||
return 1.0
|
||||
return round(float(val), 4)
|
||||
except (TypeError, ValueError):
|
||||
return 1.0
|
||||
|
||||
@classmethod
|
||||
def _graph_lock(cls, graph: Any) -> threading.RLock:
|
||||
"""Return the re-entrant lock guarding a specific graph."""
|
||||
with _graph_locks_guard:
|
||||
lock = _graph_locks.get(graph)
|
||||
if lock is None:
|
||||
lock = threading.RLock()
|
||||
_graph_locks[graph] = lock
|
||||
return lock
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Actions
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _extract_entities(self, text: str) -> str:
|
||||
"""Extract named entities from ``text``."""
|
||||
try:
|
||||
raw = self.ner_extractor.extract_entities(text) or []
|
||||
entities = [
|
||||
{
|
||||
"name": self._entity_name(e),
|
||||
"type": self._entity_type(e),
|
||||
"confidence": self._confidence(e),
|
||||
}
|
||||
for e in raw
|
||||
if self._entity_name(e)
|
||||
]
|
||||
logger.debug("extract_entities → %d entities", len(entities))
|
||||
return json.dumps({"entities": entities, "count": len(entities)})
|
||||
except Exception as exc:
|
||||
logger.warning("extract_entities failed: %s", exc)
|
||||
return json.dumps({"entities": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _extract_relations(self, text: str) -> str:
|
||||
"""Extract relationships between entities in ``text``."""
|
||||
try:
|
||||
raw = self.relation_extractor.extract_relations(text) or []
|
||||
relations = [
|
||||
{
|
||||
"source": self._relation_source(r),
|
||||
"relation": self._relation_type(r),
|
||||
"target": self._relation_target(r),
|
||||
"confidence": self._confidence(r),
|
||||
}
|
||||
for r in raw
|
||||
]
|
||||
logger.debug("extract_relations → %d relations", len(relations))
|
||||
return json.dumps({"relations": relations, "count": len(relations)})
|
||||
except Exception as exc:
|
||||
logger.warning("extract_relations failed: %s", exc)
|
||||
return json.dumps({"relations": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _add_from_text(self, text: str) -> str:
|
||||
"""
|
||||
Extract entities and relations from ``text`` and add them to the graph.
|
||||
|
||||
Duplicate nodes/edges (same id, or same source/type/target) are
|
||||
skipped so repeated calls are idempotent. Returns JSON with the
|
||||
number of nodes/edges added.
|
||||
"""
|
||||
nodes_added = 0
|
||||
edges_added = 0
|
||||
try:
|
||||
with self._graph_lock(self.graph):
|
||||
existing_nodes = {
|
||||
n.get("id") or n.get("node_id")
|
||||
for n in (
|
||||
self.graph.find_nodes() or [] # type: ignore[attr-defined]
|
||||
)
|
||||
if n.get("id") or n.get("node_id")
|
||||
}
|
||||
existing_edges = {
|
||||
(e.get("source"), e.get("type") or "related_to", e.get("target"))
|
||||
for e in (
|
||||
self.graph.find_edges() or [] # type: ignore[attr-defined]
|
||||
)
|
||||
if e.get("source") and e.get("target")
|
||||
}
|
||||
|
||||
raw_entities = self.ner_extractor.extract_entities(text) or []
|
||||
entities: List[Any] = []
|
||||
seen: set = set()
|
||||
for e in raw_entities:
|
||||
name = self._entity_name(e)
|
||||
ntype = self._entity_type(e)
|
||||
if not name or name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
entities.append(e)
|
||||
if name in existing_nodes:
|
||||
continue
|
||||
try:
|
||||
if self.graph.add_node(node_id=name, node_type=ntype):
|
||||
nodes_added += 1
|
||||
existing_nodes.add(name)
|
||||
except Exception as exc:
|
||||
logger.debug("add_node(%r) failed: %s", name, exc)
|
||||
|
||||
raw_relations = (
|
||||
self.relation_extractor.extract_relations(text, entities=entities)
|
||||
or []
|
||||
)
|
||||
for r in raw_relations:
|
||||
src = self._relation_source(r)
|
||||
tgt = self._relation_target(r)
|
||||
rtype = self._relation_type(r)
|
||||
if not src or not tgt:
|
||||
continue
|
||||
key = (src, rtype, tgt)
|
||||
if key in existing_edges:
|
||||
continue
|
||||
try:
|
||||
if self.graph.add_edge(
|
||||
source_id=src, target_id=tgt, edge_type=rtype
|
||||
):
|
||||
edges_added += 1
|
||||
existing_edges.add(key)
|
||||
except Exception as exc:
|
||||
logger.debug("add_edge(%r) failed: %s", key, exc)
|
||||
logger.debug("add_to_graph: +%d nodes, +%d edges", nodes_added, edges_added)
|
||||
return json.dumps({"nodes_added": nodes_added, "edges_added": edges_added})
|
||||
except Exception as exc:
|
||||
logger.warning("add_to_graph failed: %s", exc)
|
||||
return json.dumps({"nodes_added": 0, "edges_added": 0, "error": str(exc)})
|
||||
|
||||
def _query_graph(self, query: str) -> str:
|
||||
"""Keyword-search graph nodes by id, type and content."""
|
||||
try:
|
||||
q = (query or "").strip().lower()
|
||||
out: List[dict] = []
|
||||
seen: set = set()
|
||||
|
||||
query_method = getattr(self.graph, "query", None)
|
||||
if query_method is not None:
|
||||
for match in query_method(query) or []:
|
||||
node = match.get("node") or {}
|
||||
nid = node.get("id", "") or node.get("node_id", "")
|
||||
if not nid or nid in seen:
|
||||
continue
|
||||
seen.add(nid)
|
||||
content = match.get("content") or node.get("content", "")
|
||||
out.append(
|
||||
{
|
||||
"id": nid,
|
||||
"type": node.get("type", "") or node.get("node_type", ""),
|
||||
"label": nid,
|
||||
"content": str(content)[:500],
|
||||
"score": round(float(match.get("score") or 0.0), 4),
|
||||
}
|
||||
)
|
||||
|
||||
if q:
|
||||
for n in self.graph.find_nodes() or []: # type: ignore[attr-defined]
|
||||
if isinstance(n, dict):
|
||||
nid = n.get("id", "") or n.get("node_id", "")
|
||||
ntype = n.get("type", "") or n.get("node_type", "")
|
||||
content = str(
|
||||
n.get("content")
|
||||
or (n.get("properties") or {}).get("content", "")
|
||||
or ""
|
||||
)
|
||||
else:
|
||||
nid = getattr(n, "id", getattr(n, "label", ""))
|
||||
ntype = getattr(n, "node_type", "")
|
||||
content = str(getattr(n, "content", "") or "")
|
||||
if not nid or nid in seen:
|
||||
continue
|
||||
if q in str(nid).lower() or q in str(ntype).lower():
|
||||
seen.add(nid)
|
||||
out.append(
|
||||
{
|
||||
"id": nid,
|
||||
"type": ntype,
|
||||
"label": nid,
|
||||
"content": content[:500],
|
||||
"score": 1.0,
|
||||
}
|
||||
)
|
||||
return json.dumps({"results": out, "count": len(out)})
|
||||
except Exception as exc:
|
||||
logger.warning("query_graph failed: %s", exc)
|
||||
return json.dumps({"results": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _find_related(self, entity: str, hops: int = 1) -> str:
|
||||
"""Find concepts related to ``entity`` within ``hops`` graph hops.
|
||||
|
||||
Traversal is undirected — an edge counts as related regardless of
|
||||
direction, so both outgoing and incoming edges are honored.
|
||||
"""
|
||||
try:
|
||||
adjacency: Dict[str, List[str]] = {}
|
||||
for edge in self.graph.find_edges() or []: # type: ignore[attr-defined]
|
||||
if isinstance(edge, dict):
|
||||
src = edge.get("source")
|
||||
tgt = edge.get("target")
|
||||
else:
|
||||
src = getattr(edge, "source", None)
|
||||
tgt = getattr(edge, "target", None)
|
||||
if not src or not tgt:
|
||||
continue
|
||||
adjacency.setdefault(src, []).append(tgt)
|
||||
adjacency.setdefault(tgt, []).append(src)
|
||||
|
||||
related: List[str] = []
|
||||
frontier = [entity]
|
||||
visited = {entity}
|
||||
for _ in range(max(1, hops)):
|
||||
next_frontier: List[str] = []
|
||||
for e in frontier:
|
||||
for n in adjacency.get(e, []):
|
||||
if n in visited:
|
||||
continue
|
||||
visited.add(n)
|
||||
next_frontier.append(n)
|
||||
related.append(n)
|
||||
frontier = next_frontier
|
||||
|
||||
logger.debug("find_related('%s', hops=%d) → %d", entity, hops, len(related))
|
||||
return json.dumps(
|
||||
{"entity": entity, "related": related, "count": len(related)}
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("find_related failed: %s", exc)
|
||||
return json.dumps(
|
||||
{"entity": entity, "related": [], "count": 0, "error": str(exc)}
|
||||
)
|
||||
|
||||
# When crewai is absent there is no BaseTool to provide the public
|
||||
# ``run``/``arun`` entry points, so expose them directly. With crewai
|
||||
# installed these are left untouched so crewai's own implementations
|
||||
# (usage tracking, ``result_as_answer``) win.
|
||||
if not CREWAI_AVAILABLE:
|
||||
|
||||
def run(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool synchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
|
||||
async def arun(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool asynchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
@@ -0,0 +1,331 @@
|
||||
"""
|
||||
SemanticaKnowledgeSource — expose a Semantica ``ContextGraph`` as a CrewAI
|
||||
knowledge source.
|
||||
|
||||
Lets a ``Crew`` load the current state of a knowledge graph (nodes, edges,
|
||||
metadata) into its knowledge storage, so every agent gets retrieval access to
|
||||
graph knowledge during the kickoff.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[crewai]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from integrations.crewai import SemanticaKnowledgeSource
|
||||
>>> from semantica.context import ContextGraph
|
||||
>>> from crewai import Agent, Crew, Task
|
||||
>>> graph = ContextGraph()
|
||||
>>> graph.add_node(node_id="privacy", node_type="policy")
|
||||
>>> crew = Crew(
|
||||
... agents=[...],
|
||||
... tasks=[...],
|
||||
... knowledge_sources=[SemanticaKnowledgeSource(graph=graph)],
|
||||
... )
|
||||
|
||||
Compatibility
|
||||
-------------
|
||||
Works with ``crewai >= 0.80.0``. The ``BaseKnowledgeSource`` contract changed
|
||||
between versions (``load_content`` → ``validate_content``/``aadd``), so this
|
||||
source implements both legacy and current methods. It degrades gracefully
|
||||
when ``crewai`` is not installed: the class is still importable and carries the
|
||||
full Semantica API, but cannot be passed to a ``Crew``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: CrewAI BaseKnowledgeSource base class
|
||||
# ---------------------------------------------------------------------------
|
||||
_BaseKnowledgeSource: Any = object
|
||||
|
||||
if CREWAI_AVAILABLE:
|
||||
from crewai.knowledge.source.base_knowledge_source import (
|
||||
BaseKnowledgeSource as _BaseKnowledgeSource, # type: ignore
|
||||
)
|
||||
|
||||
|
||||
def _chunk_text_manual(text: str, chunk_size: int, chunk_overlap: int) -> List[str]:
|
||||
"""Fallback plain-text chunker for when CrewAI helpers are unavailable."""
|
||||
if not text:
|
||||
return []
|
||||
if int(chunk_size) <= 0:
|
||||
return [text]
|
||||
size = max(1, int(chunk_size))
|
||||
overlap = max(0, int(chunk_overlap))
|
||||
if len(text) <= size:
|
||||
return [text]
|
||||
step = max(1, size - overlap)
|
||||
return [text[i : i + size] for i in range(0, len(text), step)]
|
||||
|
||||
|
||||
class SemanticaKnowledgeSource(_BaseKnowledgeSource): # type: ignore[misc]
|
||||
"""
|
||||
CrewAI knowledge source backed by a Semantica ``ContextGraph``.
|
||||
|
||||
On ``add()`` the graph's nodes and edges are serialised into readable text
|
||||
and pushed through the standard CrewAI chunking / storage pipeline, making
|
||||
graph knowledge retrievable by every agent in the crew.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph:
|
||||
A ``semantica.context.ContextGraph`` to expose. A fresh in-memory
|
||||
graph is created when ``None``.
|
||||
name:
|
||||
Source name. Defaults to ``"semantica_knowledge_graph"``.
|
||||
chunk_size:
|
||||
Max characters per chunk (default 4000).
|
||||
chunk_overlap:
|
||||
Character overlap between adjacent chunks (default 200).
|
||||
"""
|
||||
|
||||
name: str = "semantica_knowledge_graph"
|
||||
graph: Any = Field(default=None, exclude=True)
|
||||
chunk_size: int = 4000
|
||||
chunk_overlap: int = 200
|
||||
had_live_state: bool = False
|
||||
reconstructed_state: bool = Field(default=False, exclude=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
graph: Any = None,
|
||||
name: Optional[str] = None,
|
||||
chunk_size: int = 4000,
|
||||
chunk_overlap: int = 200,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if CREWAI_AVAILABLE:
|
||||
# Do NOT eagerly build a graph here: pydantic calls this ``__init__``
|
||||
# during ``model_validate`` (checkpoint restore), and the eager
|
||||
# build would hide that a live graph was lost. ``model_post_init``
|
||||
# rebuilds defaults and flags ``reconstructed_state`` instead.
|
||||
super().__init__(
|
||||
graph=graph,
|
||||
name=name or "semantica_knowledge_graph",
|
||||
chunk_size=int(chunk_size),
|
||||
chunk_overlap=int(chunk_overlap),
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
if graph is None:
|
||||
from semantica.context import ContextGraph
|
||||
|
||||
graph = ContextGraph()
|
||||
super().__init__()
|
||||
self.graph = graph
|
||||
self.name = name or "semantica_knowledge_graph"
|
||||
self.chunk_size = int(chunk_size)
|
||||
self.chunk_overlap = int(chunk_overlap)
|
||||
|
||||
logger.info(
|
||||
"SemanticaKnowledgeSource initialised (crewai=%s, chunk_size=%d)",
|
||||
CREWAI_AVAILABLE,
|
||||
self.chunk_size,
|
||||
)
|
||||
self.had_live_state = True
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
"""Re-create default state after validation/deserialisation.
|
||||
|
||||
``graph`` is excluded from JSON serialisation (CrewAI checkpoints
|
||||
serialise their models via ``model_dump(mode="json")``), so a source
|
||||
restored from a checkpoint has ``None`` state until this runs.
|
||||
"""
|
||||
if self.graph is None:
|
||||
from semantica.context import ContextGraph
|
||||
|
||||
self.graph = ContextGraph()
|
||||
if self.had_live_state:
|
||||
self.reconstructed_state = True
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource: the live graph was lost during "
|
||||
"serialization/checkpoint restore — an EMPTY graph was "
|
||||
"reconstructed; re-attach the original graph before "
|
||||
"continuing"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource created a fresh in-memory "
|
||||
"ContextGraph — sources sharing knowledge must be wired to "
|
||||
"the same graph explicitly"
|
||||
)
|
||||
self.had_live_state = True
|
||||
super().model_post_init(__context)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Content extraction
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def load_content(self) -> Dict[str, str]:
|
||||
"""
|
||||
Serialise the graph into ``{id: readable_text}`` pairs.
|
||||
|
||||
Nodes are rendered with their type/content/metadata, edges with their
|
||||
source, relation type and target. This satisfies the legacy CrewAI
|
||||
``BaseKnowledgeSource.load_content`` contract.
|
||||
"""
|
||||
content: Dict[str, str] = {}
|
||||
graph = self.graph
|
||||
if graph is None:
|
||||
return content
|
||||
|
||||
try:
|
||||
for node in graph.find_nodes() or []: # type: ignore[attr-defined]
|
||||
nid = node.get("id") or node.get("node_id") or ""
|
||||
if not nid:
|
||||
continue
|
||||
parts = [
|
||||
"Entity",
|
||||
str(nid),
|
||||
"type: " + str(node.get("type", "entity")),
|
||||
]
|
||||
if node.get("content"):
|
||||
parts.append("content: " + str(node["content"]))
|
||||
if node.get("metadata"):
|
||||
try:
|
||||
import json
|
||||
|
||||
parts.append("metadata: " + json.dumps(node["metadata"]))
|
||||
except Exception:
|
||||
parts.append("metadata: " + str(node["metadata"]))
|
||||
content[str(nid)] = " | ".join(parts)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource.load_content (nodes) failed: %s", exc
|
||||
)
|
||||
|
||||
try:
|
||||
for idx, edge in enumerate(
|
||||
graph.find_edges() or [] # type: ignore[attr-defined]
|
||||
):
|
||||
src = edge.get("source")
|
||||
tgt = edge.get("target")
|
||||
if not src or not tgt:
|
||||
continue
|
||||
rel = edge.get("type") or edge.get("edge_type") or "related_to"
|
||||
weight = edge.get("weight")
|
||||
text = f"{src} -[{rel}]-> {tgt}"
|
||||
if weight is not None:
|
||||
text += f" (weight: {weight})"
|
||||
content[f"edge-{idx}"] = text
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource.load_content (edges) failed: %s", exc
|
||||
)
|
||||
|
||||
return content
|
||||
|
||||
def validate_content(self) -> Any:
|
||||
"""
|
||||
Validate that a readable graph is attached.
|
||||
|
||||
Satisfies the current CrewAI ``BaseKnowledgeSource.validate_content``
|
||||
contract.
|
||||
"""
|
||||
if self.graph is None:
|
||||
raise ValueError("SemanticaKnowledgeSource requires a ContextGraph.")
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Chunking + storage (abstract in both CrewAI generations)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _chunk(self, text: str) -> List[str]:
|
||||
"""Chunk ``text`` using CrewAI's helper when available, else manual."""
|
||||
helper = getattr(self, "_chunk_text", None)
|
||||
if helper is not None:
|
||||
try:
|
||||
return list(helper(text) or [])
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"SemanticaKnowledgeSource._chunk_text failed, falling back: %s", exc
|
||||
)
|
||||
return _chunk_text_manual(text, self.chunk_size, self.chunk_overlap)
|
||||
|
||||
def add(self) -> None:
|
||||
"""
|
||||
Process the graph into chunks and store them via CrewAI storage.
|
||||
|
||||
Sets both ``chunks`` (current CrewAI) and ``_chunks`` (legacy CrewAI)
|
||||
so either ``_save_documents`` implementation picks them up. If no
|
||||
storage has been wired (e.g. not yet attached to a ``Crew``), chunks
|
||||
are kept in memory.
|
||||
"""
|
||||
content = self.load_content()
|
||||
if not content:
|
||||
logger.debug("SemanticaKnowledgeSource.add: empty graph — nothing to store")
|
||||
return
|
||||
|
||||
chunks: List[str] = []
|
||||
for _, text in content.items():
|
||||
if text:
|
||||
chunks.extend(self._chunk(text))
|
||||
|
||||
self.chunks = chunks
|
||||
self._chunks = chunks
|
||||
|
||||
save = getattr(self, "_save_documents", None)
|
||||
if save is not None:
|
||||
if getattr(self, "storage", None) is None:
|
||||
logger.debug(
|
||||
"SemanticaKnowledgeSource.add: storage not wired — "
|
||||
"keeping chunks in memory"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
save()
|
||||
logger.info(
|
||||
"SemanticaKnowledgeSource.add: stored %d chunks", len(chunks)
|
||||
)
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"SemanticaKnowledgeSource.add: storage save FAILED (%s) — "
|
||||
"chunks are only kept in memory and agents will retrieve "
|
||||
"nothing. Configure the Crew embedder (e.g. an OpenAI "
|
||||
"embedder with OPENAI_API_KEY, or a local embedder) before "
|
||||
"running the crew.",
|
||||
exc,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"SemanticaKnowledgeSource.add: %d chunks ready in memory", len(chunks)
|
||||
)
|
||||
|
||||
async def aadd(self) -> None:
|
||||
"""
|
||||
Asynchronous variant of ``add()`` (current CrewAI contract).
|
||||
|
||||
The graph serialisation is CPU-bound, so it runs in a thread pool to
|
||||
avoid blocking the event loop.
|
||||
"""
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(None, self.add)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Inspection helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_content_summary(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Summarise what the source exposes (helpful for debugging / testing).
|
||||
"""
|
||||
content = self.load_content()
|
||||
return {
|
||||
"name": self.name,
|
||||
"source_count": len(content),
|
||||
"chunks": len(getattr(self, "chunks", []) or []),
|
||||
"crewai_available": CREWAI_AVAILABLE,
|
||||
}
|
||||
+12
-2
@@ -93,7 +93,14 @@ def _handle_tools_call(req_id: Any, params: dict) -> dict:
|
||||
result = tool["_handler"](args)
|
||||
except Exception as exc:
|
||||
log.exception("Tool %s raised an exception", name)
|
||||
return _err(req_id, _INTERNAL_ERROR, str(exc))
|
||||
# The exception's class name (e.g. "ValidationError", "TimeoutError")
|
||||
# is safe to surface — unlike str(exc), it never carries paths,
|
||||
# connection strings, or other internal detail — and lets the
|
||||
# client distinguish failure kinds without a full message.
|
||||
return _err(
|
||||
req_id, _INTERNAL_ERROR,
|
||||
f"Tool '{name}' failed ({type(exc).__name__}). See server logs for details.",
|
||||
)
|
||||
|
||||
# MCP spec: content must be a list of content items
|
||||
return _ok(req_id, {
|
||||
@@ -171,7 +178,10 @@ class SemanticaMCPServer:
|
||||
log.exception("Unhandled error in method %s", method)
|
||||
if req_id is None:
|
||||
return None
|
||||
return _err(req_id, _INTERNAL_ERROR, str(exc))
|
||||
return _err(
|
||||
req_id, _INTERNAL_ERROR,
|
||||
f"Method '{method}' failed ({type(exc).__name__}). See server logs for details.",
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def run(self) -> None:
|
||||
|
||||
@@ -53,7 +53,7 @@ plugins/
|
||||
## Prerequisites
|
||||
|
||||
```bash
|
||||
git clone https://github.com/Hawksight-AI/semantica.git
|
||||
git clone https://github.com/semantica-agi/semantica.git
|
||||
cd semantica
|
||||
pip install semantica # Python 3.10+
|
||||
```
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"name": "semantica-local",
|
||||
"owner": {
|
||||
"name": "Hawksight AI",
|
||||
"url": "https://github.com/Hawksight-AI/semantica"
|
||||
"name": "Semantica",
|
||||
"url": "https://github.com/semantica-agi/semantica"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
"author": {
|
||||
"name": "Semantica Contributors"
|
||||
},
|
||||
"homepage": "https://github.com/Hawksight-AI/semantica",
|
||||
"repository": "https://github.com/Hawksight-AI/semantica",
|
||||
"homepage": "https://github.com/semantica-agi/semantica",
|
||||
"repository": "https://github.com/semantica-agi/semantica",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"semantica",
|
||||
|
||||
+14
-5
@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "semantica"
|
||||
version = "0.6.5"
|
||||
description = "Accountability and context layer for AI agents. Context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable."
|
||||
version = "0.6.6"
|
||||
description = "Graph-Native Infrastructure for Context and Accountable AI Systems: context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable."
|
||||
readme = "README.md"
|
||||
license = { text = "MIT" }
|
||||
|
||||
@@ -85,7 +85,8 @@ dependencies = [
|
||||
"loguru>=0.7.3",
|
||||
"structlog>=22.1.0",
|
||||
"gensim>=4.4.0",
|
||||
"httpx<0.29.0"
|
||||
"httpx<0.29.0",
|
||||
"pyarrow>=14.0.0"
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
@@ -103,7 +104,7 @@ Discord = "https://discord.gg/sV34vps5hH"
|
||||
llm-openai = ["openai>=1.0.0"]
|
||||
llm-groq = ["groq>=0.4.0"]
|
||||
llm-gemini = ["google-genai>=0.1.0"]
|
||||
llm-anthropic = ["anthropic>=0.18.0"]
|
||||
llm-anthropic = ["anthropic>=0.122.0"]
|
||||
llm-ollama = ["ollama>=0.1.0"]
|
||||
llm-deepseek = ["openai>=1.0.0"]
|
||||
llm-litellm = ["litellm>=1.83.9"]
|
||||
@@ -201,6 +202,10 @@ gpu = [
|
||||
|
||||
# ---- Agentic Framework Integrations ----
|
||||
agno = ["agno>=1.0.0"]
|
||||
# crewai core provides BaseTool and BaseKnowledgeSource; crewai-tools is not
|
||||
# needed (it pulls vulnerable transitive deps like chromadb) and would only
|
||||
# duplicate the prebuilt tooling users can install separately.
|
||||
crewai = ["crewai>=0.80.0"]
|
||||
|
||||
# ---- File Watching ----
|
||||
watch = ["watchdog>=6.0.0"]
|
||||
@@ -242,6 +247,10 @@ explorer-lite = [
|
||||
]
|
||||
|
||||
# Everything (cross-platform — gpu excluded; install semantica[gpu] separately on Linux)
|
||||
# NOTE: the ``crewai`` extra is intentionally NOT in ``all``: crewai hard-requires
|
||||
# ``chromadb~=1.1.0``, which carries a pre-authentication code-injection advisory
|
||||
# (CVE-2026-45829) with no fixed release — including it here would fail the CI
|
||||
# dependency-audit/security gates. Install it explicitly via ``semantica[crewai]``.
|
||||
all = [
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,explorer]",
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,agno]"
|
||||
@@ -263,7 +272,7 @@ include = ["semantica*", "integrations*"]
|
||||
[tool.setuptools.package-data]
|
||||
# Explicit patterns are more reliable than **/* across setuptools versions.
|
||||
# static/* covers index.html / favicon; static/assets/* covers all JS/CSS chunks.
|
||||
"semantica" = ["static/*", "static/assets/*"]
|
||||
"semantica" = ["static/*", "static/assets/*", "ontology/vocabulary/*.ttl"]
|
||||
|
||||
[tool.black]
|
||||
line-length = 88
|
||||
|
||||
+16
-16
@@ -1,14 +1,14 @@
|
||||
# This file was autogenerated by uv via the following command:
|
||||
# uv pip compile -p 3.11 --extra all --generate-hashes -o requirements-ci.txt pyproject.toml
|
||||
# uv pip compile pyproject.toml --python-version 3.11 --extra all --generate-hashes -o requirements-ci.txt
|
||||
accelerate==1.14.0 \
|
||||
--hash=sha256:41b9c4377a54e0b460a959b0defa1b736e4ca0a2373252d9a539964c2afe3c8d \
|
||||
--hash=sha256:e94390c2863b873be18f623f9df48a0d8fe5eff13ea7f1a00092b0a7904888c6
|
||||
# via
|
||||
# docling-ibm-models
|
||||
# docling-slim
|
||||
agno==2.8.7 \
|
||||
--hash=sha256:6a2763eb469163f7b79ab1da6ca2f22d8619f6b9d614574f975d9c12bb4323ea \
|
||||
--hash=sha256:d49396a2062ee6994ca82695b9bd1e1b95667fec432c544afa38133e564bf090
|
||||
agno==2.9.0 \
|
||||
--hash=sha256:7777674b3931b341fad4fcf02a61b185a08588c509101348facf87feb2144c0c \
|
||||
--hash=sha256:7d9c134703e3c2798023cd57dcb9caa8e1174f6914813f9b130becfc3521a46f
|
||||
# via semantica (pyproject.toml)
|
||||
agnoctl==0.1.3 \
|
||||
--hash=sha256:6fce1d2482b1f2e0a3d14b0a7c12fbd49d8df4f0bf0a4fd9fd91753cbff5efdc \
|
||||
@@ -159,9 +159,9 @@ annotated-types==0.8.0 \
|
||||
--hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \
|
||||
--hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0
|
||||
# via pydantic
|
||||
anthropic==0.121.0 \
|
||||
--hash=sha256:6048713fa441e59e1cba8363171cd2a86273b25bd213e9c7ac70a523af88b011 \
|
||||
--hash=sha256:e79d6e08ab3376602fc9a70d4d5ea3540817c76cf7e16658bed790834e1833d6
|
||||
anthropic==0.122.0 \
|
||||
--hash=sha256:45ec906452ffae6b5f7f0c53d01f50bfb7e4ce878d7ae8e4309d13171e557e67 \
|
||||
--hash=sha256:ffec56ae96657c8d19fa575ec96f140f380c353a07ab7d61b92eb18ee6536601
|
||||
# via semantica (pyproject.toml)
|
||||
antlr4-python3-runtime==4.9.3 \
|
||||
--hash=sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b
|
||||
@@ -403,9 +403,9 @@ boto3==1.43.69 \
|
||||
--hash=sha256:4eb494d05b2bd08a7eee61b8ac4c34745c99e9bbce435c91f8d15d372dd8c2db \
|
||||
--hash=sha256:76297a0b415849c63575ae08a4f1661b2dc8ee0100f104b86f98aa69b47fa2c7
|
||||
# via semantica (pyproject.toml)
|
||||
botocore==1.43.69 \
|
||||
--hash=sha256:5caa46b740d9a886137146ffbb69edb691f702bfe74c64e85621947ae00181fd \
|
||||
--hash=sha256:b1f0e01c53d6b84ee9c184ebf3636c3b3aef85e0ae8498c74afb8734ff224f87
|
||||
botocore==1.43.73 \
|
||||
--hash=sha256:068433028e011ccbeab1dd7c46b1090c24e378397693c66e67ca571176498daa \
|
||||
--hash=sha256:0fa1e63c24b3531be3e1bc1687a88b3be9e63a430153f24edd93efc162bb1c51
|
||||
# via
|
||||
# boto3
|
||||
# s3transfer
|
||||
@@ -1731,9 +1731,9 @@ google-crc32c==1.8.0 \
|
||||
# via
|
||||
# google-cloud-storage
|
||||
# google-resumable-media
|
||||
google-genai==2.17.0 \
|
||||
--hash=sha256:6b640a2390c82b4a240873eddb9f518c6d2c33244b2de16ed3d14526a6da7f57 \
|
||||
--hash=sha256:a4835563c60aee646c9c4b261c507aa4a624710d25017012d20dc65abf3d9a54
|
||||
google-genai==2.18.1 \
|
||||
--hash=sha256:36a5949233e64a60f6cc4521bff7a76b7c569d0aa227bbe9fa642213b8a3a3b2 \
|
||||
--hash=sha256:a1e2be75c16234adc6641afd1ad4dd44218c9eec005d938bdc428585a048918a
|
||||
# via semantica (pyproject.toml)
|
||||
google-resumable-media==2.10.1 \
|
||||
--hash=sha256:224975032ddb73f7ed9e2f0f4cc08ed1b06874c52d48cc8533e3eb72980b21a0 \
|
||||
@@ -4123,9 +4123,9 @@ pooch==1.9.0 \
|
||||
--hash=sha256:de46729579b9857ffd3e741987a2f6d5e0e03219892c167c6578c0091fb511ed \
|
||||
--hash=sha256:f265597baa9f760d25ceb29d0beb8186c243d6607b0f60b83ecf14078dbc703b
|
||||
# via librosa
|
||||
portalocker==3.2.0 \
|
||||
--hash=sha256:1f3002956a54a8c3730586c5c77bf18fae4149e07eaf1c29fc3faf4d5a3f89ac \
|
||||
--hash=sha256:3cdc5f565312224bc570c49337bd21428bba0ef363bbcf58b9ef4a9f11779968
|
||||
portalocker==2.7.0 \
|
||||
--hash=sha256:032e81d534a88ec1736d03f780ba073f047a06c478b06e2937486f334e955c51 \
|
||||
--hash=sha256:a07c5b4f3985c3cf4798369631fb7011adb498e2a46d8440efc75a8f29a0f983
|
||||
# via qdrant-client
|
||||
pre-commit==4.6.2 \
|
||||
--hash=sha256:8f5d7bfb021ecdbcd9d49d89847082dd24172ccde534390081a679ad046e2441 \
|
||||
|
||||
@@ -10,7 +10,7 @@ Main exports:
|
||||
- Config: Configuration management
|
||||
"""
|
||||
|
||||
__version__ = "0.6.5"
|
||||
__version__ = "0.6.6"
|
||||
__author__ = "Semantica Contributors"
|
||||
__license__ = "MIT"
|
||||
|
||||
|
||||
@@ -1039,6 +1039,6 @@ manager = TemporalVersionManager(storage_path="large_data.db")
|
||||
## Support
|
||||
|
||||
For questions or issues:
|
||||
- GitHub Issues: https://github.com/Hawksight-AI/semantica/issues
|
||||
- GitHub Issues: https://github.com/semantica-agi/semantica/issues
|
||||
- Documentation: https://semantica.readthedocs.io
|
||||
- Community: https://discord.gg/sV34vps5hH
|
||||
|
||||
+162
-7
@@ -20,7 +20,7 @@ if sys.platform == "win32":
|
||||
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
||||
|
||||
from dataclasses import asdict, dataclass, field, is_dataclass
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PurePosixPath, PureWindowsPath
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
import yaml
|
||||
@@ -773,10 +773,22 @@ def changelog(cli_ctx: CLIContext, local_json: bool) -> None:
|
||||
_run_with_error_handling(_action)
|
||||
|
||||
|
||||
class _DeepEmbeddingFailure(Exception):
|
||||
"""A deep-probe failure from doctor's embedding checks.
|
||||
|
||||
Marks failures that happened AFTER the backend imported cleanly — model
|
||||
load, probe, or runtime problems — so the check's hint can point at the
|
||||
real remediation instead of `pip install`.
|
||||
"""
|
||||
|
||||
|
||||
@main.command()
|
||||
@click.option("--json", "local_json", is_flag=True, default=False)
|
||||
@click.option("--deep-embeddings", "deep_embeddings", is_flag=True, default=False,
|
||||
help="Also instantiate the local embedding backends and embed a probe "
|
||||
"text (catches backends that import cleanly but cannot load).")
|
||||
@click.pass_obj
|
||||
def doctor(cli_ctx: CLIContext, local_json: bool) -> None:
|
||||
def doctor(cli_ctx: CLIContext, local_json: bool, deep_embeddings: bool) -> None:
|
||||
"""Run a health check on all Semantica components and backends."""
|
||||
import importlib.metadata
|
||||
cli_ctx = _require_ctx(cli_ctx)
|
||||
@@ -787,6 +799,16 @@ def doctor(cli_ctx: CLIContext, local_json: bool) -> None:
|
||||
try:
|
||||
note = fn()
|
||||
return label, "ok", note, None
|
||||
except _DeepEmbeddingFailure as exc:
|
||||
# A deep-probe failure means the package IMPORTED fine: the pip
|
||||
# hint would be the wrong remediation for what is actually a
|
||||
# runtime/model-load problem (broken torch, failed model
|
||||
# download, missing shared libs).
|
||||
return label, "fail", str(exc), (
|
||||
"runtime/model-load failure — reinstalling the package usually "
|
||||
"does not help; check the warnings above (torch install, model "
|
||||
"download, disk space)"
|
||||
)
|
||||
except Exception as exc:
|
||||
return label, "fail", str(exc), hint
|
||||
|
||||
@@ -827,6 +849,50 @@ def doctor(cli_ctx: CLIContext, local_json: bool) -> None:
|
||||
return f"{backend} importable"
|
||||
checks.append(_check("Vector store", _vector, hint="pip install semantica[vectorstore-…]"))
|
||||
|
||||
# Embedding backends (#994): `doctor` used to report all green while
|
||||
# every local embedding backend was non-functional — import success
|
||||
# says nothing about model loading. Default checks stay cheap
|
||||
# (import + version); --deep-embeddings (or
|
||||
# SEMANTICA_DOCTOR_DEEP_EMBEDDINGS=1) instantiates the backend through
|
||||
# TextEmbedder and embeds a probe, which is the only level that
|
||||
# catches a backend that imports cleanly but cannot actually load.
|
||||
deep = deep_embeddings or os.environ.get("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", "").strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
def _embedding_backend(method: str) -> str:
|
||||
if method == "sentence_transformers":
|
||||
import sentence_transformers # noqa: F401
|
||||
note = f"importable ({importlib.metadata.version('sentence-transformers')})"
|
||||
else:
|
||||
import fastembed # noqa: F401
|
||||
note = f"importable ({importlib.metadata.version('fastembed')})"
|
||||
if not deep:
|
||||
return note
|
||||
try:
|
||||
from .embeddings import TextEmbedder
|
||||
embedder = TextEmbedder(method=method)
|
||||
if embedder.model is None and embedder.fastembed_model is None:
|
||||
raise RuntimeError(
|
||||
"model failed to load — the hash fallback is active "
|
||||
"(see warnings above); embedding quality is degraded"
|
||||
)
|
||||
probe = embedder.embed_text("semantica doctor embedding probe")
|
||||
except _DeepEmbeddingFailure:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise _DeepEmbeddingFailure(str(exc)) from exc
|
||||
return f"{note}; deep probe ok ({len(probe)}-dim)"
|
||||
|
||||
checks.append(_check(
|
||||
"Embeddings (sentence-transformers)",
|
||||
lambda: _embedding_backend("sentence_transformers"),
|
||||
hint="pip install sentence-transformers",
|
||||
))
|
||||
checks.append(_check(
|
||||
"Embeddings (fastembed)",
|
||||
lambda: _embedding_backend("fastembed"),
|
||||
hint="pip install fastembed",
|
||||
))
|
||||
|
||||
# LLM provider keys
|
||||
for provider, var in [("OpenAI", "OPENAI_API_KEY"), ("Anthropic", "ANTHROPIC_API_KEY"),
|
||||
("Groq", "GROQ_API_KEY")]:
|
||||
@@ -1708,7 +1774,43 @@ def embed_generate(cli_ctx: CLIContext, input_path: str, model: str,
|
||||
except ImportError as exc:
|
||||
raise click.ClickException(f"Embeddings module not available: {exc}") from exc
|
||||
if output:
|
||||
Path(output).write_text(json.dumps(result, default=str), encoding="utf-8")
|
||||
output_path = Path(output)
|
||||
suffix = output_path.suffix.lower()
|
||||
try:
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
arr = np.asarray(result)
|
||||
if arr.ndim == 1:
|
||||
arr = arr[np.newaxis, :]
|
||||
if arr.ndim != 2:
|
||||
raise click.ClickException(
|
||||
f"embed generate --output expects a 1-D or 2-D array, "
|
||||
f"got {arr.ndim}-D (shape {arr.shape})"
|
||||
)
|
||||
rows = [list(row) for row in arr]
|
||||
if suffix == ".parquet":
|
||||
# Schema: single 'embedding' column (list[float] per row).
|
||||
# embed index detects vector columns via
|
||||
# isinstance(df[c].iloc[0], (list, np.ndarray)).
|
||||
df = pd.DataFrame({"embedding": rows})
|
||||
df.to_parquet(output_path, index=False)
|
||||
elif suffix in (".json", ".jsonl"):
|
||||
df = pd.DataFrame({"embedding": rows})
|
||||
df.to_json(
|
||||
output_path,
|
||||
orient="records",
|
||||
lines=(suffix == ".jsonl"),
|
||||
)
|
||||
else:
|
||||
raise click.ClickException(
|
||||
f"Unsupported output format '{suffix}'. "
|
||||
"Use .parquet, .json, or .jsonl"
|
||||
)
|
||||
except ImportError as exc:
|
||||
raise click.ClickException(
|
||||
f"Missing dependency for --output: {exc}. "
|
||||
"Install pyarrow with: pip install pyarrow"
|
||||
) from exc
|
||||
_ok(cli_ctx, f"Wrote {output}")
|
||||
elif _is_json(cli_ctx, local_json):
|
||||
_jecho(result if isinstance(result, dict) else {"status": "ok"})
|
||||
@@ -3933,15 +4035,68 @@ def backup_restore(cli_ctx: CLIContext, source: str, local_dry: bool) -> None:
|
||||
|
||||
try:
|
||||
if _tf.is_tarfile(str(work_path)):
|
||||
restore_root = Path.cwd()
|
||||
restore_root = Path.cwd().resolve()
|
||||
with _tf.open(str(work_path), "r:*") as tar:
|
||||
# Dry-run listing was already handled above; extract now
|
||||
for member in tar.getmembers():
|
||||
# Strip the leading "semantica-backup/" prefix
|
||||
member.name = member.name.replace("semantica-backup/", "", 1)
|
||||
if member.name:
|
||||
tar.extract(member, path=str(restore_root))
|
||||
console.print(f" restored: {member.name}")
|
||||
if not member.name:
|
||||
continue
|
||||
|
||||
# Reject members whose resolved path escapes the
|
||||
# restore root (path traversal / absolute paths),
|
||||
# regardless of the "semantica-backup/" prefix.
|
||||
member_path = (restore_root / member.name).resolve()
|
||||
try:
|
||||
member_path.relative_to(restore_root)
|
||||
except ValueError:
|
||||
raise click.ClickException(
|
||||
f"Refusing to restore '{member.name}': "
|
||||
"path escapes the restore directory."
|
||||
)
|
||||
|
||||
# Reject symlink/hardlink members whose target
|
||||
# escapes the restore root. Checked two ways:
|
||||
# lexically (linkname itself, so an absolute path or
|
||||
# a literal ".." segment is rejected outright, with
|
||||
# no dependence on what else does or doesn't already
|
||||
# exist on disk) and by resolution (catches any
|
||||
# remaining traversal the lexical check misses).
|
||||
if member.issym() or member.islnk():
|
||||
linkname = member.linkname or ""
|
||||
linkname_parts = PurePosixPath(
|
||||
linkname.replace("\\", "/")
|
||||
).parts
|
||||
if (
|
||||
not linkname
|
||||
or os.path.isabs(linkname)
|
||||
or PureWindowsPath(linkname).is_absolute()
|
||||
or ".." in linkname_parts
|
||||
):
|
||||
raise click.ClickException(
|
||||
f"Refusing to restore '{member.name}': "
|
||||
"link target is absolute or traverses "
|
||||
"out of the archive."
|
||||
)
|
||||
link_target = (
|
||||
member_path.parent / linkname
|
||||
).resolve()
|
||||
try:
|
||||
link_target.relative_to(restore_root)
|
||||
except ValueError:
|
||||
raise click.ClickException(
|
||||
f"Refusing to restore '{member.name}': "
|
||||
"link target escapes the restore directory."
|
||||
)
|
||||
|
||||
extract_kwargs: Dict[str, Any] = {"path": str(restore_root)}
|
||||
if hasattr(_tf, "data_filter"):
|
||||
# Python >=3.12: also reject device files, and
|
||||
# further harden the traversal/ownership checks.
|
||||
extract_kwargs["filter"] = "data"
|
||||
tar.extract(member, **extract_kwargs)
|
||||
console.print(f" restored: {member.name}")
|
||||
elif src.is_dir():
|
||||
restore_root = Path.cwd()
|
||||
for f in src.rglob("*"):
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Filesystem safety helpers for human-editable Markdown persistence."""
|
||||
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def is_filesystem_link(path: Path) -> bool:
|
||||
"""Return whether *path* is a symlink, junction, or Windows reparse point."""
|
||||
if path.is_symlink():
|
||||
return True
|
||||
|
||||
isjunction = getattr(os.path, "isjunction", None)
|
||||
if isjunction is not None and isjunction(path):
|
||||
return True
|
||||
|
||||
try:
|
||||
attributes = getattr(os.lstat(path), "st_file_attributes", 0)
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return False
|
||||
|
||||
reparse_point = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
|
||||
return bool(attributes & reparse_point)
|
||||
|
||||
|
||||
def find_filesystem_link(path: Path) -> Optional[Path]:
|
||||
"""Return the first linked component in *path*, including its ancestors."""
|
||||
for candidate in (path, *path.parents):
|
||||
if is_filesystem_link(candidate):
|
||||
return candidate
|
||||
return None
|
||||
@@ -77,6 +77,7 @@ import yaml
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from ..utils.types import EntityDict, RelationshipDict
|
||||
from ._markdown_filesystem import find_filesystem_link
|
||||
|
||||
|
||||
class _UniqueKeySafeLoader(yaml.SafeLoader):
|
||||
@@ -1742,9 +1743,10 @@ class AgentMemory:
|
||||
@staticmethod
|
||||
def _write_markdown_file(file_path: Path, document: str) -> None:
|
||||
"""Atomically replace a Markdown file without following output symlinks."""
|
||||
if file_path.is_symlink():
|
||||
if find_filesystem_link(file_path) is not None:
|
||||
raise ValueError(
|
||||
f"Refusing to overwrite Markdown symbolic link: {file_path}"
|
||||
"Refusing to overwrite Markdown symbolic link or junction: "
|
||||
f"{file_path}"
|
||||
)
|
||||
|
||||
temporary_path = None
|
||||
@@ -1867,7 +1869,8 @@ class AgentMemory:
|
||||
if "\n" not in data and "\r" not in data:
|
||||
candidate = Path(data)
|
||||
try:
|
||||
candidate_exists = candidate.exists()
|
||||
candidate_is_link = find_filesystem_link(candidate) is not None
|
||||
candidate_exists = candidate_is_link or candidate.exists()
|
||||
except OSError as exc:
|
||||
error_message = (
|
||||
"Failed to inspect possible Markdown import "
|
||||
@@ -1909,62 +1912,71 @@ class AgentMemory:
|
||||
return memories
|
||||
|
||||
def _read_markdown_file_content(self, file_path: Path) -> str:
|
||||
if file_path.is_symlink():
|
||||
raise ValueError(f"Symlink Markdown import paths are rejected: {file_path}")
|
||||
if find_filesystem_link(file_path) is not None:
|
||||
raise ValueError(
|
||||
"Symlink Markdown import paths are rejected; symbolic links and "
|
||||
f"junctions are unsafe: {file_path}"
|
||||
)
|
||||
|
||||
flags = os.O_RDONLY
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
# On POSIX, O_NOFOLLOW makes os.open() fail with ELOOP if the
|
||||
# final path component is a symlink, atomically closing the TOCTOU
|
||||
# window between the is_symlink() check above and the open call.
|
||||
# On Windows, O_NOFOLLOW is not available; the is_symlink() pre-check
|
||||
# above is the only symlink defense and remains vulnerable to a narrow
|
||||
# race. The fstat()/S_ISREG guard below still rejects special files
|
||||
# (FIFOs, devices) on both platforms.
|
||||
flags |= os.O_NOFOLLOW
|
||||
nofollow_flag = getattr(os, "O_NOFOLLOW", 0)
|
||||
flags |= nofollow_flag
|
||||
|
||||
try:
|
||||
fd = os.open(str(file_path), flags)
|
||||
except OSError as exc:
|
||||
if exc.errno == getattr(errno, "ELOOP", None):
|
||||
if (
|
||||
(nofollow_flag and exc.errno == errno.ELOOP)
|
||||
or find_filesystem_link(file_path) is not None
|
||||
):
|
||||
raise ValueError(
|
||||
f"Symlink Markdown import paths are rejected: {file_path}"
|
||||
"Symlink Markdown import paths are rejected; symbolic links "
|
||||
f"and junctions are unsafe: {file_path}"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
try:
|
||||
stat_res = os.fstat(fd)
|
||||
if not stat.S_ISREG(stat_res.st_mode):
|
||||
if find_filesystem_link(file_path) is not None:
|
||||
raise ValueError(
|
||||
"Symlink Markdown import paths are rejected; symbolic links "
|
||||
f"and junctions are unsafe: {file_path}"
|
||||
)
|
||||
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
||||
raise ValueError(
|
||||
f"Markdown import path is not a regular file: {file_path}"
|
||||
)
|
||||
with open(fd, "r", encoding="utf-8", closefd=True) as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
try:
|
||||
with os.fdopen(fd, mode="r", encoding="utf-8") as source:
|
||||
fd = -1
|
||||
return source.read()
|
||||
finally:
|
||||
if fd >= 0:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
def _read_markdown_path(self, path: Path) -> List[Tuple[str, str]]:
|
||||
if path.is_symlink():
|
||||
raise ValueError(f"Symlink Markdown import paths are rejected: {path}")
|
||||
if find_filesystem_link(path) is not None:
|
||||
raise ValueError(
|
||||
"Symlink Markdown import paths are rejected; symbolic links and "
|
||||
f"junctions are unsafe: {path}"
|
||||
)
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Markdown import path does not exist: {path}")
|
||||
|
||||
if path.is_dir():
|
||||
file_paths = sorted(
|
||||
(
|
||||
file_path
|
||||
for file_path in path.iterdir()
|
||||
if file_path.is_file()
|
||||
and not file_path.is_symlink()
|
||||
and file_path.suffix.lower() in self._MARKDOWN_EXTENSIONS
|
||||
),
|
||||
key=lambda file_path: (file_path.name.casefold(), file_path.name),
|
||||
)
|
||||
file_paths = []
|
||||
for file_path in path.iterdir():
|
||||
if file_path.suffix.lower() not in self._MARKDOWN_EXTENSIONS:
|
||||
continue
|
||||
if find_filesystem_link(file_path) is not None:
|
||||
continue
|
||||
if file_path.is_file():
|
||||
file_paths.append(file_path)
|
||||
if find_filesystem_link(path) is not None:
|
||||
raise ValueError(
|
||||
"Symlink Markdown import paths are rejected; symbolic links "
|
||||
f"and junctions are unsafe: {path}"
|
||||
)
|
||||
file_paths.sort(key=lambda item: (item.name.casefold(), item.name))
|
||||
elif path.is_file():
|
||||
file_paths = [path]
|
||||
else:
|
||||
|
||||
+1589
-35
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user