mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
78fc9028a8 | ||
|
|
6b7625ef9b | ||
|
|
48a05b00a6 | ||
|
|
58b77ddcf5 | ||
|
|
e03212cd66 | ||
|
|
83c04a57d6 | ||
|
|
75b026c6dd | ||
|
|
2a303cf4da | ||
|
|
7595bad28f | ||
|
|
2d75952476 | ||
|
|
e55c03bd39 | ||
|
|
4a886d970e | ||
|
|
e1092ac507 | ||
|
|
b77e3e8c3c | ||
|
|
68f7ae3807 | ||
|
|
08a6e7c053 | ||
|
|
e6b159e5c5 | ||
|
|
7db2e2f46b | ||
|
|
3fbe3cfd2d | ||
|
|
75bc6255d4 | ||
|
|
a96f1590f1 | ||
|
|
063f447202 | ||
|
|
a1194a155d | ||
|
|
17d878cbf3 | ||
|
|
488e381247 | ||
|
|
4acd2f9c33 | ||
|
|
7cac8a6bc3 | ||
|
|
6c77594ea6 | ||
|
|
5109c6fab2 | ||
|
|
12b9694a8e | ||
|
|
49707729ad | ||
|
|
430020c7c4 | ||
|
|
43b207c1c5 | ||
|
|
55bde673c9 | ||
|
|
5c2901ae27 | ||
|
|
dae21166a1 | ||
|
|
67be421533 | ||
|
|
baf8f01f85 | ||
|
|
c58686b4ec | ||
|
|
04602a0e0e | ||
|
|
eedf1425ca | ||
|
|
d4cb14c1fb | ||
|
|
a8194dfc60 | ||
|
|
c7415f2e92 | ||
|
|
3331df28ad | ||
|
|
de5e20dc55 | ||
|
|
0f252ab355 | ||
|
|
0b77e5fe94 | ||
|
|
893b6db3c3 | ||
|
|
b8297b8077 | ||
|
|
4d37920007 | ||
|
|
6416fbb669 | ||
|
|
70aa9d01bf | ||
|
|
c53ca4e84b | ||
|
|
15171fd31a | ||
|
|
8177d88753 | ||
|
|
d94d8f6ab8 | ||
|
|
83649f6821 | ||
|
|
2f04bc01a3 | ||
|
|
5579851208 | ||
|
|
eaf51b3383 | ||
|
|
115e7965cd | ||
|
|
8639cb9f16 | ||
|
|
f1e7e64ad1 | ||
|
|
6df97cf0a0 | ||
|
|
84ce3c5155 | ||
|
|
b8175ea801 | ||
|
|
557e29ee14 | ||
|
|
c1be6dd7dc | ||
|
|
42afc06003 | ||
|
|
4513b61e40 | ||
|
|
8a4ebafb9a | ||
|
|
80b9bea0d5 | ||
|
|
5bc09a5f5a | ||
|
|
75f88b1c40 | ||
|
|
80b1cca07b | ||
|
|
1c0cebb1c3 | ||
|
|
94d0c3dc07 | ||
|
|
c0a051903f | ||
|
|
09c4b1b570 | ||
|
|
c5d13a45db | ||
|
|
611874e63e | ||
|
|
43bac6170c | ||
|
|
91d02a0f29 | ||
|
|
7c3372c062 | ||
|
|
2cfb5de43d | ||
|
|
0fa3483b96 | ||
|
|
18f1d55d77 | ||
|
|
b0080c3602 | ||
|
|
1ee3f2f214 | ||
|
|
1a3dd5038a | ||
|
|
c1154b6ed6 | ||
|
|
687a180721 | ||
|
|
bc63e962c9 | ||
|
|
9ec7959899 | ||
|
|
22bf581094 | ||
|
|
6328bfe52d | ||
|
|
81bb5f2ed8 | ||
|
|
b8e8b2f227 | ||
|
|
f821fa7e2e | ||
|
|
229cb69c50 | ||
|
|
8ef7c9f760 | ||
|
|
cab995dc97 | ||
|
|
4d88218221 | ||
|
|
918830a821 | ||
|
|
5b319560fb | ||
|
|
f29c4310a1 | ||
|
|
a2886a4e41 | ||
|
|
a0aa415fc4 | ||
|
|
ae4f1d4030 | ||
|
|
ea3416ed32 | ||
|
|
154a7347cd | ||
|
|
f2f1d6787d | ||
|
|
646c70ce63 | ||
|
|
c5981aa306 | ||
|
|
d507fda1b0 | ||
|
|
7bf7474ac1 | ||
|
|
546e27cec5 | ||
|
|
a8330874d3 | ||
|
|
1c3ac66fd9 | ||
|
|
b846ff88d4 | ||
|
|
69b79e3d67 | ||
|
|
9012492c97 | ||
|
|
6ec546b551 | ||
|
|
6002965c55 | ||
|
|
abc10bc8e0 | ||
|
|
44f585ffce | ||
|
|
f5332589d5 | ||
|
|
9a21ca9834 | ||
|
|
a169cf3fb9 | ||
|
|
656baa7aee | ||
|
|
e1725fd763 | ||
|
|
1f053e005c | ||
|
|
7ed1d49625 | ||
|
|
c94be3f9a6 | ||
|
|
142707db93 | ||
|
|
3357c14ee3 | ||
|
|
9ecae47a8a | ||
|
|
3496d62335 | ||
|
|
64f6c5cba2 | ||
|
|
ab5c12f9af | ||
|
|
fc9af2ebf8 | ||
|
|
3e9ba1b7fb | ||
|
|
51cf97765d | ||
|
|
8fa2037619 | ||
|
|
5573ab7a9f | ||
|
|
26f236923c | ||
|
|
c35899711d | ||
|
|
0a113b9702 | ||
|
|
2de6ff898a | ||
|
|
22ea189d0b | ||
|
|
30d5fef180 | ||
|
|
c85df419ae | ||
|
|
55f3ee6f84 | ||
|
|
924765b042 | ||
|
|
bde6e2d68e | ||
|
|
6f310d1d7a | ||
|
|
01bd908f86 | ||
|
|
bd35d6031b | ||
|
|
00f4e79d3e | ||
|
|
7654d8c6c7 | ||
|
|
70109133b5 | ||
|
|
53caefaf58 | ||
|
|
7ce05a3848 | ||
|
|
21f5f3d9b3 | ||
|
|
fc2083aa17 | ||
|
|
1258edfe7f | ||
|
|
5048665d35 | ||
|
|
7dce9f1b69 | ||
|
|
1b09f1ca5b | ||
|
|
03ed4b94e9 | ||
|
|
9059a44731 | ||
|
|
772d22448a | ||
|
|
b6497ace41 | ||
|
|
b094268525 | ||
|
|
e90bd048e1 | ||
|
|
8e0419c864 | ||
|
|
0d51608547 | ||
|
|
aa7b7fe525 | ||
|
|
916d3974e3 | ||
|
|
f75469f472 | ||
|
|
40b81d0582 | ||
|
|
5db0adc18a | ||
|
|
50758f6f25 | ||
|
|
2756916573 | ||
|
|
f47c730f7e | ||
|
|
721a2f0e9c | ||
|
|
dd42b7fa95 | ||
|
|
248d028b09 | ||
|
|
77a2ab7b18 | ||
|
|
c8b59b47f5 | ||
|
|
c0b6a80480 | ||
|
|
36071819b5 | ||
|
|
a4dac2342b | ||
|
|
7d272f40e8 | ||
|
|
3e5d2672ad | ||
|
|
b4b10a4928 | ||
|
|
48c58a0753 | ||
|
|
6b143ef401 | ||
|
|
49f458e927 | ||
|
|
c77184bd77 | ||
|
|
fa77f5cc47 | ||
|
|
7bddee0111 | ||
|
|
5cd4407e57 | ||
|
|
1850cdd617 | ||
|
|
5f00c00be3 | ||
|
|
dee55112ef | ||
|
|
b7ac05b6f2 | ||
|
|
d9118410bc | ||
|
|
d16db085d8 | ||
|
|
cb716cec61 | ||
|
|
712a6e6d4c | ||
|
|
b52cdd5bbe | ||
|
|
d0e018a1c9 | ||
|
|
bd8d6c5913 | ||
|
|
667e69a0c1 | ||
|
|
9c7dd16126 | ||
|
|
94adcf7ad3 | ||
|
|
80de3652cf | ||
|
|
0e1b88a593 | ||
|
|
b4f820568a | ||
|
|
8d52281cdf | ||
|
|
6a0eecbe02 | ||
|
|
aa85535d47 | ||
|
|
7d936d0f7c | ||
|
|
47531d8365 | ||
|
|
86f115d200 | ||
|
|
9c5c3c4ce0 | ||
|
|
26a5c4a1fb | ||
|
|
2adc67e25e | ||
|
|
e9e05fedbd | ||
|
|
0a8330cbb0 | ||
|
|
db4361ad46 | ||
|
|
74c093facb | ||
|
|
aae4c946ea | ||
|
|
b59211ea7f | ||
|
|
c7c7250d88 | ||
|
|
21365cb0e6 | ||
|
|
76edaeb1c0 | ||
|
|
1ad00075a3 | ||
|
|
46dcbbe731 | ||
|
|
0d447560bc | ||
|
|
5094235ce1 | ||
|
|
6f6c825f3d | ||
|
|
d293ca6009 | ||
|
|
352db64b33 | ||
|
|
bc75768afe | ||
|
|
f992504227 | ||
|
|
d41530930d | ||
|
|
692260cc76 | ||
|
|
aa58b46d4d | ||
|
|
633d485045 | ||
|
|
424b63a27d | ||
|
|
6e44d98d46 | ||
|
|
d21e5e9944 | ||
|
|
67aed43997 | ||
|
|
ac64943965 | ||
|
|
4dea295f0d | ||
|
|
6c6cb3f3b5 | ||
|
|
1ae1e6d57a | ||
|
|
495e29d543 | ||
|
|
04d2a726b9 | ||
|
|
62a027d6fd | ||
|
|
7cab35bbc0 | ||
|
|
938f846dde | ||
|
|
66be1630fa | ||
|
|
a1f835c9b2 | ||
|
|
12172d03b4 | ||
|
|
60f362817f | ||
|
|
4d1e5cf37c | ||
|
|
16893c28a4 | ||
|
|
577967a549 | ||
|
|
7fb94b6528 | ||
|
|
e197977172 | ||
|
|
ecd0a26a8d | ||
|
|
f99241ca88 | ||
|
|
dabeb0e833 | ||
|
|
9458cf5b2b | ||
|
|
3db35a6871 | ||
|
|
b1daf238ca | ||
|
|
0205ecd711 | ||
|
|
9677f25d27 | ||
|
|
4a221554de | ||
|
|
7e513a12a3 | ||
|
|
39bebe7da9 | ||
|
|
9537bf17b3 | ||
|
|
e8cb337946 | ||
|
|
840629762f | ||
|
|
df6c30653c | ||
|
|
8d3c99ba30 | ||
|
|
c2079d8e92 | ||
|
|
cc864362aa | ||
|
|
d30aea79a7 | ||
|
|
2ee4da0b47 | ||
|
|
016661463e | ||
|
|
ce914b396a | ||
|
|
b105b8ea97 | ||
|
|
6ed5aea993 | ||
|
|
80bce453c3 | ||
|
|
d102584af6 | ||
|
|
4f27d3dcae | ||
|
|
35f8c0527c | ||
|
|
28fe304f76 | ||
|
|
9eea49a070 | ||
|
|
db95cedf34 | ||
|
|
3a9c7c082f | ||
|
|
4a3cf37679 | ||
|
|
dd7b090aec | ||
|
|
eaa0f823a8 | ||
|
|
7045d7b94e | ||
|
|
8430d4a56e | ||
|
|
ea71ea1823 | ||
|
|
3a1ab1a8a6 | ||
|
|
87714ec1ad | ||
|
|
1c590c622b | ||
|
|
40d6fa05d0 | ||
|
|
ac69c27b86 | ||
|
|
a16bd9600b | ||
|
|
fd84be8b66 | ||
|
|
6cc2e67b92 | ||
|
|
2dd06aadcd | ||
|
|
86db4f923d | ||
|
|
dcd936a9ab | ||
|
|
b663c6bbbf | ||
|
|
5ab21c089e | ||
|
|
84775fdea0 | ||
|
|
2a0bc7051a | ||
|
|
8beca57238 | ||
|
|
ed44260ec3 | ||
|
|
8f09d4f57b | ||
|
|
7e05f196b5 | ||
|
|
47c7f4adbe | ||
|
|
0698ba7656 | ||
|
|
33d8f806c2 | ||
|
|
530e297d17 | ||
|
|
be2f8f8cd8 | ||
|
|
a2f10dfbdd | ||
|
|
495c2a2fbd | ||
|
|
eb8156ddb3 | ||
|
|
1c3d2b949f | ||
|
|
343d2bc418 | ||
|
|
297d959f63 | ||
|
|
161d47f4d9 | ||
|
|
99b0a517fd | ||
|
|
adb46134c4 | ||
|
|
d2d38a0509 | ||
|
|
9a21e523f0 | ||
|
|
ca11a48fef | ||
|
|
aa171c16f1 | ||
|
|
cb5e93ebc7 | ||
|
|
a256a77277 | ||
|
|
e7696f462a | ||
|
|
d6c7154fa9 | ||
|
|
b473e0bd8a | ||
|
|
b1deed5857 | ||
|
|
637bfe7314 | ||
|
|
9b7a33031c | ||
|
|
443a9b78d7 | ||
|
|
36856cc92a | ||
|
|
6e4ff0c7c5 | ||
|
|
095d7d3e52 | ||
|
|
1b706e539f |
@@ -2,4 +2,18 @@
|
||||
# Cloud Run false-positives (CKV_K8S_21/28/30) are suppressed via per-file
|
||||
# inline checkov:skip comments in deploy/gcp/cloudrun-service.yaml rather than
|
||||
# globally here, so future real Kubernetes manifests are not silently exempted.
|
||||
#
|
||||
# The knowledge-explorer Helm chart's unconditional templates (service.yaml,
|
||||
# deployment.yaml, configmap.yaml) set metadata.namespace to .Release.Namespace,
|
||||
# which is only bound at `helm install`/`helm template` time. Checkov's helm
|
||||
# framework renders the chart without a namespace override, so it always
|
||||
# resolves to "default" and trips CKV_K8S_21 even though the chart is
|
||||
# namespace-agnostic by design. Suppressed via metadata annotations
|
||||
# (checkov.io/skip1 / runterrascan.io/skip) on each resource's metadata.annotations,
|
||||
# as both Checkov and Terrascan require K8s/Helm resource-level annotations
|
||||
# rather than file-header comments.
|
||||
# deployment.yaml additionally suppresses AC_K8S_0080 and CKV_K8S_31 (seccomp) via
|
||||
# metadata.annotations on both the Deployment resource and the pod template:
|
||||
# the seccomp profile is set correctly in values.yaml and only resolves once
|
||||
# Helm actually renders `toYaml`, which static template scanning does not do.
|
||||
skip-check: []
|
||||
|
||||
@@ -70,6 +70,13 @@ updates:
|
||||
- "dependencies"
|
||||
- "github-actions"
|
||||
- "ci"
|
||||
# All our actions are SHA-pinned with a "# vX" comment; Dependabot
|
||||
# resolves the new tag's SHA and updates both the pin and the comment
|
||||
# together, so this stays the source of truth (no separate script needed).
|
||||
groups:
|
||||
github-actions:
|
||||
patterns:
|
||||
- "*"
|
||||
|
||||
# Optional dependencies (separate schedule for stability)
|
||||
- package-ecosystem: "pip"
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
> **Before you submit:** make sure you followed the [issue workflow in CONTRIBUTING.md](https://github.com/semantica-agi/semantica/blob/main/CONTRIBUTING.md#-working-on-an-existing-issue) — wait for the issue to be assigned to you before opening a PR, to avoid duplicate work.
|
||||
|
||||
## Description
|
||||
|
||||
<!-- Provide a clear description of your changes -->
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env bash
|
||||
# Verifies that every third-party GitHub Action referenced in
|
||||
# .github/workflows/*.yml and .github/workflows/*.yaml is pinned to a full
|
||||
# commit SHA (not a mutable tag
|
||||
# or branch), and that any pin's trailing "# vX" comment still matches what
|
||||
# that tag resolves to today.
|
||||
#
|
||||
# Fails closed on purpose:
|
||||
# - a `uses:` line pinned to anything other than a 40-hex-char SHA is a
|
||||
# hard failure, not a skip - this is what stops a newly-added mutable
|
||||
# tag (e.g. `uses: some/action@v1`) from slipping past unnoticed.
|
||||
# - a tag that can't be resolved via the GitHub API (rate limit, deleted
|
||||
# tag, typo) is also a hard failure rather than a warning - an
|
||||
# unverifiable pin is exactly the failure mode this check exists to
|
||||
# catch, so it must not pass silently.
|
||||
set -uo pipefail
|
||||
|
||||
fail=0
|
||||
checked=0
|
||||
|
||||
# Pattern for a third-party uses: line — stored in a variable so bash's
|
||||
# [[ =~ ]] parser never sees literal \" or \' escapes, which cause a
|
||||
# "syntax error in conditional expression: unexpected token )" at runtime.
|
||||
# Semantics: optional leading quote, owner/repo, optional subpath, @ref,
|
||||
# optional trailing quote; quote chars excluded from the ref capture group.
|
||||
USES_PATTERN='uses:[[:space:]]+["'"'"']?([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)(/[^[:space:]@"'"'"']+)?@([^[:space:]"'"'"']+)["'"'"']?'
|
||||
|
||||
while IFS=: read -r file lineno content; do
|
||||
# Local composite actions (./x) and Docker image refs (docker://...) use a
|
||||
# different pinning mechanism and aren't in scope here.
|
||||
[[ "$content" =~ uses:\ +\./ ]] && continue
|
||||
[[ "$content" =~ uses:\ +docker:// ]] && continue
|
||||
|
||||
if [[ "$content" =~ $USES_PATTERN ]]; then
|
||||
repo="${BASH_REMATCH[1]}"
|
||||
ref="${BASH_REMATCH[3]}"
|
||||
checked=$((checked + 1))
|
||||
|
||||
if [[ ! "$ref" =~ ^[0-9a-fA-F]{40}$ ]]; then
|
||||
echo "::error file=$file,line=$lineno::$repo is pinned to '$ref', not a full commit SHA. Mutable tags/branches can be silently re-pointed (see the LiteLLM/Trivy 2026 incident) - pin to a commit SHA instead."
|
||||
fail=1
|
||||
continue
|
||||
fi
|
||||
sha="$ref"
|
||||
|
||||
if [[ "$content" =~ \#[[:space:]]*([^[:space:]]+)[[:space:]]*$ ]]; then
|
||||
tag="${BASH_REMATCH[1]}"
|
||||
else
|
||||
echo "::warning file=$file,line=$lineno::$repo@$sha has no trailing '# vX' comment recording which tag it corresponds to - add one for auditability."
|
||||
continue
|
||||
fi
|
||||
|
||||
resolved=$(gh api "repos/$repo/commits/$tag" --jq '.sha' 2>/dev/null)
|
||||
if [[ -z "$resolved" ]]; then
|
||||
echo "::error file=$file,line=$lineno::Could not resolve '$repo@$tag' via the GitHub API (rate limit, deleted tag, or typo). Treating as unverifiable = failure."
|
||||
fail=1
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$resolved" != "$sha" ]]; then
|
||||
echo "::error file=$file,line=$lineno::$repo is pinned to $sha but tag '$tag' now resolves to $resolved. Update the pin or the comment."
|
||||
fail=1
|
||||
else
|
||||
echo "OK $repo@$tag -> $sha ($file:$lineno)"
|
||||
fi
|
||||
fi
|
||||
done < <(grep -rHn "uses:" .github/workflows/*.yml .github/workflows/*.yaml 2>/dev/null)
|
||||
|
||||
echo "Checked $checked action reference(s)."
|
||||
exit $fail
|
||||
@@ -13,14 +13,14 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python 3.12
|
||||
uses: actions/setup-python@v5
|
||||
- name: Set up Python 3.11
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
python-version: "3.11"
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install Dependencies
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
# pytest-benchmark --storage file://benchmarks/results --benchmark-compare
|
||||
|
||||
- name: Upload Benchmark Results
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
if: always()
|
||||
with:
|
||||
name: benchmark-report-${{ github.run_id }}
|
||||
|
||||
@@ -21,22 +21,49 @@ jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- uses: actions/setup-node@v6
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '20'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: explorer/package-lock.json
|
||||
- name: Build Explorer frontend
|
||||
- name: Install Explorer frontend dependencies
|
||||
working-directory: explorer
|
||||
run: npm ci
|
||||
- name: Test Explorer frontend
|
||||
working-directory: explorer
|
||||
run: |
|
||||
npm ci
|
||||
npm run build
|
||||
npm run test:graph-store
|
||||
npm run test:graph-workspace
|
||||
npm run test:plugin-registry
|
||||
- name: Build Explorer frontend
|
||||
working-directory: explorer
|
||||
run: npm run build
|
||||
- name: Install pinned Python dependencies
|
||||
run: |
|
||||
pip install -r requirements-ci.txt
|
||||
- name: Verify requirements-ci.txt is up to date
|
||||
run: |
|
||||
pip install uv==0.12.1
|
||||
# Re-resolve with the committed file as a constraint: upstream package
|
||||
# releases must NOT fail CI (deps only change when pyproject.toml
|
||||
# changes intentionally). Compare only version lines (pkg==ver),
|
||||
# ignoring the -c constraint comments and the `\` line continuations
|
||||
# that --generate-hashes emits.
|
||||
uv pip compile pyproject.toml --python-version 3.11 --extra all \
|
||||
--constraint requirements-ci.txt -o /tmp/requirements-ci-check.txt
|
||||
diff \
|
||||
<(grep -E '^[a-zA-Z0-9._-]+==' requirements-ci.txt | sed 's/ \\$//') \
|
||||
<(grep -E '^[a-zA-Z0-9._-]+==' /tmp/requirements-ci-check.txt)
|
||||
- run: pip install build
|
||||
- run: python -m build
|
||||
# wheel is build-time only (not in requirements-ci.txt) — install the
|
||||
# same pinned version [build-system] declares so --no-isolation works.
|
||||
- run: pip install wheel==0.48.0
|
||||
- name: Build package (no isolation — pinned deps)
|
||||
run: python -m build --no-isolation
|
||||
- name: Verify Explorer frontend is packaged
|
||||
run: |
|
||||
python - <<'PY'
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
# The CodeQL bundle download (github/codeql-action/init's "Setup CodeQL
|
||||
# tools" step) streams a ~1GB tarball from GitHub's release CDN and
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
# meaningful state carried over from a failed attempt.
|
||||
- name: Initialize CodeQL (attempt 1)
|
||||
id: codeql-init-1
|
||||
uses: github/codeql-action/init@v4
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 2)
|
||||
id: codeql-init-2
|
||||
if: steps.codeql-init-1.outcome == 'failure'
|
||||
uses: github/codeql-action/init@v4
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -52,17 +52,17 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 3)
|
||||
id: codeql-init-3
|
||||
if: steps.codeql-init-2.outcome == 'failure'
|
||||
uses: github/codeql-action/init@v4
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
languages: python
|
||||
queries: security-and-quality
|
||||
config-file: .github/codeql/codeql-config.yml
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v4
|
||||
uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4
|
||||
uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
category: "/language:python"
|
||||
upload: false
|
||||
@@ -72,7 +72,7 @@ jobs:
|
||||
# Uploads results only when Default Setup is not active.
|
||||
# If Default Setup is still enabled, this step skips gracefully
|
||||
# instead of failing the workflow with HTTP 409.
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.codeql.outputs.sarif-output }}
|
||||
category: "/language:python"
|
||||
|
||||
@@ -36,14 +36,14 @@ jobs:
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-dotnet@v5
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6
|
||||
with:
|
||||
dotnet-version: |
|
||||
5.0.x
|
||||
6.0.x
|
||||
- name: Run Microsoft Security DevOps
|
||||
uses: microsoft/security-devops-action@v1.12.0
|
||||
uses: microsoft/security-devops-action@08976cb623803b1b36d7112d4ff9f59eae704de0 # v1.12.0
|
||||
id: msdo
|
||||
with:
|
||||
# checkov is intentionally excluded from this MSDO step.
|
||||
@@ -57,11 +57,11 @@ jobs:
|
||||
# avoiding the guardian.cmd/checkov exit-code bug in the MSDO wrapper.
|
||||
tools: eslint,templateanalyzer,terrascan
|
||||
- name: Upload results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.msdo.outputs.sarifFile }}
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
}
|
||||
|
||||
- name: Upload Checkov results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: reports/checkov.sarif
|
||||
|
||||
@@ -29,11 +29,11 @@ jobs:
|
||||
name: Validate Documentation
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- uses: actions/setup-node@v6
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '20'
|
||||
- run: python docs_check.py
|
||||
@@ -44,9 +44,9 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: validate
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: actions/setup-node@v6
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
@@ -57,12 +57,12 @@ jobs:
|
||||
cd ..
|
||||
unzip -q export.zip -d site
|
||||
|
||||
- uses: actions/configure-pages@v6
|
||||
- uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6
|
||||
|
||||
- uses: actions/upload-pages-artifact@v5
|
||||
- uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5
|
||||
with:
|
||||
path: ./site
|
||||
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v5
|
||||
uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5
|
||||
|
||||
@@ -5,19 +5,28 @@ on:
|
||||
tags: ['v*']
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
environment: pypi
|
||||
concurrency:
|
||||
group: release-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
permissions:
|
||||
contents: write # for the GitHub Release
|
||||
id-token: write # for PyPI Trusted Publishing (OIDC) and attestation signing
|
||||
attestations: write # for SLSA build provenance
|
||||
# If you add another job to this workflow, give it its own explicit
|
||||
# `permissions:` block rather than relying on the workflow-level default
|
||||
# above (contents: read) - do not widen the workflow-level default.
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- uses: actions/setup-node@v6
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '20'
|
||||
cache: 'npm'
|
||||
@@ -27,8 +36,16 @@ jobs:
|
||||
run: |
|
||||
npm ci
|
||||
npm run build
|
||||
# Install the pinned dependency set (with hashes) so the sdist/wheel
|
||||
# build runs against the same versions CI tests against.
|
||||
- name: Install pinned build dependencies
|
||||
run: pip install -r requirements-ci.txt
|
||||
- run: pip install build
|
||||
- run: python -m build
|
||||
# wheel is build-time only (not in requirements-ci.txt) — install the
|
||||
# same pinned version [build-system] declares so --no-isolation works.
|
||||
- run: pip install wheel==0.48.0
|
||||
- name: Build package (no isolation — pinned deps)
|
||||
run: python -m build --no-isolation
|
||||
- name: Verify Explorer frontend is packaged
|
||||
run: |
|
||||
python - <<'PY'
|
||||
@@ -46,7 +63,11 @@ jobs:
|
||||
|
||||
print("Explorer frontend is packaged")
|
||||
PY
|
||||
- uses: softprops/action-gh-release@v3
|
||||
- name: Attest build provenance
|
||||
uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4
|
||||
with:
|
||||
subject-path: 'dist/*'
|
||||
- uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3
|
||||
with:
|
||||
files: dist/*
|
||||
- uses: pypa/gh-action-pypi-publish@release/v1
|
||||
- uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1
|
||||
|
||||
@@ -28,35 +28,71 @@ jobs:
|
||||
contents: read
|
||||
security-events: write
|
||||
actions: read
|
||||
|
||||
# Needed for the "Comment PR with Security Results" step below. Safe on
|
||||
# pull_request (not pull_request_target): GitHub always forces a
|
||||
# read-only token for PRs from forks regardless of this permission.
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
# Install the pinned dependency set FIRST so Safety scans Semantica's
|
||||
# exact CI/release dependency tree (requirements-ci.txt is generated
|
||||
# from pyproject.toml extras, so this covers the project's real deps).
|
||||
pip install -r requirements-ci.txt
|
||||
# Tooling AFTER the pinned set: installing safety/bandit/semgrep/jq
|
||||
# first lets the pinned requirements overwrite their transitive deps
|
||||
# (e.g. rich), which breaks the safety CLI at runtime.
|
||||
pip install safety bandit semgrep jq
|
||||
|
||||
|
||||
- name: Run Safety Check (Package Vulnerabilities)
|
||||
run: |
|
||||
safety check --json --output safety-report.json || true
|
||||
# NOTE: Safety 3.x repurposed --output to select a console format
|
||||
# (json/text/screen/...), not a file path. Writing JSON to a file
|
||||
# now requires --save-json; the previous `--output safety-report.json`
|
||||
# usage was silently invalid and never produced a report.
|
||||
safety check --save-json safety-report.json || true
|
||||
|
||||
# Guard 1: fail loudly if Safety exited before writing a report at all
|
||||
# (network error, API auth failure, tool crash). Without this check a
|
||||
# missing or empty file causes jq to fall back to "0", making a broken
|
||||
# scanner indistinguishable from a clean scan.
|
||||
if [ ! -s safety-report.json ]; then
|
||||
echo "::error::Safety scan produced no report (safety-report.json is missing or empty). Treating as failure — check for network errors, API auth failures, or Safety crashes in the logs above."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Checking for package vulnerabilities..."
|
||||
|
||||
# Count vulnerabilities safely
|
||||
VULNS=$(safety check --json --output /dev/stdout 2>/dev/null | jq '.vulnerabilities | length' 2>/dev/null || echo "0")
|
||||
|
||||
|
||||
# No || echo "0" fallback: if jq fails (malformed JSON, missing key,
|
||||
# vulnerabilities:null) VULNS will be empty or "null" so guard 2 below
|
||||
# catches it rather than silently treating the broken report as zero.
|
||||
VULNS=$(jq '.vulnerabilities | length' safety-report.json 2>/dev/null)
|
||||
|
||||
# Guard 2: ensure VULNS is a non-negative integer before the -gt
|
||||
# comparison. "null" (missing/null key) or "" (jq parse failure) would
|
||||
# cause bash's -gt to throw an arithmetic error and fall through to the
|
||||
# success branch — the same silent-pass bug as a missing file.
|
||||
if ! [[ "$VULNS" =~ ^[0-9]+$ ]]; then
|
||||
echo "::error::Safety report exists but 'vulnerabilities' is missing or non-numeric (got: '${VULNS}'). The report may be malformed or Safety may have written an error-only JSON. Treating as failure."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$VULNS" -gt 0 ]; then
|
||||
echo "❌ Security vulnerabilities found: $VULNS"
|
||||
echo "CI will fail to prevent merging of vulnerable dependencies"
|
||||
echo ""
|
||||
echo "Vulnerability details:"
|
||||
safety check || true
|
||||
jq -r '.vulnerabilities[] | "- \(.package_name)==\(.analyzed_version): \(.vulnerability_id) (\(.CVE // "no CVE assigned"))"' safety-report.json || true
|
||||
exit 1
|
||||
else
|
||||
echo "✅ No security vulnerabilities found"
|
||||
@@ -99,9 +135,10 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Upload Security Reports
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: security-reports
|
||||
retention-days: 14
|
||||
path: |
|
||||
safety-report.json
|
||||
bandit-report.json
|
||||
@@ -109,77 +146,91 @@ jobs:
|
||||
|
||||
- name: Comment PR with Security Results
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
|
||||
// Read safety report
|
||||
let safetyResults = '';
|
||||
try {
|
||||
const safetyData = JSON.parse(fs.readFileSync('safety-report.json', 'utf8'));
|
||||
if (safetyData.vulnerabilities && safetyData.vulnerabilities.length > 0) {
|
||||
safetyResults = `## Safety Vulnerabilities Found\\n`;
|
||||
safetyData.vulnerabilities.forEach(vuln => {
|
||||
safetyResults += `- **${vuln.package}**: ${vuln.advisory}\\n`;
|
||||
});
|
||||
} else {
|
||||
safetyResults = '## No Safety Vulnerabilities Found\\n';
|
||||
|
||||
// Renders one tool's findings as a section. `items` is already
|
||||
// the list of pre-formatted "- `thing` in `where`" strings; this
|
||||
// just handles the found/not-found/report-missing framing and
|
||||
// collapses long lists into a <details> block so the comment
|
||||
// doesn't turn into a wall of text.
|
||||
function renderSection(title, reportPath, parse) {
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
|
||||
} catch (e) {
|
||||
return [
|
||||
`### ${title}`,
|
||||
`⚠️ No report found at \`${reportPath}\` — the scan may have failed before producing output. Check the job logs.`,
|
||||
].join('\n');
|
||||
}
|
||||
} catch (e) {
|
||||
safetyResults = '## Safety scan completed\\n';
|
||||
}
|
||||
|
||||
// Read bandit report
|
||||
let banditResults = '';
|
||||
try {
|
||||
const banditData = JSON.parse(fs.readFileSync('bandit-report.json', 'utf8'));
|
||||
if (banditData.results && banditData.results.length > 0) {
|
||||
const highIssues = banditData.results.filter(issue => issue.issue_severity === 'HIGH');
|
||||
if (highIssues.length > 0) {
|
||||
banditResults = `## High Severity Security Issues Found\\n`;
|
||||
highIssues.forEach(issue => {
|
||||
banditResults += `- **${issue.test_name}**: ${issue.filename}:${issue.line_number}\\n`;
|
||||
});
|
||||
} else {
|
||||
banditResults = '## No High Severity Security Issues Found\\n';
|
||||
}
|
||||
} else {
|
||||
banditResults = '## No Bandit Issues Found\\n';
|
||||
|
||||
const items = parse(data);
|
||||
if (items.length === 0) {
|
||||
return [`### ${title}`, `✅ No findings.`].join('\n');
|
||||
}
|
||||
} catch (e) {
|
||||
banditResults = '## Bandit scan completed\\n';
|
||||
}
|
||||
|
||||
// Read semgrep report
|
||||
let semgrepResults = '';
|
||||
try {
|
||||
const semgrepData = JSON.parse(fs.readFileSync('semgrep-report.json', 'utf8'));
|
||||
if (semgrepData.results && semgrepData.results.length > 0) {
|
||||
semgrepResults = `## Security Patterns Found\\n`;
|
||||
semgrepData.results.slice(0, 10).forEach(issue => {
|
||||
semgrepResults += `- **${issue.rule_id}**: ${issue.path}\\n`;
|
||||
});
|
||||
if (semgrepData.results.length > 10) {
|
||||
semgrepResults += `- ... and ${semgrepData.results.length - 10} more\\n`;
|
||||
}
|
||||
|
||||
const lines = [`### ${title}`, `Found **${items.length}**.`, ''];
|
||||
const shown = items.slice(0, 15);
|
||||
if (items.length > 15) {
|
||||
lines.push('<details>', '<summary>Show all findings</summary>', '');
|
||||
lines.push(...items);
|
||||
lines.push('', '</details>');
|
||||
} else {
|
||||
semgrepResults = '## No Security Patterns Found\\n';
|
||||
lines.push(...shown);
|
||||
}
|
||||
} catch (e) {
|
||||
semgrepResults = '## Semgrep scan completed\\n';
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
// Create summary comment
|
||||
const comment = `# 🔒 Security Scan Results\\n\\n${safetyResults}\\n\\n${banditResults}\\n\\n${semgrepResults}\\n\\n---\\n\\n*This security scan runs automatically on source-code PRs and bi-weekly (skipped for doc/markdown-only changes).*\\n\\n📊 **Security Policy**: CI fails on vulnerabilities and HIGH severity issues.`;
|
||||
|
||||
// Post comment with error handling
|
||||
|
||||
const safetySection = renderSection(
|
||||
'Safety — dependency vulnerabilities',
|
||||
'safety-report.json',
|
||||
(data) => (data.vulnerabilities || []).map(
|
||||
(v) => `- \`${v.package_name}==${v.analyzed_version}\`: ${v.vulnerability_id}` +
|
||||
(v.CVE ? ` (${v.CVE})` : '') + ` — ${v.advisory || 'no advisory text'}`
|
||||
)
|
||||
);
|
||||
|
||||
const banditSection = renderSection(
|
||||
'Bandit — HIGH-severity code issues',
|
||||
'bandit-report.json',
|
||||
(data) => (data.results || [])
|
||||
.filter((issue) => issue.issue_severity === 'HIGH')
|
||||
.map((issue) => `- \`${issue.test_name}\` in \`${issue.filename}:${issue.line_number}\``)
|
||||
);
|
||||
|
||||
const semgrepSection = renderSection(
|
||||
'Semgrep — static analysis patterns',
|
||||
'semgrep-report.json',
|
||||
(data) => (data.results || []).map(
|
||||
(issue) => `- \`${issue.check_id}\` in \`${issue.path}:${issue.start?.line ?? '?'}\``
|
||||
)
|
||||
);
|
||||
|
||||
const comment = [
|
||||
'# 🔒 Security Scan Results',
|
||||
'',
|
||||
safetySection,
|
||||
'',
|
||||
banditSection,
|
||||
'',
|
||||
semgrepSection,
|
||||
'',
|
||||
'---',
|
||||
'',
|
||||
'*This security scan runs automatically on source-code PRs and bi-weekly (skipped for doc/markdown-only changes).*',
|
||||
'',
|
||||
'📊 **Security Policy**: CI fails on Safety vulnerabilities and Bandit HIGH-severity findings. Semgrep findings above are informational and do not block merge.',
|
||||
].join('\n');
|
||||
|
||||
try {
|
||||
await github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: comment
|
||||
body: comment,
|
||||
});
|
||||
console.log('✅ Security comment posted successfully');
|
||||
} catch (error) {
|
||||
|
||||
@@ -4,6 +4,12 @@ on:
|
||||
schedule:
|
||||
- cron: '0 0 * * 1'
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'pyproject.toml'
|
||||
- 'requirements-ci.txt'
|
||||
- '.github/workflows/security.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -12,10 +18,25 @@ jobs:
|
||||
audit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: '3.11'
|
||||
# Upgrade first: actions/setup-python's baked-in setuptools has been
|
||||
# behind known-vulnerable floors before (e.g. PYSEC-2026-3447 /
|
||||
# setuptools 75.1.0), so don't trust the preinstalled one.
|
||||
- run: python -m pip install --upgrade pip setuptools
|
||||
# Audit the pinned dependency set (requirements-ci.txt is compiled from
|
||||
# pyproject.toml with --extra all — the same coverage as the [all]
|
||||
# extra, minus the Linux-only gpu set — so this keeps scan parity with
|
||||
# CI/release builds without a time-dependent resolution). This is the
|
||||
# fix for PYSEC-2024-38 (#869): the bare-env job never had fastapi or
|
||||
# python-multipart installed to look at.
|
||||
- run: pip install -r requirements-ci.txt
|
||||
# PR runs gate on findings, since they're scoped to actual
|
||||
# pyproject.toml changes under review. The schedule/workflow_dispatch
|
||||
# runs stay non-blocking until a full pass over pre-existing findings
|
||||
# across the whole [all] tree has been done.
|
||||
- run: pip install pip-audit
|
||||
- run: pip-audit
|
||||
continue-on-error: true
|
||||
- run: pip-audit -r requirements-ci.txt
|
||||
continue-on-error: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
name: Verify Action Pins
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '.github/workflows/**'
|
||||
- '.github/scripts/verify-action-pins.sh'
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- '.github/workflows/**'
|
||||
- '.github/scripts/verify-action-pins.sh'
|
||||
schedule:
|
||||
- cron: '0 3 * * 1' # weekly, in case an upstream tag is deliberately moved
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- name: Verify pinned action SHAs match their tag comments
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: bash .github/scripts/verify-action-pins.sh
|
||||
+641
@@ -9,6 +9,647 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.6.6] - 2026-08-20
|
||||
|
||||
### Added
|
||||
|
||||
- **Semantica RDF vocabulary, and deterministic entity/relationship IRIs** (#1109, closes #1107, closes #1101) by @fabio-rovai, reviewed by @KaifAhmad1
|
||||
- Every RDF/JSON-LD export mints terms in `https://semantica.dev/ns#`, and until now nothing declared what those terms meant — the namespace 404s and no vocabulary shipped with the package, so a consumer receiving an export had no way to tell `sem:text` from a typo of it, and no closed-world checker could validate an export at all
|
||||
- `semantica/ontology/vocabulary/semantica-ns.ttl` declares the terms the exporters actually emit — drawn from the emitting call sites in `export/rdf_exporter.py`, `export/json_exporter.py` and `provenance/manager.py`, not from what a vocabulary "ought" to contain. Ships inside the package (`from semantica.ontology.vocabulary import vocabulary_turtle`) so it loads without a network round trip, and is the same document intended to be served at the namespace IRI once hosting/content-negotiation is sorted
|
||||
- `tests/ontology/test_vocabulary.py` ties the document to the code: every term a serializer can write must be declared, so adding a term to an exporter without declaring it fails the build
|
||||
- The missing-id fallback minted entity/relationship IRIs from Python's builtin `hash()`, randomised per process (`PYTHONHASHSEED`), so the same entity got a different IRI on every run and exports couldn't be diffed, deduplicated, or joined to an earlier provenance record. It also wrote `<semantica:entity_N>`, an IRI in the scheme `semantica` rather than the expansion of the declared prefix, so those nodes never joined with anything written through it. Minting now uses SHA-256 and writes a full IRI in the declared namespace; the same fix applies to the default entity/relationship types in the Turtle path
|
||||
- **Fixed during review** (Qodo): the temporal fallback minted from `source_id` only, while the main serializer accepts `source_id` or `source` — relationships using the second form hashed two empty strings, which the previous randomised `hash()` masked by making the IRI unstable anyway; once deterministic, unrelated relationships at the same list index collided on one IRI across exports. Endpoints are now resolved the same way `serialize_to_turtle` resolves them, before minting. `sem:confidence` also lost its declared `xsd:decimal` range: the N-Triples serializer types the same value `xsd:float`, and the two are disjoint, so declaring either contradicted one of the exporters (tracked in #1100) — a new `test_declared_ranges_do_not_contradict_what_the_exporters_emit` guards the whole class of that mistake
|
||||
- **Fixed in follow-up**: `serialize_to_rdfxml`'s default entity type still wrote the bare string `"semantica:Entity"` into an `rdf:resource` attribute, which (unlike a Turtle angle-bracket or an XML element name) is not namespace-expanded — the exact #1101 failure mode, just on the untested RDF/XML path. `json_exporter.py`'s `semantica:format` and `@type: "semantica:KnowledgeGraph"` were emitted but absent from both the vocabulary and the test's `EMITTED_TERMS` guard set, so the "undeclared terms fail the build" claim didn't actually cover them — both are now declared and guarded. `MANIFEST.in` didn't mirror the `pyproject.toml` package-data addition, so a source-distribution install could omit the vocabulary file. The cross-process minting-stability test replaced the subprocess's entire environment with a POSIX-only `PATH`, breaking it on Windows; now overrides only `PYTHONHASHSEED` on top of the inherited environment
|
||||
- **Also fixed, on the JSON-LD paths**: the first fix covered the Turtle, N-Triples and RDF/XML serializers, and left both JSON-LD writers interpolating the entity's own text into `f"semantica:entity/{text}"` and the endpoints into `f"semantica:rel/{source}_{target}"`. Three consequences, all live in 0.6.5: an entity whose text contained a space produced an invalid IRI, and a JSON-LD parser dropped that node in full rather than reporting it, so the entity disappeared from the export; every relationship carrying `source`/`target` rather than `source_id`/`target_id` minted the identical `semantica:rel/_`, collapsing all of them onto one node whose types and endpoints merged; and the JSON-LD `@id` disagreed with the Turtle IRI for the same entity, so the two serializations of one knowledge graph were two different graphs. Both JSON-LD writers now use `mint_entity_iri`/`mint_relationship_iri`, and `JSONExporter.export_entities`/`export_relationships` declare the `semantica` prefix their `@context` was already writing `semantica:entities` against — without it a processor reads that as an IRI in the scheme `semantica`, which is the original #1101 defect on a third path
|
||||
- `tests/export/test_jsonld_iri_minting.py` parses each export with a real JSON-LD processor and asserts the entity survives, the relationships stay distinct, no term expands into the `semantica` scheme, and the JSON-LD `@id` equals the Turtle IRI
|
||||
- 236 export and ontology tests pass
|
||||
|
||||
- **First-class CrewAI integration** (#988, closes #962) by @Shindevrp
|
||||
- New `pip install semantica[crewai]` extra (`crewai>=0.80.0`) — crewai core provides `BaseTool`/`BaseKnowledgeSource`, so `crewai-tools` is intentionally not included, and the extra is intentionally **not** part of the `all` bundle: crewai hard-requires `chromadb~=1.1.0`, which is affected by the unpatched pre-auth code-injection CVE-2026-45829 (see `integrations/crewai/README.md`)
|
||||
- `integrations/crewai/SemanticaKGTool` — a CrewAI `BaseTool` exposing 5 KG actions (`extract_entities`, `extract_relations`, `add_to_graph`, `query_graph`, `find_related`) backed by `NERExtractor` / `RelationExtractor` / `ContextGraph`; supports both sync `run()` and async `arun()`
|
||||
- `integrations/crewai/SemanticaDecisionTool` — a CrewAI `BaseTool` wrapping `AgentContext` with 5 decision-intelligence actions (`record_decision`, `find_precedents`, `trace_causal_chain`, `analyze_impact`, `check_policy`)
|
||||
- `integrations/crewai/SemanticaKnowledgeSource` — a CrewAI `BaseKnowledgeSource` that serializes a `ContextGraph` into crew knowledge storage; implements both the legacy `load_content()` and current `validate_content()`/`aadd()` contracts so it works across `crewai>=0.80.0`
|
||||
- All three classes degrade gracefully when `crewai` is not installed (still importable, full Semantica API available)
|
||||
- New `tests/integrations/crewai/`: 70 tests covering stub-based present-case behavior (Pydantic/BaseTool subclassing, every action, knowledge-source chunking/storage) plus a subprocess isolation test for the crewai-absent degradation path
|
||||
- Docs: `docs/integrations/crewai.md` page, `docs.json` Integrations nav entry, and README integration-matrix/install updates
|
||||
- **Hardened during code review**: live `graph`/`context`/extractor state is excluded from CrewAI JSON serialization (`model_dump(mode="json")`) with `model_post_init` self-healing defaults, so checkpoint/resume no longer raises `PydanticSerializationError`; `query_graph` now searches node content (not just ids/types); `trace_causal_chain` returns an explicit error instead of substituting similarity precedents when causal tracing is unavailable, and calls `trace_decision_causality(..., max_depth=...)` with the correct argument name; `find_precedents` propagates `max_precedents` as the backend `limit`; `add_to_graph` writes are serialized under a module lock so concurrent agents can't double-count duplicate adds; nameless entities are skipped instead of creating `repr()`-junk nodes
|
||||
- **Hardened during second code review**: `check_policy` rules are now coerced type-aware — `bool("false")` was truthy, so `enabled == false` reported a violation for `enabled: false`, and string datums like `"0.90"` were compared lexicographically instead of numerically; `trace_causal_chain` no longer raises `AttributeError` (which escaped the tool) when the decision context has no `knowledge_graph`, returning honest error JSON instead; knowledge-source storage failures log an actionable ERROR (a missing crew embedder otherwise silently left agents with empty retrieval); `add_to_graph` uses a per-graph re-entrant lock instead of a process-global one (independent graphs no longer serialize each other, and re-entrant extractors can't deadlock); entity/relation `confidence=None` normalizes to `1.0` instead of failing the whole extraction; added a subprocess integration test against the real `crewai` package covering `Crew`-level serialization round-trip and restore
|
||||
|
||||
- **`ContextGraph` gains retraction and purge — the graph previously had no way to remove a node or edge without discarding everything via `clear()`** (#957, closes #955) by @pravit-amp, reviewed by @KaifAhmad1
|
||||
- `retract_node()`/`retract_edge()` close an entity's validity window rather than deleting it, reusing the existing `valid_from`/`valid_until`/`state_at()` machinery: the entity drops out of `find_active_nodes()` and future `state_at()` queries going forward, but `state_at()` calls before the retraction time still return it, so decisions recorded against it stay explainable. A `("kind", id)`-keyed retraction record captures who/why/when, retrievable via `get_retraction()`/`list_retractions()`
|
||||
- `purge_node()`/`purge_edge()` are the destructive counterpart: the entity is removed outright, from history as well as the active view, for erasure obligations retraction alone cannot satisfy (e.g. GDPR Article 17). Only a tombstone remains — that a purge happened, when, and why — deliberately never the purged content, via `get_tombstone()`/`list_tombstones()`. Purge is graph-scope only: `AgentMemory` and any bound vector store are not reached, so it is one step of an erasure workflow rather than the whole of it
|
||||
- Both operations default to `cascade=True` (also touching every incident edge, and for `purge_node`, the marker node of any cross-graph link the node exits through) since leaving edges active around an inactive/removed node produces an inconsistent active view or dangling endpoints; both accept `cascade=False` for callers that want to handle edges themselves
|
||||
- Both are idempotent: retracting/purging an already-retracted/purged entity returns `False` rather than raising, and a repeat retraction preserves the original record's reason rather than overwriting it
|
||||
- Retraction/purge closing a validity window never widens an existing one — a node or edge added with `valid_until` already in the past keeps that earlier bound rather than being pushed later by a subsequent retraction time
|
||||
- Reuses the existing audit-trail path with no changes to `change_management`: `MutationRecord` already documented `REMOVE_NODE`/`REMOVE_EDGE` in its operation vocabulary; retraction now emits `UPDATE_NODE`/`UPDATE_EDGE`, purge emits `REMOVE_NODE`/`REMOVE_EDGE`, matching the documented contract. Mutation payloads are snapshotted inside the lock and the callback fires after it is released, so a callback that itself mutates the graph (e.g. `clear()`) can't observe or lose in-flight records
|
||||
- **Fixed during review** (@KaifAhmad1): `retract_edge()`/`purge_edge()` resolved "the edge" for a given `edge_id` via the first matching object only. `edge_id` is content-derived and, prior to #926, was not guaranteed unique — a graph holding two identical `add_edge()` calls had two edge objects sharing one id. A direct `retract_edge()`/`purge_edge()` call would silently leave the second duplicate untouched (still live, still active) while returning `True` and recording a tombstone/retraction that claimed the edge was fully handled; repeat `purge_edge()` calls also silently overwrote the tombstone's `reason`/`purged_at` on each partial attempt instead of no-op'ing. The same gap let `retract_node()`'s cascade skip a duplicate outright, since it checked the live `_retractions` dict mid-loop and treated the first duplicate's just-written record as proof the second was already handled. `#926` (merged) stops *new* duplicates from being created, but any graph already holding one — loaded from a save made before that fix, or built during the window before it landed — could still trigger this. Now `retract_edge()`/`purge_edge()` act on every edge matching the id under one record, and the cascade's dedup check is snapshotted before the loop starts so within-call duplicates are still closed rather than skipped. 5 new regression tests in `TestDuplicateEdgeId`
|
||||
- New `tests/context/test_context_graph_retraction.py`: 49 tests, covering retraction/purge semantics, cascade, idempotency, validity-window narrowing, id-keyspace collisions between node and edge ids, cross-graph link teardown, `clear()`/`load_from_file()` resetting retraction/tombstone state, audit-trail integration against a real `TemporalVersionManager`, mutation-emission ordering under a concurrent `clear()`, and concurrent purges
|
||||
- Full `tests/context/` suite: 533 passed
|
||||
- **`DistanceExporter.compute_pairs()` gains an opt-in `metric_errors` column to distinguish legitimate `None` results from computation failures** (#960, follow-up to #879) by @Karunasagar12
|
||||
- Previously, a `None` in `hop_count`/`weighted_distance`/`semantic_similarity`/betweenness could mean either "no path exists" or "the underlying computation raised" — logged as a warning per #879, but not otherwise surfaced, so the two cases were indistinguishable in exported CSV/JSONL/DataFrame data. `include=["metric_errors"]` now adds a `metric_errors` field per row: `""` when all requested metrics succeeded, or a comma-separated list of metric names that raised (e.g. `"hop_count,weighted_distance"`)
|
||||
- Opt-in only — default `compute_pairs()`/`to_csv()`/`to_dataframe()`/`to_jsonl()` schema is unchanged unless `"metric_errors"` is explicitly requested
|
||||
- The four metric helpers (`_betweenness`, `_hop_distance`, `_weighted_distance`, `_semantic_similarity`) now return `(value, error_name | None)` tuples internally; `compute_pairs()` aggregates the error names per row
|
||||
- **Fixed during review** (Qodo): `_betweenness()` failures weren't tracked into `metric_errors` in the initial version — centrality computation could raise and the column would still report `""`. Now returns its error tuple like the other three helpers
|
||||
- **Known limitation**: `include=["metric_errors"]` with no other metric names computes nothing, so the column is always `""` in that case — pass it alongside the metrics you want tracked, e.g. `include=["hop_count", "metric_errors"]`
|
||||
- New `tests/export/test_distance_exporter_metric_errors.py`: 6 tests covering success, single/multiple failures, opt-out, the no-path-vs-error distinction, and default-schema stability; existing `tests/export/test_distance_exporter.py` updated for the new tuple return type
|
||||
- Full `tests/export/` suite: 77 passed
|
||||
|
||||
- **`ContextGraph.to_kg_dict()`: an adapter converting a `ContextGraph`'s internal `nodes`/`edges`/`source` shape into the canonical `entities`/`relationships`/`source_id` shape `RDFExporter` and `TemporalGraphQuery` consume** (#1081) by @cxzg007
|
||||
- Previously there was no supported way to feed a `ContextGraph` into those consumers without hand-rolling the field remapping; `to_kg_dict()` does it once, with an `entities_only` option that drops relationships left dangling by the filter
|
||||
- **Fixed during review** (Qodo): null `properties`/`metadata` on a node loaded from JSON raised `TypeError` when copied — both are now guarded with `or {}`; entity ids are coerced to `str(node_id)` to match `ContextEdge`'s already-str-coerced endpoints, so valid relationships were no longer dropped by `entities_only` filtering
|
||||
- `RDFExporter`'s validator and `TemporalGraphQuery` now also accept `source_id`/`target_id` endpoints, the shape `to_kg_dict()` emits
|
||||
|
||||
### Changed
|
||||
|
||||
- **`GraphBuilder`'s 6 public methods now have Google-style docstrings** (#878, closes #876) by @cakeni
|
||||
- `semantica/kg/graph_builder.py`'s `build`, `build_single_source`, `add_temporal_edge`, `create_temporal_snapshot`, `query_temporal`, and `load_from_neo4j` — the core knowledge-graph construction API, imported directly by callers — previously had zero docstrings across all 6 methods, the only file in a 10-file audit sample with that gap, despite CONTRIBUTING.md requiring Google-style `Args`/`Returns`/`Raises`/`Example` docs for public methods. Added full docstrings for all 6, plus the previously undocumented `build_single_source`, with runnable (`# doctest: +SKIP`) usage examples
|
||||
- **Corrected during review**: `query_temporal`'s docstring claimed the query text was used to filter the graph; the implementation only records it in the result (`results = {"query": query, ...}`) with no interpretation or filtering. Corrected to state that explicitly
|
||||
- **Corrected during review**: `create_temporal_snapshot`'s docstring implied entities were filtered for validity at the snapshot timestamp like relationships are; the implementation copies all entities unfiltered and only filters `relationships` by `valid_from`/`valid_until`. Docstring now distinguishes the two
|
||||
- **Corrected during review**: `add_temporal_edge`/`create_temporal_snapshot` docstrings overclaimed numeric-timestamp support; `_parse_time()` only special-cases `str` and `datetime`, falling back to a bare `str()` cast for anything else (not true numeric parsing). Narrowed to "datetime or ISO-formatted string"
|
||||
- **Fixed along the way**: `build()`'s `**options` documented a default only for `extract`; `extract_relations`, `extract_triplets`, `ner_method`, `relation_method`, and `triplet_method` all have concrete defaults in `_extract_from_text()` (`True`, `True`, `"llm"`, `"llm"`, `"llm"`) that were left unstated, inconsistent with CONTRIBUTING.md's own docstring example of noting defaults inline
|
||||
- `python -m pytest tests/kg/test_kg.py tests/kg/test_graph_builder_external.py -q`: 45 passed
|
||||
- **`GraphBuilder` raw-text extraction now defaults to local extractors instead of LLM extraction** (#941, closes #930) by @dex0shubham
|
||||
- `GraphBuilder._extract_from_text()` defaulted `ner_method`, `relation_method`, and `triplet_method` to `"llm"`, and ran relation extraction unconditionally (`extract_relations` defaulted to `True`) — all four contradicting the defaults documented in the `build()` docstring at the time (`"ml"` / `"pattern"` / `False`), and diverging from the standalone extractors (`NERExtractor` defaults to `method="ml"`, `RelationExtractor` and `TripletExtractor` to `method="pattern"`). The practical effect was that any raw-text `build()` call silently required a configured provider, an API key, and network access
|
||||
- Defaults are now `ner_method="ml"`, `relation_method="pattern"`, `triplet_method="pattern"`, and `extract_relations=False`, matching the docstring. LLM extraction remains fully available and is now opt-in
|
||||
- **To restore the previous behaviour**, pass the methods explicitly:
|
||||
```python
|
||||
builder.build(
|
||||
sources,
|
||||
ner_method="llm",
|
||||
relation_method="llm",
|
||||
triplet_method="llm",
|
||||
extract_relations=True,
|
||||
)
|
||||
```
|
||||
- #878 landed in the meantime and resolved the same mismatch in the opposite direction, documenting the LLM values (`"llm"` / `"llm"` / `"llm"`, `extract_relations: True`) as the contract. Per the decision on #930 the code is the side that changes, so those docstring defaults are corrected here to `"ml"` / `"pattern"` / `"pattern"` / `False`, keeping #878's formatting
|
||||
- Removed the stale `# Default to LLM methods as per requirement` comment, which read as an intentional decision but did not match the documented contract
|
||||
- **Fixed along the way**: `_extract_from_text()` constructed a fresh extractor for every text, and `NERExtractor.__init__` loads its spaCy model eagerly when the method includes `"ml"` — so with the new default, a multi-document build would have reloaded the model once per source. Extractors are now built once per `(kind, method)` and reused for the lifetime of the builder, via `GraphBuilder._get_extractor()`. This path was previously unreachable by default because the old `"llm"` default never touched spaCy
|
||||
- **Fixed along the way**: `_extract_from_text()` never forwarded its extracted relations to triplet extraction — it passed only `entities=`, so `TripletExtractor` re-derived relations itself (via a method taken from `triplet_method`) whenever `relations is None`, duplicating work and producing triplets that could disagree with the relations already extracted using `relation_method`. Relations are now passed through as `relations=`; when relation extraction is disabled or fails, `None` is forwarded and `TripletExtractor` keeps its existing self-derivation behaviour
|
||||
- **Fixed along the way**: `GraphBuilder._extraction_stats` was only initialised inside `build()`, so calling `_extract_from_text()` directly raised an `AttributeError` that the extraction path's broad `except` swallowed and reported as `"Entity extraction failed"`. It is now seeded in `__init__` as well; `build()` still resets it per run
|
||||
- New regression coverage in `tests/kg/test_graph_builder_extraction_defaults.py` pinning all four defaults, verifying that no default resolves to `"llm"`, confirming explicit LLM opt-in still routes correctly, asserting extractors are constructed once across repeated texts, covering fallback method lists (e.g. `ner_method=["pattern", "ml"]`) for all three extractors, asserting relations are forwarded to triplet extraction (and that `None` is forwarded when relation extraction is disabled or fails), and running the real default path end to end with no provider mocked. Verified to fail against the pre-fix code
|
||||
- Full `kg` suite: 473 passed
|
||||
|
||||
- **Explorer graph canvas now renders edge labels** (#1013, closes #1009) by @yzxcj797
|
||||
- `GraphCanvas.tsx` had no edge-label rendering path at all; Sigma's edge-label renderer draws `data.label`, but the graph state stored the relationship type under `edgeType`, so simply enabling the renderer would have left every edge blank. `graphSceneState`'s edge reducer now maps `edgeType` onto `label` (suppressed for hidden edges)
|
||||
- Rendering is gated behind a new `edgeLabelsEnabled` entry in the Effects panel (default on), wired through the existing `GraphEffectToggle`/`GraphEffectsState` plumbing, so dense graphs can still turn labels off
|
||||
- **Fixed during review** (Qodo): two follow-up passes closed gaps the first cut left — label rendering wasn't wired through `explorationEffectsPluginPhaseC.tsx`'s Phase C variant, and toggling the effect off mid-session didn't clear already-rendered labels
|
||||
- New coverage in `explorer/tests/graphSceneState.display.test.ts`
|
||||
|
||||
- **Removed `GraphWorkspaceShell.tsx`, `GraphRuntimeStage.tsx`, and `useGraphData.ts` — a second, unused implementation of the graph-loading/error-handling logic already fixed in `GraphWorkspace.tsx`** (#984, resolves the cleanup tracked in #981 by #980's review note) by @lakshayxi
|
||||
- 1,564 lines removed; the surviving `GraphWorkspace` path is now the only implementation, so the "two copies that drifted apart" root cause #980 fixed can't recur in the copy nobody was maintaining
|
||||
|
||||
- **Explorer README and `docs/explorer-setup.md` corrected to describe the authentication 0.6.5 actually shipped**, plus a documented `/ws/graph-updates` auth note (#1040, fixes #1028) by @Kyou12138
|
||||
- Both docs still claimed the Explorer API had no built-in authentication after v0.6.5 added mandatory `SEMANTICA_API_KEY` enforcement with a `503` fail-closed default; corrected to describe the actual behavior, including that only protected routes require the key (`/api/health`/`/api/info` stay open), the non-loopback-bind CLI warning only fires in anonymous mode or when the key is unset, and `SEMANTICA_API_KEY`/`SEMANTICA_ALLOW_ANONYMOUS` are documented in the environment-variable table
|
||||
|
||||
- **CI: pinned `github/codeql-action` to current v4** (#986) by @ZohaibHassan16, and **pinned Python dependencies in `requirements-ci.txt` for reproducible CI runs** (#945) by @yunaremaia, closing the gap where an unpinned CI dependency could silently change behavior between runs
|
||||
|
||||
- **README now states up front that Semantica's explainability is system-level, not foundation-model-internal** (#1033, #1034) by @KaifAhmad1
|
||||
- Nothing in the README previously scoped what "explainable" meant, leaving readers to assume Semantica could expose or reconstruct an LLM's internal reasoning. A callout now states explicitly that Semantica explains and audits what the AI *system* did — context fed in, decisions produced, provenance, relationships, policies applied — not the model's private internal reasoning, and moved the note near the top of the README rather than leaving it implicit
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Every timestamp an export or a provenance record wrote was timezone-naive** (closes #1114) by @fabio-rovai
|
||||
- `semantica/export/` stamped with `datetime.now().isoformat()`, which reads the machine's **local** clock; `semantica/provenance/` stamped with `datetime.utcnow().isoformat()`, which reads **UTC**. Both produce a naive value and both serialize identically, so nothing downstream can tell which zone a given timestamp belongs to — the same string means two different instants depending on which module wrote it
|
||||
- In RDF the consequence is silent rather than loud. Under XSD 1.1 a value with no timezone compared against one with a timezone is indeterminate whenever the two fall inside the ±14 hour window; SPARQL turns an indeterminate comparison into an error, and `FILTER` discards errors as non-matches. A timezone-qualified query over an Oxigraph store returns an answer with every Semantica-written record quietly absent from it, which is a poor property for `prov:generatedAtTime`, `prov:startedAtTime`, `prov:endedAtTime` and `prov:atTime` to have
|
||||
- New `utc_now()`/`utc_now_iso()` in `semantica/utils/helpers.py`, exported from `semantica.utils`, and used at all 29 call sites in `export/` (`json_exporter`, `yaml_exporter`, `report_generator`, `export_provenance`) and `provenance/` (`manager`, `schemas`, `bridge_axiom`). Values now read `2026-08-19T14:19:04.229937+00:00`: one unambiguous instant, comparable against any correctly stamped value, and valid `xsd:dateTimeStamp`. `sem:exportedAt`'s range in `semantica/ontology/vocabulary/semantica-ns.ttl` is tightened from `xsd:dateTime` accordingly, and its comment no longer has to explain why the weaker range was necessary
|
||||
- `datetime.utcnow()` is deprecated as of Python 3.12 and scheduled for removal; constructing a `ProvenanceEntry` under `-W error::DeprecationWarning` on 3.13 raised, and no longer does
|
||||
- New `tests/export/test_timestamp_timezones.py` and `tests/provenance/test_timestamp_timezones.py`: offset presence on every export and provenance path, PROV-O literals valid as `xsd:dateTimeStamp`, comparison against a timezone-aware instant without `TypeError`, the Oxigraph filter that dropped the naive value (with a bound inside the indeterminate window, so the test cannot pass by accident), and the document `@id` remaining a valid IRI with `+00:00` in it. 11 of the 13 fail on the parent commit
|
||||
- **Fixed during review** (Qodo): once new entries carry `+00:00` and stored ones do not, `ProvenanceManager.query_recorded_between` and `audit_log` compared ISO timestamps as raw strings, so they ordered by spelling rather than by instant — an inclusive naive bound naming a stored offset-bearing timestamp sorted *below* it and dropped the record, and a bound written in another offset landed wherever its digits fell (`19:45+05:30` is 14:15Z, but sorted after 14:19Z). Both now compare instants through a new `to_utc_datetime()` helper that reads a missing offset as UTC, which is what the values written before this change actually were; a bound that cannot be read as a timestamp keeps the historical string comparison rather than raising on a call that used to work
|
||||
- The remaining 147 naive call sites are in `context/`, `vector_store/`, `seed/` and elsewhere, where timestamps are compared against values parsed from previously stored naive strings. Converting those without a read-side migration would raise `TypeError: can't compare offset-naive and offset-aware datetimes` on existing data, so they are deliberately left for a separate change
|
||||
|
||||
- **`split`/chunking paths bypassed the centralized spaCy model cache, reloading the model on every call** (#1042, closes #998) by @Accute9, reviewed by @Sameer6305
|
||||
- `semantica/split/methods.py`'s `split_by_sentences()` and `semantica/split/semantic_chunker.py`'s `SemanticChunker.__init__` each called `spacy.load()` directly instead of reusing the process-level cache added in #889/`semantic_extract/methods.py`'s `load_spacy_model()` — every call/construction re-paid the ~120ms model-load cost independently of `NERExtractor`, which already used the cache
|
||||
- Both now route through `load_spacy_model()`, sharing one cached `Language` instance per model name across `split_by_sentences()`, `SemanticChunker`, and `NERExtractor`; a missing model still falls back to regex/paragraph chunking without poisoning the cache for a later successful load
|
||||
- **Fixed during review** (@Sameer6305): `NERExtractor.__init__()` still had a direct `spacy.load()` call site with the same cache-bypass issue, outside the two files named in #998 but sharing the same root cause; routed through the cache alongside stale test patch targets and a strengthened cache-configuration assertion
|
||||
- **Fixed during review** (@KaifAhmad1): `SemanticChunker.__init__` only caught `OSError` around `load_spacy_model()`, while the sibling fix to `NERExtractor` in this same PR added a broader `except Exception` for a model that is installed but fails at runtime (e.g. a config incompatible with the installed spaCy version). A broken-but-present model crashed `SemanticChunker()` outright instead of degrading to fallback chunking like every other path in this PR. Added the matching `except Exception` branch, leaving `self.nlp` as `None`; new `test_semantic_chunker_falls_back_when_spacy_runtime_is_broken` mirrors the existing `NERExtractor` regression test for the same scenario
|
||||
- New `tests/split/test_spacy_model_cache.py`: cache reuse across repeated calls/instances, shared cache between `split_by_sentences()`/`SemanticChunker`/`NERExtractor`, distinct model names loading separately, missing-model fallback without poisoning the cache, and the broken-runtime fallback added above
|
||||
- `pytest tests/split/test_spacy_model_cache.py tests/split/test_splitter.py tests/split/test_chunkers.py`: all passing (3 pre-existing, unrelated `tests/test_ner_configurations.py` failures confirmed present on `main` before this PR)
|
||||
|
||||
- **`export_yaml` raised a raw `AttributeError` on list input, silently wrote empty exports for unrecognized dict keys, and graph payloads were reconciled differently by every exporter** (#958, closes #956, #952, #953) by @pravit-amp, reviewed by @Sameer6305
|
||||
- Graph payloads circulate under two vocabularies, `entities`/`relationships` and `nodes`/`edges`, and each exporter reconciled them locally with a different idiom — `LPGExporter` in particular dropped every entity whenever `nodes` was present but empty, the exact shape `JSONExporter` emits. A new `normalize_graph_payload()` in `utils/helpers.py` centralizes that decision once, adopted by `LPGExporter`, `ArangoAQLExporter`, `Neo4jCSVExporter`, and both YAML exporters; `ContextGraph.to_dict()` now round-trips through YAML correctly as a result
|
||||
- `export_yaml(records, path)` on a bare list previously failed with `AttributeError` from inside the exporter; it and the other YAML methods now reject non-mapping input with an actionable `ProcessingError` naming the expected keys, since these formats distinguish entities/relationships/triplets and guessing which one a list represents would mislabel the records
|
||||
- `export_yaml({"data": [...]}, path)` previously wrote a structurally valid file with every collection empty, no exception, no warning, and the progress log reporting a completed export. `export_semantic_network`, `export_for_pipeline`, and `export_ontology_schema` now raise `ValidationError` when the payload shares no recognized key with what the method reads, or resolves to nothing while an unread key still holds records — an empty mapping is still accepted, since a genuinely empty graph has no records to lose
|
||||
- **Breaking**: the two cases above, plus a bare list, now raise instead of returning cleanly with data silently dropped or a raw `AttributeError` from exporter internals. Migration: pass records under a recognized key (`{"entities": [...]}` / `{"nodes": [...]}` for `semantic_network`, `{"classes": [...]}` for `schema`)
|
||||
- **Fixed during review** (Qodo): progress tracking could report a completed export before the output directory existed or the file was written; `export()` now creates the directory and serializes before starting tracking, so a rejected export leaves nothing behind
|
||||
- **Fixed during review** (@Sameer6305, round 1): `normalize_graph_payload()`'s collection resolver treated any truthy value as a collection — `{"entities": "abc"}` silently became three single-character records, `{"entities": 42}` leaked a raw `TypeError` from inside `list()`. Collection values are now validated before conversion, rejecting strings/bytes/mappings/non-iterable scalars by name. Separately, `Neo4jCSVExporter._normalize_graph` called the shared resolver with `require_recognized=False`, so it alone kept accepting an unrecognized mapping as a silent empty export; the opt-out (introduced earlier in this same PR, with no other caller) was removed
|
||||
- **Fixed during review** (@Sameer6305, round 2): `YAMLSchemaExporter`'s usable-schema check could treat scalar schema metadata (`version`, `uri`, `title`, `description`) as evidence records had been exported, letting records under an unread key drop silently; and `_is_record()` accepted modules and class/type objects through the generic `__dict__` path, which would have reached exporter internals instead of failing at the boundary. Both closed, with regression coverage
|
||||
- **Fixed during final maintainer review** (before merge): four more gaps in the shared boundary that the earlier rounds didn't reach
|
||||
- `LPGExporter`/`ArangoAQLExporter` called `normalize_graph_payload()` with no type guard, so non-mapping input raised `ValidationError` from inside the resolver — while YAML and `Neo4jCSVExporter` raised `ProcessingError` for the identical mistake, per this PR's own stated contract. The `_require_mapping()` guard that already existed in `yaml_exporter.py` is now shared from `utils/helpers.py` and used by all three
|
||||
- `Neo4jCSVExporter._normalize_graph` checked `isinstance(graph, dict)`, so a non-dict `Mapping` (`MappingProxyType`, `ChainMap`) fell through to the object-attribute branch and was rejected, even though the identical payload exported fine via `LPGExporter`/`ArangoAQLExporter`/YAML. Now checks `isinstance(graph, Mapping)`
|
||||
- `normalize_graph_payload()` accepts dataclass and attribute-bearing object records (`Neo4jCSVExporter._record_to_dict` reads them), but `LPGExporter`/`ArangoAQLExporter` call `.get(...)` directly on resolved entities — an object-shaped record passed validation only to crash with a raw `AttributeError` once used, the exact failure this PR's boundary exists to prevent. Records are now converted to plain dicts at the boundary (`_coerce_records` → new `_record_to_dict`), so every consumer gets a uniform shape regardless of which reading the caller used
|
||||
- Two non-empty spellings of the same collection (e.g. `entities` and `nodes`) holding identical records in a different order were rejected as conflicting, since the check used plain list equality; a caller round-tripping through a dict-keyed cache or a set has no reason to preserve order. Comparison is now an order-independent multiset of each record's canonical JSON form
|
||||
- New regression coverage in `tests/utils/test_normalize_graph_payload.py`: exception-type parity for non-mapping input across `export_lpg`/`export_arango`/`export_neo4j_csv`, dataclass-record conversion verified end-to-end through the same three exporters, `Neo4jCSVExporter` accepting a `MappingProxyType` payload, and reordered-alias equality (plus a duplicate-count case confirming the multiset check still catches real conflicts); 4 existing tests updated to assert the corrected dict-conversion behavior instead of the previous object passthrough
|
||||
- `pytest tests/export tests/utils tests/context tests/test_export_module.py tests/test_export_methods_wrapper.py tests/test_notebooks_simulation.py`: 718 passed, 4 skipped (up from 641 passed, 62 subtests at PR submission); `black`/`isort`/`flake8 --max-line-length=88` clean on every line this PR touches; `python -m build`: succeeds
|
||||
|
||||
- **`ContextGraph.add_edge` had no dedupe — identical edges were stored repeatedly under one shared edge ID, and re-ingest doubled the edge set** (#926, closes #922) by @pravit-amp
|
||||
- `_add_internal_edge` appended to `self.edges`, `edge_type_index`, and `_adjacency` unconditionally, with no check for an edge already present. Edge identity is content-derived (`_resolve_edge_identity` builds `edge_id` from `source_id`/`target_id`/`edge_type`/`weight`/`metadata`/`valid_from`/`valid_until`), so two identical `add_edge` calls produced two edge objects sharing one `edge_id` — the graph already considered them the same edge, it just kept both copies. `self.nodes` already deduped by ID; edges did not, so `stats()["edge_count"]` inflated, `density()` could exceed its mathematical maximum of `1.0`, and a refresh/restore job calling `build_from_entities_and_relationships()` (or reloading a saved graph) doubled the edge set on every cycle
|
||||
- Added an `edge_id -> ContextEdge` index (`_edge_index`), mirroring how `self.nodes` dedupes by node ID. `_add_internal_edge` now returns `False` when the `edge_id` already exists, checked before touching `edges`/`edge_type_index`/`_adjacency` and before firing the mutation callback, so a repeat `add_edge` is a silent no-op with no phantom `ADD_EDGE` audit event
|
||||
- Genuinely parallel edges are unaffected: differing type/weight/metadata/validity still produce distinct content-derived `edge_id`s, so multigraph semantics are preserved
|
||||
- Both state-reset paths (`load_from_file()` and `clear()`) also clear `_edge_index`
|
||||
- New tests: repeat `add_edge` is a no-op, parallel edges with distinct attributes are preserved, re-ingest via `build_from_entities_and_relationships()` stays at one edge, and `clear()` resets the dedupe index
|
||||
- `pytest tests/context/test_context.py`: 31 passed
|
||||
|
||||
- **`POST /api/enrich/extract` returned 503 on every request; the whole `/api/decisions*` family returned 500 as soon as a decision existed** (#886, closes #883, closes #884, closes #889) by @joseedson18jc, reviewed by @Sameer6305
|
||||
- `semantica/explorer/routes/enrich.py` imported `extract_entities`/`extract_relations` from `semantica.semantic_extract.methods`, names that module never defined (only per-strategy variants like `extract_entities_ml` exist) — the `except ImportError` handler reported this as `"semantic_extract module not available"`, masking a wiring bug as a missing dependency. The route now calls `NamedEntityRecognizer`/`RelationExtractor` directly and forwards extracted entities into relation extraction instead of re-deriving them
|
||||
- `ContextGraph.record_decision()` stores `timestamp` as `datetime.now().timestamp()` (a float), while `DecisionResponse.timestamp` was typed `Optional[str]`; passing the value through unconverted failed pydantic validation on every decision route (`/api/decisions`, `/{id}`, `/{id}/chain`, `/{id}/precedents`, `/{id}/compliance`). Added a `field_validator(mode="before")` on `DecisionResponse` normalizing float/int/datetime inputs to ISO-8601
|
||||
- Folds in the fix for #889: `extract_entities_ml`/`extract_relations_similarity`/`extract_relations_dependency` called `spacy.load()` on every invocation (~120ms of a ~132ms call, ~60x the actual extraction work). Added a process-level, lock-guarded `load_spacy_model()` cache in `semantic_extract/methods.py`, keyed by model name; failed loads are not cached, and the separate `get_nlp_model()` cache (different `disable=` pipeline config for similarity work) is kept independent to avoid handing one caller's spaCy pipeline to another
|
||||
- **Fixed during review** (@Sameer6305): capped previously-unbounded input text on `/api/enrich/extract`; tightened the route's exception handling
|
||||
- **Fixed during review** (@KaifAhmad1): the timestamp validator's `math.isfinite()` guard only rejected NaN/inf — a finite-but-out-of-range epoch (e.g. milliseconds mistakenly stored instead of seconds, such as `1723600000000`) still raised an uncaught `OverflowError`/`OSError` from `datetime.fromtimestamp()`, reintroducing an unhandled 500 on `/api/decisions*` for exactly the class of bug this PR closes. Now caught and re-raised as a `ValueError`. Also excluded `bool` from the numeric branch (`isinstance(True, int)` is `True` in Python, so `timestamp=True` was silently coerced to epoch 1 instead of being rejected)
|
||||
- New/updated tests: `tests/explorer/test_explorer_api.py` (`TestRecordedDecisions`, extraction coverage, 4 new `TestDecisionResponseTimestampValidator` cases for the range/bool fixes), `tests/semantic_extract/test_spacy_model_cache.py` (6 tests)
|
||||
- `pytest tests/explorer tests/semantic_extract/test_spacy_model_cache.py`: 266 passed
|
||||
|
||||
- **Explorer UI hid backend failures: graph load hung forever, landing page always showed "System Online"** (#980, closes #977) by @ZohaibHassan16, reviewed by @Sameer6305
|
||||
- `GraphWorkspace.tsx` only destructured `{ data, isLoading, isFetching }` from `useLoadGraph()`, ignoring the `isError`/`error`/`refetch` that `useQuery` (`retry: 0`) already returned. Combined with `GraphLoadingOverlay` having no error prop and `showLoadingOverlay` staying true whenever `loadingProgress` held a stale frame, a backend-down or failed fetch left the graph workspace stuck on the last progress frame indefinitely, with no error message and no way to recover short of a full page reload
|
||||
- `GraphLoadingOverlay` now accepts `error`/`onRetry` and renders an error card with the real fetch error message and a Retry button (`refetch()`) instead of the stuck progress UI
|
||||
- The landing page's `WelcomeScreen` replaced its hardcoded `ready: boolean` (and hardcoded "System Online" text) with a real `checking` / `online` / `offline` status derived from the same connectivity probe already driving the 4th metric card, so the status dot, text, and metric can no longer drift apart or lie about connectivity
|
||||
- **Smaller fixes bundled in the same PR**: search results are now dismissible (previously stayed open indefinitely, pushing the graph down); relevance scores display as rounded whole numbers instead of `96.900`/`138.000`; added a debounced (250ms) typeahead combobox to graph search with arrow-key navigation, `aria-activedescendant`, and Escape-to-close, using the existing `/api/graph/search` endpoint
|
||||
- **Fixed during review** (Qodo): the typeahead's debounced fetch had no `AbortController`, so a fast-typing user could have a stale suggestion response resolve after a newer one, replacing correct suggestions with outdated ones. In-flight requests are now aborted on every re-debounce and when the query is cleared after a selection
|
||||
- **Noted during review** (@Sameer6305): `GraphWorkspaceShell.tsx` contains a third, unused implementation of the same graph-loading/error-handling logic this PR fixes — the issue itself named "two copies that drifted apart" as the root cause the original bug slipped through. Deliberately left out of this PR's scope and tracked separately in #981 rather than blocking this fix
|
||||
- `npx tsc -b`: clean; `test:graph-store`/`test:graph-workspace`/`test:plugin-registry`: 42 passed; `npm run build`: succeeds
|
||||
|
||||
- **Markdown import hardened against TOCTOU symlink races during file reads** (#932, closes #856) by @lakshanmuruganandam, with fixes by @Sameer6305
|
||||
- `AgentMemory._read_markdown_path` read files via `Path.read_text()` after a `Path.is_symlink()` pre-check, leaving a time-of-check/time-of-use window: a path validated as a regular file could be swapped for a symlink before the actual read, causing the importer to follow the link and read an unintended target
|
||||
- Reads now go through a new `_read_markdown_file_content()` helper: the path is opened via low-level `os.open()` with `os.O_NOFOLLOW` on platforms that support it (POSIX), so a symlink substituted after validation fails atomically with `ELOOP` instead of being followed; the resulting file descriptor is then verified with `os.fstat()`/`stat.S_ISREG()` to reject non-regular files (FIFOs, devices) even after a successful open
|
||||
- Directory imports now also exclude symlinked entries from the file listing (`not file_path.is_symlink()`), consistent with the single-file path already rejecting them
|
||||
- **Known limitation**: Windows has no `os.O_NOFOLLOW`, so on that platform the only defense is the earlier `is_symlink()` pre-check, leaving a narrow TOCTOU window; documented inline rather than implying a stronger cross-platform guarantee than the implementation provides
|
||||
- New `tests/context/test_agent_memory_markdown.py` coverage: rejecting a symlinked path at both the private helper and the public `import_data()` API, silently excluding symlinked entries during directory import, and the `fstat()`/`S_ISREG` guard against non-regular files (mocked FIFO)
|
||||
- `pytest tests/context/test_agent_memory_markdown.py`: 46 passed, 4 skipped (symlink-creation tests skip on Windows without `SeCreateSymbolicLinkPrivilege`)
|
||||
|
||||
- **`VectorManager.maintain_store()`/`collect_statistics()` crashed with `AttributeError` on persistent `VectorStore` backends** (#914, closes #855) by @yunaremaia, with fixes by @Sameer6305
|
||||
- Both methods accessed `store.vectors`/`store.metadata` directly, which are only initialized for the `inmemory` backend — any persistent backend (FAISS, Qdrant, Pinecone, Milvus, SQLite, PgVector, Weaviate) crashed immediately. Same root cause as the #839/#843/#845/#848 cluster, but `VectorManager` operates on a `VectorStore` instance from the outside, so the fix needed a public accessor rather than another internal guard
|
||||
- Added a backend-agnostic `VectorStore.count()`: the `inmemory` backend counts its local dict; persistent backends delegate to a `count()` on the wrapped backend store when one exists, or raise `NotImplementedError` — following the `get_vector()`/`get_metadata()` precedent from #843, a missing/uninitialized backend store is never silently reported as an empty, healthy store
|
||||
- `maintain_store()` and `collect_statistics()` now go through `store.count()` instead of touching `.vectors`/`.metadata`
|
||||
- **Fixed during review** (@Sameer6305): the initial version had `count()` implemented at the dispatch level only, with no shipped backend actually providing one, and `maintain_store()` manufactured a vacuous `metadata_count == vector_count` tautology for persistent backends (always reporting `healthy: True` without checking anything). Added real `count()` implementations to `FAISSStore` (`len(index.vector_ids)` — FAISS has no delete path, so this list is always consistent with the index), `SQLiteVecStore`, and `PgVectorStore` (both via `SELECT COUNT(*)`); `Qdrant`/`Pinecone`/`Milvus`/`Weaviate` continue to raise `NotImplementedError` since none of them guarantee a cheap, reliable synchronous count. `maintain_store()` now reports `metadata_count: None` for persistent backends instead of the fabricated equality, with `healthy` meaning "store is reachable," not "metadata verified"
|
||||
- Two earlier Qodo findings (a count() path that silently returned 0 for a missing backend store, and an unvalidated `hasattr` check that could raise `TypeError` on a mis-shaped adapter) were fixed before this review — replaced with `NotImplementedError` and a `getattr`+`callable()` capability check, respectively
|
||||
- New `tests/vector_store/test_vector_manager_persistent.py`: dispatch-level tests for `count()` (inmemory, delegation, missing backend, non-callable `count`, mis-shaped adapter), full `VectorManager` inmemory semantics including divergence detection, persistent-backend dispatch tests, and backend-specific tests against real/mocked FAISS, SQLite (`sqlite-vec`, skipped if unavailable), and PgVector stores
|
||||
- Core `vector_store` suite: 40 passed
|
||||
|
||||
- **`ContextGraph.get_node_property`/`get_node_attributes` "not found" contract clarified; `add_node_attribute` mutation-callback exception safety fixed** (#882, closes #877) by @ZohaibHassan16
|
||||
- `get_node_property` returned `None` for both "node missing" and "property missing" with no way to distinguish them, and `get_node_attributes` returned `{}` for a missing node while its siblings disagreed on the not-found signal (`get_node_property`/`find_node` → `None`, `get_edge_data` → `{}`). Both now accept a `default=` parameter matching `dict.get()`'s convention, defaulting to their historical return values (`None` and `{}` respectively) for backward compatibility. Callers that need to disambiguate "node missing" from "value legitimately absent" can pass a private sentinel as `default`
|
||||
- Added Google-style docstrings to `get_node_property`, `get_node_attributes`, `get_edge_data`, and `find_node` documenting each method's not-found contract, addressing #877's "sibling not-found contract undocumented" gap
|
||||
- **Corrected during review**: the PR as submitted claimed to fix `add_node_attribute` firing its `mutation_callback` "outside `with self._lock`, without holding the lock," but the diff only removed a stray blank line — the callback call remained outside the lock, unchanged. Further investigation found this was not actually a bug: `self._lock` is a `threading.RLock`, and the same release-the-lock-before-invoking-the-callback pattern is used deliberately in `_add_internal_node`/`_add_internal_edge` elsewhere in this class, avoiding holding the lock for the duration of an arbitrary user-supplied callback. The real inconsistency was that, unlike those two siblings, `add_node_attribute`'s callback call wasn't wrapped in `try/except` — a raising callback propagated uncaught here but was caught and logged there. Now wrapped the same way (`except Exception as e: self.logger.warning(...)`)
|
||||
- 13 tests covering happy path, missing node, missing property, sentinel disambiguation, falsy-zero, callback firing/non-firing, and (added during review) a raising callback no longer propagating out of `add_node_attribute`
|
||||
- `pytest tests/context/test_context.py -q`: 27 passed
|
||||
|
||||
- **Three `tests/normalize/` tests failed for reasons unrelated to the normalize implementations: a missing optional-dependency skip guard, an incomplete chardet allowlist, and a UTC/local timezone mismatch** (#881, closes #860) by @aoright
|
||||
- `test_detect_language`/`test_detect_with_confidence` in `tests/normalize/test_language_detector.py` asserted on real `langdetect` output with no skip guard, even though `langdetect` is an optional dependency absent from `pyproject.toml` that `LanguageDetector` already degrades gracefully without (`LANGDETECT_AVAILABLE = False`, falls back to `default_language`) — any environment without it failed both tests unconditionally, including a fresh CI run without optional extras installed. Both are now gated with `@unittest.skipUnless(LANGDETECT_AVAILABLE, ...)`
|
||||
- `test_detect_encoding` in `tests/normalize/test_encoding_handler.py` asserted `chardet.detect()`'s result against a 3-name allowlist (`iso-8859-1`/`windows-1252`/`latin-1`); on a short Latin-1 sample, chardet is free to return other compatible single-byte codepages (e.g. `windows-1253`), which fails the allowlist and then cascades into `test_convert_to_utf8` decoding the bytes as Greek instead of the original text. The test now uses a longer, unambiguous Latin-1 corpus and asserts that the detected encoding round-trip-decodes the original text instead of matching a fixed name list; `test_convert_to_utf8` now passes `source_encoding="latin-1"` explicitly rather than relying on chardet's heuristic auto-detection
|
||||
- `test_normalize_date_relative` in `tests/normalize/test_date_normalizer.py` compared `RelativeDateProcessor`'s local-clock-based `"today"` (`datetime.now()`, naive, UTC-normalized after the fact by `convert_to_utc()`) against a separately-computed UTC reference date — failing intermittently in any timezone east of UTC whenever the local and UTC dates diverge for part of the day. The test now patches `datetime.now()` to a fixed reference time, making the assertion independent of host timezone
|
||||
- `pytest tests/normalize`: 77 passed, 2 skipped (`langdetect` not installed); `black`/`isort`/`flake8 --max-line-length=88` clean on all three changed files. Test-only change; no production code touched
|
||||
|
||||
- **MCP server reported a stale `0.4.0` version instead of the installed package version** (#870, closes #863) by @oiahoon
|
||||
- `semantica/mcp_server/__init__.py` hardcoded `"version": "0.4.0"` in both the MCP `initialize` response (`SERVER_INFO`) and the `semantica://schema/info` resource, regardless of the actual installed `semantica` version — every MCP client (Claude Desktop, Windsurf, Cline, Continue, VS Code Copilot, etc.) showed the wrong server version. Both surfaces now derive from `semantica.__version__`, the package's authoritative version source, so they can no longer drift from `pyproject.toml`
|
||||
- New regression coverage in `tests/test_mcp_server_version.py`, including `!= "0.4.0"` canaries and a cross-surface consistency check
|
||||
- **Fixed along the way**: the separate root-level `mcp/` package (`mcp/__init__.py`, `mcp/server.py`, `mcp/resources/registry.py`) — a companion MCP server implementation not included in the built distribution, but documented in `mcp/__init__.py` as a supported way to run against Claude Desktop/Windsurf/etc. from a source checkout — had the same three hardcoded `0.4.0` literals; fixed the same way, with matching regression tests in `tests/test_mcp_package_version.py`
|
||||
|
||||
- **`VectorStore._filter_by_metadata()` `AttributeError` on all persistent backends** (#857, closes #849) by @TaherTadpatri
|
||||
- `_filter_by_metadata()` iterated `self.metadata` directly, which only exists on the `inmemory` backend — any persistent backend (`faiss`, `qdrant`, `pinecone`, `milvus`, `pgvector`, `sqlite`, `weaviate`) crashed with `AttributeError` on `filter_decisions(query=None, ...)` / metadata-only filtering. Filtering is now delegated to a native `filter_by_metadata()` implemented on each backend store, using backend-native payload/SQL/JSON filtering (Qdrant `scroll()`, Pinecone `query()`, Milvus expression filters, PostgreSQL JSONB, SQLite `json_extract()`, Weaviate collection filters)
|
||||
- **Fixed along the way**: `PineconeStore.get_index()` and `filter_by_metadata()` called a nonexistent `self.describe_index_stats()` on the store itself (the method only exists on the `PineconeIndex` wrapper returned by `self.index`); the resulting `AttributeError` was silently swallowed, so dimension auto-detection always failed quietly. Now correctly calls `self.index.describe_index_stats()`
|
||||
- **Fixed along the way**: `PineconeStore.filter_by_metadata()` probed for filter-only matches using an all-zero dummy query vector, which Pinecone rejects for cosine-metric indexes — the library's own default — making metadata-only filtering silently non-functional out of the box. Now uses a unit vector instead
|
||||
- **Fixed along the way**: `PgVectorStore.filter_by_metadata()`'s list-filter branch formatted boolean values with `str(v)` (`'True'`/`'False'`), never matching PostgreSQL JSONB's lowercase `'true'`/`'false'` text rendering, even though the equivalent scalar-filter branch already handled this correctly
|
||||
- **Fixed along the way**: list-valued metadata fields (e.g. `{"tags": ["python", "js"]}`) could never match a list filter on the SQLite or PostgreSQL backends, because both extracted the whole array as its JSON/text representation instead of matching individual elements — silently diverging from the in-memory backend's set-intersection semantics. SQLite now uses `json_each()` over a `json_type`-guarded array/scalar wrapper; PostgreSQL now uses the `?|` "any array element" operator alongside the existing scalar `= ANY(...)` path
|
||||
- **Fixed along the way**: `FAISSStore.filter_by_metadata(limit=0)` returned one result instead of zero, because the limit check ran after appending the current match
|
||||
- **Fixed along the way**: `MilvusStore`'s metadata expression builder rendered `NaN`/`Infinity` filter values as bare unquoted tokens, producing an invalid Milvus expression whose server-side rejection was then swallowed by a broad `except`, indistinguishable from "no matches"; these values are now rejected up front with a clear `ValidationError`
|
||||
- New/expanded test coverage in `tests/vector_store/test_backend_metadata_filtering.py` (all 7 backends, including the Pinecone dimension/zero-vector, PgVector boolean-list, FAISS `limit=0`, and Milvus `NaN` regressions) and `tests/vector_store/test_sqlite_vec_store.py` (new `TestSQLiteVecStoreFilterByMetadata`, run against the real `sqlite-vec` extension, including the array-vs-scalar intersection case)
|
||||
|
||||
- **`DistanceExporter` silently swallowed metric computation failures, exporting `None` values indistinguishable from a legitimate "no path" result** (#879, closes #874) by @AmirF194
|
||||
- `_betweenness`, `_hop_distance`, `_weighted_distance`, and `_semantic_similarity` each caught `Exception` and returned their sentinel (`None`/`{}`) with no logging; a failed computation and a real "no path exists" looked identical in exported CSV/JSONL/DataFrame data. All four now log a `warning` with `exc_info=True` before returning the sentinel; exported row shape and values are unchanged
|
||||
- **Fixed along the way**: the module logger was built with `get_logger(__name__)`, which double-prefixed it to `semantica.semantica.export.distance_exporter` — a name `setup_logging()` never configures — so this module's logging (including a pre-existing `logger.debug` call) was silent regardless. Now uses `get_logger("export.distance_exporter")`, matching every other exporter in the module
|
||||
- New regression coverage in `tests/export/test_distance_exporter.py`: warnings fire on exception for all four helpers, exported sentinel values/shape stay unchanged, and the legitimate "no KG backend" `None` path still logs nothing
|
||||
- Full `tests/export/` suite: 71 passed
|
||||
|
||||
- **`explain_violations` rendered hardcoded placeholders (`min_count=1`, `max_count=1`) instead of the SHACL shape's real constraint values, and misused the violation message text as the datatype/class value** (#1094) by @cxzg007
|
||||
- `_run_pyshacl` never read `sh:minCount`/`sh:maxCount`/`sh:datatype`/`sh:class` back from the violation's `sh:sourceShape`, so every plain-English explanation was wrong regardless of what the shape actually declared. `SHACLViolation` now carries those four fields (also exposed via `to_dict()`), populated by back-referencing `sh:sourceShape`; `explain_violations` renders the real values, falling back to `"?"` when a value is genuinely absent
|
||||
- **Known limitation**: `sh:qualifiedMinCount`/`sh:qualifiedMaxCount` are not handled yet and still fall back to the `"?"` placeholder
|
||||
- New regression tests cover both the rendering path and the `sh:sourceShape` back-reference (skipped when `pyshacl`/`rdflib` are absent)
|
||||
|
||||
- **Entity merging silently dropped `entity_id` aliases, and exact-match entity resolution had three correctness gaps** (#1086, #1026) by @T1mn
|
||||
- `entity_merger.py`/`merge_strategy.py`/`entity_resolver.py` used inconsistent logic for extracting an entity's id across the merge path, so a merged entity could lose the `entity_id` aliases that let later lookups find it under its old identity. A new `semantica/utils/entity_ids.py` unifies id extraction across all three call sites
|
||||
- `EntityResolver`'s exact-match path is now honored rather than silently falling through to fuzzy matching in some cases; entities with no identifier are preserved instead of being dropped, and blank exact-match names are ignored rather than matching every other blank name
|
||||
- New/expanded coverage in `tests/kg/test_entity_pipeline.py` and `tests/kg/test_entity_resolver_exact.py`
|
||||
|
||||
- **`flatten_dict()` silently collided keys when a flattened path from one branch matched a literal key already present at the target depth** (#1062) by @shahzaib-ahmadcs
|
||||
- Two differently-shaped inputs could flatten to the same output key, with the second write silently overwriting the first — no error, no warning, just a dropped value. Collisions are now detected and handled explicitly instead of overwriting
|
||||
|
||||
- **`ExcelParser.__init__` raised `NameError` on every instantiation — `get_progress_tracker()` was called but never imported** (#1016, closes #1014) by @pravit-amp
|
||||
- Same defect as the one fixed for `SimilarityCalculator` in #530, this time in `semantica/parse/excel_parser.py`; the existing test imported the class but never constructed it, so nothing caught the missing import. Added construction coverage for every parser exported from `semantica.parse`, driven off `__all__` so future additions are covered automatically, living outside `test_parse_comprehensive.py` (whose `setUp` mocks `get_progress_tracker` into each module and would mock away the exact interaction under test)
|
||||
|
||||
- **Graph analytics (`centrality_calculator.py`, `community_detector.py`, `connectivity_analyzer.py`) dropped isolated nodes and diverged on how each computed its working view of the graph** (#1011) by @T1mn
|
||||
- Each analyzer had its own ad hoc logic for building the node/edge set it operated over, and none of them included nodes with no edges — a node with zero connections simply vanished from centrality scores, community assignments, and connectivity reports instead of appearing with a zero/singleton value. A new shared `semantica/kg/_graph_view.py` centralizes graph-view construction (including node fallbacks and community payload shaping) for all three analyzers, which are now ~250 lines lighter combined
|
||||
- New `tests/kg/test_analytics_node_scope.py` covering isolated-node presence across all three analyzers
|
||||
|
||||
- **Explorer fired temporal-bounds and snapshot requests before the graph itself had loaded, tripling failed requests when the backend was down and leaving the timeline scrubber with nothing to scrub** (#1003) by @lakshayxi
|
||||
- Two new predicate functions gate the temporal effects on the graph having actually loaded (an empty graph still counts as loaded); confirmed against a downed backend that this cuts three failing requests per page load down to one
|
||||
|
||||
- **`SeedDataManager.load_from_database()` never actually reached the database, and connection failures were mislabeled as a missing optional dependency** (#995, closes #973) by @yzxcj797
|
||||
- `DBIngestor.execute_query`/`export_table` need the connection string as their first positional argument; `load_from_database()` only passed it into the constructor's config dict, which those methods never read, so every call raised `TypeError` before connecting. Also split the combined `except (ImportError, OSError)` handling apart — a genuine connection failure was reported as `"module not available"`, sending debugging in the wrong direction; `OSError` now propagates as an actual failure, chained via `from e`
|
||||
|
||||
- **SPARQL `CONSTRUCT` detection matched inside a leading `#`-comment, misclassifying `SELECT`/`ASK` queries as `CONSTRUCT` across all four SPARQL backends** (#951) by @pravit-amp
|
||||
- `CONSTRUCT_QUERY_RE` skipped comments with a bare `\#[^\n]*`, whose backtracking `*` let a `# CONSTRUCT ...` comment line "swallow" the real query-form keyword on the next line for a query like `# CONSTRUCT ...\nSELECT ...`. The mistaken `CONSTRUCT` classification sent `Accept: text/turtle` and tried to parse a SELECT/ASK response body as Turtle, failing with a misleading parse error. The regex now requires a comment to reach a line terminator (LF or CR, per the SPARQL grammar) before matching
|
||||
|
||||
- **`k_shortest_paths` mutated caller-visible graph state during traversal and ignored direction when excluding already-used edges** (#1000) by @T1mn
|
||||
- `semantica/kg/path_finder.py`'s search left side effects behind after returning, and edge exclusion during Yen's-algorithm-style path removal didn't respect the traversal direction of directed graphs, letting a later search see edges that should have been available. Both fixed; new coverage in `tests/kg/test_path_finder.py`
|
||||
|
||||
- **`trace_decision_causality()` ignored explicitly recorded causal edges, inferring causes only from shared NER entities plus timestamp ordering** (#983) by @hsd2514
|
||||
- A `CAUSED`/`INFLUENCED`/`PRECEDENT_FOR` edge added via `add_causal_relationship()` had no effect on the trace — when entity extraction found nothing in common between two decisions, `trace_decision_chain()` came back empty even with an explicit edge stored in the graph. Explicit causal edges are now traversed first as ground truth, with entity/timestamp inference kept as an additive fallback for pairs with no explicit link; edges whose source has no decision record (e.g. a graph restored via `from_dict`) are skipped so a stale edge can't abort the trace
|
||||
|
||||
- **`RepoIngestor`'s module-level DNS resolve cache had no lock, raising `RuntimeError: OrderedDict mutated during iteration` under concurrent `ingest_repository()` calls** (#979) by @manjunathbhaskar
|
||||
- `_REPO_HOST_RESOLVE_CACHE` is a shared `OrderedDict` read, written, and pruned by every thread with no synchronization — reliably reproduced with 32 threads hammering resolution under a low TTL and small cache cap. Now guarded by a lock
|
||||
|
||||
- **`GraphBuilder` didn't remap relationship endpoints after entity resolution merged nodes, leaving relationships pointing at ids that no longer existed in the resolved graph** (#978) by @T1mn
|
||||
- New coverage in `tests/kg/test_graph_builder_external.py`; a follow-up commit hardens the remapping against edge cases found during review
|
||||
|
||||
- **Explorer's dev server esbuild target didn't match the browser targets the production build declares**, occasionally producing dev-only syntax errors on older browsers (#966) by @le-czs
|
||||
- `explorer/vite.config.ts` now sets the dev esbuild target explicitly to match
|
||||
|
||||
- **`normalize`'s number normalizer accepted currency symbols without validating them against the surrounding text, and an earlier fix's currency-code matching wasn't token-bounded** (#940) by @Mr-Neutr0n, reviewed by @ZohaibHassan16
|
||||
- Symbol currencies are now validated before being accepted; currency codes are matched on token boundaries so a code embedded inside a longer token no longer false-positives
|
||||
|
||||
- **`ContextGraph.to_dict()` was the one reader on the class that didn't hold `self._lock`, raising `RuntimeError: dictionary changed size during iteration` under a concurrent writer and risking a torn snapshot otherwise** (#929) by @pravit-amp
|
||||
- Every other reader (`stats()`, `density()`, `find_nodes()`, `find_edges()`, `get_neighbors()`, `get_nodes_by_label()`, `state_at()`, `save_to_file()`) already took the lock after it was introduced; `to_dict()` predated that change and was missed. `save_to_file()` was safe only incidentally, since it builds its payload inline under its own lock rather than delegating to `to_dict()`
|
||||
|
||||
- **`PipelineWithProvenance` had a broken import and no working `run()` method** (#862) by @Karunasagar12
|
||||
- `from .pipeline import Pipeline` failed because `Pipeline` lives in `pipeline_builder.py`, not a nonexistent `pipeline.py` — fixed to `from .pipeline_builder import Pipeline`. The class also had no `run()`; it now delegates to `ExecutionEngine.execute_pipeline()`, the intended execution path for a built `Pipeline`. The constructor now accepts a built `Pipeline` instance directly
|
||||
|
||||
### Security
|
||||
|
||||
- **Tarball restore path traversal, latent SQL injection, DNS-rebinding TOCTOU in the shared SSRF guard, stored XSS in report generation, and unvalidated SPARQL object IRIs in AnzoStore** (#1079) by @KaifAhmad1
|
||||
- `semantica backup restore`'s tar extraction (`cli.py`) stripped only the literal `semantica-backup/` prefix and called `tar.extract()` with no path-containment check, no symlink/hardlink validation, and (on Python <3.12) no extraction filter — a crafted archive member (`../../<file>`, or a symlink pointing outside the restore root) could write arbitrary files above the restore directory. Every member is now validated for resolved-path containment before extraction, symlink/hardlink targets are rejected both lexically (absolute path, `..` segments) and by resolution, and `filter="data"` is applied on Python ≥3.12
|
||||
- `DataExporter.export_table_data()` (`db_ingestor.py`) was missing the `text` import from `sqlalchemy` — a `NameError` that made the method non-functional, but latently: the query it built from raw f-string interpolation of `table_name`/`schema`/`where`/`order_by` was already injectable, so fixing the import alone (without also fixing the injection) would have silently armed it. Both are fixed together: the import is restored, `table_name`/`schema` are now validated against a strict identifier allowlist, and `where`/`order_by` are checked against a blocklist (statement separators, comments, UNION, DDL/DML keywords, time-based blind-injection primitives, schema-enumeration terms). This is a blocklist, not a grammar — it closes the concrete UNION-exfiltration path and common injection primitives, but a boolean-blind subquery using none of the blocked keywords could still get through; `where`/`order_by` must be treated as trusted/operator input, not exposed to untrusted end users, and the docstrings now say so explicitly
|
||||
- `request_with_ssrf_guard()` (`ssrf.py`) validated a hostname's resolved IPs, then let the underlying HTTP client re-resolve the same hostname independently at connect time — a low-TTL or DNS-rebinding answer could differ between the two lookups, so a hostname that validated as public could still connect to a private/internal address. Ported the IP-pinning pattern already used by `explorer/routes/ontology.py`'s `_make_pinned_session` into the shared ingest guard: the one resolution that decides accept/reject is now also the one the connection is pinned to, via a custom `HTTPAdapter` that presents the real hostname over TLS SNI / Host header while connecting only to the validated IPs. Also closes the RFC 6598 Carrier-Grade NAT gap noted as a known limitation in #905/#868: `100.64.0.0/10` is now in `BLOCKED_NETWORKS`
|
||||
- `ReportGenerator._generate_html()` (`export/report_generator.py`) f-string-interpolated report title/summary/metrics into HTML with no escaping — an ingested entity or document whose content flowed into a report (e.g. `<img src=x onerror=...>`) executed as stored XSS when the report was opened. All interpolated values are now `html.escape()`d
|
||||
- `AnzoStore._format_object_for_sparql()` (`triplet_store/anzo_store.py`) validated the subject/predicate of a triplet via `sparql_escaping.validate_uri()` before interpolating them into a SPARQL `INSERT DATA` clause, but delegated the **object** position to a separate formatter that wrapped it as `<{obj}>` without the same validation — an object value containing `>`/`}`/`{`/`"` could close the intended `<...>` token early and inject additional SPARQL Update operations. The Blazegraph/RDF4J backends were hardened for the equivalent gap previously; Anzo's object position now goes through the same `validate_uri()` check
|
||||
- Also hardened in the same pass: Apache AGE's `create_index()` `index_type` parameter is now allowlisted (was interpolated raw into a `USING` clause); Neo4j's `limit` is now explicitly validated (raises `ValidationError` for non-integer input instead of falling through to a generic `ProcessingError`); the `ffprobe` metadata-extraction subprocess call is guarded against a filename starting with `-` being parsed as an option; the MCP server no longer echoes raw exception text to JSON-RPC clients, logging full details server-side and returning a generic message plus the exception class name instead
|
||||
- **Fixed during review** (@KaifAhmad1): the SSRF IP-pinning change introduced a connection-pool leak of its own — `requests.Session.mount()` silently drops whatever adapter it replaces without closing it, so a multi-hop redirect chain on a reused session leaked one pooled connection per hop. Pinned adapters are now tagged and explicitly closed before being replaced, both per-hop and on final restore
|
||||
- **Fixed during review** (@KaifAhmad1): mounting a pinned adapter and setting a Host header on a caller-supplied `Session` is not inherently thread-safe — two guarded calls sharing the same session from different threads could interleave their mount/restore cycles. Added a per-session lock (`_get_session_lock`) so concurrent guarded calls on the same session now serialize instead of racing; verified with a two-thread test showing correct serialization and zero cross-contamination of per-request Host headers
|
||||
- **Fixed during automated PR review** (Qodo): `export_table_data()`'s new identifier/fragment validation raised `ValidationError` from inside a `try` whose blanket `except Exception` re-wrapped it as `ProcessingError`, masking the distinction between "bad input" and "the export itself failed" that callers rely on elsewhere in this module. Added the `except ValidationError: raise` guard already used by its sibling methods
|
||||
- **Fixed during automated PR review** (Qodo): on a hop where IP pinning doesn't apply (`allow_private_ips=True`), `_apply_connection_pin()` unconditionally popped the session's `Host` header instead of restoring whatever it was before pinning touched it — a caller-supplied session carrying its own legitimate `Host` override (e.g. fronting a private endpoint under a different name) had that override silently dropped for the in-flight request, only reappearing afterward via the outer `finally` restore. It now restores the session's own pre-call header state (set back if present, popped only if it was truly absent) instead of always popping
|
||||
- **Fixed during automated PR review** (Qodo): the `where`/`order_by` blocklist matched keywords/punctuation inside properly quoted string literals and identifiers too, so legitimate data like `status = 'union'` or `name = 'a--b'` was rejected as if it were SQL syntax. The blocklist now runs against a copy with quoted-literal contents masked out (`_mask_sql_literals`) — a malformed/unterminated quote sequence doesn't match the masking pattern and is left fully exposed to the blocklist, so this closes false positives without opening a masking-based bypass; the fragment actually used in the query is unchanged
|
||||
- Re-ran each finding's proof-of-concept (or an equivalent adversarial test) against the fix and confirmed it is blocked: tar path/symlink traversal (both lexical and resolved-path forms), SQL UNION exfiltration and identifier breakout, DNS-rebinding TOCTOU (including under a configured `HTTP_PROXY`, which the pinning adapter also rejects outright since a proxy would resolve DNS itself), stored XSS, and the AnzoStore SPARQL injection
|
||||
- `pytest tests/ingest/`: 266 passed, 2 skipped (10 pre-existing failures unrelated to this change — identical failure set confirmed on unmodified `main`); full regression sweep across `graph_store`, `export`, `triplet_store`, `parse`, and backup/restore: 313 passed
|
||||
|
||||
- **`Authorization`/`Proxy-Authorization` credentials could leak to a different origin across HTTP redirects, and several ingest paths bypassed the shared SSRF/redirect guard entirely** (#1067, closes #947) by @Sameer6305, reviewed by @KaifAhmad1
|
||||
- `request_with_ssrf_guard()` previously only stripped sensitive headers from per-request `kwargs["headers"]` on a cross-origin redirect; session-level `Authorization`/`Proxy-Authorization` headers, `session.auth`, and `session.trust_env` (`.netrc` lookup) could all still resurrect credentials on the hop to a foreign origin. All five credential sources are now stripped case-insensitively, kept stripped for the remainder of a multi-hop redirect chain (no resurrection even if a later hop returns to the original host), and unconditionally restored via `finally` — including on exceptions and redirect-limit errors
|
||||
- `MCPClient._send_request_http()` and `PublicAPIIngestor.detect_public_api()`/`ingest_public_api()` called `httpx.post()`/`requests.post()`/`session.request()` directly, bypassing `request_with_ssrf_guard()` entirely. Both now route through the shared guard, including when `validate_no_auth=False`
|
||||
- `SeedDataManager.load_from_api()` mutated the caller-supplied `headers` dict in place when adding an API-key `Authorization` header, silently leaking the key back into a dict the caller might reuse elsewhere. Now copies before modifying
|
||||
- **Fixed during review** (@KaifAhmad1): `allow_private_ips=True` (used to let MCP servers run on localhost/internal networks) was applied to every redirect hop, not just the operator-configured host — a compromised or malicious MCP server could 302-redirect to an internal address (e.g. `169.254.169.254` cloud metadata) and the guard would follow it unchecked, defeating the SSRF protection this PR otherwise adds. Added `allow_private_ips_on_redirect` to `request_with_ssrf_guard()`: a redirect target inherits the original host's private-IP trust only when it matches that host; any other host falls back to strict validation. `MCPClient` now pins `allow_private_ips_on_redirect=False`, so only same-host redirects on a trusted MCP server keep working — a cross-host hop into private address space is blocked
|
||||
- **Fixed during review** (@KaifAhmad1): `detect_public_api()` only caught `requests.exceptions.RequestException`, but `request_with_ssrf_guard()` raises `ValidationError` (a disjoint hierarchy) for SSRF-blocked hosts, blocked redirect targets, missing `Location`, or exceeded redirect limits — unlike its sibling `ingest_public_api()`, which already caught it. Callers (including `is_public_api()`) got an undocumented raw `ValidationError` instead of `ProcessingError`, and the error-logging call was skipped. Now catches `(ValidationError, ProcessingError)` and re-raises, matching the sibling method
|
||||
- **Fixed during review** (@KaifAhmad1): `detect_public_api()`/`ingest_public_api()` forwarded `**options` into `request_with_ssrf_guard(..., session=self.session, allow_private_ips=self.allow_private_ips, **request_options)` without stripping `session`/`allow_private_ips` from `request_options` first — a caller passing either through the per-call `**options` (a plausible mistake, since `allow_private_ips` is also a documented constructor-level knob) got a raw `TypeError: got multiple values for keyword argument`. Both are now popped from `request_options` before the call
|
||||
- New regression coverage added during review: `TestAllowPrivateIpsOnRedirect` (cross-host redirect into private space blocked, same-host redirect trust preserved, default behavior unchanged for existing callers that don't pass the new kwarg) and `TestMCPClientAuthRedirect::test_redirect_to_private_ip_is_blocked`/`test_same_host_redirect_on_private_mcp_server_is_not_blocked` in `tests/ingest/test_auth_header_redirect_security.py`; `test_detect_public_api_propagates_ssrf_validation_error` and duplicate-kwarg regression tests for both methods in `tests/ingest/test_public_api_ingestor.py`
|
||||
- `pytest tests/ingest/test_auth_header_redirect_security.py tests/ingest/test_public_api_ingestor.py tests/test_seed_manager.py tests/ingest/test_submodules.py tests/ingest/test_cookbook_integration.py`: 111 passed
|
||||
|
||||
- **`FeedIngestor`/`FeedMonitor` (RSS/Atom feed ingestion) had no SSRF protection, allowing requests to internal/private network targets** (#928, closes #927) by @ZohaibHassan16
|
||||
- `FeedIngestor.ingest_feed()`, `discover_feeds()` (link-tag fetch, common-path HEAD probe, and feed-validation GET), and `FeedMonitor.check_updates()` all called `requests.get()`/`requests.head()` directly with default redirect-following and no scheme allowlist or private/loopback/link-local IP validation — despite `semantica/ingest/ssrf.py`'s `request_with_ssrf_guard()` already existing and being used by `web_ingestor.py`/`api_ingestor.py`. `ingest_feed()`'s own URL check only verified `urlparse(url).scheme`/`.netloc` were non-empty, never that the scheme was http/https or that the resolved target IP was safe. Reachable via the public `ingest_feed()`/`ingest()` entry points with any caller-supplied feed URL
|
||||
- All 5 call sites now route through `request_with_ssrf_guard()`, which validates scheme (http/https only) and resolved IP before the request, and re-validates every redirect `Location` before following it — closing both the direct-IP and redirect-chain SSRF paths. Added an `allow_private_ips` config option to both `FeedIngestor` and `FeedMonitor`, consistent with the other ingestors
|
||||
- **Fixed during review** (Qodo): `test_discover_feeds_empty` mocked `requests.get`, which no longer executes now that the code path goes through `request_with_ssrf_guard()` (backed by `requests.request`) — the test was passing without exercising the real code. Corrected to mock `requests.request` and `socket.getaddrinfo`
|
||||
- `pytest tests/ingest/test_feed_ingestor.py`: 12/12 passed. Independently reproduced the issue's own PoC (`FeedIngestor().ingest_feed("http://127.0.0.1:8765/feed.xml")` against a live local server) and confirmed it now raises `ValidationError` instead of succeeding
|
||||
- **Known limitation carried over from `discover_feeds()`'s pre-existing design**: its common-path and feed-validation loops use a blanket `except Exception: continue`, which now also silently absorbs `ValidationError` from a blocked candidate URL the same way it already absorbed network failures — the request is still correctly blocked before reaching the network, so this is not an SSRF bypass, just a missed opportunity to log "blocked as SSRF target" distinctly from "unreachable"
|
||||
|
||||
- **`RepoIngestor` clone surface hardened against GitPython URL/option injection** (#905, closes #868) by @pravit-amp
|
||||
- `RepoIngestor.ingest_repository()` passed the caller-supplied repository URL and arbitrary `**options` straight through to `git.Repo.clone_from()` on a `GitPython>=3.1.50` floor predating hardening for `ext::`-style transport helpers and `$VAR`/`${VAR}` environment-variable expansion in clone URLs — unvalidated clone options (`upload_pack`, `multi_options`, `template`, `config`, `env`, ...) could be abused for command execution, and unvalidated hostnames allowed SSRF against internal services (e.g. cloud metadata endpoints)
|
||||
- `GitPython` floor raised to `>=3.1.58`
|
||||
- Clone options passed to `clone_from()` are now allowlisted to `{depth, branch, single_branch, no_tags}`; anything else raises `ValidationError` before the clone is attempted
|
||||
- Repository URLs are validated before cloning: scheme allowlist (`https`, `http`, `git`, `ssh`), rejection of `$VAR`/`${VAR}` tokens, and hostname resolution with every returned address screened against private/loopback/link-local/unspecified ranges. scp-like SSH remotes (`user@host:path`) are recognized and normalized to `ssh://` before the clone call
|
||||
- **Fixed during review** (@Sameer6305): the SSRF check originally used `ip.is_reserved`, which flags the NAT64 Well-Known Prefix (`64:ff9b::/96`, RFC 6052) as reserved — falsely blocking `github.com` and other public hosts on IPv6-only/dual-stack networks using NAT64. Narrowed the block list to private/loopback/link-local/unspecified only
|
||||
- **Fixed during review** (@Sameer6305): local filesystem repository paths (`git clone /path/to/local/repo`) were being treated as remote URLs and rejected outright; local paths now bypass network validation entirely since they make no network requests and carry no SSRF risk
|
||||
- **Known limitation**: the SSRF host check does not classify RFC 6598 Carrier-Grade NAT space (`100.64.0.0/10`) as blocked — Python's `ipaddress.IPv4Address.is_private` does not cover that range, so a hostname resolving into it (e.g. some Kubernetes/CNI pod networks) would not be caught. Follow-up recommended to add it explicitly alongside the existing private/loopback/link-local checks
|
||||
- `pytest tests/ingest/test_repo_ingestor_security.py -v`: 44 passed
|
||||
|
||||
- **HTTP response header injection via `node_id`, unbounded-memory DoS in link prediction, and unsanitized imported node IDs in the Explorer** (#912) by @Sunil56224972
|
||||
- `semantica/explorer/routes/provenance.py`'s `GET /api/provenance/report` f-string-interpolated the `node_id` query parameter directly into the `Content-Disposition` response header; a `\r\n`-bearing `node_id` could inject arbitrary response headers (`Set-Cookie` session fixation, `Content-Type` override for reflected XSS). Fixed with `_safe_content_disposition_filename()`, which strips `\r`, `\n`, `\x00`, `"`, `\` and length-caps the value before interpolation
|
||||
- `POST /api/enrich/links` (link prediction) loaded up to 999,999 nodes with no cap or concurrency guard, then scored every candidate — a single request could consume ~1.6 GB RAM, and concurrent requests compounded that with no limit. Capped the candidate pool at 10,000 nodes (`413` if exceeded) and added an `asyncio.Semaphore(2)`, mirroring the SPARQL DoS fix in #898
|
||||
- `POST /api/import` stored uploaded JSON/CSV node IDs verbatim; since provenance reports reflect `node_id` into `Content-Disposition`, an attacker could upload a node with a CRLF-bearing ID once and trigger the header-injection chain above for every subsequent viewer. Added `_sanitize_import_node_id()`, applied to node and edge `source_id`/`target_id` fields on both the JSON and CSV import paths
|
||||
- **Corrected during review**: the JSON import path had a second, unsanitized branch — any uploaded node object already carrying a `"properties"` key (the shape this app's own `/api/export` produces, and already used elsewhere in the test suite) was appended to the graph as-is, bypassing `_sanitize_import_node_id()` entirely and leaving the stored-header-injection chain open via a one-line payload (`{"id": "<crlf>", "properties": {}}`). That branch now sanitizes `id` before storing
|
||||
- **Corrected during review**: the link-prediction cap checked `total` only *after* calling `session.get_nodes()`/`get_edges()`, which normalize the graph's *entire* matching node/edge set before applying `limit` — so the guard ran after the expensive work it was meant to prevent had already happened, on every request regardless of graph size. Added `GraphSession.get_raw_counts()`, an O(1) check against the raw `len(graph.nodes)`/`len(graph.edges)` collections, and moved the size check ahead of the normalizing calls
|
||||
- **Corrected during review**: 5 of the original PR's 22 regression tests asserted that literal words like `"Set-Cookie"`/`"Content-Type"` disappeared from the sanitized value — the sanitizer only strips `\r\n\x00"\\`, not letters, so those assertions failed against the PR's own fix as submitted. Corrected to assert on the property that actually blocks header injection (no `\r`/`\n` survives), and added end-to-end tests that exercise the real `/api/import` → `/api/provenance/report` route chain (not just the standalone sanitizer function) so the `properties`-key bypass has regression coverage
|
||||
- Full `explorer` suite: 241 passed; `tests/test_security_regression_pr2.py`: 30 passed
|
||||
|
||||
- **`fastapi`/`python-multipart` floors in the `explorer` extra allowed PYSEC-2024-38 (CVE-2024-24762 / GHSA-2jv5-9r88-3w3p, `python-multipart` ReDoS)** (#871, closes #869) by @agu2347
|
||||
- `explorer` declared `fastapi>=0.100.0` and `python-multipart>=0.0.6`; both floors resolve to versions carrying a ReDoS in `python-multipart`'s `Content-Type` header option parser (`parse_options_header`), reachable by any endpoint that accepts form/multipart data — an attacker-crafted header option can stall the event loop for minutes
|
||||
- **Corrected during review**: the original fix raised only `fastapi>=0.109.1`, leaving `python-multipart>=0.0.6` unchanged. `python-multipart` is declared as its own direct dependency in the `explorer` extra rather than pulled in transitively via `fastapi[all]`, so a bare `fastapi` install enforces no `python-multipart` floor at all — the vulnerable `0.0.6` could still resolve with `fastapi>=0.109.1` in place. Floors raised to `fastapi>=0.109.2` / `python-multipart>=0.0.7`, the first versions of each that exclude the vulnerable range
|
||||
- **Fixed along the way**: the `Security` workflow's `pip-audit` job ran only on a weekly schedule with `continue-on-error: true`, against a bare Python environment with none of Semantica's optional extras installed — it would never have seen `fastapi`/`python-multipart` regardless of which floor was pinned. `security-scan.yml`'s Safety check has the same blind spot (`pip install -e ".[llm-litellm]"` only, never `[explorer]`). `pip-audit` now also runs on `pull_request` when `pyproject.toml` changes, installs `semantica[all]`, and fails the build on any finding for that trigger; the schedule/`workflow_dispatch` runs stay non-blocking pending a full pass over any pre-existing findings across the whole `[all]` tree
|
||||
- **Caught by the new gate on its first run**: `python -m pip install -e ".[all]"` pulled in `setuptools==79.0.1`, vulnerable to CVE-2026-59890/GHSA-h35f-9h28-mq5c/PYSEC-2026-3447 (Unicode-normalization bypass of `MANIFEST.in` exclude/prune patterns on macOS APFS/HFS+, letting excluded files leak into a built sdist), fixed in `83.0.0`. `[build-system] requires` had the exact same too-permissive-floor pattern this whole entry is about (`setuptools>=61.0`), and `actions/setup-python`'s baked-in `setuptools` isn't governed by that pin at all since it's outside any isolated build. Bumped `[build-system] requires` to `setuptools>=83.0.0`, and the `Security` workflow now runs `pip install --upgrade pip setuptools` before auditing so the scanned environment can't have a stale ambient copy regardless of what governs it
|
||||
- Full `explorer` suite: 241 passed
|
||||
|
||||
- **`SeedDataManager.load_from_api()` made unguarded HTTP requests, with no SSRF protection at all** (#942) by @ZohaibHassan16
|
||||
- `load_from_api()` called `requests.get()` directly instead of going through `semantica/ingest/ssrf.py`'s `request_with_ssrf_guard()`, unlike every other ingestor in this module — a caller-supplied `api_url` could target internal/private network addresses with no validation. Now routes through the shared guard, gaining redirect validation and bounded DNS resolution for free
|
||||
- **Follow-up** (#959, closes #943) by @yunaremaia: added an `allow_private_ips` opt-in (parsed via the shared `parse_bool` helper) for trusted internal deployments that legitimately need to load from a private-network API, while keeping the guard's block-by-default behavior for everyone else
|
||||
|
||||
## [0.6.5] - 2026-08-11
|
||||
|
||||
### Added
|
||||
|
||||
- **Embedded Oxigraph backend for `TripletStore`** (#838, closes #834) by @Linxiushen
|
||||
- Added `OxigraphStore` (`semantica/triplet_store/oxigraph_store.py`), an in-process SPARQL 1.1 store via the optional `pyoxigraph` dependency — no external server (Blazegraph/Jena/RDF4J/Anzo) required, fixing the confusing plain connection-error failure `TripletStore` previously produced with no server running (no local Docker daemon, no Java, CI, or a fresh laptop)
|
||||
- Runs fully in memory by default, or persists to a local directory via `TripletStore(backend="oxigraph", path=...)`; reopening the same directory resumes existing data
|
||||
- Full CRUD, native batch loading (`Store.extend`), named-graph scoping (`graph=` on add/query), and SPARQL SELECT/ASK/CONSTRUCT/DESCRIBE result mapping matching the existing backend contract; reuses `sparql_escaping.py` for datatype-IRI resolution instead of reimplementing it, and preserves RDF literal datatype/language metadata across writes, reads, and query results
|
||||
- New optional `semantica[tripletstore-oxigraph]` extra (`pyoxigraph>=0.5.0`), included in the `all` extra; the import is lazy, so `TripletStore` and the rest of Semantica keep working without `pyoxigraph` installed
|
||||
- Wired into `TripletStore` (`backend="oxigraph"`, added to `SUPPORTED_BACKENDS` and `NAMED_GRAPH_CAPABLE_BACKENDS`) and exported from `semantica.triplet_store`; README, module reference, glossary, and usage guide updated with install/configuration examples
|
||||
- **Fixed along the way**: a missing `pyoxigraph` install surfaced as a generic wrapped `ProcessingError` instead of the underlying `ImportError` and its install hint, because `TripletStore._initialize_store_backend()`'s broad `except Exception` caught and rewrapped it; `ImportError` is now re-raised as-is so the `pip install "semantica[tripletstore-oxigraph]"` hint reaches the caller
|
||||
- New integration tests in `tests/triplet_store/test_oxigraph_store.py` covering persistence/reopen, named-graph isolation, SELECT/ASK/CONSTRUCT result shapes, and the missing-dependency error message; skipped automatically when `pyoxigraph` isn't installed, and not yet exercised in CI since it doesn't install the optional extra or run the Python test suite
|
||||
|
||||
- **PROV-O trust blockers and general spec completeness for `ProvenanceManager`** (#825) by @KaifAhmad1
|
||||
- **Invalidation instead of hard delete**: new `ProvenanceManager.invalidate(entity_id, agent_id, reason=None)` tombstones an entry — archives its pre-invalidation state under a stable versioned key, then appends the invalidated entry (`invalidated`, `invalidated_at_time`, `invalidated_by`, `invalidation_reason`) — instead of mutating or deleting it, so an audit can prove a fact existed, was reviewed, and was retracted. `ProvenanceManager.clear()` remains the bulk dev/test store-reset utility it always was; it was not repurposed
|
||||
- **Hash-chained integrity**: every entry now carries `sequence_id`/`previous_checksum`, chaining it to the entry immediately before it in insertion order. New `ProvenanceManager.verify_chain()` walks the chain and reports any break, including a row hard-deleted directly from the underlying table — something a lone per-row SHA-256 checksum can never detect on its own. `compute_checksum()` now also covers `agent_id`/`agent_type`, the lineage-link fields, and the invalidation fields, closing several fields that previously weren't tamper-evident
|
||||
- **Typed Agent/Activity**: `agent_id` was a dead field — no `track_*` method read it from kwargs, so it was always the `"semantica"` default regardless of what callers passed; fixed, and paired with new `AgentRecord(id, agent_type, is_automated)` / `ActivityRecord(id, activity_type, started_at_time, ended_at_time)` dataclasses (pass via `agent=`/`activity=` kwargs) so a human reviewer, an LLM call, and an automated pipeline stage are now distinguishable, and activities carry real start/end timing. Wired through all 18 `*_provenance.py` wrapper modules and `track_entity`/`track_relationship`/`track_chunk`/`track_property_source`
|
||||
- **Versioning vs. derivation split**: new `previous_version_id` ("this corrects a prior version of the same fact") and `derived_from_id` ("this was derived from a different source entity") fields, additive alongside the legacy combined `parent_entity_id` so existing readers are unaffected
|
||||
- **Downstream lineage traversal**: new `get_descendants()`/`trace_descendants()` (reverse BFS in both `InMemoryStorage` and `SQLiteStorage`), closing the gap flagged in `semantica/explorer/routes/provenance.py` where `direction="downstream"` was dead code with no reverse lookup to feed it; the Explorer's `/api/provenance` lineage response now merges both directions
|
||||
- **W3C PROV-O qualified relations**: `export_prov()` now emits `prov:qualifiedAssociation`/`hadRole` (distinguishing "approved by" from "generated by" for sign-off workflows), `qualifiedGeneration`/`Generation`, `qualifiedUsage`/`Usage`, `qualifiedDerivation`/`Derivation`, `qualifiedInvalidation`/`Invalidation`, `wasAssociatedWith` (Activity→Agent), `actedOnBehalfOf` (Agent→Agent delegation), and `wasInformedBy` (Activity→Activity, via a new `informed_by=[...]` kwarg), alongside the existing plain triples
|
||||
- **Bitemporal + Bundle support**: `revision_type`/`supersedes`/`valid_from`/`valid_until` fields (plain caller-supplied passthrough, matching the deprecated `kg.ProvenanceTracker`'s actual contract) plus new `revision_history()` and `query_recorded_between()` methods, closing the two "no direct equivalent yet" rows in `docs/migration/kg-provenance-tracker.md`; `bundle_id` emits `prov:Bundle`/`hadMember` membership triples to partition provenance by source/dataset/ingestion-run
|
||||
- **Configurable, interlinked namespace**: `export_prov(base_uri=...)` / `--base-uri` CLI flag, defaulting to a new `ProvenanceManager.DEFAULT_BASE_URI` (`https://semantica.dev/ns#`) that `RDFExporter`'s `NamespaceManager` and `OWLExporter`'s default `ontology_uri` now both reuse, so KG-exported, OWL-exported, and PROV-exported URIs for the same `entity_id` co-resolve instead of three independently-hardcoded placeholder domains
|
||||
- New CLI commands: `semantica provenance invalidate|verify-chain|descendants`
|
||||
- **Fixed along the way**: `track_entities_batch()` silently absorbed batch-level typed kwargs (`agent_id`, `entity_type`, `activity_id`) into the opaque `metadata` JSON blob instead of forwarding them, so the documented banking example in `docs/guides/provenance.md` never actually worked as written
|
||||
- **Fixed along the way**: `compute_checksum()` had to exclude `entity_id` itself from the hash — `track_entity()`'s versioning archives a prior value by copying it to a new key (`"X"` → `"X:v:<timestamp>"`), and hashing `entity_id` meant that legitimate relabel permanently orphaned any other entry that had already chained its `previous_checksum` from the pre-relabel value, surfacing as a false-positive "broken chain." Archival and invalidation are now always a pure relabel (unchanged checksum/sequence position) followed by a fresh chained append, never an in-place mutation of an already-chained entry
|
||||
- **Fixed along the way**: `InMemoryStorage.get_chain_head()` ignored the already-committed chain head whenever the current transaction had staged any entries, understating the head and corrupting the next append's chain link
|
||||
- **Fixed along the way**: several new `ProvenanceEntry` fields were initially wired into the dataclass and `export_prov()` but not into `SQLiteStorage`'s DDL/INSERT/row-mapping — `InMemoryStorage` stores the dataclass directly so it masked the gap. Added a permanent regression test (`test_all_fields_round_trip_through_sqlite`) asserting every field survives a SQLite round trip, to catch this class of bug for any future field additions
|
||||
- Flagged, not fixed (separate, pre-existing issues independent of #825): `semantica/pipeline/pipeline_provenance.py` imports a nonexistent module and wraps a `Pipeline` dataclass with no `run()` method, so `PipelineWithProvenance` has never worked; most of the 18 wrapper modules' backing classes are themselves missing or incomplete (e.g. `context.context_manager`, `deduplication.deduplicator`, `normalize.normalizer` don't exist; `EmbeddingGenerator` exists but has no `.embed()`); `kg_provenance.py` passes `entity_type` inside its `metadata={}` dict instead of as a top-level `track_entity()` kwarg across most of its ~30 call sites, so it never actually populates the real field
|
||||
- Extensive new test coverage across `tests/provenance/test_manager.py`, `test_schemas.py`, and `test_storage.py` (invalidation, hash-chain verification including a simulated hard-delete-detection case and an interleaved-chaining stress test, agent/activity typing, versioning/derivation split, downstream lineage, qualified export triples, bitemporal methods, Bundle export, and namespace interlinking)
|
||||
|
||||
- **Altair Anzo triplet store backend** (#813) by @KaifAhmad1
|
||||
- Added `AnzoStore` (`semantica/triplet_store/anzo_store.py`), a fourth peer to `BlazegraphStore`/`RDF4JStore`/`JenaStore` speaking plain SPARQL 1.1 over HTTP — no new dependency, since Anzo has no official Python SDK but needs none
|
||||
- The one structural difference from the existing backends: Anzo addresses data by a dataset/graphmart **URI** (`dataset_uri`, required) rather than a short namespace/repository name, so the endpoint path (`<endpoint>/sparql/<store_type>/<url-encoded_dataset_uri>`) percent-encodes it; `store_type` defaults to `"graphmart"` and can be set to `"dataset"`
|
||||
- Reuses the shared `sparql_escaping.py` literal-escaping, datatype-IRI resolution, and CONSTRUCT-detection helpers rather than reimplementing them, matching `BlazegraphStore`'s CONSTRUCT/bindings `execute_sparql` contract exactly
|
||||
- Wired into `TripletStore` (`backend="anzo"`, added to `SUPPORTED_BACKENDS` and `NAMED_GRAPH_CAPABLE_BACKENDS`) and `config.py` (`TRIPLET_STORE_ANZO_ENDPOINT` env var / `anzo_endpoint` config key), and exported from `semantica.triplet_store`
|
||||
- 32 new tests in `tests/triplet_store/test_anzo_store.py` (mocked HTTP, no live Anzo instance needed), including dataset-URI percent-encoding cases that don't apply to the other backends
|
||||
- Bulk loading uses SPARQL `INSERT DATA` (the same approach `BlazegraphStore` uses) rather than Anzo's separate HTTP Client Interface, keeping the `bulk_load()` contract identical across backends
|
||||
|
||||
- **Comprehensive unit and security test suite for the `/api/sparql` Explorer route** (#773) by @Sameer6305
|
||||
- Added `tests/explorer/test_sparql_route.py` (34 tests) covering the SPARQL Explorer route (`semantica/explorer/routes/sparql.py`), which executes arbitrary SPARQL queries against an in-memory rdflib projection of the live graph and previously had zero test coverage
|
||||
- Verified read-only allowlist enforcement against write and mutation queries (`INSERT DATA`, `DELETE DATA`, `DELETE WHERE`, `DROP ALL`, `CLEAR ALL`, `LOAD`, `CREATE GRAPH`, `MODIFY`, comments, and multi-statement injections like `SELECT ... ; DROP ALL`), confirming rejected queries short-circuit before any graph is built or queried
|
||||
- Verified resource-limiting behavior, confirming row capping (`_SPARQL_MAX_ROWS`) truncates results and sets `truncated: true`, query timeout (`_SPARQL_TIMEOUT_S`) returns a clean error message without crashing, and concurrency semaphore (`_SPARQL_MAX_CONCURRENT`) prevents thread starvation under load
|
||||
- Verified RDF projection fidelity for node properties and edge relationships, and error formatting for malformed SPARQL syntax with line and column extraction
|
||||
- Follow-up review fixes (#805): extracted the duplicated row-cap-and-truncate loop (previously copy-pasted between the `CONSTRUCT`/`DESCRIBE` and `SELECT` branches) into a shared `_cap_rows()` helper so the `_SPARQL_MAX_ROWS` cap is enforced identically by both; added `test_row_cap_truncates_construct_results`, since the truncation path for `CONSTRUCT`/`DESCRIBE` results had no direct test coverage even though `SELECT` truncation did
|
||||
|
||||
- **Global default persistent storage for `ProvenanceManager`, plus a working `provenance` CLI** (#795, #802) by @Sameer6305 and @KaifAhmad1
|
||||
- Every ingestion/processing module (`kg_provenance.py`, `pipeline_provenance.py`, and 20+ other call sites) instantiated its own `ProvenanceManager()` with no `storage_path`, so all of them silently fell back to `InMemoryStorage` and the SQLite audit trail was never actually written. `ProvenanceManager.set_default_storage_path(path)` now sets a class-level default that every no-arg instantiation picks up, and `Semantica.__init__` wires `config.provenance.storage_path` into it automatically during orchestrator init
|
||||
- Added the thread-safe `default_storage_path(path)` context manager (`semantica.provenance.default_storage_path`) for test isolation — it stacks nested overrides and guarantees restoration of the previous default on exit, even on exception, so tests can't leak global state into each other
|
||||
- Fixed `ProvenanceManager.__init__` raising `TypeError` on the CLI's `config=` kwarg, and implemented the four methods the CLI already called but that didn't exist on the class: `lineage()`, `audit_log()`, `export_prov()` (W3C PROV-O turtle/ntriples/jsonld via `rdflib`), and `check()` — unblocking `semantica provenance lineage|audit|export|check` end-to-end
|
||||
- Follow-up review fixes: `track_entity` no longer aliases a caller-supplied `used_entities` list (it copied the reference and later mutated it in place via `.append()`, which could corrupt a list the caller still held); removed dead fallback branches in `orchestrator.py`/`manager.py` left over from not realizing `Config.get()` already resolves dotted paths; added a `--dry-run` option to `provenance audit` to match `provenance export` (previously only the global `--dry-run` flag worked, not a local one); and `provenance check --strict` no longer prints a green "✓" success line immediately before failing — a failing check now renders as a warning before the `ClickException` is raised
|
||||
|
||||
- **Markdown round-trip export/import for `AgentMemory`** (#765, #786) by @SaurabhScripts and @Sameer6305
|
||||
- `AgentMemory.export(format="markdown")` and `import_data(format="markdown")` add a human-editable, diff-friendly alternative to the existing JSON/dict serialization: one Markdown file per memory item, with `id`, `created_at`, `updated_at`, and `type`/`kind` in required YAML frontmatter and the memory content as the Markdown body
|
||||
- Exporting without a `destination` returns a single memory as a Markdown string; exporting a set requires a destination directory and writes one stable, content-hashed filename per memory ID, so re-exporting an unchanged set is byte-for-byte idempotent
|
||||
- Importing upserts by ID: unknown IDs create new memories, known IDs replace them atomically (local state and vector store are only mutated after the whole batch validates cleanly), and unchanged re-imports are a deterministic no-op
|
||||
- Malformed frontmatter, duplicate IDs within an import batch, and duplicate YAML keys are all rejected before any memory is mutated, with actionable error messages
|
||||
- Export refuses to overwrite symbolic links and replaces files atomically; import safely compares timezone-aware and timezone-naive timestamps so retention, recency sorting, and date filters stay correct across both
|
||||
- Entities and relationships round-trip as memory-local provenance only — Markdown import intentionally does not write into `ContextGraph`, matching the MVP scope agreed on in #765
|
||||
- Documented the file contract and workflow in `docs/reference/context.md`; 43 new tests in `tests/context/test_agent_memory_markdown.py` cover round-trip losslessness, idempotency, validation errors, rollback on failure, and vector-store sync ordering
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`PipelineWithProvenance` raised `ModuleNotFoundError` on import and `AttributeError` on `.run()`** (#858, closes #858) by @Karunasagar12
|
||||
- `from .pipeline import Pipeline` failed because `semantica/pipeline/pipeline.py` does not exist; corrected to `from .pipeline_builder import Pipeline`
|
||||
- `.run()` called `self._pipeline.run()` on the `Pipeline` dataclass, which has no such method; replaced with `self._engine.execute_pipeline(self._pipeline, ...)` delegating to `ExecutionEngine`
|
||||
- Constructor now accepts a built `Pipeline` instance (from `PipelineBuilder.build()`) instead of `**config`; the old `Pipeline(**config)` internal construction was invalid and never functional
|
||||
- Replaced deprecated `datetime.utcnow()` with `datetime.now(timezone.utc)` in `run()`
|
||||
|
||||
- **`VectorStore.search_vectors()` returned inconsistent result shapes across backend implementations** (#853, closes #845) by @Sameer6305, reviewed by @KaifAhmad1
|
||||
- Every built-in backend (FAISS, Milvus, pgvector, Pinecone, Qdrant, SQLite-vec, Weaviate, in-memory) now returns the same canonical `SearchResult` shape (`id`, `score`, `metadata`, `vector`, `distance`), instead of some backends omitting `vector`/`metadata`/`distance` or, for Weaviate, returning a backend-specific `properties` key instead of `metadata`
|
||||
- Added a `SearchResult` `TypedDict` (`semantica/vector_store/vector_store.py`, exported from `semantica.vector_store`) documenting the contract; `metadata` now always defaults to `{}` rather than being absent, and `id` accepts `Union[str, int]` to accommodate Milvus/Qdrant's native integer IDs without casting
|
||||
- **Review fix**: the score-normalization formula added for Pinecone and Qdrant (`1.0 / (1.0 + max(0.0, 1.0 - score))`) clamped every raw score `>= 1.0` to an identical `1.0`, silently collapsing result ranking whenever the raw score could exceed 1 — which happens routinely for dot-product-metric indexes (unbounded), as opposed to cosine (bounded to `[-1, 1]`). Replaced with `(score / (1 + |score|) + 1) / 2`, which is strictly monotonic and bounded in `(0, 1)` for any real input, so ranking order is preserved regardless of metric or vector normalization
|
||||
- Added `test_qdrant_unbounded_dot_product_scores_preserve_ranking` and `test_pinecone_unbounded_dotproduct_scores_preserve_ranking` (`tests/vector_store/test_search_result_schema.py`) asserting normalized scores stay strictly ordered and bounded for raw scores well above 1.0, the case the original formula silently collapsed and the existing tests (which only used scores `< 1`) never exercised
|
||||
- Left out of scope, per the original PR: Weaviate's `similarity_search()` still isn't wired into `VectorStore.search_vectors()`'s backend dispatch; Milvus's collection schema still has no metadata column so its results always return `metadata: {}`; and `include_vectors` support (populating the `vector` field) is not yet implemented for any backend
|
||||
|
||||
- **`DecisionEmbeddingPipeline.find_similar_decisions()` crashed with `AttributeError` for any `VectorStore` backend other than `inmemory`** (#842, closes #839) by @Sameer6305
|
||||
- `_get_candidate_embeddings()` iterated `VectorStore.vectors`/`VectorStore.metadata` directly, internal dicts only populated for `backend="inmemory"`; every persistent backend (FAISS, Pinecone, Qdrant, Milvus, ...) raised `AttributeError`. It now fetches candidates via the backend-agnostic `VectorStore.search_vectors()`, reading metadata via a `res.get("metadata") or res.get("payload")` fallback for backends that key it differently
|
||||
- Backends such as FAISS don't return the raw vector for each hit; `find_similar_decisions()` and `_find_semantic_similar()` now fall back to the search-provided score (normalized from `distance` when present) as the semantic similarity for those candidates instead of computing cosine similarity against a zero placeholder vector
|
||||
- `get_decision_statistics()` had the identical bug iterating `store.metadata.values()`; it now returns a limited stats payload with an explanatory `warning` field for backends that don't expose a full in-memory metadata dict, instead of crashing
|
||||
- **Fixed along the way**: `_get_candidate_embeddings()`'s expand-and-retry loop (which widens the search pool when post-filtering leaves too few matches) discarded every candidate it had found once the pool hit its cap (`limit * 10`) without ever collecting `limit` matches or getting a short page back from the backend — the loop fell through without executing the branch that assigns results, silently returning `[]` even when matching candidates existed. It now falls back to the last batch collected instead of dropping it
|
||||
- Added end-to-end regression tests against real `inmemory` and `faiss` backends (no mocks) plus a targeted unit test for the expand-and-retry loop's fallback behavior
|
||||
|
||||
- **`QdrantStore.search_vectors()` returned results keyed by `"payload"` instead of `"metadata"`** (#841, closes #840) by @divyankshah
|
||||
- `QdrantCollection.search_points()` built its result dicts as `{"id", "score", "payload"}`, while `PineconeStore.search_vectors()` and every other backend consumed by `HybridSearch` use `"metadata"`. This silently dropped Qdrant metadata from results and made `HybridSearch.filter_by_metadata()` reject every candidate whenever a filter was applied, since it looks up `result["metadata"]` and got nothing back
|
||||
- Normalized `search_points()` to return `"metadata"` instead of `"payload"`, matching the existing convention; no other module reads the old key, so the rename is a straight fix rather than a partial one
|
||||
- Extended `tests/vector_store/test_vector_store_deepdive.py::test_qdrant_store` to assert the returned key is `"metadata"` (not `"payload"`) and that `HybridSearch.filter_by_metadata()` correctly matches against Qdrant results end-to-end
|
||||
|
||||
- **Explorer Temporal panel never rendered after clicking the toolbar button** (#830, #836) by @Sameer6305
|
||||
- The panel stayed permanently stuck on "Loading temporal…" in `npm run dev`, with repeating "Maximum update depth exceeded" errors in the browser console. Two independent render loops were responsible:
|
||||
- **Diagnostics state churn**: `handleDiagnosticsChange` unconditionally called `setGraphDiagnosticsState` on every invocation. `buildEffectAvailability` (inside `GraphCanvas`'s diagnostics `useEffect`) always returns a new object, so each call scheduled a re-render that immediately retriggered the effect. Fixed by comparing the incoming snapshot field-by-field against the last accepted value via `lastDiagnosticsRef` before calling `setState`
|
||||
- **scrubberTime churn**: React 18 concurrent mode re-ran `TimelinePanel`'s `useEffect` with a structurally-new `Date` object for the same timestamp when speculative renders discarded `useMemo` caches, causing repeated `setScrubberTime` calls that propagated into `temporalState` churn and retriggered the diagnostics effect. Fixed by deduplicating by millisecond value via `onTimeChange`/`lastScrubberMsRef`
|
||||
- **Bonus**: `temporal-overlay`'s `shouldLoad` predicate was changed to gate strictly on `panelState["temporal-panel"]`, removing the `|| temporalState?.currentTime` branch that caused eager loading on every scrubber update and continuously cancelled in-flight `load()` completions
|
||||
- **Bonus**: `temporalState` removed from the plugin-loading `useEffect` dependency array; predicates extracted into `pluginRegistryPredicates.ts` and wired through `GraphWorkspace.tsx` so regression tests exercise the production code rather than a local copy
|
||||
- The `scrubberTime`-churn fix was also applied to the equivalent (but currently unused/unmounted) `GraphWorkspaceShell.tsx`, which shares the same `TimelinePanel` integration pattern but does not have the diagnostics-churn code path
|
||||
- **Follow-up review fix**: the diagnostics dedup's `structureLayer` comparison now also covers `disabledReason`, `curveCount`, `bridgeCurveCount`, and `backboneCurveCount` (previously only `cacheKey`/`lastDrawAt`/`enabled` were compared, so a pure `disabledReason` transition could leave the dev-only diagnostics panel stale)
|
||||
- **Follow-up review fix**: `test:graph-store`, `test:graph-workspace`, and the new `test:plugin-registry` regression test are now run in CI (`.github/workflows/ci.yml`) — previously none of the Explorer frontend's `node --test` suites executed anywhere in CI, only `npm run build`, so this fix's own regression coverage (and all prior frontend test coverage) provided no protection against silent regressions
|
||||
- **`HybridSearch.search()` crashed with `AttributeError` for any `VectorStore` backend other than `inmemory`** (#833, #837) by @KaifAhmad1
|
||||
- `HybridSearch.search()` read `self.vector_store.vectors` directly, an internal dict `VectorStore` only populates for `backend="inmemory"`; every other backend (faiss, weaviate, qdrant, milvus, pinecone, pgvector, sqlite) raised `AttributeError`, making `HybridSearch` unusable against any real store. It now delegates to `VectorStore.search_vectors()` (the backend-agnostic public API) for non-inmemory backends, applies `metadata_filter` as a post-filter over the returned candidates, and normalizes results to a consistent `{id, score, distance, metadata}` shape
|
||||
- **Fixed along the way**: `vector_ids` could stay `None` when callers passed explicit `vectors`/`metadata` without `vector_ids`, crashing downstream list indexing — now defaulted to generated positional IDs
|
||||
- **Fixed along the way**: a `query_vector` passed as a plain list crashed backend stores (e.g. `FAISSStore.search_similar`) that call `.ndim` on it — now normalized to a numpy array up front
|
||||
- **Fixed along the way**: `VectorStore.store_vectors()` silently dropped metadata for FAISS (and any `add_vectors`-only backend) because it called `add_vectors(vectors, **options)` without forwarding `metadata`, even though `FAISSStore.add_vectors()` accepts it — this blocked `HybridSearch`'s metadata filtering from ever matching anything on FAISS
|
||||
- **Follow-up review fixes**: the legacy `top_k` kwarg was read but left in `options`, then forwarded via `**options` into `VectorStore.search_vectors()`, colliding with backends (sqlite, pgvector) that pass an explicit `top_k=k` to their own `search()` and raising `TypeError: got multiple values for keyword argument 'top_k'` — now popped instead of just read; `VectorStore.search_vectors()`'s dispatch only recognized backend methods named `search`/`search_similar`, so delegation still hit `NotImplementedError` for qdrant/milvus/pinecone, which name their method `search_vectors()` with a differently-named count parameter (`limit` vs `k`) — added a third dispatch branch that binds the count positionally so it works regardless of the backend's parameter name; a missing `distance` in backend-delegated results defaulted to the raw `score`, silently reusing the local path's cosine-similarity convention (`distance = 1 - score`) even for backends using unrelated metrics (L2, inner product) — now left as `None` instead of a fabricated, metric-inconsistent value
|
||||
- Verified across all 7 supported backends: `inmemory`/`faiss`/`sqlite` work live end-to-end; `pgvector`'s dispatch reaches `PgVectorStore.add()`/`.search()` (blocked only by no Postgres server in the verification sandbox); `qdrant`/`milvus`/`pinecone` now reach their real `search_vectors()` method instead of crashing, though their storage side (`store_vectors()`) still doesn't recognize `insert_vectors`/`upsert_vectors`, and `weaviate` remains entirely unwired (`add_objects`/`query_vectors`) on both sides — both are separate, pre-existing gaps independent of this fix, left for a follow-up
|
||||
|
||||
- **`VectorStore.store_vectors()` silently dropped metadata for FAISS (and any `add_vectors`-only) backend** (#832, #835) by @KaifAhmad1
|
||||
- `store_vectors()` fell into a branch that called `self._backend_store.add_vectors(vectors, **options)` without `metadata` whenever the backend exposed `add_vectors()` but neither `add()` nor `store_vectors()` — true for `FAISSStore`, the backend most real usage configures for genuine ANN search. Every caller that stores vectors with metadata (e.g. `AgentMemory._store_memory_vector()`, used internally by `AgentContext.store()`) lost that metadata once it reached FAISS, with no error or warning
|
||||
- Downstream, `ContextRetriever._retrieve_from_vector()` recovers a result's text via `metadata.get("content", "")`, which was always `""` for any vector stored this way; `_rank_and_merge()` then embedded that empty string, tripping `TextEmbedder.embed_text()`'s empty-text rejection and masking the real bug as a spurious `TextEmbedder` failure recorded by the progress tracker
|
||||
- `store_vectors()` now forwards `metadata` to `add_vectors()`, but only when the backend's `add_vectors()` signature actually accepts it (checked via `inspect.signature`, accepting either an explicit `metadata` parameter or a `**kwargs` catch-all), so a future/custom backend with a stricter signature raises no `TypeError`
|
||||
- **Follow-up review fix**: the `inspect.signature()` probe is wrapped in `try/except (ValueError, TypeError)`, consistent with the identical pattern already used in `ProvenanceManager.trace_lineage()`, so signature introspection failing on an unusual callable can no longer abort `store_vectors()` before it even attempts to call the backend
|
||||
|
||||
- **`AgnoDecisionKit.check_policy` silently treated unevaluable policy rules as compliant** (#778, #822) by @Sameer6305
|
||||
- `_eval_rule()` previously `return`ed `True` when a rule referenced a field missing from the decision payload, or when the rule string didn't match the expected `<field> <op> <value>` format — the docstring's claim that exceptions never silently return `compliant=True` didn't cover this, since neither path raised
|
||||
- Both cases now raise `ValueError` instead, which routes through `check_policy`'s existing exception handler and records a `warnings` entry (e.g. `"Could not evaluate rule 'minimum_score >= 0.9': rule references undefined field 'minimum_score'"`) instead of disappearing with no signal
|
||||
- `violations`/`compliant` are unaffected — an unevaluable rule is not counted as a violation, since it's genuinely unknown whether it would have passed; this matches the existing `compliant`/`violations`/`warnings` shape already used by `ContextGraph.enforce_decision_policy`
|
||||
- This is additive: `warnings` was already part of the return contract and populated for other exception cases, so no caller that only checks `compliant` is affected, and no existing test asserts `warnings == []` for a payload that hits either of these paths
|
||||
- **Follow-up review fix**: `check_policy` decoded `policy_rules` with `json.loads` and iterated the result without checking it was actually a list; a JSON-encoded bare string (e.g. `policy_rules='"confidence >= 0.7"'`) decodes to a `str`, so iterating it evaluated one "rule" per character — combined with the fix above, an 18-character rule string produced 17 warnings instead of being treated as the single rule it was meant to be. A decoded string is now wrapped as a single-element rule list; any other non-list shape (number, object, etc.) or non-string list element now produces exactly one `warnings` entry instead of silently misbehaving or being iterated character-by-character
|
||||
- **Follow-up review fix**: `_eval_rule` used `data.get(field) is None` to detect a missing field, which can't distinguish a genuinely absent key from a key explicitly present with a JSON `null` value — both produced the same "undefined field" warning, misdiagnosing nullable fields. Field presence is now checked with `field not in data` first; a present-but-`null` value now raises a distinct `"field {field!r} is null — cannot evaluate rule"` message instead of the misleading "undefined field" one
|
||||
- **Follow-up review fix**: `check_policy` only checked that `decision_data` was valid JSON, not that it decoded to an object. When it decoded to a list, `field not in data` silently became list-*membership* testing instead of a key check (e.g. `"confidence" not in ["confidence", 0.95]` is `False`), so a matching rule fell through to `data["confidence"]`, which raised a raw, confusing `TypeError: list indices must be integers or slices, not str` instead of any meaningful diagnostic; numbers/strings/bools produced similarly opaque `TypeError`s. `check_policy` now rejects any `decision_data` that doesn't decode to a JSON object upfront with a single clear `violations` entry, the same way it already rejects malformed JSON
|
||||
- Added 15 tests to `tests/integrations/agno/test_decision_kit.py` covering the missing-field case (the issue's traced example), the malformed-rule-string case, the bare-JSON-string `policy_rules` amplification case, non-list/non-string `policy_rules` shapes, the missing-key-vs-null-value distinction, non-object `decision_data` shapes (list/number/string/bool/null), and regression checks confirming normal rule evaluation on present fields is unchanged
|
||||
|
||||
- **No cycle detection for SKOS concepts at write time** (#774, #819) by @mikemikimike, reviewed by @Sameer6305 and @KaifAhmad1
|
||||
- Added cycle detection (`validate_skos_hierarchy`) for `skos:broader` and `skos:narrower` relationships in `ContextGraph.add_edge()` and `ContextGraph.add_edges()`, preventing direct 2-node cycles, self-loops, and multi-hop hierarchy cycles
|
||||
- Added `GraphSession.add_nodes_and_edges()` to validate SKOS hierarchy edges upfront under lock before node insertion, preventing partial-write leaks where nodes remain after a cyclic edge is rejected
|
||||
- Updated vocabulary, ontology (`/api/ontology/load`, `/api/ontology/create`), and JSON/CSV import routes to use `add_nodes_and_edges()` and return HTTP 422 with actionable error messages when a cycle is detected
|
||||
- Follow-up fix by @KaifAhmad1: `validate_skos_hierarchy()` previously re-walked *every* SKOS hierarchy edge already in the graph on each write, so one pre-existing cycle anywhere (e.g. legacy data) blocked all unrelated future writes; it now only traverses concepts touched by the edges being written, while still checking against existing edges for cycles that span old and new data
|
||||
- Follow-up fix by @KaifAhmad1: in `/api/ontology/load`, `except HTTPException: raise` was unreachable because a broader `except Exception` clause above it already matched `HTTPException`, so a 422 raised after a successful `OntologyIngestor` parse was silently swallowed and reprocessed via the fallback RDF parser; reordered the clauses so the deliberate 422 always propagates
|
||||
- Follow-up fix (#775): `/api/ontology/{uri}/refresh` was missed by the original sweep and still called `session.add_nodes()` then `session.add_edges()` as two independent operations, so a cyclic SKOS edge rejected by `add_edges()` left the nodes from the preceding `add_nodes()` call committed to the graph; switched to `session.add_nodes_and_edges()` with the same `except ValueError` → HTTP 422 handling already used by `/api/ontology/load` and `/api/ontology/create`. Audited every other `add_nodes()`/`add_edges()` pairing in the repo (`GraphStore`, `graph_builder.py`, `agent_memory.py`, `context_graph.py.load()`, `enrich.py`) — none share `GraphSession`'s SKOS-cycle-validation write path, so none were changed
|
||||
|
||||
- **Agno `_AgentScopedStore.upsert_memory` silently swallowed decision recording failures** (#779)
|
||||
- `upsert_memory()` now logs `logger.warning("[%s] record_decision failed: %s", self._role, exc, exc_info=True)` when `record_decision()` fails, matching the error-logging convention used for `store()` in the same method with traceback context preserved
|
||||
- Preserves graceful fallback behavior: `record_decision()` remains optional and `upsert_memory()` continues without propagating the exception
|
||||
- Added regression coverage in `tests/integrations/agno/test_shared_context.py` for both `store()` and `record_decision()` warning paths
|
||||
|
||||
- **`AgnoDecisionKit`/`AgnoKGToolkit` silently swallowed Agno tool registration failures** (#780, #818) by @Sameer6305 and @KaifAhmad1
|
||||
- Removed the `try/except: pass` wrapped around `self.register(fn)` in both toolkits' `__init__`; when Agno is installed, a registration failure now propagates immediately instead of leaving the toolkit half-registered with no signal to the caller
|
||||
- Graceful degradation when Agno isn't installed (`AGNO_AVAILABLE=False`) is unchanged — `_tools` is still populated so callers can introspect available tools without the package
|
||||
- Fixed a related duplicate-entry bug: `self._tools` was appended to unconditionally *before* `register()` ran, which could double-count a tool when Agno's own `Toolkit.register()` also tracks it in `self._tools`
|
||||
- This is a behavior change for callers that construct these toolkits expecting instantiation to always succeed — audited: no in-repo call site relies on the old silent-failure behavior
|
||||
- Expanded `tests/integrations/agno/test_decision_kit.py` and `test_kg_toolkit.py` with coverage for registration invocation counts, failure propagation, graceful degradation, and no-duplicate-`_tools` assertions
|
||||
|
||||
- **`ProvenanceManager` tracking methods silently swallowed failures without logging and returned fabricated entries** (#783)
|
||||
- `track_relationship()`, `track_chunk()`, and `track_property_source()` now return `Optional[ProvenanceEntry]` (`None` on storage failure, consistent with #782's `track_entity` fix) instead of a fabricated populated object
|
||||
- `_save_entry()` now always logs on any storage failure, including previously-silent per-item batch failures
|
||||
- `track_entities_batch()` and `track_chunks_batch()`'s rare block-level transaction failures are now logged too
|
||||
- `source_tracker.py`'s `track_sources_batch()` no longer counts failed tracking calls in its stats
|
||||
|
||||
- **MCP `handle_get_causal_chain` returned an empty-but-valid-looking response when both `CausalChainAnalyzer` and the graph fallback were unavailable** (#781, #817) by @Sameer6305 and @KaifAhmad1
|
||||
- Returns an explicit `{"error": "Causal chain analysis is not supported on this graph backend", "chain": []}` instead of `{"chain": [], "count": 0, "direction": ...}`, letting clients distinguish "unsupported" from a legitimately empty chain
|
||||
- The fallback path now introspects `graph.get_causal_chain`'s signature to forward `direction`/`max_depth` (or a `depth` kwarg, or nothing, depending on what the backend accepts) instead of always calling with just `decision_id`, matching the primary analyzer path's behavior
|
||||
- Hardened input handling: non-dict `args`, non-string `decision_id` (previously a latent `AttributeError` on `.strip()`), and `max_depth` clamped to `(0, 100]` with a safe default on invalid input
|
||||
- Added `tests/test_mcp_decisions_causal_chain.py` (11 tests) covering the unsupported-backend, fallback-forwarding, and validation/exception paths across multiple backend signature shapes
|
||||
- **Follow-up review fix**: the signature-detection try/except previously caught the *actual call*'s exceptions in the same block used for introspection failures, so a genuine bug inside a backend's `get_causal_chain` (raising an unrelated `TypeError`) was misread as a signature mismatch and the backend was invoked a second time with identical arguments before the real error surfaced. Signature introspection and the resulting call are now split into separate try/excepts so a successfully-introspected call is made exactly once; added `test_internal_typeerror_calls_backend_only_once` to lock this in
|
||||
|
||||
- **`ProvenanceManager.track_entity` persisted partial history and returned fabricated entries on storage failure** (#782, #816) by @Sameer6305 and @KaifAhmad1
|
||||
- `track_entity()`'s two-step write (history archive + primary update) is now atomic — if either write fails, the whole operation rolls back via the existing #807 `transaction()` mechanism, instead of silently persisting a partial state
|
||||
- `track_entity()`'s return type is now `Optional[ProvenanceEntry]`: on failure it returns a safe deep copy of the pre-failure existing entry (if one existed) or `None` (if this was a brand-new, never-successfully-tracked entity) — never a fabricated object claiming values that were never actually persisted
|
||||
- This is a behavior change for callers that inspect the return value without checking for `None` first — audited: 0 of 47 production call sites in the repo currently dereference the return value, so this is safe today, but any NEW caller must handle `None`
|
||||
- `InMemoryStorage` gained real transactional rollback (staging-buffer based) to match this guarantee — previously `transaction()` was a no-op
|
||||
|
||||
- **`ProvenanceManager` duplicated the same checksum/persist/exception-swallow block across 4 tracking methods** (#784, #815) by @Sameer6305 and @KaifAhmad1
|
||||
- Consolidated the repeated `entry.checksum = compute_checksum(entry)` / `try: self.storage.store(entry) except Exception: pass` block used by `track_entity`, `track_relationship`, `track_chunk`, and `track_property_source` into a single `ProvenanceManager._save_entry()` helper, preserving the existing graceful-failure behavior and the batch `_conn`/re-raise semantics from #807
|
||||
- Added 4 regression tests (`tests/provenance/test_manager.py`) covering storage-failure swallowing for each of the four tracking methods, none of which had coverage for this path before
|
||||
- **Follow-up review fix**: the initial refactor of `track_entity`'s exception fallback (the branch that runs when a failure happens *before* the entry is built, e.g. a retrieve error inside the atomic transaction) routed through `_save_entry()`, which made a new `self.storage.store(entry)` call outside the already-failed transaction — a real behavioral change from the original code (which only computed a checksum on that path) that could have reintroduced the exact race #807's `BEGIN IMMEDIATE` transaction serialization was meant to prevent. Reverted that branch to only compute the checksum, and added `test_track_entity_pre_build_failure_fallback_skips_store` asserting `storage.store` is never called on that path
|
||||
|
||||
- **`SQLiteStorage` and `ProvenanceManager` connection churn, non-atomic writes, and batch tracking overhead** (#807) by @Sameer6305
|
||||
- Scoped a single SQLite connection to the full duration of each public storage method call (`track_entity()`, `store()`, `retrieve_all()`, `clear()`) instead of opening independent connections per internal SQL statement, reducing connection churn by ~67% while closing the handle before the public method returns to preserve Windows filesystem unlink safety
|
||||
- Implemented the `SQLiteStorage.transaction()` context manager with Write-Ahead Logging (`PRAGMA journal_mode=WAL`), `busy_timeout=5000`, `synchronous=NORMAL`, and immediate write transactions (`BEGIN IMMEDIATE`), ensuring concurrent read-modify-write sequences (including history version ID generation) are serialized without lock contention or data loss
|
||||
- Added block-level transaction sharing to `track_entities_batch()` and `track_chunks_batch()`, reducing SQLite commit overhead by ~99.9% for large batches and deferring `tracked_count` increments until successful commit so rolled-back items are never reported as successes
|
||||
- Preserved 100% backward compatibility for custom storage backends overriding `trace_lineage(self, entity_id)` by inspecting signatures dynamically before passing `max_depth`, and optimized BFS lineage queries with batched IN-clause lookups per frontier level
|
||||
- **Follow-up fix**: `retrieve()` and `trace_lineage()` were initially routed through `transaction()` too, so plain reads took the same `BEGIN IMMEDIATE` writer lock as read-modify-write calls, serializing every read behind every other read/write and defeating the WAL concurrency this PR was meant to add. They now use a dedicated `_read_connection()` (configured, no explicit `BEGIN`) so reads no longer contend for the writer lock
|
||||
- **Follow-up fix**: `track_entity()`/`track_chunk()` caught all internal storage exceptions unconditionally, so when called from `track_entities_batch()`/`track_chunks_batch()`'s shared per-block transaction, a single item's storage failure (e.g. non-JSON-serializable metadata) was swallowed inside the call and never surfaced to the batch loop's per-item `except`, inflating `tracked_count` for entries that were never persisted. Both methods now re-raise when invoked with a shared `_conn` (batch context) while still degrading gracefully on standalone calls, so batch counts match what's actually committed
|
||||
- Added 8 dedicated regression tests in `tests/provenance/test_sqlite_storage_performance_807.py` covering PRAGMA configuration, Windows unlink safety, batch transaction sharing, BFS `max_depth`, rollback count accuracy, custom storage backward compatibility, concurrent read-modify-write serialization, and connection cleanup guards on configuration error
|
||||
|
||||
- **Closed remaining `ProvenanceManager` storage-failure test-coverage gaps identified by a #785 audit** (#785)
|
||||
- An audit of `tests/provenance/` (filed against a claim that zero tests exercised `storage.store()` failures) found #782/#783/#784/#807 had already closed most of the gap, but two residual surfaces had no test: `track_relationship()`, `track_chunk()`, and `track_property_source()`'s storage-failure-swallowing contract (returns `None`, logs, persists nothing) was only verified against `InMemoryStorage`, never `SQLiteStorage`; and `track_chunks_batch()` had no test for per-item `_save_entry` failure logging or for the block-level transaction-failure log message, even though `track_entities_batch()` had both
|
||||
- No production code changed — #782/#783/#784/#807 already implemented the correct behavior; this closes the coverage gap proving it holds on both backends
|
||||
- Added `test_track_relationship_storage_error_swallowed_sqlite`, `test_track_chunk_storage_error_swallowed_sqlite`, `test_track_property_source_storage_error_swallowed_sqlite`, `test_chunks_batch_logs_per_item_failure_memory`, and `test_track_chunks_batch_block_level_transaction_failure_logs` to `tests/provenance/test_manager.py`
|
||||
- Read-path failure coverage (`get_lineage()`/`trace_lineage()`/`get_provenance()`/`clear()` propagating a raised storage exception) remains untested and is a candidate for a follow-up issue, since none of those methods currently wrap the underlying storage call in a try/except
|
||||
|
||||
- **Explorer's Provenance UI used a naive 2-hop graph traversal instead of the audit-grade `ProvenanceManager` backend** (#792, #809) by @Sameer6305
|
||||
- `semantica/explorer/routes/provenance.py` never imported or called `ProvenanceManager` (`semantica/provenance/manager.py`); `/api/provenance` and `/api/provenance/report` built their lineage response entirely from a naive 2-hop networkx traversal over the live graph instead of querying the SQLite-backed, checksummed audit log. Both endpoints now query `session.provenance_manager.get_lineage(node_id)` first, and a new `_transform_audit_lineage()` maps the W3C PROV-O entries into the exact `{"nodes": [...], "edges": [...]}` shape `LineageDiagram.tsx` already expects — no frontend changes required
|
||||
- Falls back to the original 2-hop traversal, never a 500: no audit records for a node, a `ProvenanceManager` storage failure (corrupted DB, permissions), or a failed SHA-256 integrity check on any entry in the lineage chain all degrade cleanly to the naive path. A new `source: "audit" | "graph_traversal"` field on the response discloses which path actually served the data
|
||||
- `ProvenanceManager.get_lineage()` now returns `integrity_verified`, computed by re-verifying every entry's checksum before it's trusted; a single tampered or corrupted entry anywhere in the lineage chain now falls the *entire* response back to graph traversal rather than serving partially-verified audit data
|
||||
- Replaced an initial classmethod-based `ProvenanceManager.set_default_storage_path()` approach (caught in review before merge — it would have let any two sessions/apps in the same process silently share and overwrite each other's storage path, including across unrelated test runs) with `provenance_storage_path` threaded through `GraphSession.__init__` and `create_app(...)`, so each session's `ProvenanceManager` is independently scoped
|
||||
- Disclosed limitation: `ProvenanceManager.trace_lineage()`/`get_lineage()` only walk `parent_entity_id`/`used_entities` backward, so the audit path currently surfaces upstream lineage only — the naive fallback remains the only source for downstream/descendant relationships until `ProvenanceManager` gains a reverse lookup
|
||||
- New `tests/explorer/test_provenance_manager_wiring.py` (8 tests): the audit path via a real multi-hop `track_entity()` chain, empty-record fallback, simulated storage-failure degradation (asserts `200`, not `500`), checksum-tamper fallback, evidence-field preservation, `create_app()` storage-path wiring, and cross-session storage isolation, confirmed order-invariant across `tests/explorer/` and `tests/provenance/` in both execution orders
|
||||
|
||||
- **`POST /shacl/validate` and the `/health` SHACL dimension never ran live SHACL validation** (#772, #804) by @Sameer6305 and @KaifAhmad1
|
||||
- `/shacl/validate` had no data graph to validate submitted shapes against — only a Turtle syntax check. Added `_data_graph_turtle_for_uri()`, which serializes the loaded ontology's nodes/edges into an RDF/Turtle instance graph (CURIE resolution across owl/rdfs/skos/dct/dc, arbitrary node-property projection, typed individuals) and wires both `/shacl/validate` and the `/health` SHACL dimension to `OntologyEngine.validate_graph()` via pySHACL, returning real `conforms`/violations instead of a hardcoded `status="unavailable"` stub
|
||||
- Fixed a cross-ontology namespace leak in `_node_belongs_to_ontology`: its prefix fallback (`_extract_namespace()`) split only on the last `/`, so sibling ontologies sharing a domain (e.g. `.../onto-a` and `.../onto-b`) could match entities across ontologies that shouldn't be related; fixed by comparing against the full URI stem via the new `_ontology_namespace()` helper
|
||||
- Added resource guardrails to `/shacl/validate` to close a DoS risk flagged in review: a submitted-Turtle byte cap (`SEMANTICA_MAX_SHACL_TURTLE_BYTES`, default 256 KB), a parsed-triple cap (`SEMANTICA_MAX_SHACL_TRIPLES`, default 1,000), a validation timeout (`SEMANTICA_MAX_SHACL_TIMEOUT`, default 15s), and a global concurrency semaphore (`SEMANTICA_MAX_SHACL_CONCURRENCY`, default 4)
|
||||
- Fixed `HealthDimension.status` being set to `"error"` on a real (non-`ImportError`) validation exception, which isn't a valid value on that model — Pydantic construction raised and turned the whole `/health` endpoint into a 422 on any real bug; now reports `status="critical"` (already a valid value) with a regression test forcing this exact path
|
||||
- Follow-up review fixes: reverted an unrelated regression that had crept into this PR — `POST /api/ontology/create` had gone back to silently swallowing `OntologyEngine.from_data`/`from_text` failures into a near-empty "minimal" ontology instead of raising `HTTPException(500)`, undoing the earlier #770/#787 fix for the same endpoint (and breaking `TestOntologyCreateFailures`, which wasn't run before this PR's initial merge request); `sh:Warning`/`sh:Info`-severity pySHACL results were silently dropped from the `/shacl/validate` response — a shape using non-`Violation` severities could report `conforms=False` with an empty `violations` list and no explanation, so warnings/infos are now folded into the response's `violations` array; and `/health` was independently re-fetching and re-truncation-checking the same ontology's nodes/edges once for the generated SHACL shapes and once for the data graph — both now share a single fetch via `_fetch_analysis_graph()`
|
||||
- New regression tests: `TestOntologyCreateFailures` (pre-existing, now passing again), `test_shacl_validate_surfaces_warning_severity_results`, `test_health_dedupes_node_edge_fetch`, plus the existing 26-test `tests/explorer/test_ontology_subissue3.py` suite (28/28 passing) and the pre-existing `tests/ontology/` suite (83/83 passing)
|
||||
|
||||
- **Neptune cookbook CloudFormation stack exposed the database port to the entire internet and had no network audit trail** ([code scanning alert #28](https://github.com/semantica-agi/semantica/security/code-scanning/28), [#26](https://github.com/semantica-agi/semantica/security/code-scanning/26), [#27](https://github.com/semantica-agi/semantica/security/code-scanning/27), `AC_AWS_0276`/`AC_AWS_0369`/`AC_AWS_0148`) by @KaifAhmad1
|
||||
- `cookbook/introduction/neptune-setup.yaml`'s security group let anyone on `0.0.0.0/0` reach the Neptune Bolt/OpenCypher port (8182); it now requires a `ClientCidr` parameter (CIDR-validated, no default) so the stack can't be created without the deployer explicitly scoping access to their own IP or VPN/office range
|
||||
- Added `AWS::EC2::FlowLog` plus a dedicated CloudWatch Logs group and IAM role so all traffic in the stack's VPC is now logged
|
||||
- Left the account-wide IAM password policy check (`AC_AWS_0148`) unimplemented as a stack resource on purpose: `AWS::IAM::AccountPasswordPolicy` is an account singleton, and wiring it into a disposable per-learner tutorial stack would mean creating or deleting this stack also mutates or removes the account's real password policy — suppressed with a documented `ts:skip=AC_AWS_0148` explaining why, rather than "fixed"
|
||||
- Updated `21_Amazon_Neptune_Store.ipynb`'s `aws cloudformation create-stack` instructions, prerequisites, and cost table to match the new required `ClientCidr` parameter and flow-log line item
|
||||
|
||||
- **Follow-up to the knowledge-explorer Helm chart default-namespace/seccomp scanner findings reopening** ([code scanning alert #846](https://github.com/semantica-agi/semantica/security/code-scanning/846), [#847](https://github.com/semantica-agi/semantica/security/code-scanning/847), [#848](https://github.com/semantica-agi/semantica/security/code-scanning/848), [#68](https://github.com/semantica-agi/semantica/security/code-scanning/68), [#63](https://github.com/semantica-agi/semantica/security/code-scanning/63), `CKV_K8S_21`/`AC_K8S_0086`/`AC_K8S_0080`) by @KaifAhmad1
|
||||
- The `checkov.io/skip1` metadata annotation added previously (see the `CKV_K8S_21` entry below) evidently isn't being honored by the Microsoft Defender for DevOps scan — the same finding reopened under new alert numbers on the current `main`. Added the more standard `# checkov:skip=CKV_K8S_21` and `# ts:skip=AC_K8S_0086` inline comments at the top of `templates/deployment.yaml`, `templates/service.yaml`, and `templates/configmap.yaml` as a second suppression path (matching the convention already used in `deploy/gcp/cloudrun-service.yaml`), plus `# ts:skip=AC_K8S_0080` on `templates/deployment.yaml` for the seccomp finding, which trips for the same root cause: terrascan's static template scan never resolves `{{ toYaml .Values.podSecurityContext }}`, even though `values.yaml` sets `seccompProfile.type: RuntimeDefault` correctly
|
||||
- Confirmed the `deploy/kubernetes/*` (non-Helm) manifests already had TLS and seccomp configured correctly, so no code change was needed there for the corresponding alerts (#61 and the non-Helm seccomp finding) — expected to close on the next scan
|
||||
- Documented both suppression mechanisms and the reasoning in `.checkov.yaml`
|
||||
- Residual risk: this environment could not run checkov/terrascan locally to confirm the inline comments are actually honored during a Helm-rendered scan; if the alerts are still open after the next scan, the reliable fallback is splitting the CI checkov/terrascan invocation so `deploy/helm/` is scanned with these specific checks excluded via `--skip-check` instead of relying on in-file suppression
|
||||
|
||||
- **`react-hooks/set-state-in-effect` cascading renders across 12 Explorer workspace files** (#769, #796) by @Sameer6305 and @KaifAhmad1
|
||||
- Replaced synchronous `setState` calls inside `useEffect` bodies with React's recommended "adjust state during render" pattern (`if (x !== prevX) { setPrevX(x); ...setState... }`) across `OntologyWorkspace`, `ManageWorkspace`, `LineageWorkspace`, and `GraphWorkspace`, and inlined async data-fetching effects with `ignore` flags to prevent race conditions and stale writes after unmount
|
||||
- Fixed a regression the inlining itself introduced: `AlignmentsTab.tsx`, `KGOverviewTab.tsx`, `OntologyManager.tsx`, and `VersionsTab.tsx` each duplicated their existing fetch callback (`reload` / `fetchOverview` / `fetchRegistry` / `loadVersions`+`loadProposals`) into a second, inline copy for the mount effect, and the copy silently dropped the `setError`/`flashMsg` calls the original had — re-introducing, on the very first page load, the exact error-swallowing behavior that #767/#790 had already fixed for these same files. The inline copies now mirror the original's error handling (including `207` partial-success messages) exactly
|
||||
- Fixed `LineageDiagram.tsx` only clearing the previously-rendered nodes/edges when the new `activeId` was falsy instead of on every id change, so switching directly between two lineage views briefly kept showing the *previous* view's stale diagram instead of clearing before the new fetch resolved
|
||||
- `GraphWorkspace.tsx` and `GraphLoadingOverlay.tsx` still have unrelated `react-hooks/set-state-in-effect` violations outside this PR's 12-file scope (confirmed via `npx eslint .`); left as follow-up work rather than expanding this PR further
|
||||
|
||||
- **Checkov flagged the knowledge-explorer Helm chart for using the default Kubernetes namespace** ([code scanning alert #779](https://github.com/semantica-agi/semantica/security/code-scanning/779), [#778](https://github.com/semantica-agi/semantica/security/code-scanning/778), [#777](https://github.com/semantica-agi/semantica/security/code-scanning/777), `CKV_K8S_21`) by @KaifAhmad1
|
||||
- `templates/service.yaml`, `templates/deployment.yaml`, and `templates/configmap.yaml` all already set `metadata.namespace` to `{{ .Release.Namespace }}`, which is only bound at `helm install`/`helm template` time; Checkov's helm framework renders the chart without a namespace override, so it always resolves to `default` and trips `CKV_K8S_21` even though the chart is namespace-agnostic by design
|
||||
- Added a `checkov.io/skip1: CKV_K8S_21` metadata annotation to each of the three files to suppress the scanner artifact false-positive properly in Helm templates, and documented the reasoning in `.checkov.yaml`
|
||||
|
||||
- **No React error boundaries around lazy-loaded Explorer workspaces — a single render error crashed the whole app** (#768, #794) by @Sameer6305
|
||||
- Added an `ErrorBoundary` class component (`explorer/src/ErrorBoundary.tsx`) and wrapped each lazy-loaded workspace's `<Suspense>` block in `App.tsx` with it, keyed on the active sub-view so navigating away from and back to a crashed tab remounts it cleanly
|
||||
- Failed retries are capped at 3 before the fallback UI switches from "Try Again" to a "Reload Application" dead-end, preventing infinite retry loops on deterministic crashes; raw error/stack details are logged via `console.error` only and never rendered into the fallback UI
|
||||
- Fixed the retry counter so it resets after a retry actually succeeds and stays error-free for a few seconds, instead of never resetting (which could permanently exhaust the retry budget on unrelated, individually-recoverable transient errors) or resetting on the very next commit (which could fire prematurely while `Suspense` was still showing its fallback)
|
||||
|
||||
- **Explorer frontend workspaces silently swallowed network/server errors** (#767, #790) by @Sameer6305
|
||||
- `ShaclStudio.tsx`, `VersionsTab.tsx`, `SKOSVocabularyManager.tsx`, `EntityResolutionTab.tsx`, `LineageDiagram.tsx`, `DecisionWorkspace.tsx`, `KGOverviewTab.tsx`, `OntologyManager.tsx`, `OntologySearch.tsx`, `ReasoningWorkspace.tsx`, and `SparqlWorkspace.tsx` now render a visible error banner instead of only `console.error()`-ing failed fetches
|
||||
- Added explicit `response.status === 207` (Multi-Status) handling across these workspaces so partial backend failures surface a warning instead of reading as a full success (`response.ok` is `true` for all 2xx codes, including 207)
|
||||
- Added defensive JSON parsing so an unexpected non-JSON (e.g. HTML 500) response body no longer crashes the app with `SyntaxError: Unexpected token < in JSON`
|
||||
- Fixed `KGOverviewTab.tsx` dropping the `/api/graph/nodes` partial-success warning whenever `/api/graph/stats` also returned 207 — both warnings are now shown (appended) instead of one being silently discarded
|
||||
- Fixed `HealthTab.tsx`'s registry load still using a bare `.catch(() => {})` that swallowed errors identically to the pattern fixed elsewhere in this same folder; failures now populate the existing error banner
|
||||
- Fixed `AlignmentsTab.tsx`'s `reload()` using `Promise.allSettled` but never handling the `"rejected"` branches for the registry/alignments fetches, so both failures previously vanished with no error surfaced and no logging
|
||||
|
||||
- **`tests/explorer/test_explorer_api.py` failed with `TypeError: Client.__init__() got an unexpected keyword argument 'app'` on current httpx** (#788, #789) by @Sameer6305
|
||||
- `httpx>=0.28.0` removed the `app=` kwarg that Starlette's `TestClient` relies on to wrap a FastAPI app for testing; `httpx` wasn't pinned anywhere in `pyproject.toml`, so different environments could independently resolve an incompatible transitive version and hit the same break
|
||||
- Added an explicit `httpx<0.28.0` constraint to the main `[project.dependencies]` array (not just a dev extra), so it applies globally across production, dev, and CI installs
|
||||
- Without the pin, the full test suite fails to even complete collection (fails immediately on `tests/explorer/test_vocabulary.py` with the same `TestClient` error); with it, `tests/explorer/test_explorer_api.py` goes from 7 failed/12 passed/58 errors to 77 passed, 0 errors
|
||||
|
||||
- **Explorer backend routes returned HTTP 200 with error/empty bodies on failure, defeating frontend error handling** (#770, #787) by @Sameer6305 and @KaifAhmad1
|
||||
- `GET /api/temporal/patterns` now raises `HTTPException(500)` on a genuine computation failure instead of silently returning an empty-but-valid `TemporalPatternResponse`; the `ImportError` fallback (optional `kg` extra not installed) is unchanged and still degrades gracefully to an empty list
|
||||
- `POST /api/ontology/create` now raises `HTTPException(500)` when ontology generation fails in either the `sample_data` or `schema_text` mode, instead of silently falling back to a partial/minimal ontology with a misleading `nodes_added` count
|
||||
- `GET /api/analytics` sets `response.status_code = 207` (Multi-Status) when some, but not all, of the requested metrics fail, and raises `HTTPException(500)` when every requested metric fails — a plain 2xx (including 207) reads as success to callers that only check `response.ok`, so an all-failed request now surfaces as a hard error rather than a body full of `{"error": ...}`
|
||||
- Added regression tests covering all three failure paths (`test_patterns_failure_returns_500`, `test_analytics_partial_failure_returns_207`, `test_analytics_total_failure_returns_500`, and two `TestOntologyCreateFailures` cases)
|
||||
|
||||
### Security
|
||||
|
||||
- **DNS check-then-use hardening for the ontology URL fetcher, and a remaining object-IRI validation gap** (#916, follow-up to GHSA-8c7v-62gr-hj6g and GHSA-8vgg-8mr4-r236) by @KaifAhmad1
|
||||
- **DNS check-then-use (TOCTOU) window**: GHSA-8c7v-62gr-hj6g's own fix description flagged this as a secondary gap — `_validate_fetch_url()` resolved and validated a hostname once, but `_fetch_url_sync()` then let `requests` resolve the same hostname again independently at connect time. A low-TTL or rebinding DNS answer could differ between the two lookups, reopening the SSRF window the validation exists to close
|
||||
- `_validate_fetch_url()` now returns the validated IP, and a new `_make_pinned_session()` builds a per-hop `requests.Session` whose connection pool is pinned directly to that IP — bypassing DNS resolution for the connection entirely — while explicitly restoring the real hostname as the outgoing HTTP `Host` header and, for HTTPS, the TLS SNI `server_hostname`/`assert_hostname`, so the connection reaches the validated IP but still presents (and is verified against) the real hostname's identity, keeping virtual hosting and certificate validation correct
|
||||
- Caught during implementation: an earlier draft set urllib3's `_dns_host` post-construction, assuming (as in some urllib3 releases) that it was decoupled from `host`. In the version this project installs (2.7.0), `host` is a property that reads/writes `_dns_host` directly, so that approach would have silently changed the Host header too — caught by an end-to-end test against a real local server before landing, rather than shipping. Verified with real (non-mocked) local HTTP and HTTPS servers, the latter using a generated self-signed certificate to prove SNI/cert-hostname verification checks the real hostname rather than the pinned IP, plus a negative control confirming a hostname/cert mismatch is still correctly rejected, not silently bypassed
|
||||
- **Object-IRI validation gap** (GHSA-8vgg-8mr4-r236 follow-up, distinct from the object-branch fix already shipped in #911): a triplet object already wrapped in `<...>` skipped `sparql_escaping.validate_uri()` in both `blazegraph_store.py` and `rdf4j_store.py`'s `_format_object_for_sparql`/`_format_object_for_ntriples`, only checking the inner content for a literal space or `>` — the pre-wrapped and unwrapped branches now validate identically
|
||||
- **Fixed along the way** (caught in automated review across two follow-up rounds): `_validate_fetch_url()` originally pinned to only the first resolved IP, so a hostname with multiple A/AAAA records would fail outright if that specific address was unreachable — it now returns every validated IP and `_make_pinned_session()` falls back through all of them, verified by pinning to a genuinely unreachable address followed by a working one and confirming the fetch still succeeds; the test HTTPS server allowed TLSv1/TLSv1.1 by not setting a minimum version, now pinned to TLSv1.2; and when an HTTP(S) proxy applied, pinning was silently skipped in favor of the unpinned path — proxies are now disabled outright for this fetcher (`session.trust_env = False`, so `HTTP_PROXY`/`HTTPS_PROXY` env vars are never consulted) with a fail-closed 502 backstop if a proxy is ever forced onto the session some other way, verified by pointing `HTTP_PROXY` at an address that would fail if actually used and confirming the fetch still succeeds directly
|
||||
- New `tests/explorer/test_ontology_dns_pinning.py` (12 tests: real local HTTP/HTTPS servers including 2 real-TLS checks, multi-IP fallback success/failure, and no-proxy-trust verification — gracefully skipped without the optional `cryptography` package where applicable); updated `tests/explorer/test_ontology_ssrf.py` for the new per-hop session construction; 4 new tests in `tests/triplet_store/test_sparql_injection.py` for the object-IRI fix. Full `explorer` + `triplet_store` suite: 572 passed
|
||||
|
||||
- **Missing Origin validation on the `/ws/graph-updates` WebSocket handshake** (#917, GHSA-4643-wpgq-w329) by @KaifAhmad1
|
||||
- `CORSMiddleware` doesn't cover WebSocket handshakes at all (Starlette's CORS support only wraps HTTP), so under `SEMANTICA_ALLOW_ANONYMOUS=true` — the mode `docker-compose.dev.yml` ships — the anonymous-mode key bypass accepted a `/ws/graph-updates` connection from any origin. Loopback binding isn't a boundary against a browser: any page the operator has open can still reach `ws://localhost:8000/ws/graph-updates` directly, and `ConnectionManager.broadcast` sends every `graph_mutation` to every connected socket with no per-connection scoping. Combined with `/api/import` accepting `multipart/form-data` (a CORS-safelisted content type that skips preflight), a hostile page could write to the graph over REST and read the result back over the unauthenticated WebSocket
|
||||
- Not affected: any deployment with `SEMANTICA_API_KEY` configured — the handshake already rejects without a valid key in that mode. This was an anonymous-mode-only, development-configuration exposure
|
||||
- Fix: check the handshake's `Origin` header against `app.state.explorer_settings['allowed_origins']` — the same list `CORSMiddleware` already enforces for HTTP — before the key check. A missing `Origin` (native/CLI clients, which never set the header) is still allowed through, since the browser is the only threat this closes
|
||||
- 4 new tests in `tests/explorer/test_explorer_auth.py`: hostile Origin rejected under anonymous mode; hostile Origin rejected even with a correct key (Origin is checked first, so a leaked key alone can't hijack the socket); an allowlisted Origin still connects; a missing Origin still connects. Full `explorer` suite: 226 passed
|
||||
|
||||
- **Polynomial-time ReDoS in the SPARQL route's `_PREFIX_DECL` regex** (#915, CodeQL `py/polynomial-redos`) by @Sameer6305
|
||||
- The prior pattern's trailing `\s*` overlapped with the preceding `<[^>]*>` IRI-body match on inputs containing no closing `>` (e.g. `base<` followed by thousands of `!<` repetitions), forcing the regex engine to explore every possible split between the two quantifiers — O(n²) backtracking reachable from `req.query` via `_is_read_only_query()`
|
||||
- Fixed by making the two quantifiers character-disjoint: horizontal whitespace only (`[ \t]`, never overlapping the IRI body) instead of `\s*`, and excluding CR/LF from the IRI body (`[^>\r\n]*`) so it can never span a line boundary. Independently verified: the exact pathological payload (`base<` + `!<` × 5,000/20,000) scales linearly (0.238ms → 0.841ms for 4x input, not the ~16x a surviving quadratic blowup would show)
|
||||
- Added `_SPARQL_MAX_QUERY_LEN = 10_000` as defense-in-depth, checked in `execute_sparql()` before any regex work so a future pattern regression stays bounded regardless
|
||||
- Two correctness regressions raised in review were checked and did not reproduce: comment-then-prefix stripping order means an inline comment after a `PREFIX` line (`PREFIX ex: <...> # comment`) is already gone by the time `_PREFIX_DECL` runs, verified directly against the pipeline; and the allowlist's `.sub()`-based cleaning only ever affects the yes/no decision, never the query actually sent to `graph.query()` — so even the narrow case of a multi-line string literal that happens to start a line with the literal text `PREFIX` or `BASE` can only cause a legitimate query to be wrongly rejected, never let something malicious through, since rdflib's parser still gates whatever actually executes
|
||||
- 20 new/updated tests in `tests/explorer/test_sparql_route.py` and `tests/test_security_regression.py` (inline prologues, CRLF line endings, multi-line CRLF prefix chains, oversized-query rejection). 225 `explorer` + 82 SPARQL-specific tests passing
|
||||
|
||||
- **SPARQL injection via unvalidated triplet IRIs** (#911, GHSA-8vgg-8mr4-r236) by @KaifAhmad1
|
||||
- `Triplet.subject`/`.predicate` (and, in some builders, `.object`) were interpolated directly into SPARQL update/query strings in the Blazegraph and RDF4J stores, and into a SELECT filter in the Jena store. A subject containing `>` closes the `<...>` IRI token early, so the rest of the value is parsed as more SPARQL. Entity names are document text in the normal ingest pipeline, so anyone whose content gets processed could append operations like `CLEAR ALL`, running with the application's store credentials
|
||||
- Applied the existing `sparql_escaping.validate_uri` (already used by `anzo_store.py`, the one backend that was already hardened — this generalizes its approach rather than inventing a new one) at every subject/predicate/object interpolation site: `blazegraph_store.py`'s `_build_insert_data`, `_triplets_to_rdf`, `bulk_load`'s `graph` option, `get_triplets`'s filter, and `delete_triplet`; `rdf4j_store.py`'s `_triplets_to_ntriples`, `get_triplets`'s filter, and `delete_triplet`; `jena_store.py`'s `get_triplets`'s filter (the only vulnerable site there — `add_triplets`/`delete_triplet` already use rdflib's native `Graph.add`/`.remove` with `URIRef` rather than building query strings)
|
||||
- **Fixed along the way** (caught in review, by @ZohaibHassan16): `_format_object_for_sparql`'s URI branch — used when a triplet's *object* is itself a URI rather than a literal — only checked for spaces and `>` inline instead of running the same `validate_uri` check applied to subject/predicate, leaving the object position as a narrower but real gap in both Blazegraph and RDF4J. Also fixed test flakiness in `RDF4JStore`'s test fixtures, which weren't mocking `_connect()` and so were making real network calls
|
||||
- New `tests/triplet_store/test_sparql_injection.py` (12+ tests) reproducing the advisory's own injection payload (`http://example.com/a> ... ; CLEAR ALL ; INSERT DATA { ...`) against all three backends' write and read paths, asserting the malicious query is never built or sent. Full triplet_store suite: 330+ tests passing
|
||||
- Side note, not part of this fix: found that `jena_store.py`'s `get_triplets()` builds syntactically invalid SPARQL for its WHERE-clause filters (missing a `FILTER()`/separator before the equality conditions) — a pre-existing correctness bug, unrelated to the injection fix, left alone here and worth a separate follow-up
|
||||
|
||||
- **Cypher injection via unvalidated node labels, relationship types, and property keys** (#910, GHSA-482h-hw99-h62p) by @KaifAhmad1
|
||||
- Node labels and property keys passed to `create_node`/`create_relationship` were interpolated directly into Cypher strings in the Neptune, Neo4j, and FalkorDB graph stores. Property *values* are parameterized, but labels and keys can't be bound as query parameters, and nothing validated them — so a document-derived entity type or property name (the normal ingest path) could close the current Cypher token early and append arbitrary statements (e.g. `DETACH DELETE`), running with the application's database credentials
|
||||
- New shared `semantica/graph_store/query_sanitize.py`: `sanitize_identifier()` generalizes `age_store.py`'s existing `_sanitize_label`/`_sanitize_rel_type` (the only backend that already validated this) into a helper the other backends import without an import cycle with `graph_store.py`/`methods.py`
|
||||
- Applied at every label/relationship-type/property-key interpolation site in `amazon_neptune.py`, `neo4j_store.py`, `falkordb_store.py`, `graph_store.py` (`degree_centrality`'s own query builder), and `methods.py` (`update_relationship`'s own query builder) — covers `create_node`, `create_nodes`, `create_relationship`, `get_nodes`, `get_relationships`, `get_neighbors`, `shortest_path`, `update_node`, `create_index`, and all relationship-type filters across the three backends
|
||||
- **Fixed along the way** (caught in review, by @Sameer6305): `depth`/`max_depth` path-length parameters are meant to be integers, but `Neo4jStore.get_neighbors()`/`shortest_path()` interpolated them into the Cypher variable-length-path syntax (`*1..{depth}`) without coercion — unlike the Neptune/FalkorDB equivalents, which already cast to `int()`. A string `depth` (e.g. `"1]->(x) DETACH DELETE x //"`) reached the query verbatim. Added the same `int()` coercion Neptune/FalkorDB already had, plus `GraphStore.get_neighbors()`'s `hops`/`depth` alias resolution
|
||||
- New `tests/graph_store/test_cypher_injection.py` (unit tests on `sanitize_identifier` plus the labels/keys/rel-types injection payload run against Neptune/Neo4j/FalkorDB `create_node`/`create_relationship`, asserting the malicious query is never built or sent) and the depth-coercion regression above; plus additions to `tests/test_graph_store.py` (`degree_centrality`) and `tests/test_graph_store_methods.py` (`update_relationship`). Full graph_store suite: 224+ tests passing
|
||||
|
||||
- **4 critical/high vulnerabilities in the Explorer API and vector store: RCE, SSRF, XXE, and DoS, plus Cypher/SPARQL injection hardening found along the way** (#898) by @Sunil56224972
|
||||
- **[CWE-502] Arbitrary code execution via `pickle.load()`**: `VectorStore.save()`/`load()` used `pickle` for the on-disk `store_data.pkl`; a crafted `.pkl` file placed in the store directory (file upload, shared filesystem, or supply-chain compromise) could execute arbitrary code on deserialization. Replaced with JSON — vectors and metadata are fully JSON-serializable, so nothing is lost — and `load()` now refuses any legacy `.pkl` file it finds with a migration error rather than deserializing it
|
||||
- **[CWE-918] SSRF via redirect bypass in `ontology.py`'s URL fetcher**: `_validate_fetch_url()` correctly blocked private/loopback/reserved addresses on the caller-supplied URL, but `_fetch_url_sync()` fetched with `allow_redirects=True`, so a validated *public* first hop could 302 to `http://169.254.169.254/...` (cloud instance metadata) or an internal service, and `requests` followed it with no re-check. Redirects are now followed manually, capped at 5 hops, with `_validate_fetch_url()` re-run against every hop's target — including relative `Location` headers, resolved via `urljoin()` before validation — and every response (redirect or final) is explicitly closed to avoid leaking connections back to the pool
|
||||
- **[CWE-611] XXE injection in the RDF/XML parser**: `_safe_parse_rdf()` depended on `defusedxml` for XXE protection, but `defusedxml` wasn't declared in `pyproject.toml`'s `explorer` extra, so it was silently absent in normal installs and the code fell back to a bare warning plus unsafe parsing — a crafted RDF/XML ontology with an external entity could read arbitrary server files. Added `defusedxml>=0.7.1` to the extra, and `_safe_parse_rdf()` now fails closed: it raises rather than parsing untrusted RDF/XML if `defusedxml` isn't importable, replacing an earlier regex-based DOCTYPE-stripping fallback that was reviewed and rejected as bypassable
|
||||
- **[CWE-770] DoS via unbounded SPARQL graph materialization**: `_build_rdflib_graph()` loaded up to 999,999 nodes and 999,999 edges into memory per query, and with up to 4 concurrent SPARQL requests permitted, an attacker could exhaust server memory. Added a 50,000 node/edge cap (`_SPARQL_MAX_GRAPH_NODES`); oversized graphs now return a clean error instead of attempting materialization
|
||||
- **Cypher injection via Apache AGE's `graph_name` and `$$`-delimiter breakout**: `graph_name` was interpolated unvalidated into `cypher('{graph_name}', $$ ... $$)`, and raw Cypher query text containing `$$` could close AGE's dollar-quoted string delimiter early and append arbitrary SQL. `graph_name` is now validated against the same identifier allowlist `age_store.py` already used for labels/relationship types, and any query containing `$$` is rejected outright
|
||||
- **SPARQL Explorer route (`/api/sparql`) hardened against comment/PREFIX-hiding bypass**: `_is_read_only_query()` now strips comments and PREFIX/BASE declarations before checking the leading keyword, and additionally scans the full query body for SPARQL Update keywords (INSERT/DELETE/DROP/LOAD/CLEAR/CREATE/COPY/MOVE/ADD) — so `SELECT ... ; DROP ALL` is now rejected by the keyword scan itself rather than relying solely on rdflib's parser
|
||||
- **Fixed along the way** (maintainer follow-up, addressing automated review findings and a regression introduced across several rounds of iteration on the original fix):
|
||||
- `VectorStore.save()`'s numpy handling used `list(v)` for the JSON fallback path, which produces `numpy.float32` elements that `json.dump()` can't serialize — changed to `v.tolist()`
|
||||
- the SPARQL graph-size `ValueError` was raised outside `execute_sparql()`'s exception handling and surfaced as an unhandled 500 instead of a clean API error — moved inside
|
||||
- every streamed `requests` response in the ontology redirect loop, including the one actually read and returned, is now closed in a `finally` block — a connection-pool leak that a rework of the redirect logic had briefly reintroduced after an earlier fix
|
||||
- a later commit meant to add opt-in API-key auth (`explorer/auth.py`, gated on `EXPLORER_API_KEY`) instead **replaced and silently disabled** the `Depends(require_auth)` enforcement already merged into `main` for GHSA-j4mq-hprp-987v (Critical — unauthenticated Explorer API), removed the `/ws/graph-updates` handshake check, and — unlike `require_auth` — failed *open* (allowed all requests) whenever its key was unset. Merging that version would have silently reverted an already-fixed Critical CVE the moment this branch landed. Removed `explorer/auth.py`; restored the per-router `Depends(require_auth)` wiring and the WebSocket auth check; kept the one genuine improvement in that commit (adding `X-API-Key` to the CORS `allow_headers` list) by folding it into the existing CORS config
|
||||
- the new SPARQL keyword-scan's comment-stripping regex (`#[^\n]*`) also matched the `#` inside standard RDF namespace IRIs (e.g. `.../1999/02/22-rdf-syntax-ns#`), corrupting any query with a normal `rdf:`/`rdfs:`-style `PREFIX` declaration — caught because the hardening's own bundled tests failed against two of its own cases. Fixed by only treating `#` as a comment-start at line-start or after whitespace; the companion `PREFIX`/`BASE` regex was also fixed to accept bare `BASE <...>` declarations, which have no prefix-name token between the keyword and the IRI
|
||||
- New/updated regression tests: `tests/explorer/test_ontology_ssrf.py` (redirect re-validation, relative-redirect resolution, response closing, redirect-cap enforcement), `tests/test_security_regression.py` (Cypher/SPARQL injection, XXE, numpy serialization, SSRF redirect handling), plus additions to `tests/explorer/test_sparql_route.py`, `tests/vector_store/test_vector_store.py`, and `tests/explorer/test_explorer_auth.py`
|
||||
- Note: the Cypher-injection hardening here is scoped to `age_store.py`'s `graph_name`/`$$` breakout, found while reviewing this PR. The broader label/property-key/relationship-type injection across the Neptune, Neo4j, and FalkorDB backends (GHSA-482h-hw99-h62p, #910) and the triplet-store SPARQL injection across Blazegraph/RDF4J/Jena (GHSA-8vgg-8mr4-r236, #911) are covered by separate, still-open PRs, as is the unauthenticated-Explorer-API fix referenced above (GHSA-j4mq-hprp-987v, #909, already merged)
|
||||
|
||||
- **CI/CD supply-chain hardening against mutable-tag Action compromise (LiteLLM/Trivy-class attack)** (#824) by @KaifAhmad1
|
||||
- Every third-party GitHub Action across all 8 workflows is now pinned to a full commit SHA instead of a mutable tag (`@v7` → `@3d3c42e... # v7`), closing the exact vector used against LiteLLM in March 2026 (a compromised Trivy Action tag stole a long-lived publishing token)
|
||||
- Added `verify-action-pins.yml` + `.github/scripts/verify-action-pins.sh`: a CI check that fails closed on any `uses:` reference that isn't a full SHA (catching a newly introduced mutable tag, not just auditing existing pins) and re-verifies every pin against the GitHub API on each workflow change, on push to `main`, and weekly; an unresolvable API lookup is treated as a failure rather than a silent skip
|
||||
- `release.yml`: scoped `permissions` to the job level (workflow default is now `contents: read`), added a `concurrency` group so simultaneous tag pushes can't race the publish job, and added SLSA build provenance attestation (`actions/attest-build-provenance`) for every released wheel
|
||||
- Created a protected `pypi` GitHub Environment (required reviewer, restricted to `v*` tag deployments) and enabled branch protection on `main` (required PR review with stale-approval dismissal, required status checks, no force-push/deletion, required conversation resolution) — PyPI publishing already used Trusted Publishing (OIDC) with no long-lived token
|
||||
- Grouped Dependabot's `github-actions` updates into a single PR
|
||||
|
||||
- **`security-scan.yml`'s Safety dependency-vulnerability check was silently non-functional** (#824) by @KaifAhmad1
|
||||
- `safety check --json --output safety-report.json` is invalid in Safety 3.x (`--output` now selects a console format, not a file path); the command errored on every run, swallowed by `|| true`, so no report was ever produced and the job always fell back to a generic "scan completed" message with the vulnerability count hardcoded to 0
|
||||
- Switched to `--save-json`, the correct flag for writing a JSON report to disk; also fixed `vuln.package` → `vuln.package_name` and Semgrep's `issue.rule_id` → `issue.check_id` (both produced `undefined` in the PR comment)
|
||||
- The job never installed Semantica's own dependencies before scanning, so Safety was auditing the scanner tools' own transitive deps, not the project's; added `pip install -e ".[llm-litellm]"` so the actual dependency tree — including the LiteLLM extra — is what gets scanned
|
||||
- Rewrote the PR-comment builder: every line previously used `\\n` inside JS template literals, which renders as the literal text `\n` rather than a newline, producing an unreadable wall of text; now builds real line arrays and collapses long finding lists into a `<details>` block
|
||||
- Added the `pull-requests: write` permission the comment-posting step was missing (silently failing via its own try/catch on every prior run)
|
||||
|
||||
- **`pypdf2==3.0.1` removed (CVE-2023-36464)** (#824) by @KaifAhmad1
|
||||
- Surfaced by the Safety fix above: PyPDF2 is a discontinued project (merged into `pypdf`) permanently frozen at the vulnerable 3.0.1 with no patched release possible. `grep -rn "import PyPDF2"` found zero real usages anywhere in the codebase — it was only referenced in docstrings describing a `PyPDF2.PdfReader()` fallback for PDF parsing that was never actually implemented (`pdfplumber` does the real work). Removed the dependency and corrected the stale docstrings in `parse/__init__.py`, `parse/methods.py`, `parse/pdf_parser.py`, and `ingest/email_ingestor.py`
|
||||
|
||||
- **10 Bandit B324 false positives suppressed (non-cryptographic MD5 use)** (#824) by @KaifAhmad1
|
||||
- Surfaced by the same Safety fix restoring a working CI gate: Bandit's HIGH-severity check was blocking on 10 pre-existing `hashlib.md5()` calls, all generating short deterministic cache keys, entity IDs, or IRI suffixes from non-secret input — none used for passwords, tokens, or verifying untrusted data
|
||||
- Bandit's own message suggests `usedforsecurity=False`, but that keyword argument needs Python 3.9+ and `pyproject.toml` declares `requires-python = ">=3.8"`; used a targeted `# nosec B324` with a one-line justification instead, which suppresses only this check with no runtime behavior change on any supported Python version
|
||||
|
||||
## [0.6.0] - 2026-07-21
|
||||
|
||||
### Added
|
||||
|
||||
+84
-13
@@ -2,20 +2,58 @@
|
||||
|
||||
Thank you for your interest in contributing! Every contribution, no matter how small, is valuable. 🎉
|
||||
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/Hawksight-AI/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/semantica-agi/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
|
||||
> **New to contributing?** Start with a [`good first issue`](https://github.com/Hawksight-AI/semantica/labels/good%20first%20issue) or join our [Discord](https://discord.gg/sV34vps5hH) community.
|
||||
> **New to contributing?** Start with a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue) or join our [Discord](https://discord.gg/sV34vps5hH) community.
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
1. Find a [`good first issue`](https://github.com/Hawksight-AI/semantica/labels/good%20first%20issue)
|
||||
2. [Fork Semantica](https://github.com/Hawksight-AI/semantica/fork) & clone the repository
|
||||
1. Find a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue)
|
||||
2. [Fork Semantica](https://github.com/semantica-agi/semantica/fork) & clone the repository
|
||||
3. Make your changes
|
||||
4. Submit a pull request!
|
||||
|
||||
**Need help?** Join [Discord](https://discord.gg/sV34vps5hH) or [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
**Need help?** Join [Discord](https://discord.gg/sV34vps5hH) or [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions)
|
||||
|
||||
---
|
||||
|
||||
## 🗂️ Working on an Existing Issue
|
||||
|
||||
If you want to work on an open GitHub issue, please follow these steps to keep things coordinated and avoid duplicate effort:
|
||||
|
||||
1. **Check the issue.** Look at the issue's assignees and recent comments. If someone is already actively working on it, consider a different issue or ask in the comments whether help is welcome.
|
||||
|
||||
2. **Comment if you'd like the issue reserved.** Leaving a comment like *"I'd like to take this on"* is the fastest way to get assigned, but it isn't required — maintainers can also assign an issue directly to a contributor (e.g., based on recent activity in the repo) without waiting for a comment first.
|
||||
|
||||
3. **Wait for assignment.** A maintainer will assign the issue when appropriate, whether or not a comment was left. Please wait for this before investing significant time in implementation, as priorities and approaches can shift.
|
||||
|
||||
4. **Create a branch and implement.** Once assigned, fork the repository (if you haven't already), create a dedicated branch, and begin your work.
|
||||
|
||||
```bash
|
||||
git checkout -b fix/short-description # or feature/short-description
|
||||
```
|
||||
|
||||
5. **Open a focused PR and link the issue.** When you're ready, open a pull request and reference the issue in the description (e.g., `Closes #123`). Keep the PR scoped to the work described in the issue.
|
||||
|
||||
> **Why this matters:** Assignment (with or without a comment) helps maintainers track who is working on what and prevent two contributors from solving the same problem independently. It also gives you a chance to align on the expected approach before writing code.
|
||||
|
||||
Not sure where to start? Try a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue) or ask in [Discord](https://discord.gg/sV34vps5hH).
|
||||
|
||||
---
|
||||
|
||||
## 🔀 Duplicate PRs & Issue Priority
|
||||
|
||||
When more than one pull request targets the same issue, maintainers triage using this order of priority. These rules decide between PRs that are otherwise following the [assignment workflow above](#-working-on-an-existing-issue) — opening a PR before being assigned doesn't grant priority on its own, and an unassigned PR can still be closed as a duplicate once someone else is assigned to the issue.
|
||||
|
||||
1. **Contributor-raised issue with an existing PR.** If the person who opened the issue has also opened a PR for it, that PR is prioritized (they still need to be assigned before it's merged).
|
||||
2. **Maintainer-raised issue with a claim comment.** If we opened the issue and someone has commented asking to work on it, we assign it to them and check their PR before picking up any other PR for the same issue.
|
||||
3. **No prior assignment or comment.** If multiple PRs exist and no one was assigned or claimed the issue first, priority goes to whichever contributor has the most consistent activity in the repo over the last 60 days (e.g., merged PRs, substantive reviews, or issue triage participation) — not just PR volume.
|
||||
4. **Late duplicate PRs.** If a PR is opened after another contributor has already been assigned to the issue, we close the duplicate early rather than let it sit open, and point the author to another open issue (or ask them to check `main` for newly opened ones). This avoids contributors spending time updating a PR that won't be merged.
|
||||
5. **Overlapping scope.** If a PR covers multiple issues, or there's genuine overlap between competing PRs, maintainers discuss it on [Discord](https://discord.gg/sV34vps5hH) before deciding rather than resolving it unilaterally.
|
||||
|
||||
**Why this matters:** it keeps triage predictable, avoids wasted contributor effort on PRs that won't merge, and helps retain active contributors.
|
||||
|
||||
---
|
||||
|
||||
@@ -78,7 +116,7 @@ Thank you for your interest in contributing! Every contribution, no matter how s
|
||||
|
||||
**What:** Report bugs you find
|
||||
|
||||
**How:** Use the [bug report template](https://github.com/Hawksight-AI/semantica/issues/new?template=bug_report.md)
|
||||
**How:** Use the [bug report template](https://github.com/semantica-agi/semantica/issues/new?template=bug_report.md)
|
||||
|
||||
**Include:** Description, steps to reproduce, expected vs actual behavior, environment details
|
||||
|
||||
@@ -88,7 +126,7 @@ Thank you for your interest in contributing! Every contribution, no matter how s
|
||||
|
||||
**What:** Suggest new features or improvements
|
||||
|
||||
**How:** Use the [feature request template](https://github.com/Hawksight-AI/semantica/issues/new?template=feature_request.md)
|
||||
**How:** Use the [feature request template](https://github.com/semantica-agi/semantica/issues/new?template=feature_request.md)
|
||||
|
||||
**Include:** Problem statement, proposed solution, use cases
|
||||
|
||||
@@ -108,7 +146,7 @@ Thank you for your interest in contributing! Every contribution, no matter how s
|
||||
|
||||
**What:** Help others in the community
|
||||
|
||||
**Where:** [Discord](https://discord.gg/sV34vps5hH), [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
**Where:** [Discord](https://discord.gg/sV34vps5hH), [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions)
|
||||
|
||||
**Examples:** Answer questions, review PRs, share your projects
|
||||
|
||||
@@ -135,12 +173,12 @@ Thank you for your interest in contributing! Every contribution, no matter how s
|
||||
|
||||
### 1. Fork & Clone
|
||||
|
||||
First, [fork Semantica](https://github.com/Hawksight-AI/semantica/fork) on GitHub, then:
|
||||
First, [fork Semantica](https://github.com/semantica-agi/semantica/fork) on GitHub, then:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/your-username/semantica.git
|
||||
cd semantica
|
||||
git remote add upstream https://github.com/Hawksight-AI/semantica.git
|
||||
git remote add upstream https://github.com/semantica-agi/semantica.git
|
||||
```
|
||||
|
||||
### 2. Set Up Environment
|
||||
@@ -157,6 +195,39 @@ pip install -e ".[dev]"
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
### Pinned CI dependencies
|
||||
|
||||
`requirements-ci.txt` pins every transitive dependency at exact versions so CI,
|
||||
security scans, and release builds install the same packages every run (the
|
||||
Python equivalent of `explorer/package-lock.json` + `npm ci`). It is a
|
||||
**separate build environment**: every package carries a SHA-256 hash
|
||||
(`--generate-hashes`), so installs are reproducible and supply-chain safe —
|
||||
never install into your local dev environment from it.
|
||||
|
||||
Regenerate it after changing `pyproject.toml` dependencies:
|
||||
|
||||
```bash
|
||||
pip install uv==0.12.1
|
||||
uv pip compile pyproject.toml --python-version 3.11 --extra all --generate-hashes -o requirements-ci.txt
|
||||
```
|
||||
|
||||
The `all` extra is the repo's cross-platform dependency set (GPU extras like
|
||||
`faiss-gpu`/`cupy` are excluded and installed separately on Linux — see
|
||||
`pyproject.toml`). Keep the pinned `uv` version in sync with CI so regeneration
|
||||
is deterministic.
|
||||
|
||||
CI's staleness check re-resolves with the committed lockfile as a constraint
|
||||
and compares version lines only: upstream package releases never fail CI —
|
||||
the lockfile changes only when `pyproject.toml` changes intentionally.
|
||||
|
||||
CI fails if `requirements-ci.txt` is stale relative to `pyproject.toml`
|
||||
(the version-line comparison detects new/removed/changed dependencies).
|
||||
|
||||
Build-system pins: `[build-system].requires` is pinned to exact versions
|
||||
(`setuptools==84.0.0`, `wheel==0.48.0`) and release builds run
|
||||
`python -m build --no-isolation` against the lockfile — no unpinned
|
||||
build-time isolation anywhere.
|
||||
|
||||
### 3. Create Branch
|
||||
|
||||
```bash
|
||||
@@ -327,8 +398,8 @@ result = instance.method()
|
||||
## 🆘 Getting Help
|
||||
|
||||
- 💬 [Discord](https://discord.gg/sV34vps5hH) - Real-time chat
|
||||
- 💭 [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions) - Q&A
|
||||
- 🐛 [GitHub Issues](https://github.com/Hawksight-AI/semantica/issues) - Bug reports
|
||||
- 💭 [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions) - Q&A
|
||||
- 🐛 [GitHub Issues](https://github.com/semantica-agi/semantica/issues) - Bug reports
|
||||
|
||||
**Before asking:** Check existing documentation, search issues/discussions, review cookbook examples
|
||||
|
||||
@@ -363,4 +434,4 @@ This project follows a [Code of Conduct](CODE_OF_CONDUCT.md). Be respectful and
|
||||
|
||||
Every contribution matters - whether it's a single line of code, a typo fix, a helpful answer, or a bug report. We appreciate you! 🙏
|
||||
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/Hawksight-AI/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/semantica-agi/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
|
||||
+1
-1
@@ -101,7 +101,7 @@ When using the all-contributors bot, use these codes:
|
||||
- `infra` - Infrastructure
|
||||
- `maintenance` - Maintenance
|
||||
|
||||
See [all-contributors specification](https://allcontributors.org/docs/en/emoji-key) for complete list.
|
||||
See [all-contributors specification](https://github.com/all-contributors/all-contributors#emoji-key) for complete list.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
recursive-include semantica/static *
|
||||
recursive-include semantica/ontology/vocabulary *.ttl
|
||||
|
||||
@@ -2,6 +2,16 @@
|
||||
|
||||
<img src="Semantica Logo.png" alt="Semantica" width="420"/>
|
||||
|
||||
<div style="display:flex; gap:10px; align-items:center; flex-wrap:wrap;">
|
||||
<a href="https://trendshift.io/repositories/18986?utm_source=repository-badge&utm_medium=badge&utm_campaign=badge-repository-18986" target="_blank" rel="noopener noreferrer">
|
||||
<img src="https://trendshift.io/api/badge/repositories/18986" alt="semantica-agi/semantica | Trendshift" width="250" height="55"/>
|
||||
</a>
|
||||
|
||||
<a href="https://trendshift.io/repositories/18986?utm_source=trendshift-badge&utm_medium=badge&utm_campaign=badge-trendshift-18986" target="_blank" rel="noopener noreferrer">
|
||||
<img src="https://trendshift.io/api/badge/trendshift/repositories/18986/weekly?language=Python" alt="semantica-agi/semantica | Trendshift" width="250" height="55"/>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
### Graph-Native Infrastructure for Context and Accountable AI Systems
|
||||
|
||||
#### *The Open Source Palantir for AI Agents*
|
||||
@@ -20,6 +30,10 @@
|
||||
|
||||
[](https://getsemantica.ai/) [](https://docs.getsemantica.ai/) [](https://discord.gg/sV34vps5hH) [](https://x.com/BuildSemantica) [](https://www.youtube.com/watch?v=QfnNZg4-dZA) [](CHANGELOG.md)
|
||||
|
||||
```bash
|
||||
pip install semantica
|
||||
```
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
@@ -46,9 +60,12 @@ Most AI agents act without a trail. They store embeddings, not meaning: context
|
||||
|
||||
Semantica sits underneath your LLM, vector store, and agent framework as a deterministic infrastructure layer: no LLM required for graph construction, reasoning, or provenance.
|
||||
|
||||
> ⚠️ **System-level explainability, not foundation-model explainability.** Semantica does not expose or reconstruct what happens *inside* the LLM — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. Semantica explains what's *outside* the model: the context and data fed in, the decision produced, its provenance, relevant relationships, applied policies, and the full execution trail.
|
||||
|
||||
**Who it's for:**
|
||||
|
||||
- **AI/ML platform teams** shipping agents that make consequential decisions and need structured, queryable context built from fragmented raw data, not just a vector index
|
||||
- **Data platform teams on Databricks or Snowflake** who need to turn tables already sitting in Unity Catalog or a Snowflake warehouse into a governed, lineage-tracked knowledge graph, without exporting that data to a third-party SaaS first
|
||||
- **Compliance, risk, and audit teams** who need a straight answer to "why did the AI do that?" in a format a regulator will actually accept
|
||||
- **Regulated enterprises** (finance, healthcare, legal, government, defense) that can't ship a black box, and can't send their data to someone else's SaaS to get one
|
||||
- **Platform and infra engineers** who want the KG, reasoning, and provenance stack self-hosted and swappable, not locked to one vendor's backend
|
||||
@@ -66,10 +83,11 @@ Semantica sits underneath your LLM, vector store, and agent framework as a deter
|
||||
- **Full Auditability:** W3C PROV-O provenance on every fact, with audit trails exportable to JSON, CSV, or RDF
|
||||
- **Deterministic Reasoning:** Forward chaining, Rete network, Datalog, and SPARQL with fully explainable paths, not black boxes
|
||||
- **Knowledge Pipeline:** Multi-source ingestion, entity-aware chunking, NER/relation/event extraction, and knowledge graph construction, with semantic deduplication and provenance-preserving merges throughout
|
||||
- **Enterprise Data Platforms:** Native connectors for Databricks (Unity Catalog + Delta Lake, PAT/OAuth M2M auth, catalog/schema/table/lineage introspection) and Snowflake (warehouse/database/schema, key-pair and OAuth auth), so tables already living in your lakehouse or warehouse become graph nodes with provenance, not another export/import hop
|
||||
- **Graph Analytics:** Centrality, community detection, link prediction, and shortest-path queries over the graph you just built
|
||||
- **Polyglot Graph Storage:** Native RDF (Blazegraph, Apache Jena, Eclipse RDF4J via SPARQL) and Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune via Cypher), plus vector stores, all swappable without touching your code
|
||||
- **Polyglot Graph Storage:** Native RDF (embedded Oxigraph, Blazegraph, Apache Jena, Eclipse RDF4J via SPARQL) and Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune via Cypher), plus vector stores, all swappable without touching your code
|
||||
- **Visualization:** Explore any graph, ontology, or timeline in an interactive browser workbench
|
||||
- **Drop-in Integrations:** Native Agno support, a full-featured MCP server, a comprehensive CLI, a REST API, and plugins across major editors
|
||||
- **Drop-in Integrations:** Native Agno and CrewAI support, a full-featured MCP server, a comprehensive CLI, a REST API, and plugins across major editors
|
||||
|
||||
---
|
||||
|
||||
@@ -124,7 +142,7 @@ compliant = graph.check_decision_rules({"category": "vendor_selection"}) # poli
|
||||
```bash
|
||||
semantica doctor
|
||||
# Python 3.11.9 pass
|
||||
# semantica 0.6.0 pass
|
||||
# semantica 0.6.6 pass
|
||||
# faiss vector store pass
|
||||
# Config file pass ~/.semantica/config.yaml
|
||||
```
|
||||
@@ -154,7 +172,7 @@ Sources → Ingest → Parse → Normalize → Split → Extract → Conflict De
|
||||
- **Extract → Conflict Detection → Deduplication:** NER, relations, events, triplets; conflicting facts flagged and resolved before they merge
|
||||
- **Knowledge Graph:** `GraphBuilder` constructs the graph; bi-temporal facts and full graph analytics (centrality, communities, link prediction) run on top of it
|
||||
- **Ontology · Reasoning · Provenance · Decisions:** the intelligence layer sitting on the KG, with SHACL/OWL governance, Rete/Datalog/SPARQL inference, W3C PROV-O lineage, and first-class decision records
|
||||
- **Storage:** polyglot by design, with RDF triple stores (Blazegraph, Apache Jena, Eclipse RDF4J), Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune), and vector stores, all swappable without touching your code
|
||||
- **Storage:** polyglot by design, with RDF triple stores (embedded Oxigraph, Blazegraph, Apache Jena, Eclipse RDF4J), Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune), and vector stores, all swappable without touching your code
|
||||
- **Outputs:** export (RDF, OWL, Parquet, Cypher, JSON-LD), interactive visualization, and access via REST API, MCP server, or CLI
|
||||
|
||||
**→ [Full Mermaid diagrams for the pipeline and the decision intelligence lifecycle](ARCHITECTURE.md)**
|
||||
@@ -285,17 +303,10 @@ graph.add_causal_relationship(d1, d2, relationship_type="CAUSED")
|
||||
prov.track_entity("patient_P4821", source="ehr/medication_orders_2024.json",
|
||||
metadata={"extractor": "NamedEntityRecognizer"})
|
||||
|
||||
# Export W3C PROV-O for regulator submission - RDFExporter expects
|
||||
# {"entities": [...], "relationships": [...]}, so map ContextGraph.to_dict()'s
|
||||
# {"nodes": [...], "edges": [...]} shape onto it first
|
||||
graph_dict = graph.to_dict()
|
||||
kg = {
|
||||
"entities": [{"id": n["id"], "type": n["type"], "text": n["content"]} for n in graph_dict["nodes"]],
|
||||
"relationships": [
|
||||
{"source_id": e["source"], "target_id": e["target"], "type": e["type"]}
|
||||
for e in graph_dict["edges"]
|
||||
],
|
||||
}
|
||||
# Export W3C PROV-O for regulator submission - to_kg_dict() is the official
|
||||
# adapter that emits the {"entities": [...], "relationships": [...]} /
|
||||
# source_id shape RDFExporter expects, so no manual field mapping is needed
|
||||
kg = graph.to_kg_dict()
|
||||
RDFExporter().export(kg, "audit_trail.ttl", format="turtle")
|
||||
```
|
||||
|
||||
@@ -309,7 +320,7 @@ Every module below is independently importable, with working code samples verifi
|
||||
|
||||
| Module | What it does |
|
||||
| --- | --- |
|
||||
| [`semantica.ingest`](#semanticaingest-multi-source-ingestion) | Files, web, databases, APIs, streams, email, Git, Parquet, Snowflake, MCP |
|
||||
| [`semantica.ingest`](#semanticaingest-multi-source-ingestion) | Files, web, databases, APIs, streams, email, Git, Parquet, Databricks, Snowflake, MCP |
|
||||
| [`semantica.semantic_extract`](#semanticasemantic_extract-ner-relations-events-triplets) | NER, relation extraction, event detection, triplet generation |
|
||||
| [`semantica.kg`](#semanticakg-knowledge-graph-construction--analysis) | Graph construction, centrality, communities, link prediction |
|
||||
| [`semantica.reasoning`](#semanticareasoning-forward-chaining-rete-datalog-sparql) | Forward chaining, Rete, Datalog, SPARQL, fully explainable |
|
||||
@@ -360,9 +371,38 @@ rows = DBIngestor().ingest_database(
|
||||
)
|
||||
```
|
||||
|
||||
**Supported sources:** Local files (PDF, DOCX, PPTX, HTML, TXT, CSV, JSON, YAML, Excel, XML) · Web pages · RSS/Atom feeds · REST APIs · Databases (PostgreSQL, MySQL, SQLite, Oracle, SQL Server) · Parquet datasets · Snowflake · Git repositories · Email (IMAP/POP3) · Message streams (Kafka, RabbitMQ, Kinesis, Pulsar) · MCP resources · Apache Arrow/Feather/IPC (`ArrowIngestor`)
|
||||
```python
|
||||
# Enterprise data platforms - pull tables straight out of your lakehouse
|
||||
# or warehouse, with lineage, instead of exporting to CSV first
|
||||
from semantica.ingest import DatabricksIngestor, SnowflakeIngestor
|
||||
|
||||
Elasticsearch and Google Drive ingestion also ship (`ElasticIngestor`, `GDriveIngestor`) but aren't re-exported from the top-level `semantica.ingest` namespace yet — import them directly: `from semantica.ingest.elastic_ingestor import ElasticIngestor`.
|
||||
# pip install "semantica[db-databricks]"
|
||||
databricks = DatabricksIngestor(
|
||||
host="https://adb-xxx.azuredatabricks.net",
|
||||
token="dapi-xxxxxxxx", # or client_id/client_secret for OAuth M2M
|
||||
http_path="/sql/1.0/warehouses/xxxxxxxx",
|
||||
catalog="main",
|
||||
)
|
||||
customers = databricks.ingest_table("customers", limit=10_000)
|
||||
sales = databricks.ingest_query("SELECT * FROM sales WHERE region = 'EMEA'")
|
||||
table_lineage = databricks.get_table_lineage("customers", catalog="main", schema="default") # Unity Catalog lineage
|
||||
|
||||
# pip install semantica[db-snowflake]
|
||||
snowflake = SnowflakeIngestor(
|
||||
account="myaccount",
|
||||
user="myuser",
|
||||
password="mypassword", # or private_key=... for key-pair; use authenticator="oauth", token=... for OAuth
|
||||
warehouse="COMPUTE_WH",
|
||||
database="MYDB",
|
||||
)
|
||||
orders = snowflake.ingest_table("ORDERS", limit=10_000)
|
||||
```
|
||||
|
||||
> **Security Note:** Never hardcode credentials (`token`, `password`, `private_key`) in production code; pass them via environment variables (e.g., `DATABRICKS_TOKEN`, `SNOWFLAKE_PASSWORD`) or a secrets manager.
|
||||
|
||||
**Supported sources:** Local files (PDF, DOCX, PPTX, HTML, TXT, CSV, JSON, YAML, Excel, XML) · Web pages · RSS/Atom feeds · REST APIs · Databases (PostgreSQL, MySQL, SQLite, Oracle, SQL Server) · Parquet datasets · Databricks (Unity Catalog + Delta Lake) · Snowflake · Git repositories · Email (IMAP/POP3) · Message streams (Kafka, RabbitMQ, Kinesis, Pulsar) · MCP resources · Apache Arrow/Feather/IPC (`ArrowIngestor`)
|
||||
|
||||
DuckDB, Elasticsearch, Google Drive, HuggingFace, MongoDB, and Pandas ingestion also ship (`DuckDBIngestor`, `ElasticIngestor`, `GDriveIngestor`, `HuggingFaceIngestor`, `MongoIngestor`, `PandasIngestor`) but aren't re-exported from the top-level `semantica.ingest` namespace yet — import them directly: `from semantica.ingest.duckdb_ingestor import DuckDBIngestor`.
|
||||
|
||||
</details>
|
||||
|
||||
@@ -840,20 +880,14 @@ fact = BiTemporalFact(
|
||||
recorded_at=datetime(2024, 3, 5),
|
||||
)
|
||||
|
||||
# Query facts valid within a time window - query_time_range() expects
|
||||
# {"relationships": [...]} with source_id/target_id keys, which differs from
|
||||
# ContextGraph.to_dict()'s {"nodes", "edges"} shape, so map it first
|
||||
graph_dict = graph.to_dict()
|
||||
kg_relationships = {
|
||||
"relationships": [
|
||||
{**e, "source_id": e["source"], "target_id": e["target"]}
|
||||
for e in graph_dict["edges"]
|
||||
]
|
||||
}
|
||||
# Query facts valid within a time window - to_kg_dict() is the official
|
||||
# adapter that emits {"entities", "relationships"} with source_id/target_id
|
||||
# keys, the shape query_time_range() expects (no manual mapping required)
|
||||
kg = graph.to_kg_dict()
|
||||
|
||||
tq = TemporalGraphQuery()
|
||||
facts_in_window = tq.query_time_range(
|
||||
kg_relationships, query="valid_facts", start_time="2024-01-01", end_time="2024-12-31"
|
||||
kg, query="valid_facts", start_time="2024-01-01", end_time="2024-12-31"
|
||||
)
|
||||
|
||||
# Normalize natural language temporal expressions - returns a (start, end) range
|
||||
@@ -1110,7 +1144,8 @@ if report.valid:
|
||||
| **Ontology Hub** | SHACL Studio · visual editor · cross-ontology alignments · health dashboard |
|
||||
| **Vector Store** | FAISS · Pinecone · Weaviate · Qdrant · Milvus · PgVector · hybrid + filtered search |
|
||||
| **Graph Databases (LPG)** | Neo4j · FalkorDB · Apache AGE · AWS Neptune |
|
||||
| **Triple Stores (RDF)** | Blazegraph · Apache Jena · Eclipse RDF4J · unified `TripletStore` interface · SPARQL query & bulk load |
|
||||
| **Triple Stores (RDF)** | Oxigraph (embedded) · Blazegraph · Apache Jena · Eclipse RDF4J · unified `TripletStore` interface · SPARQL query & bulk load |
|
||||
| **Enterprise Data Platforms** | Databricks (`DatabricksIngestor`: Unity Catalog + Delta Lake, PAT/OAuth M2M, table/query ingestion, catalog/schema/table/lineage introspection) · Snowflake (`SnowflakeIngestor`: warehouse/database/schema, password/key-pair/OAuth auth) |
|
||||
| **LLM Providers** | **All already supported today:** OpenAI (GPT-4o, o1, o3) · Anthropic (Claude) · Google Gemini · Mistral · Meta Llama · Groq · Cohere · Azure OpenAI · AWS Bedrock · Ollama · DeepSeek · Perplexity · Together AI · Fireworks AI · Replicate · HuggingFace · via `semantica.llms` and LiteLLM |
|
||||
|
||||
---
|
||||
@@ -1151,7 +1186,7 @@ Start with `semantica`, verify with `doctor`, build a graph, and explore the com
|
||||
|
||||
## Integrations
|
||||
|
||||
Native plugin bundles for Claude Code, Cursor, Codex, Windsurf, Cline, Continue, VS Code, and OpenClaw; a full-featured MCP server for any MCP-compatible client; a comprehensive REST API; and first-class Agno support for multi-agent shared context. Every major LLM provider is already supported via `semantica.llms` and LiteLLM: OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Azure, Bedrock, Ollama, DeepSeek, HuggingFace, and more.
|
||||
Native plugin bundles for Claude Code, Cursor, Codex, Windsurf, Cline, Continue, VS Code, and OpenClaw; a full-featured MCP server for any MCP-compatible client; a comprehensive REST API; and first-class Agno and CrewAI support for agentic frameworks. Every major LLM provider is already supported via `semantica.llms` and LiteLLM: OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Azure, Bedrock, Ollama, DeepSeek, HuggingFace, and more.
|
||||
|
||||
MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
|
||||
@@ -1265,6 +1300,11 @@ MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
<strong>Agno</strong><br/>
|
||||
<sub>First-class · <code>pip install semantica[agno]</code></sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/crewAIInc/crewAI"><img src="https://github.com/crewAIInc.png?size=120" alt="CrewAI" width="48" height="48" /></a><br/>
|
||||
<strong>CrewAI</strong><br/>
|
||||
<sub>First-class · <code>pip install semantica[crewai]</code></sub>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th colspan="8" align="left">Already Supported via REST API & MCP</th>
|
||||
@@ -1281,11 +1321,6 @@ MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
<sub>REST API · MCP</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/crewAIInc/crewAI"><img src="https://github.com/crewAIInc.png?size=120" alt="CrewAI" width="48" height="48" /></a><br/>
|
||||
<strong>CrewAI</strong><br/>
|
||||
<sub>REST API · MCP</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/run-llama/llama_index"><img src="https://github.com/run-llama.png?size=120" alt="LlamaIndex" width="48" height="48" /></a><br/>
|
||||
<strong>LlamaIndex</strong><br/>
|
||||
<sub>REST API · MCP</sub>
|
||||
@@ -1316,11 +1351,6 @@ MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
<sub>Dedicated toolkit</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/crewAIInc/crewAI"><img src="https://github.com/crewAIInc.png?size=120" alt="CrewAI" width="48" height="48" /></a><br/>
|
||||
<strong>CrewAI</strong><br/>
|
||||
<sub>Dedicated toolkit</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/run-llama/llama_index"><img src="https://github.com/run-llama.png?size=120" alt="LlamaIndex" width="48" height="48" /></a><br/>
|
||||
<strong>LlamaIndex</strong><br/>
|
||||
<sub>Dedicated toolkit</sub>
|
||||
@@ -1436,12 +1466,18 @@ For contributor / dev-server setup: **[explorer/README.md: Local Setup Guide](ex
|
||||
|
||||
---
|
||||
|
||||
## What's New in v0.6.0
|
||||
## What's New in v0.6.6
|
||||
|
||||
- **Named-Graph Support for `JenaStore`:** Migrated onto `rdflib.Dataset(default_union=False)`, completing cross-backend named-graph parity across Blazegraph, RDF4J, and Jena; `add_triplets()` gains a `graph=` option
|
||||
- **SPARQL CONSTRUCT Query Templates:** Parameterized, injection-safe `CONSTRUCT` templates extended from Blazegraph-only to RDF4J and Jena, plus pipeline integration via the `construct_template` step type
|
||||
- **Databricks Connector:** `DatabricksIngestor` for Unity Catalog + Delta Lake ingestion, with PAT/OAuth M2M auth, table/query ingestion, and catalog/schema/table/lineage introspection. Install with `pip install "semantica[db-databricks]"`
|
||||
- **SQLite Vector Store Backend:** `SQLiteVecStore`, a disk-backed local vector store on `sqlite-vec`'s `vec0` virtual tables, with Cosine/L2 metrics, metadata filtering, and WAL mode. Install with `pip install semantica[vectorstore-sqlite]`
|
||||
**Security release — upgrading is strongly recommended.** Fixes for a privately disclosed batch of vulnerabilities spanning backup/restore, database export, outbound requests, and triplet-store backends, plus SSRF hardening across ingestion:
|
||||
|
||||
- **Tarball restore path traversal**: `semantica backup restore` now validates every archive member for path containment and rejects symlink/hardlink escapes before extraction
|
||||
- **Latent SQL injection in `DataExporter.export_table_data()`**: table/schema names are now identifier-allowlisted and `where`/`order_by` fragments are blocklist-checked
|
||||
- **DNS-rebinding TOCTOU in the shared SSRF guard**: the resolved IP that passes validation is now the one the connection is pinned to, closing the check-then-use race (also closes the `100.64.0.0/10` CGNAT gap)
|
||||
- **Stored XSS in HTML report generation** and **unvalidated SPARQL object IRIs in AnzoStore** (SPARQL injection): both now escape/validate before interpolation
|
||||
- **`Authorization`/`Proxy-Authorization` credential leakage across redirects**, plus **SSRF gaps in `FeedIngestor`/`FeedMonitor`, `RepoIngestor`, and the MCP/public-API ingest paths**: all now route through the shared, redirect-safe SSRF guard
|
||||
- **HTTP response header injection and an unbounded-memory DoS** in the Explorer API, and a **`fastapi`/`python-multipart` ReDoS** (PYSEC-2024-38): floors raised, inputs sanitized, candidate pools capped
|
||||
|
||||
Also ships: **first-class CrewAI integration** (`semantica[crewai]`, extraction/decision tools + a knowledge source), **`ContextGraph` retraction and purge** (GDPR-style erasure without a full `clear()`), a declared **Semantica RDF vocabulary with deterministic entity/relationship IRIs** (stable, diffable exports), and **timezone-aware timestamps** across `export/` and `provenance/`.
|
||||
|
||||
→ [Full release notes](RELEASE_NOTES.md) · [Changelog](CHANGELOG.md)
|
||||
|
||||
@@ -1459,6 +1495,8 @@ Semantica is designed for environments where AI outputs must be explainable, aud
|
||||
- **Cybersecurity:** Threat attribution, incident response timelines, and IOC provenance tracking
|
||||
- **Autonomous Systems:** Decision logs, safety validation, and explainable AI for certification
|
||||
|
||||
> ⚠️ **This is system-level explainability, not foundation-model explainability.** Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. What Semantica explains is *outside* the model: the context and data fed in, the decision produced, its provenance, the relevant relationships, the policies applied, and the full execution trail. In short, Semantica explains and audits what the AI system did, not the LLM's private internal reasoning.
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
@@ -1470,11 +1508,13 @@ pip install semantica[all] # everything
|
||||
|
||||
```bash
|
||||
pip install semantica[agno] # Agno multi-agent integration
|
||||
pip install semantica[crewai] # CrewAI integration
|
||||
pip install semantica[llm-litellm] # OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Bedrock, Ollama, DeepSeek, and more
|
||||
pip install semantica[graph-neo4j] # Neo4j graph store (LPG)
|
||||
pip install semantica[graph-falkordb] # FalkorDB graph store (LPG)
|
||||
pip install semantica[graph-apache-age] # Apache AGE graph store (LPG)
|
||||
pip install semantica[graph-amazon-neptune] # AWS Neptune graph store (LPG)
|
||||
pip install semantica[tripletstore-oxigraph] # Embedded in-memory/on-disk RDF store
|
||||
# RDF triple stores (Blazegraph, Apache Jena, Eclipse RDF4J) need no extra:
|
||||
# semantica.triplet_store talks SPARQL over HTTP using the core `requests` dependency
|
||||
pip install semantica[vectorstore-qdrant] # Qdrant vector store
|
||||
|
||||
+98
-3
@@ -24,7 +24,7 @@ Security vulnerabilities should be reported privately to prevent potential explo
|
||||
|
||||
### 2. Report Security Issue
|
||||
|
||||
Create a [GitHub Security Advisory](https://github.com/semantica-agi/semantica/security/advisories/new) or contact us through [GitHub Issues](https://github.com/semantica-agi/semantica/issues) with "[SECURITY]" prefix.
|
||||
Create a [GitHub Security Advisory](https://github.com/semantica-agi/semantica/security/advisories/new) or contact us via the security email listed in `SUPPORT.md`.
|
||||
|
||||
Include the following information:
|
||||
|
||||
@@ -37,7 +37,7 @@ Include the following information:
|
||||
|
||||
### 3. Response Timeline
|
||||
|
||||
- **Initial Response**: Within 48 hours
|
||||
- **Initial Response**: Within 24 hours for critical issues; within 48 hours for non-critical issues
|
||||
- **Status Update**: Within 7 days
|
||||
- **Resolution**: Depends on severity and complexity
|
||||
|
||||
@@ -112,6 +112,101 @@ We regularly update dependencies to address security vulnerabilities. However, y
|
||||
- Be cautious with external API calls
|
||||
- Implement proper authentication and authorization
|
||||
|
||||
## CI/CD Supply-Chain Security
|
||||
|
||||
Semantica's build and release pipeline is explicitly hardened against
|
||||
CI/CD supply-chain attacks — the class of attack behind the March 2026
|
||||
LiteLLM/Trivy incident, where a compromised third-party Action with a
|
||||
**mutable tag** was used to steal a long-lived publishing token, after which
|
||||
malicious packages were pushed straight to PyPI without ever touching the
|
||||
source repository. Every control below maps directly to closing one step of
|
||||
that attack chain.
|
||||
|
||||
### Immutable build inputs
|
||||
|
||||
- **Risk**: a tag (`@v4`, `@release/v1`) is re-pointed by a compromised upstream maintainer or account, silently changing what every consumer's CI runs.
|
||||
**Control**: every third-party GitHub Action in every workflow is pinned to a full 40-character commit SHA, with the human-readable tag kept only as a trailing comment (e.g. `actions/checkout@3d3c42e... # v7`).
|
||||
- **Risk**: a SHA pin drifts out of sync with its own comment over time, or is mistyped.
|
||||
**Control**: `verify-action-pins.yml` fails closed on any `uses:` reference that isn't a full commit SHA (catching a newly added mutable tag, not just auditing existing pins), resolves every pinned tag via the GitHub API on each workflow change, on every push to `main`, and weekly, and fails if the SHA no longer matches the tag it claims to be — an API lookup that can't be resolved is treated as a failure, not a silent skip.
|
||||
- **Risk**: manually re-pinning ~15 actions across 8 workflow files on every upstream release is error-prone.
|
||||
**Control**: Dependabot (`github-actions` ecosystem) opens a grouped PR that bumps the SHA *and* the tag comment together whenever an action releases — pins never require hand-editing.
|
||||
|
||||
### Publishing pipeline (highest-privilege path)
|
||||
|
||||
- **Risk**: a long-lived `PYPI_TOKEN` sitting in repo/org secrets is exfiltrated by any compromised step.
|
||||
**Control**: PyPI publishing uses Trusted Publishing (OIDC) (`id-token: write`) — there is no long-lived PyPI credential anywhere in this repository to steal.
|
||||
- **Risk**: a compromised CI run publishes to PyPI with no human in the loop.
|
||||
**Control**: the publish job runs only inside a protected `pypi` GitHub Environment with a required human reviewer — every release needs manual approval in the Actions UI before it runs.
|
||||
- **Risk**: the release job could be triggered from an arbitrary branch/ref.
|
||||
**Control**: the `pypi` environment's deployment-branch policy is restricted to `v*` tags only.
|
||||
- **Risk**: a scanner or unrelated job inherits publish-level credentials.
|
||||
**Control**: `release.yml` sets `permissions: contents: read` at the workflow level; `contents: write` / `id-token: write` / `attestations: write` are granted only to the release job, never workflow-wide.
|
||||
- **Risk**: two tag pushes race through the publish pipeline simultaneously.
|
||||
**Control**: `concurrency: group: release-${{ github.ref }}` serializes releases per tag.
|
||||
- **Risk**: a consumer can't verify a wheel on PyPI actually came from this repo's CI.
|
||||
**Control**: SLSA build provenance is attested for every release via `actions/attest-build-provenance`, producing a signed, verifiable record of the exact commit and workflow run that produced the artifact (checkable with `gh attestation verify`).
|
||||
|
||||
### Repository controls
|
||||
|
||||
- **Risk**: unreviewed or force-pushed changes land on `main`.
|
||||
**Control**: `main` requires 1 approving PR review (stale approvals dismissed on new pushes), resolved conversations, and blocks force-pushes and branch deletion.
|
||||
- **Risk**: a PR merges without its security/CI checks passing.
|
||||
**Control**: merges require the `build`, `Analyze Python` (CodeQL), and `security-scan` checks to pass, in strict mode (checks must be re-run against the latest `main`).
|
||||
- **Risk**: a compromised scanner job reaches secrets or write access.
|
||||
**Control**: scanning jobs (`CodeQL`, `security-scan.yml`, `security.yml`, `defender-for-devops.yml`) run with read-only, least-privilege permissions (typically `contents: read` + `security-events: write` only) and never share a job, environment, or secret scope with the publish job.
|
||||
- **Risk**: secrets are committed accidentally.
|
||||
**Control**: GitHub secret scanning and push protection are both enabled at the repository level, rejecting pushes that contain recognizable credential patterns before they land in history.
|
||||
|
||||
## Automated Security Scanning
|
||||
|
||||
Every scan below runs continuously in CI, not just at release time:
|
||||
|
||||
- **CodeQL** (`security-and-quality` query pack) — Python source: injection, unsafe deserialization, and other code-level vulnerability classes. Runs in `codeql.yml` on every push/PR to `main` and weekly.
|
||||
- **Bandit** — Python-specific security anti-patterns (hardcoded secrets, unsafe `eval`/`pickle`, weak crypto, etc.); CI fails on any HIGH-severity finding. Runs in `security-scan.yml` on every push/PR to `main` and twice weekly.
|
||||
- **Semgrep** (`p/security` ruleset) — cross-language static-analysis security patterns. Runs in `security-scan.yml` on every push/PR to `main` and twice weekly.
|
||||
- **Safety** — known CVEs in Semantica's own installed dependencies, including optional LLM-provider extras such as LiteLLM; CI fails on any match. Runs in `security-scan.yml` on every push/PR to `main` and twice weekly.
|
||||
- **pip-audit** — independent, PyPA-maintained vulnerability database cross-check against installed dependencies (Safety and pip-audit use different advisory sources, so both run). Runs in `security.yml` weekly.
|
||||
- **Microsoft Defender for DevOps** (`eslint`, `templateanalyzer`, `terrascan`) — JavaScript/TypeScript lint-security rules and infrastructure-as-code misconfigurations. Runs in `defender-for-devops.yml` on every push/PR to `main` and weekly.
|
||||
- **Checkov** — Kubernetes, Helm, Dockerfile, GitHub Actions, and secrets-pattern IaC scanning; results upload to the same Security tab as CodeQL. Runs in `defender-for-devops.yml` on every push/PR to `main` and weekly.
|
||||
- **GitGuardian** — secret-detection check on every pull request, installed as a GitHub App integration (not a repo-local workflow). Runs on every PR.
|
||||
- **GitHub secret scanning + push protection** — blocks known credential patterns before they're pushed, and continuously scans existing history. Platform-level, continuous.
|
||||
- **Dependabot** — version/security PRs for Python, Docker, and GitHub Actions dependencies, grouped where relevant to reduce review noise. Configured in `.github/dependabot.yml`, runs weekly for security-relevant packages and monthly for docs dependencies.
|
||||
- **`verify-action-pins.yml`** — enforces that every Action reference is a full commit SHA (failing on a newly introduced mutable tag) and confirms each SHA still matches the tag it claims to be. Runs on every workflow change, every push to `main`, and weekly.
|
||||
|
||||
All SARIF-producing scanners (CodeQL, Checkov, Microsoft Defender) publish
|
||||
findings to the repository's **Security → Code scanning alerts** tab, giving
|
||||
a single audit trail across tools rather than scattered per-tool reports.
|
||||
|
||||
### Adopting this posture in a fork or downstream deployment
|
||||
|
||||
Teams standing up their own instance of Semantica, or forking it for an
|
||||
internal/regulated deployment, can reuse this posture directly:
|
||||
|
||||
1. Keep Dependabot's `github-actions` ecosystem entry — it is what keeps
|
||||
SHA pins current without manual maintenance.
|
||||
2. Re-run `verify-action-pins.yml` after re-pointing the repository's Actions
|
||||
at your own mirrors, if you do so.
|
||||
3. If you publish your own PyPI package from a fork, configure your own
|
||||
Trusted Publishing trust relationship on PyPI (Trusted Publishing is
|
||||
scoped to a specific `owner/repo` + workflow filename) and your own
|
||||
protected environment with your own required reviewers — these are not
|
||||
transferable from this repository.
|
||||
4. Branch protection, environment protection, and repository secret
|
||||
scanning are repository *settings*, not workflow files — cloning or
|
||||
forking the repo does **not** copy them. They must be re-applied via
|
||||
the GitHub UI or API on the new repository.
|
||||
5. GitHub secret scanning and push protection are repository settings that
|
||||
don't carry over to a fork either — re-enable both under the new
|
||||
repository's Security settings, not just Dependabot.
|
||||
6. GitGuardian runs as a GitHub App installation scoped to this specific
|
||||
repository, not a workflow file — a fork gets no secret-detection
|
||||
coverage from it until the app is installed separately on the new repo.
|
||||
7. CodeQL's `upload-sarif` step in `codeql.yml` only runs meaningfully if
|
||||
Default Setup is *not* already enabled for the repository (it's designed
|
||||
to skip gracefully otherwise) — check whether Default Setup or Advanced
|
||||
Setup is active on the new repository and adjust expectations for where
|
||||
CodeQL findings show up accordingly.
|
||||
|
||||
## Dependency Security Policy
|
||||
|
||||
### Regular Updates
|
||||
@@ -156,7 +251,7 @@ We appreciate responsible disclosure. Security researchers who help us improve t
|
||||
|
||||
For security-related questions or concerns:
|
||||
|
||||
- **GitHub Issues**: [Create an issue](https://github.com/semantica-agi/semantica/issues) with "[SECURITY]" prefix
|
||||
- **Private Reporting**: Please do not report vulnerabilities in public issues.
|
||||
- **GitHub Security Advisories**: [Report vulnerability](https://github.com/semantica-agi/semantica/security/advisories/new)
|
||||
|
||||
## Additional Resources
|
||||
|
||||
@@ -3,81 +3,7 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Amazon Neptune Graph Store\n",
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook covers the Amazon Neptune Database integration in Semantica. Amazon Neptune is a fully managed graph database service that supports both property graphs (via OpenCypher/Gremlin) and RDF graphs (via SPARQL).\n",
|
||||
"\n",
|
||||
"### Key Features\n",
|
||||
"\n",
|
||||
"- **IAM Authentication**: Secure access using AWS SigV4 signatures via AuthManager\n",
|
||||
"- **OpenCypher Support**: Query using standard OpenCypher syntax\n",
|
||||
"- **Bolt Protocol**: Uses Neo4j Bolt driver for efficient binary communication\n",
|
||||
"- **Native ~id Support**: Leverages Neptune's native element ID handling\n",
|
||||
"- **Full CRUD Operations**: Create, read, update, delete nodes and relationships\n",
|
||||
"- **Automatic Retry**: Built-in retry logic with exponential backoff for transient errors\n",
|
||||
"\n",
|
||||
"### Prerequisites\n",
|
||||
"\n",
|
||||
"- An Amazon Neptune Database cluster\n",
|
||||
"- AWS credentials configured (boto3, environment variables, or IAM role)\n",
|
||||
"- Network access to your Neptune cluster (VPC, security groups)\n",
|
||||
"\n",
|
||||
"#### Quick Setup with CloudFormation\n",
|
||||
"\n",
|
||||
"If you don't have a Neptune cluster, use the provided CloudFormation template to create one with a public endpoint and IAM authentication:\n",
|
||||
"\n",
|
||||
"```bash\n",
|
||||
"# Deploy the Neptune stack (takes ~15-20 minutes)\n",
|
||||
"aws cloudformation create-stack \\\n",
|
||||
" --stack-name semantica-neptune \\\n",
|
||||
" --template-body file://neptune-setup.yaml \\\n",
|
||||
" --capabilities CAPABILITY_NAMED_IAM\n",
|
||||
"\n",
|
||||
"# Wait for stack creation to complete\n",
|
||||
"aws cloudformation wait stack-create-complete --stack-name semantica-neptune\n",
|
||||
"\n",
|
||||
"# Get the outputs (endpoint, port, credentials)\n",
|
||||
"aws cloudformation describe-stacks --stack-name semantica-neptune \\\n",
|
||||
" --query 'Stacks[0].Outputs' --output table\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"The template creates:\n",
|
||||
"- VPC with public subnets and Internet Gateway\n",
|
||||
"- Neptune cluster (`db.t3.medium`) with IAM authentication enabled\n",
|
||||
"- IAM user with least-privilege access for OpenCypher queries\n",
|
||||
"- Security group allowing Bolt protocol (port 8182) access\n",
|
||||
"\n",
|
||||
"> ⚠️ **Security Note**: This template creates an IAM User with static access keys for simplicity in demo/test environments. For production use, we recommend IAM Roles (EC2 instance roles, ECS task roles, Lambda execution roles) which provide temporary credentials that are automatically rotated. The secret access key in the Cloudformation outputs is provided in plaintext to simplify initial setup - in production, use AWS Secrets Manager.\n",
|
||||
"\n",
|
||||
"**Outputs:**\n",
|
||||
"- `NeptuneEndpoint` - Cluster hostname (use as `NEPTUNE_ENDPOINT`)\n",
|
||||
"- `NeptunePort` - 8182 (use as `NEPTUNE_PORT`)\n",
|
||||
"- `AwsAccessKeyId` - IAM user access key (use as `AWS_ACCESS_KEY_ID`)\n",
|
||||
"- `AwsSecretAccessKey` - IAM user secret key in **plaintext** (use as `AWS_SECRET_ACCESS_KEY`)\n",
|
||||
"- `AwsRegion` - Deployment region (use as `AWS_REGION`)\n",
|
||||
"\n",
|
||||
"**Cleanup:**\n",
|
||||
"```bash\n",
|
||||
"aws cloudformation delete-stack --stack-name semantica-neptune\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"**Estimated Monthly Cost (approximately 100-105 USD/month at 100% utilization):**\n",
|
||||
"\n",
|
||||
"| Resource | Cost (USD) |\n",
|
||||
"| --- | --- |\n",
|
||||
"| Neptune db.t3.medium instance | ~96/month (0.132/hr) |\n",
|
||||
"| Storage (10 GB) | ~1/month |\n",
|
||||
"| I/O requests | ~1-5/month |\n",
|
||||
"| Public IPv4 address | ~3.60/month (0.005/hr) |\n",
|
||||
"| VPC, subnets, route tables, Internet Gateway, IAM | No Additional Charge |\n",
|
||||
"\n",
|
||||
"> **Free Tier**: New Neptune users get 30 days free (750 hours of db.t3.medium, 10M I/Os, 1 GB storage). Delete the stack when not in use to avoid charges.\n",
|
||||
"\n",
|
||||
"---"
|
||||
]
|
||||
"source": "# Amazon Neptune Graph Store\n\n## Overview\n\nThis notebook covers the Amazon Neptune Database integration in Semantica. Amazon Neptune is a fully managed graph database service that supports both property graphs (via OpenCypher/Gremlin) and RDF graphs (via SPARQL).\n\n### Key Features\n\n- **IAM Authentication**: Secure access using AWS SigV4 signatures via AuthManager\n- **OpenCypher Support**: Query using standard OpenCypher syntax\n- **Bolt Protocol**: Uses Neo4j Bolt driver for efficient binary communication\n- **Native ~id Support**: Leverages Neptune's native element ID handling\n- **Full CRUD Operations**: Create, read, update, delete nodes and relationships\n- **Automatic Retry**: Built-in retry logic with exponential backoff for transient errors\n\n### Prerequisites\n\n- An Amazon Neptune Database cluster\n- AWS credentials configured (boto3, environment variables, or IAM role)\n- Network access to your Neptune cluster (VPC, security groups)\n- Your public IP address or VPN/office CIDR (run `curl ifconfig.me` to find your public IP), used below to restrict database access\n\n#### Quick Setup with CloudFormation\n\nIf you don't have a Neptune cluster, use the provided CloudFormation template to create one with a public endpoint and IAM authentication:\n\n```bash\n# Deploy the Neptune stack (takes ~15-20 minutes)\n# Replace 203.0.113.25/32 with your own public IP (run `curl ifconfig.me` to find it)\n# or your office/VPN CIDR. This restricts who can reach the database on the\n# network level - never widen it to 0.0.0.0/0 outside of a short-lived local experiment.\naws cloudformation create-stack \\\n --stack-name semantica-neptune \\\n --template-body file://neptune-setup.yaml \\\n --parameters ParameterKey=ClientCidr,ParameterValue=203.0.113.25/32 \\\n --capabilities CAPABILITY_NAMED_IAM\n\n# Wait for stack creation to complete\naws cloudformation wait stack-create-complete --stack-name semantica-neptune\n\n# Get the outputs (endpoint, port, credentials)\naws cloudformation describe-stacks --stack-name semantica-neptune \\\n --query 'Stacks[0].Outputs' --output table\n```\n\nThe template creates:\n- VPC with public subnets, Internet Gateway, and VPC Flow Logs (to CloudWatch Logs)\n- Neptune cluster (`db.t3.medium`) with IAM authentication enabled\n- IAM user with least-privilege access for OpenCypher queries\n- Security group allowing Bolt protocol (port 8182) access only from the `ClientCidr` you specify\n\n> ⚠️ **Security Note**: This template creates an IAM User with static access keys for simplicity in demo/test environments. For production use, we recommend IAM Roles (EC2 instance roles, ECS task roles, Lambda execution roles) which provide temporary credentials that are automatically rotated. The secret access key in the Cloudformation outputs is provided in plaintext to simplify initial setup - in production, use AWS Secrets Manager. The `ClientCidr` parameter is required (no default) precisely so the database is never silently exposed to the whole internet.\n\n**Outputs:**\n- `NeptuneEndpoint` - Cluster hostname (use as `NEPTUNE_ENDPOINT`)\n- `NeptunePort` - 8182 (use as `NEPTUNE_PORT`)\n- `AwsAccessKeyId` - IAM user access key (use as `AWS_ACCESS_KEY_ID`)\n- `AwsSecretAccessKey` - IAM user secret key in **plaintext** (use as `AWS_SECRET_ACCESS_KEY`)\n- `AwsRegion` - Deployment region (use as `AWS_REGION`)\n\n**Cleanup:**\n```bash\naws cloudformation delete-stack --stack-name semantica-neptune\n```\n\n**Estimated Monthly Cost (approximately 100-105 USD/month at 100% utilization):**\n\n| Resource | Cost (USD) |\n| --- | --- |\n| Neptune db.t3.medium instance | ~96/month (0.132/hr) |\n| Storage (10 GB) | ~1/month |\n| I/O requests | ~1-5/month |\n| Public IPv4 address | ~3.60/month (0.005/hr) |\n| VPC Flow Logs (CloudWatch Logs) | ~1-2/month depending on traffic |\n| VPC, subnets, route tables, Internet Gateway, IAM | No Additional Charge |\n\n> **Free Tier**: New Neptune users get 30 days free (750 hours of db.t3.medium, 10M I/Os, 1 GB storage). Delete the stack when not in use to avoid charges.\n\n---"
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -722,4 +648,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,14 @@
|
||||
# ts:skip=AC_AWS_0148 IAM password policy is an AWS-account-wide singleton, not a
|
||||
# per-stack resource. Managing it here would mean every learner who deploys or
|
||||
# deletes this cookbook stack also mutates (or removes) their account's password
|
||||
# policy as a side effect. Account password policy should be set once, out of
|
||||
# band, by the account owner - not by a disposable tutorial stack.
|
||||
AWSTemplateFormatVersion: '2010-09-09'
|
||||
Description: >
|
||||
Amazon Neptune cluster with public endpoint, IAM authentication, and least-privilege
|
||||
IAM user for Semantica cookbook. Uses db.t3.medium (most cost-effective Neptune instance type).
|
||||
Network access to the Bolt/OpenCypher port is restricted to an operator-supplied CIDR
|
||||
(see ClientCidr) - do not widen this to 0.0.0.0/0 outside of a short-lived local experiment.
|
||||
|
||||
Parameters:
|
||||
EnvironmentName:
|
||||
@@ -9,6 +16,16 @@ Parameters:
|
||||
Default: semantica-neptune
|
||||
Description: Environment name prefix for resource naming
|
||||
|
||||
ClientCidr:
|
||||
Type: String
|
||||
Description: >-
|
||||
CIDR block allowed to reach the Neptune Bolt/OpenCypher endpoint (port 8182) - e.g. your
|
||||
workstation's public IP as "x.x.x.x/32", or your office/VPN CIDR. Required: there is no
|
||||
default, so you must explicitly choose a range. Passing 0.0.0.0/0 is possible but exposes
|
||||
the database to the entire internet and is strongly discouraged beyond a brief local test.
|
||||
AllowedPattern: '^((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])/(3[0-2]|[12]?[0-9])$'
|
||||
ConstraintDescription: Must be a valid IPv4 CIDR block with octets 0-255 and prefix 0-32, e.g. 203.0.113.25/32
|
||||
|
||||
Resources:
|
||||
# =============================================================================
|
||||
# VPC & NETWORKING
|
||||
@@ -87,6 +104,57 @@ Resources:
|
||||
RouteTableId: !Ref PublicRouteTable
|
||||
SubnetId: !Ref PublicSubnet2
|
||||
|
||||
# =============================================================================
|
||||
# VPC FLOW LOGS
|
||||
# =============================================================================
|
||||
|
||||
FlowLogGroup:
|
||||
Type: AWS::Logs::LogGroup
|
||||
Properties:
|
||||
LogGroupName: !Sub /aws/vpc/${EnvironmentName}-flow-logs
|
||||
RetentionInDays: 30
|
||||
|
||||
FlowLogRole:
|
||||
Type: AWS::IAM::Role
|
||||
Properties:
|
||||
RoleName: !Sub ${EnvironmentName}-flow-log-role
|
||||
AssumeRolePolicyDocument:
|
||||
Version: '2012-10-17'
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Principal:
|
||||
Service: vpc-flow-logs.amazonaws.com
|
||||
Action: sts:AssumeRole
|
||||
Policies:
|
||||
- PolicyName: flow-log-publish
|
||||
PolicyDocument:
|
||||
Version: '2012-10-17'
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- logs:CreateLogGroup
|
||||
- logs:DescribeLogGroups
|
||||
- logs:DescribeLogStreams
|
||||
Resource: "*"
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- logs:CreateLogStream
|
||||
- logs:PutLogEvents
|
||||
Resource: !GetAtt FlowLogGroup.Arn
|
||||
|
||||
VPCFlowLog:
|
||||
Type: AWS::EC2::FlowLog
|
||||
Properties:
|
||||
ResourceType: VPC
|
||||
ResourceId: !Ref VPC
|
||||
TrafficType: ALL
|
||||
LogDestinationType: cloud-watch-logs
|
||||
LogGroupName: !Ref FlowLogGroup
|
||||
DeliverLogsPermissionArn: !GetAtt FlowLogRole.Arn
|
||||
Tags:
|
||||
- Key: Name
|
||||
Value: !Sub ${EnvironmentName}-vpc-flow-log
|
||||
|
||||
# =============================================================================
|
||||
# SECURITY GROUP
|
||||
# =============================================================================
|
||||
@@ -95,14 +163,14 @@ Resources:
|
||||
Type: AWS::EC2::SecurityGroup
|
||||
Properties:
|
||||
GroupName: !Sub ${EnvironmentName}-neptune-sg
|
||||
GroupDescription: Security group for Neptune cluster - allows Bolt protocol access
|
||||
GroupDescription: Security group for Neptune cluster - allows Bolt protocol access from ClientCidr only
|
||||
VpcId: !Ref VPC
|
||||
SecurityGroupIngress:
|
||||
- IpProtocol: tcp
|
||||
FromPort: 8182
|
||||
ToPort: 8182
|
||||
CidrIp: 0.0.0.0/0
|
||||
Description: Allow Bolt protocol access from anywhere
|
||||
CidrIp: !Ref ClientCidr
|
||||
Description: Allow Bolt/OpenCypher protocol access from the operator-specified CIDR
|
||||
SecurityGroupEgress:
|
||||
- IpProtocol: -1
|
||||
CidrIp: 0.0.0.0/0
|
||||
|
||||
@@ -9,7 +9,10 @@ flyctl launch --copy-config --config deploy/fly/fly.toml --no-deploy
|
||||
# Fly.io private networking uses .internal hostnames — do not use localhost
|
||||
# unless FalkorDB is a co-located process inside the same Machine.
|
||||
flyctl secrets set FALKORDB_HOST=<falkordb-app-name>.internal FALKORDB_PORT=6379
|
||||
flyctl secrets set SEMANTICA_API_KEY=$(openssl rand -hex 32)
|
||||
flyctl deploy --config deploy/fly/fly.toml
|
||||
```
|
||||
|
||||
Change `app` in `fly.toml` before launch if the default app name is already taken.
|
||||
|
||||
Fly apps get a public `*.fly.dev` URL by default, so `SEMANTICA_API_KEY` is required — without it the Explorer refuses every protected route (503) rather than serving anonymously. Pass the same value as the `X-API-Key` header from any client that talks to the deployed API.
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# checkov:skip=CKV_K8S_21:Namespace is bound via .Release.Namespace at helm install/template time; this chart is namespace-portable by design.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
@@ -5,6 +6,9 @@ metadata:
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "knowledge-explorer.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
runterrascan.io/skip: '[{"rule": "AC_K8S_0086", "comment": "Namespace is bound via .Release.Namespace at helm install time"}]'
|
||||
checkov.io/skip1: CKV_K8S_21=Namespace bound via .Release.Namespace at helm install/template time
|
||||
data:
|
||||
{{- range $key, $value := .Values.env }}
|
||||
{{ $key }}: {{ $value | quote }}
|
||||
|
||||
@@ -5,6 +5,10 @@ metadata:
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "knowledge-explorer.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
runterrascan.io/skip: '[{"rule": "AC_K8S_0086", "comment": "Namespace is bound via .Release.Namespace at helm install time"}, {"rule": "AC_K8S_0080", "comment": "seccompProfile RuntimeDefault is set in values.yaml (podSecurityContext)"}]'
|
||||
checkov.io/skip1: CKV_K8S_21=Namespace bound via .Release.Namespace at helm install/template time
|
||||
checkov.io/skip2: CKV_K8S_31=seccompProfile RuntimeDefault set in values.yaml
|
||||
spec:
|
||||
{{- if not .Values.autoscaling.enabled }}
|
||||
replicas: {{ .Values.replicaCount }}
|
||||
@@ -19,8 +23,10 @@ spec:
|
||||
{{- include "knowledge-explorer.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
{{- with .Values.podAnnotations }}
|
||||
annotations:
|
||||
runterrascan.io/skip: '[{"rule": "AC_K8S_0080", "comment": "seccompProfile RuntimeDefault is set in values.yaml (podSecurityContext)"}]'
|
||||
checkov.io/skip1: CKV_K8S_31=seccompProfile RuntimeDefault set in values.yaml
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# checkov:skip=CKV_K8S_21:Namespace is bound via .Release.Namespace at helm install/template time; this chart is namespace-portable by design.
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
@@ -5,6 +6,9 @@ metadata:
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "knowledge-explorer.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
runterrascan.io/skip: '[{"rule": "AC_K8S_0086", "comment": "Namespace is bound via .Release.Namespace at helm install time"}]'
|
||||
checkov.io/skip1: CKV_K8S_21=Namespace bound via .Release.Namespace at helm install/template time
|
||||
spec:
|
||||
type: {{ .Values.service.type }}
|
||||
ports:
|
||||
|
||||
@@ -9,7 +9,10 @@ railway add --database redis
|
||||
railway variable --set "FALKORDB_HOST=${{Redis.REDISHOST}}"
|
||||
railway variable --set "FALKORDB_PORT=${{Redis.REDISPORT}}"
|
||||
railway variable --set "ALLOWED_ORIGINS=https://${{RAILWAY_PUBLIC_DOMAIN}}"
|
||||
railway variable --set "SEMANTICA_API_KEY=$(openssl rand -hex 32)"
|
||||
railway up
|
||||
```
|
||||
|
||||
The Redis plugin variables are wired to the requested FalkorDB env names for deployment compatibility. The Explorer currently reads these settings but does not persist graph state to FalkorDB.
|
||||
|
||||
Railway exposes this service on a public domain, so `SEMANTICA_API_KEY` is required — without it the Explorer refuses every protected route (503) rather than serving anonymously. Pass the same value as the `X-API-Key` header from any client that talks to the deployed API.
|
||||
|
||||
@@ -9,3 +9,5 @@ render blueprint apply deploy/render/render.yaml
|
||||
```
|
||||
|
||||
After creation, update `ALLOWED_ORIGINS` in the Render dashboard if you attach a custom domain.
|
||||
|
||||
`SEMANTICA_API_KEY` is auto-generated by the blueprint (`generateValue: true`) since this service gets a public `onrender.com` URL — without it the Explorer refuses every protected route (503) rather than serving anonymously. Find the generated value in the Render dashboard's environment tab and pass it as the `X-API-Key` header from any client that talks to the deployed API.
|
||||
|
||||
@@ -20,6 +20,8 @@ services:
|
||||
type: keyvalue
|
||||
name: semantica-explorer-redis
|
||||
property: port
|
||||
- key: SEMANTICA_API_KEY
|
||||
generateValue: true
|
||||
|
||||
- type: keyvalue
|
||||
name: semantica-explorer-redis
|
||||
|
||||
@@ -16,6 +16,8 @@ services:
|
||||
ALLOWED_ORIGINS: http://localhost:5173,http://127.0.0.1:5173,http://localhost:8000,http://127.0.0.1:8000
|
||||
FALKORDB_HOST: falkordb
|
||||
FALKORDB_PORT: "6379"
|
||||
# Local dev only: this compose file is not for public exposure.
|
||||
SEMANTICA_ALLOW_ANONYMOUS: "true"
|
||||
volumes:
|
||||
- ./semantica:/app/semantica
|
||||
- ./pyproject.toml:/app/pyproject.toml:ro
|
||||
|
||||
@@ -8,6 +8,11 @@ services:
|
||||
FALKORDB_HOST: falkordb
|
||||
FALKORDB_PORT: "6379"
|
||||
ALLOWED_ORIGINS: ${ALLOWED_ORIGINS:-http://localhost:8000,http://127.0.0.1:8000}
|
||||
# Required for API access - the Explorer refuses all protected routes
|
||||
# (503) until this is set. Generate one with `openssl rand -hex 32`.
|
||||
SEMANTICA_API_KEY: ${SEMANTICA_API_KEY:-}
|
||||
# Trusted local-only setups only: bypasses the API key entirely.
|
||||
SEMANTICA_ALLOW_ANONYMOUS: ${SEMANTICA_ALLOW_ANONYMOUS:-false}
|
||||
depends_on:
|
||||
falkordb:
|
||||
condition: service_started
|
||||
|
||||
@@ -23,7 +23,7 @@ Loads data from any source into the pipeline as a unified `SourceDocument`.
|
||||
| Parquet | `ingest.ParquetIngestor` | PyArrow, Hive-style partitions (v0.5.0) |
|
||||
| XML | `ingest.XMLIngestor` | XXE-safe lxml, XSD/DTD validation (v0.5.0) |
|
||||
| Web pages | `ingest.WebIngestor` | Configurable depth, link filtering |
|
||||
| SQL / Snowflake | `ingest.DBIngestor` / `ingest.SnowflakeIngestor` | Custom SQL, schema introspection |
|
||||
| SQL / Snowflake / Databricks | `ingest.DBIngestor` / `ingest.SnowflakeIngestor` / `ingest.DatabricksIngestor` | Custom SQL, schema introspection, Unity Catalog lineage |
|
||||
| Kafka / streams | `ingest.StreamIngestor` | Real-time feed ingestion |
|
||||
| Email | `ingest.EmailIngestor` | IMAP/SMTP with attachment extraction |
|
||||
| Repositories | `ingest.RepoIngestor` | Git repos, code structure |
|
||||
|
||||
@@ -18,7 +18,7 @@ Find your goal below. The **Module** column is your import path; **Key class** i
|
||||
| Crawl a website | `ingest` | `WebIngestor` |
|
||||
| Load Parquet files or partitioned datasets | `ingest` | `ParquetIngestor` |
|
||||
| Ingest XML with schema validation | `ingest` | `XMLIngestor` |
|
||||
| Ingest from SQL, Snowflake, Kafka, or email | `ingest` | `DBIngestor`, `SnowflakeIngestor`, `StreamIngestor` |
|
||||
| Ingest from SQL, Snowflake, Databricks, Kafka, or email | `ingest` | `DBIngestor`, `SnowflakeIngestor`, `DatabricksIngestor`, `StreamIngestor` |
|
||||
| Extract clean text and tables from a document | `parse` | `DocumentParser` |
|
||||
| Parse complex PDFs with OCR or multi-column layout | `parse` | `DoclingParser` |
|
||||
| Chunk text for embedding or RAG | `split` | `TextSplitter` |
|
||||
|
||||
+8
-8
@@ -13,33 +13,33 @@ icon: "quote-left"
|
||||
<Tab title="BibTeX">
|
||||
```bibtex
|
||||
@software{semantica2026,
|
||||
title = {Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering},
|
||||
author = {Hawksight AI},
|
||||
title = {Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems},
|
||||
author = {Semantica},
|
||||
year = {2026},
|
||||
url = {https://github.com/semantica-agi/semantica},
|
||||
version = {0.6.0},
|
||||
version = {0.6.6},
|
||||
doi = {10.5281/zenodo.XXXXXXX}
|
||||
}
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="APA">
|
||||
Hawksight AI. (2026). *Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering* (Version 0.6.0) \[Computer software\]. https://github.com/semantica-agi/semantica
|
||||
Semantica. (2026). *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems* (Version 0.6.6) \[Computer software\]. https://github.com/semantica-agi/semantica
|
||||
</Tab>
|
||||
<Tab title="MLA">
|
||||
Hawksight AI. *Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering*. Version 0.6.0, GitHub, 2026, https://github.com/semantica-agi/semantica.
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.6, GitHub, 2026, https://github.com/semantica-agi/semantica.
|
||||
</Tab>
|
||||
<Tab title="Chicago">
|
||||
Hawksight AI. *Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering*. Version 0.6.0. GitHub, 2026. https://github.com/semantica-agi/semantica.
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.6. GitHub, 2026. https://github.com/semantica-agi/semantica.
|
||||
</Tab>
|
||||
<Tab title="IEEE">
|
||||
Hawksight AI, "Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering," Version 0.6.0, GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica
|
||||
Semantica, "Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems," Version 0.6.6, GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
|
||||
## Acknowledgment Text
|
||||
|
||||
> "This work uses Semantica (Hawksight AI, 2026), an open-source framework for semantic layer construction and knowledge engineering."
|
||||
> "This work uses Semantica (2026), an open-source graph-native infrastructure framework for context and accountable AI systems, providing Context Graphs, knowledge graphs, and full decision provenance."
|
||||
|
||||
|
||||
## Share Your Research
|
||||
|
||||
@@ -16,6 +16,9 @@ At its core, Semantica adds a **context and accountability layer** on top of you
|
||||
- **Accountability Layer** — Provenance tracking, decision intelligence, conflict detection, and W3C PROV-O compliance make every claim in your AI stack auditable and explainable.
|
||||
- **Extension Layer** — `PluginRegistry` and `MethodRegistry` let you replace or augment any component: ingestors, extractors, reasoning engines, backends: without changing framework code.
|
||||
|
||||
<Warning>
|
||||
**This is system-level explainability, not foundation-model explainability.** Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. What Semantica explains is *outside* the model: the context and data fed in, the decision produced, its provenance, the relevant relationships, the policies applied, and the full execution trail. In short, Semantica explains and audits *what the AI system did*, not the foundation model's private internal reasoning.
|
||||
</Warning>
|
||||
|
||||
## Knowledge Graphs
|
||||
|
||||
|
||||
@@ -102,6 +102,7 @@
|
||||
"group": "Integrations",
|
||||
"pages": [
|
||||
"integrations/agno",
|
||||
"integrations/crewai",
|
||||
"integrations/docling",
|
||||
"integrations/snowflake",
|
||||
"integrations/databricks"
|
||||
|
||||
@@ -162,7 +162,7 @@ semantica-explorer --graph my_graph.json --no-browser
|
||||
```
|
||||
|
||||
<Warning>
|
||||
`--host 0.0.0.0` makes Explorer reachable on every network interface. The server has no built-in authentication. Only use this on a trusted private network.
|
||||
`--host 0.0.0.0` makes Explorer reachable on every network interface. Since v0.6.5 the Explorer API requires `SEMANTICA_API_KEY` (sent as the `X-API-Key` header) and fails closed with `503` when unconfigured; unauthenticated access is only possible when `SEMANTICA_ALLOW_ANONYMOUS=true` is set explicitly. Only use this on a trusted private network.
|
||||
</Warning>
|
||||
|
||||
|
||||
|
||||
+12
-2
@@ -17,7 +17,7 @@ icon: "circle-question"
|
||||
| API key required? | Optional: pattern extraction works with no keys |
|
||||
| Works with LangChain / LlamaIndex? | Yes: Semantica is a layer on top, not a replacement |
|
||||
| Production-ready? | Yes: 1,000+ tests, v0.5.0 ships with 12 security fixes |
|
||||
| Latest version? | **v0.6.0** (July 2026) |
|
||||
| Latest version? | **v0.6.6** (August 2026) |
|
||||
| Local LLMs? | Yes: Ollama via LiteLLM, HuggingFaceLLM for air-gapped |
|
||||
|
||||
|
||||
@@ -52,6 +52,16 @@ Semantica works alongside these frameworks, not against them.
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Does Semantica explain an LLM's internal reasoning or chain-of-thought?" icon="triangle-exclamation">
|
||||
|
||||
No. This is **system-level explainability, not foundation-model explainability**. Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system.
|
||||
|
||||
What Semantica explains is *outside* the model: what context and data were used, what decision was produced, the provenance behind it, the relevant relationships, the policies applied, and the resulting decision trail.
|
||||
|
||||
In short: Semantica explains and audits *what the AI system did* — not the foundation model's private internal reasoning.
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Is Semantica free?" icon="tag">
|
||||
|
||||
Yes: MIT licensed, no vendor lock-in, no paywalled features. Some capabilities require third-party API keys (e.g., OpenAI embeddings, Groq inference), but Semantica itself is always free and open source.
|
||||
@@ -129,7 +139,7 @@ If you're on an older version, install extras individually: `pip install "semant
|
||||
| :-------- | :------- |
|
||||
| **Files** | PDF, DOCX, HTML, JSON, CSV, Excel, PPTX, Parquet (v0.5.0), XML (v0.5.0), archives |
|
||||
| **Web** | `WebIngestor` crawl, RSS feeds, sitemaps |
|
||||
| **Databases** | PostgreSQL, MySQL, Snowflake via `DBIngestor` / `SnowflakeIngestor` |
|
||||
| **Databases** | PostgreSQL, MySQL, Snowflake, Databricks via `DBIngestor` / `SnowflakeIngestor` / `DatabricksIngestor` |
|
||||
| **NoSQL** | MongoDB via `MongoIngestor`, DuckDB via `DuckDBIngestor` |
|
||||
| **Streams** | Kafka, real-time ingestion via `StreamIngestor` |
|
||||
| **Protocols** | MCP (Model Context Protocol) via `MCPIngestor` |
|
||||
|
||||
@@ -42,7 +42,7 @@ icon: "rocket"
|
||||
Verify installation:
|
||||
```python
|
||||
import semantica
|
||||
print(semantica.__version__) # 0.6.0
|
||||
print(semantica.__version__) # 0.6.6
|
||||
```
|
||||
</Check>
|
||||
</Step>
|
||||
|
||||
+1
-1
@@ -149,7 +149,7 @@ A database optimized for storing and querying graph-structured data using node a
|
||||
A retrieval strategy combining vector similarity search with keyword or metadata filtering: higher accuracy than either approach alone.
|
||||
|
||||
**Triplet Store**
|
||||
A database designed specifically for storing and querying RDF `(subject, predicate, object)` triples. Semantica supports Blazegraph, Apache Jena, and RDF4J.
|
||||
A database designed specifically for storing and querying RDF `(subject, predicate, object)` triples. Semantica supports embedded Oxigraph as well as Blazegraph, Apache Jena, and RDF4J.
|
||||
|
||||
**Vector Store**
|
||||
A database optimized for storing and searching high-dimensional embedding vectors by similarity. Semantica supports FAISS, Pinecone, Weaviate, Qdrant, Milvus, and PgVector.
|
||||
|
||||
@@ -50,6 +50,7 @@ Use the ingest module when your data lives outside Semantica and you need to bri
|
||||
- **Web content** — public documentation sites, regulatory publication pages, news feeds, or any URL you can crawl.
|
||||
- **REST APIs** — internal platforms (SIEM, EDR, ITSM, CRM), threat intelligence feeds, or any paginated HTTP endpoint.
|
||||
- **Databases** — existing SQL databases where relevant records can be fetched with a targeted query.
|
||||
- **Enterprise data platforms** — tables already living in a Databricks lakehouse (Unity Catalog + Delta Lake) or a Snowflake warehouse, without exporting to CSV first.
|
||||
- **Live streams** — Kafka or other message brokers where you need to process events as they arrive.
|
||||
- **Git repositories** — source code, documentation, or configuration files tracked in version control.
|
||||
|
||||
@@ -298,6 +299,89 @@ for bundle in stix_xml_files:
|
||||
print(f"{bundle.source_path}: {len(bundle.elements)} elements parsed")
|
||||
```
|
||||
|
||||
## Source 6 — Enterprise Data Platforms (Databricks & Snowflake)
|
||||
|
||||
`DatabricksIngestor` and `SnowflakeIngestor` return wrapper objects (`DatabricksData` / `SnowflakeData`) whose `.data` field is `List[Dict]` — the same list-of-dicts row shape that `DBIngestor.execute_query()` returns directly, without a wrapper. The same "transform to text, then store" pattern from Source 3 applies: pull only the tables and columns you need with a targeted query, then build a sentence per record before handing it to `AgentContext.store()`.
|
||||
|
||||
```python
|
||||
from semantica.ingest import DatabricksIngestor
|
||||
|
||||
# Unity Catalog + Delta Lake — PAT or OAuth M2M auth
|
||||
databricks = DatabricksIngestor(
|
||||
host="https://adb-xxx.azuredatabricks.net",
|
||||
token="dapi-xxxxxxxx",
|
||||
http_path="/sql/1.0/warehouses/xxxxxxxx",
|
||||
catalog="main",
|
||||
)
|
||||
|
||||
# .data is List[Dict] — one dict per row, same shape as DBIngestor.execute_query()
|
||||
customers = databricks.ingest_query(
|
||||
"SELECT customer_id, name, industry, arr FROM main.default.customers "
|
||||
"WHERE churn_risk_score > 0.7"
|
||||
)
|
||||
customer_texts = [
|
||||
f"Customer {r['customer_id']} ({r['name']}, {r['industry']}): "
|
||||
f"ARR ${r['arr']:,}, flagged high churn risk"
|
||||
for r in customers.data
|
||||
]
|
||||
|
||||
# Unity Catalog lineage — build Table --DEPENDS_ON--> Table edges directly from
|
||||
# Unity Catalog's own lineage tracking, instead of re-deriving them from query logs
|
||||
lineage = databricks.get_table_lineage("customers", catalog="main", schema="default")
|
||||
lineage_texts = [
|
||||
f"Table main.default.customers depends on {upstream}"
|
||||
for upstream in lineage["upstream"]
|
||||
]
|
||||
```
|
||||
|
||||
```python
|
||||
from semantica.ingest import SnowflakeIngestor
|
||||
|
||||
snowflake = SnowflakeIngestor(
|
||||
account="myaccount",
|
||||
user="myuser",
|
||||
password="mypassword", # or private_key=... for key-pair; use authenticator="oauth", token=... for OAuth
|
||||
warehouse="COMPUTE_WH",
|
||||
database="ANALYTICS",
|
||||
schema="PUBLIC",
|
||||
)
|
||||
|
||||
# Snowflake uppercases unquoted identifiers, so unquoted columns come back
|
||||
# as ORDER_ID, PRODUCT, etc. unless the source table quotes them lowercase
|
||||
orders = snowflake.ingest_query(
|
||||
"SELECT order_id, product, region, amount FROM orders "
|
||||
"WHERE order_date >= DATEADD(day, -30, CURRENT_DATE())"
|
||||
)
|
||||
order_texts = [
|
||||
f"Order {r['ORDER_ID']}: {r['PRODUCT']} in {r['REGION']}, ${r['AMOUNT']}"
|
||||
for r in orders.data
|
||||
]
|
||||
```
|
||||
|
||||
Feed the resulting text lists into `AgentContext.store()` exactly like any other structured source:
|
||||
|
||||
```python
|
||||
from semantica.context import AgentContext, ContextGraph
|
||||
from semantica.vector_store import VectorStore
|
||||
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
context = AgentContext(
|
||||
vector_store = VectorStore(backend="faiss"),
|
||||
knowledge_graph = graph,
|
||||
)
|
||||
|
||||
context.store(
|
||||
customer_texts + lineage_texts + order_texts,
|
||||
extract_entities=True,
|
||||
extract_relationships=True,
|
||||
)
|
||||
print(f"Enterprise data graph: {graph.stats()['node_count']} nodes")
|
||||
```
|
||||
|
||||
For authentication details (PAT vs. OAuth M2M for Databricks; password vs. key-pair vs. OAuth for Snowflake), schema/catalog introspection, and troubleshooting, see the dedicated [Databricks Integration](../integrations/databricks) and [Snowflake Integration](../integrations/snowflake) guides.
|
||||
|
||||
> **Security Note:** Never hardcode credentials (`token`, `password`, `private_key`) in production code; pass them via environment variables (e.g., `DATABRICKS_TOKEN`, `SNOWFLAKE_PASSWORD`) or a secrets manager.
|
||||
|
||||
## Combining All Five Sources
|
||||
|
||||
Once you have text from each source, `AgentContext.store()` accepts a flat list of strings. Semantica embeds and indexes them together — the context graph has no concept of which string came from which source unless you add metadata explicitly.
|
||||
@@ -831,3 +915,5 @@ print(f"Compliance graph: {graph.stats()['node_count']} nodes, "
|
||||
- [Context Graphs](context-graphs) — storing and querying the entities you ingest as a typed property graph
|
||||
- [Semantic Extraction](semantic-extraction) — NER, relation extraction, and triplet extraction from ingested text
|
||||
- [Provenance](provenance) — tracking the origin document, confidence score, and ingestion timestamp for every extracted entity
|
||||
- [Databricks Integration](../integrations/databricks) — Unity Catalog setup, PAT/OAuth M2M authentication, and lineage introspection
|
||||
- [Snowflake Integration](../integrations/snowflake) — warehouse setup and password/key-pair/OAuth authentication
|
||||
|
||||
@@ -83,9 +83,13 @@ prov = ProvenanceManager(storage=SQLiteStorage("audit.db"))
|
||||
|
||||
For any regulated deployment — security operations, clinical data, financial risk — use `storage_path`. A SQLite file can be backed up, versioned, and queried with standard tools without requiring a server.
|
||||
|
||||
<Note>
|
||||
`SQLiteStorage` automatically configures Write-Ahead Logging (`WAL`), `busy_timeout=5000`, and `synchronous=NORMAL`, and executes read-modify-write operations (like `track_entity()`) in atomic immediate transactions (`BEGIN IMMEDIATE`); plain reads (`retrieve()`, `trace_lineage()`) use a separate connection without an explicit write lock so they don't serialize behind writers. Furthermore, `ProvenanceManager` automatically supports custom storage backends overriding only `trace_lineage(self, entity_id)` without requiring `max_depth` in their signature.
|
||||
</Note>
|
||||
|
||||
## Recording provenance when ingesting data
|
||||
|
||||
The moment data enters your graph is the moment provenance must be recorded. `track_entity()` captures the source document, the timestamp, the operator or pipeline that ran the extraction, a verbatim quote from the source, and a confidence score. It returns a `ProvenanceEntry` with a SHA-256 checksum computed automatically.
|
||||
The moment data enters your graph is the moment provenance must be recorded. `track_entity()` captures the source document, the timestamp, the operator or pipeline that ran the extraction, a verbatim quote from the source, and a confidence score. It returns an `Optional[ProvenanceEntry]` (`ProvenanceEntry` on success, or `None` if storage fails on a brand-new entity) with a SHA-256 checksum computed automatically.
|
||||
|
||||
```python
|
||||
# Ingesting CVE-2024-3400 from NVD and a commercial feed
|
||||
@@ -629,12 +633,29 @@ Every `ProvenanceEntry` maps directly to W3C PROV-O terms. If your compliance te
|
||||
| :--- | :--- | :--- |
|
||||
| `prov:Entity` | `entity_id` | The tracked object — entity, chunk, relationship, or property |
|
||||
| `prov:Activity` | `activity_id` | The process that produced it — `"ner_extraction"`, `"bureau_parsing"` |
|
||||
| `prov:Agent` | `agent_id` | Who ran the activity — pipeline name, analyst ID |
|
||||
| `prov:wasDerivedFrom` | `parent_entity_id` | The previous version of this entity — enables version chaining |
|
||||
| `prov:Agent` / `prov:Person` / `prov:SoftwareAgent` / `prov:Organization` | `agent_id`, `agent_type`, `is_automated` | Who — or what — ran the activity, and whether a human was directly accountable |
|
||||
| `prov:qualifiedAssociation` + `prov:hadRole` | `role` | The agent's role for this specific entity — `"generator"` (default), `"approver"`, `"reviewer"` — for sign-off/four-eyes workflows |
|
||||
| `prov:wasDerivedFrom` | `parent_entity_id` (legacy combined field) | The previous version or source of this entity |
|
||||
| — | `previous_version_id` | This entry corrects/replaces a prior version of the *same* fact |
|
||||
| `prov:wasDerivedFrom` | `derived_from_id` | This entry was derived from a *different* source entity |
|
||||
| `prov:used` | `used_entities` | Entity IDs consumed to produce this one |
|
||||
| `prov:generatedAtTime` | `timestamp` | ISO datetime, auto-set to `datetime.utcnow()` at write time |
|
||||
| `prov:qualifiedInvalidation` | `invalidated`, `invalidated_at_time`, `invalidated_by`, `invalidation_reason` | A retraction/correction recorded as a tombstone via `ProvenanceManager.invalidate()`, never a hard delete |
|
||||
| `prov:startedAtTime` / `prov:endedAtTime` | `activity_started_at_time`, `activity_ended_at_time` | Typed Activity timing — pass an `ActivityRecord` via the `activity=` kwarg to set these together with `activity_id` |
|
||||
| `prov:qualifiedGeneration`/`Generation`, `qualifiedUsage`/`Usage`, `qualifiedDerivation`/`Derivation` | (derived from the fields above) | Additive qualified forms of `wasGeneratedBy`/`used`/`wasDerivedFrom`, emitted automatically alongside the plain triples |
|
||||
| `prov:wasAssociatedWith` | (derived from `agent_id`) | Direct Activity→Agent link, distinct from the Entity→Agent `wasAttributedTo` |
|
||||
| `prov:actedOnBehalfOf` | `acted_on_behalf_of` | Agent→Agent delegation — e.g. an automated agent acting on behalf of the human/organization that authorized it |
|
||||
| `prov:wasInformedBy` | `informed_by_activities` (pass as `informed_by=[...]`) | Chains this entry's activity to prior activities it was informed by (e.g. a pipeline stage informed by the stage before it) |
|
||||
| `prov:Bundle` + `prov:hadMember` | `bundle_id` | Groups entries by source/dataset/ingestion-run (membership triples, not true RDF named-graph partitioning) |
|
||||
| — | `valid_from`, `valid_until`, `revision_type`, `supersedes` | Bitemporal fields merged from the deprecated `kg.ProvenanceTracker` — always caller-supplied (never auto-computed), surfaced via `ProvenanceManager.revision_history()`, which falls back to timestamp-based derivation for entries that don't set them explicitly |
|
||||
|
||||
The `checksum` field is not part of the PROV-O standard — it is Semantica's tamper-detection extension. Every entry's SHA-256 is computed from its content fields at write time and can be recomputed at any time to verify the record has not been modified.
|
||||
`previous_version_id` and `derived_from_id` are additive alongside `parent_entity_id` — existing code reading `parent_entity_id` keeps working unchanged, while new code gets the two relations disambiguated.
|
||||
|
||||
The `checksum` field is not part of the PROV-O standard — it is Semantica's tamper-detection extension. Every entry's SHA-256 now also incorporates `previous_checksum` (the prior entry's checksum, by insertion order via `sequence_id`), chaining every entry to the one before it. `ProvenanceManager.verify_chain()` walks the full chain and reports any break — including a row that was hard-deleted from the underlying table, which a lone per-row checksum can't detect on its own.
|
||||
|
||||
Note: the banking example above passes `agent_id="credit_data_service_v2"` to `track_entities_batch()` — this now actually populates the entry's `agent_id` field (previously a bug caused batch-level typed kwargs like `agent_id`/`entity_type`/`activity_id` to be silently absorbed into the opaque `metadata` blob instead).
|
||||
|
||||
`export_prov()` mints entity/agent/activity URIs under `ProvenanceManager.DEFAULT_BASE_URI` (`https://semantica.dev/ns#` by default — the same namespace `RDFExporter`'s `NamespaceManager` uses for its `"semantica"` prefix, so KG-exported and PROV-exported URIs for the same `entity_id` co-resolve) unless overridden via `export_prov(base_uri=...)` or the CLI's `--base-uri` option.
|
||||
|
||||
## Related Guides
|
||||
|
||||
|
||||
+9
-1
@@ -3,6 +3,10 @@ title: "Semantica"
|
||||
description: "The Accountability and Context Layer for AI: Context Graphs · Decision Intelligence · Full Provenance"
|
||||
---
|
||||
|
||||
```bash
|
||||
pip install semantica
|
||||
```
|
||||
|
||||
Your AI agent just made a decision. Now someone needs to explain it.
|
||||
|
||||
*What did it know at the time? Which facts shaped the outcome? Where did those facts come from? Has it made the same call before: and did that go well?*
|
||||
@@ -188,7 +192,11 @@ decision_id = context.record_decision(
|
||||
|
||||
## Built for Where Mistakes Have Consequences
|
||||
|
||||
Semantica was designed for domains where every decision must be explainable and every fact must be traceable:
|
||||
Semantica was designed for domains where every decision must be explainable and every fact must be traceable.
|
||||
|
||||
<Warning>
|
||||
**This is system-level explainability, not foundation-model explainability.** Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. What Semantica explains is *outside* the model: the context and data fed in, the decision produced, its provenance, the relevant relationships, the policies applied, and the full execution trail. See [Core Concepts](concepts) for the full scope note.
|
||||
</Warning>
|
||||
|
||||
**Healthcare & Life Sciences**
|
||||
- Clinical decision support with full audit trails
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
---
|
||||
title: "CrewAI Integration"
|
||||
description: "Give CrewAI crews a shared semantic knowledge graph, decision intelligence, and graph-based retrieval via three drop-in components."
|
||||
icon: "users"
|
||||
---
|
||||
|
||||
> Three drop-in components that bring Semantica's knowledge graph and decision intelligence into any CrewAI crew.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install "semantica[crewai]"
|
||||
```
|
||||
|
||||
Requires `crewai >= 0.80.0`. If `crewai` is not installed, the integration still imports — every class carries the full Semantica API and degrades gracefully, but cannot be passed to a `Crew`.
|
||||
|
||||
## Components at a Glance
|
||||
|
||||
- **SemanticaKGTool** — `Agent(tools=[…])`: 5 KG construction/query actions: extract entities, extract relations, add to graph, query graph, find related.
|
||||
- **SemanticaDecisionTool** — `Agent(tools=[…])`: 5 decision intelligence actions: record decisions, find precedents, trace causal chains, analyze impact, check policies.
|
||||
- **SemanticaKnowledgeSource** — `Crew(knowledge_sources=[…])`: Serializes a `ContextGraph` into CrewAI knowledge storage so every agent gets retrieval access to the graph.
|
||||
|
||||
## Component Details
|
||||
|
||||
<Tabs>
|
||||
<Tab title="SemanticaKGTool">
|
||||
Lets agents actively **build and query** a shared `ContextGraph` mid-reasoning.
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKGTool
|
||||
|
||||
graph = ContextGraph()
|
||||
|
||||
analyst = Agent(
|
||||
role="Knowledge Analyst",
|
||||
goal="Build and explore a knowledge graph from documents",
|
||||
backstory="You map entities and relationships into a shared graph.",
|
||||
tools=[SemanticaKGTool(graph=graph)],
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[analyst],
|
||||
tasks=[Task(
|
||||
description="Extract and link key entities from the brief",
|
||||
expected_output="JSON",
|
||||
agent=analyst,
|
||||
)],
|
||||
)
|
||||
crew.kickoff()
|
||||
```
|
||||
|
||||
| Tool | Description |
|
||||
| :------ | :------------- |
|
||||
| `extract_entities` | Extract named entities from `text` |
|
||||
| `extract_relations` | Extract relationships between entities in `text` |
|
||||
| `add_to_graph` | Extract entities/relations from `text` and add them to the shared graph |
|
||||
| `query_graph` | Keyword-search the graph by node id, type, and content using `query` |
|
||||
| `find_related` | Find concepts related to `entity` within `hops` hops |
|
||||
|
||||
All actions return JSON so agents get parseable results.
|
||||
|
||||
**Sharing a graph:** the tool reads/writes whatever `graph` you pass in. When no `graph` is given, a fresh in-memory `ContextGraph()` is created (and a warning is logged) — two tool instances that each auto-create their own graph do **not** share knowledge. Pass the same `ContextGraph` to every agent that must share state.
|
||||
</Tab>
|
||||
<Tab title="SemanticaDecisionTool">
|
||||
Exposes Semantica's decision intelligence as a native CrewAI tool, backed by `AgentContext`.
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from integrations.crewai import SemanticaDecisionTool
|
||||
|
||||
planner = Agent(
|
||||
role="Decision Planner",
|
||||
goal="Make grounded, precedented decisions",
|
||||
backstory="You record decisions and validate them against policy.",
|
||||
tools=[SemanticaDecisionTool()],
|
||||
)
|
||||
|
||||
crew = Crew(agents=[planner], tasks=[...])
|
||||
```
|
||||
|
||||
When no `AgentContext` is passed, one is created in-memory with `decision_tracking=True` and its own `ContextGraph`, so decision actions work out of the box (a warning is logged — pass the same `AgentContext` to every agent that must share decision state). Missing optional fields in `record_decision` fall back to `category="general"`, `reasoning="agent decision"`, and `outcome="recorded"`. `find_precedents` returns up to `max_precedents` results. If a knowledge graph cannot trace causality, `trace_causal_chain` returns an explicit error rather than substituting similarity-based results.
|
||||
|
||||
| Tool | Description |
|
||||
| :------ | :------------- |
|
||||
| `record_decision` | Record a decision with reasoning, outcome, and confidence |
|
||||
| `find_precedents` | Search for similar past decisions |
|
||||
| `trace_causal_chain` | Trace the causal chain from a decision |
|
||||
| `analyze_impact` | Assess downstream influence of a decision |
|
||||
| `check_policy` | Validate a proposed decision against policy rules |
|
||||
</Tab>
|
||||
<Tab title="SemanticaKnowledgeSource">
|
||||
Gives **every agent in the crew** retrieval access to a `ContextGraph`.
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKnowledgeSource
|
||||
|
||||
graph = ContextGraph()
|
||||
graph.add_node(node_id="privacy", node_type="policy", content="...")
|
||||
|
||||
researcher = Agent(
|
||||
role="Policy Researcher",
|
||||
goal="Answer questions from the knowledge base",
|
||||
backstory="You retrieve from graph knowledge to answer accurately.",
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[researcher],
|
||||
tasks=[...],
|
||||
knowledge_sources=[SemanticaKnowledgeSource(graph=graph)],
|
||||
)
|
||||
```
|
||||
|
||||
On kickoff the graph's nodes and edges are serialized, chunked, and stored through CrewAI's knowledge pipeline.
|
||||
|
||||
> **Embedder required:** storing chunks goes through CrewAI's knowledge pipeline, which needs an embedder to be configured. Set `Crew(embedder=...)` (or provide the default credentials CrewAI falls back to, e.g. `OPENAI_API_KEY`). If no working embedder is configured, storage fails, an ERROR is logged, and agents will retrieve **nothing** — the crew still runs, but its knowledge queries return empty.
|
||||
|
||||
**Compatibility:** CrewAI's `BaseKnowledgeSource` contract changed between `0.80.x` and current releases (`load_content()` → `validate_content()`/`aadd()`). `SemanticaKnowledgeSource` implements both legacy and current methods, so it works across `crewai>=0.80.0`.
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
## Checkpoints & Serialization
|
||||
|
||||
CrewAI serializes tools and knowledge sources to JSON for checkpointing/resume. Live Semantica state (`ContextGraph`, `AgentContext`, extractors) is **excluded from that serialization** — a restored tool/source comes back with a fresh in-memory `ContextGraph` and logs a warning. Until you re-attach the live graph/context, the restored objects answer queries against an **empty** graph, so re-wire them after resuming (e.g. `restored_tool.graph = live_graph`) before agents continue.
|
||||
|
||||
## API Reference
|
||||
|
||||
```python
|
||||
from integrations.crewai import (
|
||||
SemanticaKGTool, # BaseTool: KG construction/query actions
|
||||
SemanticaDecisionTool, # BaseTool: decision intelligence actions
|
||||
SemanticaKnowledgeSource, # BaseKnowledgeSource: graph → crew knowledge
|
||||
CREWAI_AVAILABLE, # bool: True if crewai is installed
|
||||
)
|
||||
```
|
||||
|
||||
All three classes are usable without `crewai` installed: they carry the full Semantica API and degrade gracefully.
|
||||
|
||||
## See Also
|
||||
|
||||
- [Context Module](../reference/context) — AgentContext and ContextGraph backing the integration.
|
||||
- [Semantic Extraction](../reference/semantic_extract) — NERExtractor / RelationExtractor used by SemanticaKGTool.
|
||||
- [LLMs](../reference/llms) — Configure LLM providers for your crew's agents.
|
||||
- [Vector Store](../reference/vector_store) — Vector backend used by SemanticaDecisionTool.
|
||||
@@ -17,8 +17,8 @@ Every method on `kg.ProvenanceTracker` now emits a `DeprecationWarning` on use,
|
||||
| `track_entity(entity_id, source, metadata)` | `track_entity(entity_id, source, metadata)` | Same call shape. `ProvenanceManager` additionally auto-links each update to its prior version via `parent_entity_id`. |
|
||||
| `get_all_sources(entity_id)` | `get_all_sources(entity_id)` | Field name differs: the `kg` tracker returns each record's time under `"recorded_at"`; `ProvenanceManager` returns `"timestamp"`. |
|
||||
| `clear(entity_id=None)` | `clear()` | `ProvenanceManager.clear()` clears all provenance data; there is no per-entity clear yet. |
|
||||
| `query_recorded_between(start, end)` | *No direct equivalent yet* | Filter the entries returned by `get_lineage()` / `trace_lineage()` client-side in the meantime. |
|
||||
| `revision_history(fact_id)` | *No direct equivalent yet* | `get_lineage(fact_id)["lineage_chain"]` returns the full chain of `ProvenanceEntry` records but not in the same versioned shape. |
|
||||
| `query_recorded_between(start, end)` | `query_recorded_between(start, end)` | Same call shape; filters by `timestamp` (ISO 8601 string comparison) across all tracked entries, not just one entity. |
|
||||
| `revision_history(fact_id)` | `revision_history(fact_id)` | Same call shape and return shape (`version`, `valid_from`, `valid_until`, `recorded_at`, `author`, optional `revision_type`/`supersedes`) — walks the entity's `previous_version_id` chain rather than a flat per-entity dict. |
|
||||
| `export_audit_log(fact_ids, format)` | *No direct equivalent yet* | Build the export from `get_lineage()` output, or serialize `get_statistics()` for a summary view. |
|
||||
|
||||
Methods with no direct equivalent are not planned to be reimplemented on `kg.ProvenanceTracker` — they will need a small adapter in caller code, or a feature request against `ProvenanceManager` if you rely on them heavily.
|
||||
|
||||
+14
-6
@@ -31,7 +31,7 @@ Semantica is organized into **27 modules** across six logical layers. Each modul
|
||||
Loads data from files, web, databases, and streams into a unified `SourceDocument` format.
|
||||
|
||||
```python
|
||||
from semantica.ingest import FileIngestor, WebIngestor, ParquetIngestor, XMLIngestor
|
||||
from semantica.ingest import FileIngestor, WebIngestor, ParquetIngestor, XMLIngestor, DatabricksIngestor
|
||||
|
||||
# Files: PDF, DOCX, CSV, Excel, PPTX, JSON, HTML, archives
|
||||
ingestor = FileIngestor()
|
||||
@@ -39,18 +39,26 @@ documents = ingestor.ingest_directory("data/")
|
||||
|
||||
# Web crawl
|
||||
web_ingestor = WebIngestor()
|
||||
pages = web_ingestor.ingest_urls(["https://example.com"])
|
||||
page = web_ingestor.ingest_url("https://example.com")
|
||||
|
||||
# Parquet: single file, partitioned directory, Hive-style (v0.5.0)
|
||||
parquet = ParquetIngestor()
|
||||
sources = parquet.ingest("data/events.parquet")
|
||||
|
||||
# XML with XSD/DTD validation, namespace handling (v0.5.0)
|
||||
xml = XMLIngestor(validate_xsd="schema.xsd")
|
||||
sources = xml.ingest("data/records/")
|
||||
xml = XMLIngestor()
|
||||
sources = xml.ingest("data/records/", schema_path="schema.xsd")
|
||||
|
||||
# Enterprise lakehouse/warehouse — Unity Catalog + Delta Lake, or a Snowflake warehouse
|
||||
databricks = DatabricksIngestor(host="...", token="...", http_path="...")
|
||||
customers = databricks.ingest_table("customers")
|
||||
```
|
||||
|
||||
**Available ingestors:** `FileIngestor`, `WebIngestor`, `ParquetIngestor`, `XMLIngestor`, `RESTIngestor`, `PublicAPIIngestor`, `DBIngestor`, `DuckDBIngestor`, `ElasticIngestor`, `EmailIngestor`, `FeedIngestor`, `GDriveIngestor`, `HuggingFaceIngestor`, `MCPIngestor`, `MongoIngestor`, `OntologyIngestor`, `PandasIngestor`, `RepoIngestor`, `SnowflakeIngestor`, `StreamIngestor`
|
||||
**Available ingestors:** `FileIngestor`, `WebIngestor`, `ParquetIngestor`, `XMLIngestor`, `RESTIngestor`, `PublicAPIIngestor`, `DBIngestor`, `DatabricksIngestor`, `SnowflakeIngestor`, `EmailIngestor`, `FeedIngestor`, `MCPIngestor`, `OntologyIngestor`, `RepoIngestor`, `StreamIngestor`, `ArrowIngestor`, `CloudStorageIngestor`
|
||||
|
||||
<Note>
|
||||
`DuckDBIngestor`, `ElasticIngestor`, `GDriveIngestor`, `HuggingFaceIngestor`, `MongoIngestor`, and `PandasIngestor` also ship but aren't re-exported from the top-level `semantica.ingest` namespace yet — import them directly, e.g. `from semantica.ingest.duckdb_ingestor import DuckDBIngestor`.
|
||||
</Note>
|
||||
|
||||
### Parse
|
||||
|
||||
@@ -243,7 +251,7 @@ store.add_triplets(subject, predicate, obj)
|
||||
results = store.sparql("SELECT ?s ?p ?o WHERE { ?s ?p ?o }")
|
||||
```
|
||||
|
||||
**Backends:** Blazegraph, Apache Jena, RDF4J
|
||||
**Backends:** Oxigraph (embedded), Blazegraph, Apache Jena, RDF4J
|
||||
|
||||
|
||||
## Quality Assurance
|
||||
|
||||
@@ -586,6 +586,51 @@ history = memory.get_conversation_history(conversation_id="conv_001", max_items=
|
||||
| `max_memory_size` | `int` | `10000` | Max items before LRU eviction |
|
||||
| `retention_policy` | `str` | `"unlimited"` | `"N_days"` (e.g. `"30_days"`) or `"unlimited"` |
|
||||
|
||||
### Markdown Round Trips
|
||||
|
||||
`AgentMemory` can export human-editable Markdown and import the edited files back.
|
||||
Each file contains one memory item, with required metadata in YAML frontmatter and
|
||||
the memory content in the Markdown body:
|
||||
|
||||
```markdown
|
||||
---
|
||||
id: mem_compliance_rule
|
||||
created_at: '2026-07-22T09:00:00+00:00'
|
||||
updated_at: '2026-07-22T10:30:00+00:00'
|
||||
type: compliance
|
||||
tags:
|
||||
- trading
|
||||
- approval
|
||||
---
|
||||
|
||||
All trades must be pre-approved.
|
||||
```
|
||||
|
||||
```python
|
||||
from pathlib import Path
|
||||
|
||||
# A single selected memory can be returned as Markdown text.
|
||||
document = memory.export(format="markdown", type="compliance")
|
||||
|
||||
# Export a memory set as one stable Markdown file per item.
|
||||
memory.export(format="markdown", destination="memory_export/")
|
||||
|
||||
# New IDs create memories; existing IDs are updated in place.
|
||||
count = memory.import_data(Path("memory_export/"), format="markdown")
|
||||
```
|
||||
|
||||
The required frontmatter fields are `id`, `created_at`, `updated_at`, and either
|
||||
`type` or `kind`. Optional metadata can be edited at the top level. Imports reject
|
||||
malformed or duplicate fields before changing memory, and re-importing unchanged
|
||||
files is idempotent. Memory-local `entities` and `relationships` are preserved as
|
||||
provenance but are not applied to `ContextGraph` by Markdown import. Use a dedicated
|
||||
export directory: matching files are overwritten, but unrelated or stale Markdown
|
||||
files are not deleted automatically. Export refuses to overwrite symbolic links and
|
||||
uses atomic file replacement. Timestamp offsets are preserved in Markdown and
|
||||
normalized to UTC only for comparisons, so aware and local-naive records can be
|
||||
queried together safely. Vector-store writes are deferred until the in-memory import
|
||||
commits; adapter synchronization remains best-effort and logs failures.
|
||||
|
||||
|
||||
## PolicyEngine
|
||||
|
||||
|
||||
@@ -258,11 +258,16 @@ Full interactive docs at `http://localhost:8000/docs`. All endpoints accept and
|
||||
| `/api/vocabulary/hierarchy` | `GET` | Concept hierarchy tree |
|
||||
| `/api/vocabulary/import` | `POST` | Import SKOS/RDF vocabulary file |
|
||||
|
||||
SKOS hierarchy writes reject cycles in both `skos:broader` and
|
||||
`skos:narrower` relationships. Vocabulary imports validate the complete
|
||||
batch before adding nodes, while direct graph/session edge writes apply
|
||||
the same invariant at the graph storage boundary.
|
||||
|
||||
**SPARQL:**
|
||||
|
||||
| Endpoint | Method | Description |
|
||||
| :-------- | :------ | :----------- |
|
||||
| `/api/sparql` | `POST` | Execute a SPARQL SELECT or ASK query |
|
||||
| `/api/sparql` | `POST` | Execute a read-only SPARQL query (`SELECT`, `ASK`, `CONSTRUCT`, or `DESCRIBE`); `CONSTRUCT`/`DESCRIBE` return triples as `subject`, `predicate`, `object` columns, and `ASK` returns a `result` boolean column |
|
||||
|
||||
</Accordion>
|
||||
<Accordion title="Decisions, Provenance, Annotations & Export">
|
||||
|
||||
@@ -203,6 +203,13 @@ export_lpg(graph, "import.cypher", method="cypher")
|
||||
exporter = SemanticNetworkYAMLExporter()
|
||||
exporter.export(graph, "graph.yaml")
|
||||
```
|
||||
|
||||
The YAML exporters read `entities`/`relationships`/`triplets` (with
|
||||
`nodes`/`edges` accepted as aliases, so `ContextGraph.to_dict()` exports
|
||||
directly). A non-empty mapping supplying none of them raises
|
||||
`ValidationError` rather than writing a file with every collection empty,
|
||||
as does one whose collection value is not a list of records
|
||||
(`{"entities": "abc"}`).
|
||||
</Tab>
|
||||
<Tab title="Graph DB Import">
|
||||
**LPGExporter** writes Cypher `CREATE` statements for Neo4j and Memgraph:
|
||||
@@ -236,6 +243,12 @@ export_lpg(graph, "import.cypher", method="cypher")
|
||||
|
||||
Both exporters write to a file and return `None`.
|
||||
|
||||
`LPGExporter`, `ArangoAQLExporter`, and `Neo4jCSVExporter` resolve mapping
|
||||
payloads on the same terms as the YAML exporters above, so an unrecognized
|
||||
or malformed mapping is rejected instead of exported as an empty graph.
|
||||
`Neo4jCSVExporter` still reads graph *objects* off their
|
||||
`nodes`/`entities` and `edges`/`relationships` attributes.
|
||||
|
||||
<Warning>
|
||||
**`ArangoAQLExporter.export()` and `LPGExporter.export()` write to a file and return `None`.** They do not return the AQL/Cypher string. Write to a file and read it back if you need the string.
|
||||
</Warning>
|
||||
|
||||
@@ -6,7 +6,7 @@ icon: "database"
|
||||
|
||||
**`semantica.ingest`** is the **universal entry point** for loading data into Semantica:
|
||||
|
||||
- 15+ ingestion adapters: files, web, SQL, Snowflake, Kafka, MCP, Git repos, email
|
||||
- 15+ ingestion adapters: files, web, SQL, Databricks, Snowflake, Kafka, MCP, Git repos, email
|
||||
- PyArrow Parquet with column selection and partitioned dataset support
|
||||
- XXE-safe lxml XML with optional XSD schema validation
|
||||
- `ingest()` unified dispatcher: auto-detects source type from path or URL
|
||||
@@ -29,6 +29,7 @@ icon: "database"
|
||||
| `SnowflakeIngestor` | Snowflake data warehouse queries and table exports |
|
||||
| `DatabricksIngestor` | Databricks Unity Catalog metadata, Delta table queries, and lineage |
|
||||
| `ParquetIngestor` | Apache Parquet files and partitioned datasets with column selection |
|
||||
| `ArrowIngestor` | Apache Arrow IPC and Feather file processing |
|
||||
| `XMLIngestor` | XXE-safe XML parsing with optional XSD schema validation |
|
||||
| `EmailIngestor` | IMAP/POP3 email ingestion with attachment extraction |
|
||||
| `OntologyIngestor` | OWL/RDF/Turtle ontology file ingestion |
|
||||
|
||||
@@ -155,13 +155,15 @@ prop_entry = manager.track_property_source(
|
||||
|
||||
### Batch Tracking
|
||||
|
||||
Batch tracking methods process items in blocks (default `batch_size=1000`) inside a shared transaction per block. Only entities or chunks that successfully commit to storage are added to the returned count, preventing rolled-back entries from inflating success counts.
|
||||
|
||||
```python
|
||||
entities = [
|
||||
{"id": "entity_1", "confidence": 0.9},
|
||||
{"id": "entity_2", "confidence": 0.85},
|
||||
]
|
||||
count = manager.track_entities_batch(entities, source="doc_1")
|
||||
# Returns the number of entities successfully tracked
|
||||
# Returns the number of entities successfully tracked and committed
|
||||
|
||||
chunks = [
|
||||
{"id": "chunk_0", "start_index": 0, "end_index": 500},
|
||||
@@ -219,10 +221,10 @@ cleared = manager.clear()
|
||||
|
||||
| Method | Returns | Description |
|
||||
| :------ | :------- | :----------- |
|
||||
| `track_entity(entity_id, source, metadata, **kwargs)` | `ProvenanceEntry` | Record entity provenance; checksum set automatically |
|
||||
| `track_relationship(relationship_id, source, metadata, **kwargs)` | `ProvenanceEntry` | Record relationship provenance |
|
||||
| `track_chunk(chunk_id, source_document, ...)` | `ProvenanceEntry` | Record chunk provenance with char offsets |
|
||||
| `track_property_source(entity_id, property_name, value, source)` | `ProvenanceEntry` | Record property-level source attribution |
|
||||
| `track_entity(entity_id, source, metadata, **kwargs)` | `Optional[ProvenanceEntry]` | Record entity provenance atomically; returns `ProvenanceEntry` on success, or `None`/existing entry on storage failure |
|
||||
| `track_relationship(relationship_id, source, metadata, **kwargs)` | `Optional[ProvenanceEntry]` | Record relationship provenance; returns `ProvenanceEntry` on success, or `None` on storage failure |
|
||||
| `track_chunk(chunk_id, source_document, ...)` | `Optional[ProvenanceEntry]` | Record chunk provenance with char offsets; returns `ProvenanceEntry` on success, or `None` on storage failure |
|
||||
| `track_property_source(entity_id, property_name, value, source)` | `Optional[ProvenanceEntry]` | Record property-level source attribution; returns `ProvenanceEntry` on success, or `None` on storage failure |
|
||||
| `track_entities_batch(entities, source)` | `int` | Batch-track entities; returns success count |
|
||||
| `track_chunks_batch(chunks, source_document)` | `int` | Batch-track chunks; returns success count |
|
||||
| `get_lineage(entity_id)` | `Dict[str, Any]` | Full lineage as aggregated dict |
|
||||
@@ -234,7 +236,7 @@ cleared = manager.clear()
|
||||
|
||||
## ProvenanceEntry Fields
|
||||
|
||||
`ProvenanceEntry` is the core dataclass. Every tracking method returns one:
|
||||
`ProvenanceEntry` is the core dataclass. Every tracking method returns one on success (or `None` on storage failure):
|
||||
|
||||
```python
|
||||
from semantica.provenance import ProvenanceEntry
|
||||
@@ -322,6 +324,9 @@ manager = ProvenanceManager(storage_path="provenance.db")
|
||||
|
||||
`SQLiteStorage` creates the database and indexes automatically on first use.
|
||||
|
||||
- **Atomicity & Concurrency**: Configures Write-Ahead Logging (`PRAGMA journal_mode=WAL`), `PRAGMA busy_timeout=5000`, and `PRAGMA synchronous=NORMAL`. Read-modify-write methods (`track_entity()`, `store()`) open a single connection and execute inside an immediate write transaction (`BEGIN IMMEDIATE`), ensuring these sequences are serialized across concurrent connections without leaving open file handles across calls. Plain reads (`retrieve()`, `trace_lineage()`) use a separate connection with no explicit write lock, so concurrent reads don't serialize behind writers or each other.
|
||||
- **Backward Compatibility**: Custom storage subclasses overriding `trace_lineage(self, entity_id)` remain backward compatible; `ProvenanceManager` inspects the override signature and automatically calls it with one argument if `max_depth` is unsupported.
|
||||
|
||||
## Tamper-Evident Checksums
|
||||
|
||||
`compute_checksum` and `verify_checksum` are auto-used by `track_entity` and all other tracking methods. You can also call them directly:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
title: "Triplet Store Module"
|
||||
description: "RDF triple storage with SPARQL queries and bulk loading: Blazegraph, Apache Jena, and RDF4J."
|
||||
description: "Embedded and server-backed RDF storage with SPARQL queries and bulk loading."
|
||||
icon: "table"
|
||||
---
|
||||
|
||||
@@ -16,14 +16,15 @@ icon: "table"
|
||||
| `BlazegraphStore` | Blazegraph REST API: SPARQL 1.1 Update, namespace management |
|
||||
| `JenaStore` | Apache Jena: rdflib-backed, SPARQL read support via remote endpoint |
|
||||
| `RDF4JStore` | Eclipse RDF4J: REST API, transaction support |
|
||||
| `OxigraphStore` | Embedded SPARQL 1.1 store with in-memory and on-disk modes |
|
||||
|
||||
## What You Get
|
||||
|
||||
- **TripletStore** — Unified interface across Blazegraph, Apache Jena, and RDF4J: swap backends with one parameter.
|
||||
- **TripletStore** — Unified interface across embedded Oxigraph, Blazegraph, Apache Jena, and RDF4J: swap backends with one parameter.
|
||||
- **SPARQL** — Full SPARQL SELECT, ASK, CONSTRUCT, and UPDATE query support via `execute_query()`.
|
||||
- **Bulk Loading** — `add_triplets()` batches writes with configurable batch size, retry logic, and progress tracking.
|
||||
- **SKOS Vocabulary** — Built-in helpers: `add_skos_concept()` and `get_skos_concepts()` for controlled vocabulary management.
|
||||
- **Named Graphs** — Blazegraph and RDF4J support named graph scoping via `graph=` on `execute_query()`.
|
||||
- **Named Graphs** — Oxigraph, Blazegraph, and RDF4J support named graph scoping via `graph=` on `execute_query()`.
|
||||
- **Delta Computation** — `compute_delta(old_graph_uri, new_graph_uri)` returns added and removed triples between two named graph snapshots.
|
||||
|
||||
## Getting Started
|
||||
@@ -117,6 +118,25 @@ for row in result.bindings:
|
||||
## Backends
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Oxigraph">
|
||||
```bash
|
||||
pip install "semantica[tripletstore-oxigraph]"
|
||||
```
|
||||
|
||||
```python
|
||||
# In-memory: no server process or files required
|
||||
store = TripletStore(backend="oxigraph")
|
||||
|
||||
# Persistent: reopen the same directory to reuse the data
|
||||
persistent_store = TripletStore(
|
||||
backend="oxigraph",
|
||||
path="./data/knowledge-graph",
|
||||
)
|
||||
```
|
||||
|
||||
**Best for:** local development, CI, desktop applications, and persistent
|
||||
single-process workloads without external infrastructure.
|
||||
</Tab>
|
||||
<Tab title="Blazegraph">
|
||||
```bash
|
||||
pip install requests
|
||||
@@ -172,6 +192,7 @@ for row in result.bindings:
|
||||
|
||||
| Backend | License | Named Graphs | Write via | Best For |
|
||||
| :------- | :------- | :------------ | :--------- | :-------- |
|
||||
| Oxigraph | Apache 2.0 / MIT | Yes | Embedded native API | Local, CI, on-disk |
|
||||
| Blazegraph | Open source | Yes | SPARQL Update REST | High triple count, SPARQL 1.1 |
|
||||
| Apache Jena | Apache 2.0 | No (rdflib backend) | rdflib in-process | Local dev, read queries |
|
||||
| RDF4J | Eclipse 1.0 | Yes | REST API N-Triples | Enterprise Java, transactions |
|
||||
@@ -180,7 +201,9 @@ for row in result.bindings:
|
||||
</Tabs>
|
||||
|
||||
<Tip>
|
||||
**Use Apache Jena for development, Blazegraph for production.** Jena initializes with rdflib in-memory: no server required for local testing. Switch to Blazegraph for high-throughput persistent workloads by changing `backend=`.
|
||||
**Use Oxigraph for zero-infrastructure development and local persistence.**
|
||||
Switch to a server-backed store for distributed production deployments by
|
||||
changing `backend=`.
|
||||
</Tip>
|
||||
|
||||
## Triplet Object
|
||||
@@ -364,10 +387,10 @@ while True:
|
||||
|
||||
## Named Graph Scoping
|
||||
|
||||
Blazegraph and RDF4J support named graphs. Scope `execute_query()` to a named graph with the `graph=` parameter:
|
||||
Oxigraph, Blazegraph, and RDF4J support named graphs. Scope `execute_query()` to a named graph with the `graph=` parameter:
|
||||
|
||||
```python
|
||||
# Add a triplet: named graph stored in metadata or backend-specific API
|
||||
# Add a triplet to a named graph
|
||||
from semantica.semantic_extract.types import Triplet
|
||||
|
||||
t = Triplet(
|
||||
@@ -375,7 +398,7 @@ t = Triplet(
|
||||
predicate="http://example.org/p",
|
||||
object="http://example.org/b",
|
||||
)
|
||||
store.add_triplet(t) # named graph targeting requires backend-specific API
|
||||
store.add_triplet(t, graph="http://example.org/graph1")
|
||||
|
||||
# Query a named graph via FROM clause in SPARQL
|
||||
result = store.execute_query("""
|
||||
@@ -393,11 +416,14 @@ result = store.execute_query("""
|
||||
```
|
||||
|
||||
<Note>
|
||||
Named graph support is only available for Blazegraph and RDF4J backends. The `graph=` parameter is silently ignored for the Jena backend.
|
||||
Named graph query scoping is available for Oxigraph, Blazegraph, and RDF4J.
|
||||
The `graph=` query parameter is silently ignored for the Jena backend.
|
||||
</Note>
|
||||
|
||||
<Tip>
|
||||
**Use named graphs to isolate sources.** Pass `graph="http://example.org/source_A"` to `execute_query()` to scope a query to a specific named graph. Blazegraph and RDF4J support named graphs; Jena (rdflib backend) does not.
|
||||
**Use named graphs to isolate sources.** Pass `graph="http://example.org/source_A"`
|
||||
to writes and `execute_query()` to scope both storage and retrieval. Oxigraph,
|
||||
Blazegraph, and RDF4J support named graph query scoping.
|
||||
</Tip>
|
||||
|
||||
## Bulk Loading
|
||||
|
||||
+10
-1
@@ -63,7 +63,9 @@ semantica-explorer --graph my_graph.json --no-browser
|
||||
python -m semantica.explorer --graph my_graph.json
|
||||
```
|
||||
|
||||
> **Security note:** The Explorer API has no built-in authentication. The default `--host 127.0.0.1` binds to localhost only, so it is not reachable from other machines on your network. If you bind to `0.0.0.0`, all graph data is readable and writable by any host that can reach the port. The CLI will print a warning in that case.
|
||||
> **Security note:** Since v0.6.5 the Explorer API requires an API key on protected routes. Set the `SEMANTICA_API_KEY` environment variable and send it as the `X-API-Key` header; without a configured key, protected routes fail closed with `503` rather than serving anonymously. To opt into unauthenticated access for local development only, set `SEMANTICA_ALLOW_ANONYMOUS=true` explicitly. (`/api/health` and `/api/info` are intentionally unauthenticated.)
|
||||
>
|
||||
> The default `--host 127.0.0.1` binds to localhost only, so it is not reachable from other machines on your network. If you bind to `0.0.0.0`, all graph data is readable and writable by any host that can reach the port (subject to API-key auth). The CLI prints a warning when binding to a non-loopback host in anonymous mode or when `SEMANTICA_API_KEY` is unset.
|
||||
|
||||
---
|
||||
|
||||
@@ -148,6 +150,8 @@ This writes the compiled assets to `../semantica/static/`. The Python server the
|
||||
| --- | --- | --- |
|
||||
| `EXPLORER_CORS_ORIGINS` | `http://localhost:5173,http://127.0.0.1:5173` | Comma-separated list of allowed CORS origins |
|
||||
| `EXPLORER_CORS_CREDENTIALS` | `false` | Set to `true` to allow credentialed cross-origin requests (only needed behind an authenticating reverse proxy) |
|
||||
| `SEMANTICA_API_KEY` | *(unset)* | API key required on protected routes since v0.6.5; send it as the `X-API-Key` header. When unset, protected routes fail closed with `503`. |
|
||||
| `SEMANTICA_ALLOW_ANONYMOUS` | `false` | Set to `true` to opt into unauthenticated access (local development only). |
|
||||
|
||||
---
|
||||
|
||||
@@ -251,6 +255,11 @@ Vite automatically tries the next available port and prints the actual URL in th
|
||||
- Confirm the backend exposes the `/ws/graph-updates` WebSocket endpoint.
|
||||
- Check DevTools → Network → WS tab for the connection status and error code.
|
||||
- Ensure the backend version matches the frontend — mixing major versions can cause protocol mismatches.
|
||||
- **Authentication:** `/ws/graph-updates` enforces the same API key as the REST routes. Browsers cannot set custom headers on a WebSocket handshake, so pass the key as a query parameter instead:
|
||||
```
|
||||
ws://127.0.0.1:8000/ws/graph-updates?api_key=<your-key>
|
||||
```
|
||||
Non-browser clients (native apps, scripts) may send it as the `X-API-Key` header. A missing or incorrect key results in close code `4401`; if `SEMANTICA_API_KEY` is unset and `SEMANTICA_ALLOW_ANONYMOUS` is not `true`, the connection is also rejected. Note that API keys in URLs appear in server logs — prefer the header for non-browser clients.
|
||||
|
||||
---
|
||||
|
||||
|
||||
Generated
+104
-360
@@ -37,7 +37,7 @@
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@vitejs/plugin-react": "^4.3.0",
|
||||
"babel-plugin-react-compiler": "^1.0.0",
|
||||
"eslint": "^9.39.4",
|
||||
"eslint": "^10.8.0",
|
||||
"eslint-plugin-react-hooks": "^7.0.1",
|
||||
"eslint-plugin-react-refresh": "^0.5.2",
|
||||
"globals": "^17.4.0",
|
||||
@@ -836,81 +836,44 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@eslint/config-array": {
|
||||
"version": "0.21.2",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.2.tgz",
|
||||
"integrity": "sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==",
|
||||
"version": "0.23.5",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz",
|
||||
"integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@eslint/object-schema": "^2.1.7",
|
||||
"@eslint/object-schema": "^3.0.5",
|
||||
"debug": "^4.3.1",
|
||||
"minimatch": "^3.1.5"
|
||||
"minimatch": "^10.2.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
"node": "^20.19.0 || ^22.13.0 || >=24"
|
||||
}
|
||||
},
|
||||
"node_modules/@eslint/config-helpers": {
|
||||
"version": "0.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz",
|
||||
"integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==",
|
||||
"version": "0.7.0",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.7.0.tgz",
|
||||
"integrity": "sha512-DObd/KKUsU+FaFv4PLxSRenpXfQWmPXXP3pPZ6/K1PCrMu2vQpMDMuQe/BqYeoLcz8ro0bVDF1RxOJgfVEdhUw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@eslint/core": "^0.17.0"
|
||||
"@eslint/core": "^1.2.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
"node": "^20.19.0 || ^22.13.0 || >=24"
|
||||
}
|
||||
},
|
||||
"node_modules/@eslint/core": {
|
||||
"version": "0.17.0",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz",
|
||||
"integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==",
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz",
|
||||
"integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@types/json-schema": "^7.0.15"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@eslint/eslintrc": {
|
||||
"version": "3.3.5",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.5.tgz",
|
||||
"integrity": "sha512-4IlJx0X0qftVsN5E+/vGujTRIFtwuLbNsVUe7TO6zYPDR1O6nFwvwhIKEKSrl6dZchmYBITazxKoUYOjdtjlRg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ajv": "^6.14.0",
|
||||
"debug": "^4.3.2",
|
||||
"espree": "^10.0.1",
|
||||
"globals": "^14.0.0",
|
||||
"ignore": "^5.2.0",
|
||||
"import-fresh": "^3.2.1",
|
||||
"js-yaml": "^4.1.1",
|
||||
"minimatch": "^3.1.5",
|
||||
"strip-json-comments": "^3.1.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://opencollective.com/eslint"
|
||||
}
|
||||
},
|
||||
"node_modules/@eslint/eslintrc/node_modules/globals": {
|
||||
"version": "14.0.0",
|
||||
"resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz",
|
||||
"integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
"node": "^20.19.0 || ^22.13.0 || >=24"
|
||||
}
|
||||
},
|
||||
"node_modules/@eslint/js": {
|
||||
@@ -927,27 +890,27 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@eslint/object-schema": {
|
||||
"version": "2.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz",
|
||||
"integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==",
|
||||
"version": "3.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz",
|
||||
"integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
"node": "^20.19.0 || ^22.13.0 || >=24"
|
||||
}
|
||||
},
|
||||
"node_modules/@eslint/plugin-kit": {
|
||||
"version": "0.4.1",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz",
|
||||
"integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==",
|
||||
"version": "0.7.2",
|
||||
"resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.2.tgz",
|
||||
"integrity": "sha512-+CNAzxglkrpNf/kKywqQfk74QjtceuOE7Qm+AF8miRvPF/wmmK5+OJOgVh3AVTT3RP2mH3+FOaxlE5v72owk0A==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@eslint/core": "^0.17.0",
|
||||
"@eslint/core": "^1.2.1",
|
||||
"levn": "^0.4.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
"node": "^20.19.0 || ^22.13.0 || >=24"
|
||||
}
|
||||
},
|
||||
"node_modules/@humanfs/core": {
|
||||
@@ -1602,6 +1565,13 @@
|
||||
"@types/d3-selection": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/esrecurse": {
|
||||
"version": "4.3.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz",
|
||||
"integrity": "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/estree": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz",
|
||||
@@ -1849,45 +1819,6 @@
|
||||
"typescript": ">=4.8.4 <6.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": {
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz",
|
||||
"integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "18 || 20 || >=22"
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": {
|
||||
"version": "5.0.6",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz",
|
||||
"integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"balanced-match": "^4.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": "18 || 20 || >=22"
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": {
|
||||
"version": "10.2.5",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz",
|
||||
"integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==",
|
||||
"dev": true,
|
||||
"license": "BlueOak-1.0.0",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^5.0.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": "18 || 20 || >=22"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/typescript-estree/node_modules/semver": {
|
||||
"version": "7.7.4",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz",
|
||||
@@ -1943,19 +1874,6 @@
|
||||
"url": "https://opencollective.com/typescript-eslint"
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz",
|
||||
"integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": "^20.19.0 || ^22.13.0 || >=24"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://opencollective.com/eslint"
|
||||
}
|
||||
},
|
||||
"node_modules/@vitejs/plugin-react": {
|
||||
"version": "4.7.0",
|
||||
"resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz",
|
||||
@@ -2016,9 +1934,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/acorn": {
|
||||
"version": "8.16.0",
|
||||
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz",
|
||||
"integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==",
|
||||
"version": "8.17.0",
|
||||
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.17.0.tgz",
|
||||
"integrity": "sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
@@ -2039,9 +1957,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/ajv": {
|
||||
"version": "6.14.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz",
|
||||
"integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==",
|
||||
"version": "6.15.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz",
|
||||
"integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -2055,29 +1973,6 @@
|
||||
"url": "https://github.com/sponsors/epoberezkin"
|
||||
}
|
||||
},
|
||||
"node_modules/ansi-styles": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
|
||||
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"color-convert": "^2.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/argparse": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
|
||||
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
|
||||
"dev": true,
|
||||
"license": "Python-2.0"
|
||||
},
|
||||
"node_modules/attr-accept": {
|
||||
"version": "2.2.5",
|
||||
"resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.2.5.tgz",
|
||||
@@ -2098,11 +1993,14 @@
|
||||
}
|
||||
},
|
||||
"node_modules/balanced-match": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
|
||||
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz",
|
||||
"integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "18 || 20 || >=22"
|
||||
}
|
||||
},
|
||||
"node_modules/baseline-browser-mapping": {
|
||||
"version": "2.10.20",
|
||||
@@ -2118,14 +2016,16 @@
|
||||
}
|
||||
},
|
||||
"node_modules/brace-expansion": {
|
||||
"version": "1.1.14",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz",
|
||||
"integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==",
|
||||
"version": "5.0.8",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz",
|
||||
"integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"balanced-match": "^1.0.0",
|
||||
"concat-map": "0.0.1"
|
||||
"balanced-match": "^4.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": "20 || >=22"
|
||||
}
|
||||
},
|
||||
"node_modules/browserslist": {
|
||||
@@ -2162,16 +2062,6 @@
|
||||
"node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
|
||||
}
|
||||
},
|
||||
"node_modules/callsites": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
|
||||
"integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/caniuse-lite": {
|
||||
"version": "1.0.30001788",
|
||||
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001788.tgz",
|
||||
@@ -2193,49 +2083,12 @@
|
||||
],
|
||||
"license": "CC-BY-4.0"
|
||||
},
|
||||
"node_modules/chalk": {
|
||||
"version": "4.1.2",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
|
||||
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ansi-styles": "^4.1.0",
|
||||
"supports-color": "^7.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/chalk?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/classcat": {
|
||||
"version": "5.0.5",
|
||||
"resolved": "https://registry.npmjs.org/classcat/-/classcat-5.0.5.tgz",
|
||||
"integrity": "sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/color-convert": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
|
||||
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"color-name": "~1.1.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=7.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/color-name": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/commander": {
|
||||
"version": "2.20.3",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
|
||||
@@ -2253,13 +2106,6 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/concat-map": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
|
||||
"integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/convert-source-map": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz",
|
||||
@@ -2447,9 +2293,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/dompurify": {
|
||||
"version": "3.4.11",
|
||||
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.11.tgz",
|
||||
"integrity": "sha512-zhlUV12GsaRzMsf9q5M254YhA4+VuF0fG+QFqu6aYpoGlKtz+w8//jBcGVYBgQkR5GHjUomejY84AV+/uPbWdw==",
|
||||
"version": "3.4.13",
|
||||
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.13.tgz",
|
||||
"integrity": "sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ==",
|
||||
"license": "(MPL-2.0 OR Apache-2.0)",
|
||||
"peer": true,
|
||||
"optionalDependencies": {
|
||||
@@ -2529,33 +2375,33 @@
|
||||
}
|
||||
},
|
||||
"node_modules/eslint": {
|
||||
"version": "9.39.4",
|
||||
"resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.4.tgz",
|
||||
"integrity": "sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==",
|
||||
"version": "10.8.0",
|
||||
"resolved": "https://registry.npmjs.org/eslint/-/eslint-10.8.0.tgz",
|
||||
"integrity": "sha512-nuKKvN+oIBO0koN7Tm7dlkmnkc21mtt0QJLwAKzjLq14y6lRTdVG36MZHJ8eQHwdJMwZbQNMlPOYedMq/oVJvQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"workspaces": [
|
||||
"packages/*"
|
||||
],
|
||||
"dependencies": {
|
||||
"@eslint-community/eslint-utils": "^4.8.0",
|
||||
"@eslint-community/regexpp": "^4.12.1",
|
||||
"@eslint/config-array": "^0.21.2",
|
||||
"@eslint/config-helpers": "^0.4.2",
|
||||
"@eslint/core": "^0.17.0",
|
||||
"@eslint/eslintrc": "^3.3.5",
|
||||
"@eslint/js": "9.39.4",
|
||||
"@eslint/plugin-kit": "^0.4.1",
|
||||
"@eslint-community/regexpp": "^4.12.2",
|
||||
"@eslint/config-array": "^0.23.5",
|
||||
"@eslint/config-helpers": "^0.7.0",
|
||||
"@eslint/core": "^1.2.1",
|
||||
"@eslint/plugin-kit": "^0.7.2",
|
||||
"@humanfs/node": "^0.16.6",
|
||||
"@humanwhocodes/module-importer": "^1.0.1",
|
||||
"@humanwhocodes/retry": "^0.4.2",
|
||||
"@types/estree": "^1.0.6",
|
||||
"ajv": "^6.14.0",
|
||||
"chalk": "^4.0.0",
|
||||
"cross-spawn": "^7.0.6",
|
||||
"debug": "^4.3.2",
|
||||
"escape-string-regexp": "^4.0.0",
|
||||
"eslint-scope": "^8.4.0",
|
||||
"eslint-visitor-keys": "^4.2.1",
|
||||
"espree": "^10.4.0",
|
||||
"esquery": "^1.5.0",
|
||||
"eslint-scope": "^9.1.2",
|
||||
"eslint-visitor-keys": "^5.0.1",
|
||||
"espree": "^11.2.0",
|
||||
"esquery": "^1.7.0",
|
||||
"esutils": "^2.0.2",
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"file-entry-cache": "^8.0.0",
|
||||
@@ -2565,8 +2411,7 @@
|
||||
"imurmurhash": "^0.1.4",
|
||||
"is-glob": "^4.0.0",
|
||||
"json-stable-stringify-without-jsonify": "^1.0.1",
|
||||
"lodash.merge": "^4.6.2",
|
||||
"minimatch": "^3.1.5",
|
||||
"minimatch": "^10.2.5",
|
||||
"natural-compare": "^1.4.0",
|
||||
"optionator": "^0.9.3"
|
||||
},
|
||||
@@ -2574,7 +2419,7 @@
|
||||
"eslint": "bin/eslint.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
"node": "^20.19.0 || ^22.13.0 || >=24"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://eslint.org/donate"
|
||||
@@ -2619,48 +2464,50 @@
|
||||
}
|
||||
},
|
||||
"node_modules/eslint-scope": {
|
||||
"version": "8.4.0",
|
||||
"resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz",
|
||||
"integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==",
|
||||
"version": "9.1.2",
|
||||
"resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz",
|
||||
"integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==",
|
||||
"dev": true,
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"@types/esrecurse": "^4.3.1",
|
||||
"@types/estree": "^1.0.8",
|
||||
"esrecurse": "^4.3.0",
|
||||
"estraverse": "^5.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
"node": "^20.19.0 || ^22.13.0 || >=24"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://opencollective.com/eslint"
|
||||
}
|
||||
},
|
||||
"node_modules/eslint-visitor-keys": {
|
||||
"version": "4.2.1",
|
||||
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz",
|
||||
"integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==",
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz",
|
||||
"integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
"node": "^20.19.0 || ^22.13.0 || >=24"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://opencollective.com/eslint"
|
||||
}
|
||||
},
|
||||
"node_modules/espree": {
|
||||
"version": "10.4.0",
|
||||
"resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz",
|
||||
"integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==",
|
||||
"version": "11.2.0",
|
||||
"resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz",
|
||||
"integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==",
|
||||
"dev": true,
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"acorn": "^8.15.0",
|
||||
"acorn": "^8.16.0",
|
||||
"acorn-jsx": "^5.3.2",
|
||||
"eslint-visitor-keys": "^4.2.1"
|
||||
"eslint-visitor-keys": "^5.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
"node": "^20.19.0 || ^22.13.0 || >=24"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://opencollective.com/eslint"
|
||||
@@ -2972,16 +2819,6 @@
|
||||
"graphology-types": ">=0.23.0"
|
||||
}
|
||||
},
|
||||
"node_modules/has-flag": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
|
||||
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/hermes-estree": {
|
||||
"version": "0.25.1",
|
||||
"resolved": "https://registry.npmjs.org/hermes-estree/-/hermes-estree-0.25.1.tgz",
|
||||
@@ -3018,23 +2855,6 @@
|
||||
"node": ">= 4"
|
||||
}
|
||||
},
|
||||
"node_modules/import-fresh": {
|
||||
"version": "3.3.1",
|
||||
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz",
|
||||
"integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"parent-module": "^1.0.0",
|
||||
"resolve-from": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/imurmurhash": {
|
||||
"version": "0.1.4",
|
||||
"resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
|
||||
@@ -3081,29 +2901,6 @@
|
||||
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/js-yaml": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.2.0.tgz",
|
||||
"integrity": "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/puzrin"
|
||||
},
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/nodeca"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"argparse": "^2.0.1"
|
||||
},
|
||||
"bin": {
|
||||
"js-yaml": "bin/js-yaml.js"
|
||||
}
|
||||
},
|
||||
"node_modules/jsesc": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz",
|
||||
@@ -3198,13 +2995,6 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/lodash.merge": {
|
||||
"version": "4.6.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
|
||||
"integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/loose-envify": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
|
||||
@@ -3256,16 +3046,19 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/minimatch": {
|
||||
"version": "3.1.5",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz",
|
||||
"integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==",
|
||||
"version": "10.2.5",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz",
|
||||
"integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"license": "BlueOak-1.0.0",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
"brace-expansion": "^5.0.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
"node": "18 || 20 || >=22"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/mnemonist": {
|
||||
@@ -3306,9 +3099,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/nanoid": {
|
||||
"version": "3.3.11",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz",
|
||||
"integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==",
|
||||
"version": "3.3.16",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz",
|
||||
"integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -3412,19 +3205,6 @@
|
||||
"mnemonist": "^0.39.2"
|
||||
}
|
||||
},
|
||||
"node_modules/parent-module": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
|
||||
"integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"callsites": "^3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/path-exists": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
|
||||
@@ -3510,9 +3290,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/postcss": {
|
||||
"version": "8.5.10",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz",
|
||||
"integrity": "sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==",
|
||||
"version": "8.5.23",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz",
|
||||
"integrity": "sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -3530,7 +3310,7 @@
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"nanoid": "^3.3.11",
|
||||
"nanoid": "^3.3.16",
|
||||
"picocolors": "^1.1.1",
|
||||
"source-map-js": "^1.2.1"
|
||||
},
|
||||
@@ -3712,16 +3492,6 @@
|
||||
"integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/resolve-from": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
|
||||
"integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/rollup": {
|
||||
"version": "4.60.2",
|
||||
"resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.2.tgz",
|
||||
@@ -3832,32 +3602,6 @@
|
||||
"integrity": "sha512-HTEHMNieakEnoe33shBYcZ7NX83ACUjCu8c40iOGEZsngj9zRnkqS9j1pqQPXwobB0ZcVTk27REb7COQ0UR59w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/strip-json-comments": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
|
||||
"integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/supports-color": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
|
||||
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"has-flag": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/tinyglobby": {
|
||||
"version": "0.2.16",
|
||||
"resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz",
|
||||
|
||||
@@ -9,7 +9,8 @@
|
||||
"lint": "eslint .",
|
||||
"preview": "vite preview",
|
||||
"test:graph-store": "node --test tests/graphStore.multi-edge.test.mjs",
|
||||
"test:graph-workspace": "node --import tsx --test tests/graphSceneState.display.test.ts"
|
||||
"test:graph-workspace": "node --import tsx --test tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts",
|
||||
"test:plugin-registry": "node --import tsx --test tests/pluginRegistry.temporal.test.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@monaco-editor/react": "^4.7.0",
|
||||
@@ -41,7 +42,7 @@
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@vitejs/plugin-react": "^4.3.0",
|
||||
"babel-plugin-react-compiler": "^1.0.0",
|
||||
"eslint": "^9.39.4",
|
||||
"eslint": "^10.8.0",
|
||||
"eslint-plugin-react-hooks": "^7.0.1",
|
||||
"eslint-plugin-react-refresh": "^0.5.2",
|
||||
"globals": "^17.4.0",
|
||||
|
||||
+88
-51
@@ -1,4 +1,4 @@
|
||||
import { lazy, Suspense, useEffect, useState, type ReactNode } from 'react';
|
||||
import { lazy, Suspense, useEffect, useState, type ReactNode } from 'react';
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
|
||||
import {
|
||||
ArrowRight,
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
ShieldCheck,
|
||||
type LucideIcon,
|
||||
} from 'lucide-react';
|
||||
import { ErrorBoundary } from './ErrorBoundary';
|
||||
|
||||
const DecisionWorkspace = lazy(() => import('./workspaces/DecisionWorkspace/DecisionWorkspace').then((module) => ({ default: module.DecisionWorkspace })));
|
||||
const DiffMergeWorkspace = lazy(() => import('./workspaces/DiffMergeWorkspace/DiffMergeWorkspace').then((module) => ({ default: module.DiffMergeWorkspace })));
|
||||
@@ -66,6 +67,14 @@ type GraphStatsPayload = {
|
||||
edges?: number;
|
||||
};
|
||||
|
||||
type ConnectionStatus = 'checking' | 'online' | 'offline';
|
||||
|
||||
const CONNECTION_STATUS_LABEL: Record<ConnectionStatus, string> = {
|
||||
checking: 'Connecting…',
|
||||
online: 'System Online',
|
||||
offline: 'Backend Unreachable',
|
||||
};
|
||||
|
||||
const queryClient = new QueryClient();
|
||||
|
||||
const PREVIEW_DOTS = Array.from({ length: 42 }, (_, i) => ({
|
||||
@@ -718,19 +727,34 @@ const shellStyles = `
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
margin-bottom: 24px;
|
||||
--status-color: #4cc38a;
|
||||
--status-shadow-a: 0 0 0 3px rgba(76, 195, 138, 0.22), 0 0 12px rgba(76, 195, 138, 0.5);
|
||||
--status-shadow-b: 0 0 0 5px rgba(76, 195, 138, 0.1), 0 0 20px rgba(76, 195, 138, 0.35);
|
||||
}
|
||||
|
||||
.landing-status-bar[data-status='checking'] {
|
||||
--status-color: #f2b66d;
|
||||
--status-shadow-a: 0 0 0 3px rgba(242, 182, 109, 0.22), 0 0 12px rgba(242, 182, 109, 0.5);
|
||||
--status-shadow-b: 0 0 0 5px rgba(242, 182, 109, 0.1), 0 0 20px rgba(242, 182, 109, 0.35);
|
||||
}
|
||||
|
||||
.landing-status-bar[data-status='offline'] {
|
||||
--status-color: #ff7b72;
|
||||
--status-shadow-a: 0 0 0 3px rgba(255, 123, 114, 0.22), 0 0 12px rgba(255, 123, 114, 0.5);
|
||||
--status-shadow-b: 0 0 0 5px rgba(255, 123, 114, 0.1), 0 0 20px rgba(255, 123, 114, 0.35);
|
||||
}
|
||||
|
||||
.landing-status-dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 999px;
|
||||
background: #4cc38a;
|
||||
box-shadow: 0 0 0 3px rgba(76, 195, 138, 0.22), 0 0 12px rgba(76, 195, 138, 0.5);
|
||||
background: var(--status-color);
|
||||
box-shadow: var(--status-shadow-a);
|
||||
animation: landing-pulse 2.4s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.landing-status-text {
|
||||
color: #4cc38a;
|
||||
color: var(--status-color);
|
||||
font: 700 11px/1 "JetBrains Mono", monospace;
|
||||
letter-spacing: 0.1em;
|
||||
text-transform: uppercase;
|
||||
@@ -1322,8 +1346,8 @@ const shellStyles = `
|
||||
}
|
||||
|
||||
@keyframes landing-pulse {
|
||||
0%, 100% { box-shadow: 0 0 0 3px rgba(76, 195, 138, 0.22), 0 0 12px rgba(76, 195, 138, 0.5); }
|
||||
50% { box-shadow: 0 0 0 5px rgba(76, 195, 138, 0.1), 0 0 20px rgba(76, 195, 138, 0.35); }
|
||||
0%, 100% { box-shadow: var(--status-shadow-a); }
|
||||
50% { box-shadow: var(--status-shadow-b); }
|
||||
}
|
||||
|
||||
.workspace-loading {
|
||||
@@ -1493,10 +1517,10 @@ function WelcomeScreen({
|
||||
onOpenDecisions: () => void;
|
||||
onOpenManage: () => void;
|
||||
}) {
|
||||
const [stats, setStats] = useState<{ nodes: number | null; edges: number | null; ready: boolean }>({
|
||||
const [stats, setStats] = useState<{ nodes: number | null; edges: number | null; status: ConnectionStatus }>({
|
||||
nodes: null,
|
||||
edges: null,
|
||||
ready: false,
|
||||
status: 'checking',
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
@@ -1506,31 +1530,32 @@ function WelcomeScreen({
|
||||
.then((response) => (response.ok ? response.json() as Promise<GraphStatsPayload> : null))
|
||||
.then((payload) => {
|
||||
if (!payload) {
|
||||
setStats((current) => ({ ...current, ready: false }));
|
||||
setStats((current) => ({ ...current, status: 'offline' }));
|
||||
return;
|
||||
}
|
||||
|
||||
setStats({
|
||||
nodes: getNumberStat(payload, ['node_count', 'nodeCount', 'nodes']),
|
||||
edges: getNumberStat(payload, ['edge_count', 'edgeCount', 'edges']),
|
||||
ready: true,
|
||||
status: 'online',
|
||||
});
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
return;
|
||||
}
|
||||
setStats((current) => ({ ...current, ready: false }));
|
||||
setStats((current) => ({ ...current, status: 'offline' }));
|
||||
});
|
||||
|
||||
return () => controller.abort();
|
||||
}, []);
|
||||
|
||||
const isOnline = stats.status === 'online';
|
||||
const metrics: LandingMetric[] = [
|
||||
{ label: 'Knowledge nodes', value: formatMetric(stats.nodes, 'Live'), tone: 'cyan' },
|
||||
{ label: 'Relationships mapped', value: formatMetric(stats.edges, 'Ready'), tone: 'mint' },
|
||||
{ label: 'Graph modes', value: '3', tone: 'amber' },
|
||||
{ label: stats.ready ? 'Dataset online' : 'Ready to explore', value: stats.ready ? 'Active' : 'Standby', tone: 'rose' },
|
||||
{ label: isOnline ? 'Dataset online' : 'Ready to explore', value: isOnline ? 'Active' : 'Standby', tone: 'rose' },
|
||||
];
|
||||
|
||||
const secondaryLaunchers: LandingAction[] = [
|
||||
@@ -1573,9 +1598,9 @@ function WelcomeScreen({
|
||||
{/* ── Hero ── */}
|
||||
<section className="landing-hero">
|
||||
<div className="landing-copy">
|
||||
<div className="landing-status-bar">
|
||||
<div className="landing-status-bar" data-status={stats.status}>
|
||||
<div className="landing-status-dot" />
|
||||
<span className="landing-status-text">System Online</span>
|
||||
<span className="landing-status-text">{CONNECTION_STATUS_LABEL[stats.status]}</span>
|
||||
<div className="landing-status-divider" />
|
||||
<span className="landing-status-version">Semantica v2 · Semantic Intelligence</span>
|
||||
</div>
|
||||
@@ -1800,14 +1825,16 @@ export default function App() {
|
||||
</>
|
||||
}
|
||||
>
|
||||
<Suspense fallback={<WorkspaceFallback />}>
|
||||
{exploreView === 'graph' ? (
|
||||
<GraphWorkspace
|
||||
externalFocusNodeId={graphFocusRequest?.nodeId}
|
||||
externalFocusToken={graphFocusRequest?.token}
|
||||
/>
|
||||
) : <VocabularyWorkspace />}
|
||||
</Suspense>
|
||||
<ErrorBoundary key={`explore-${exploreView}`}>
|
||||
<Suspense fallback={<WorkspaceFallback />}>
|
||||
{exploreView === 'graph' ? (
|
||||
<GraphWorkspace
|
||||
externalFocusNodeId={graphFocusRequest?.nodeId}
|
||||
externalFocusToken={graphFocusRequest?.token}
|
||||
/>
|
||||
) : <VocabularyWorkspace />}
|
||||
</Suspense>
|
||||
</ErrorBoundary>
|
||||
</WorkspaceShell>
|
||||
);
|
||||
}
|
||||
@@ -1829,9 +1856,11 @@ export default function App() {
|
||||
</>
|
||||
}
|
||||
>
|
||||
<Suspense fallback={<WorkspaceFallback />}>
|
||||
{analyzeView === 'reasoning' ? <ReasoningWorkspace /> : <SparqlWorkspace />}
|
||||
</Suspense>
|
||||
<ErrorBoundary key={`analyze-${analyzeView}`}>
|
||||
<Suspense fallback={<WorkspaceFallback />}>
|
||||
{analyzeView === 'reasoning' ? <ReasoningWorkspace /> : <SparqlWorkspace />}
|
||||
</Suspense>
|
||||
</ErrorBoundary>
|
||||
</WorkspaceShell>
|
||||
);
|
||||
}
|
||||
@@ -1843,9 +1872,11 @@ export default function App() {
|
||||
subtitle="Inspect decision chains, causal context, and precedent matches."
|
||||
kicker="Decision Intelligence"
|
||||
>
|
||||
<Suspense fallback={<WorkspaceFallback />}>
|
||||
<DecisionWorkspace />
|
||||
</Suspense>
|
||||
<ErrorBoundary key="decisions">
|
||||
<Suspense fallback={<WorkspaceFallback />}>
|
||||
<DecisionWorkspace />
|
||||
</Suspense>
|
||||
</ErrorBoundary>
|
||||
</WorkspaceShell>
|
||||
);
|
||||
}
|
||||
@@ -1873,12 +1904,14 @@ export default function App() {
|
||||
</>
|
||||
}
|
||||
>
|
||||
<Suspense fallback={<WorkspaceFallback />}>
|
||||
{enrichView === 'import' ? <ImportExportWorkspace /> :
|
||||
enrichView === 'merge' ? <DiffMergeWorkspace /> :
|
||||
enrichView === 'resolve' ? <EntityResolutionTab /> :
|
||||
<RegistryTab />}
|
||||
</Suspense>
|
||||
<ErrorBoundary key={`enrich-${enrichView}`}>
|
||||
<Suspense fallback={<WorkspaceFallback />}>
|
||||
{enrichView === 'import' ? <ImportExportWorkspace /> :
|
||||
enrichView === 'merge' ? <DiffMergeWorkspace /> :
|
||||
enrichView === 'resolve' ? <EntityResolutionTab /> :
|
||||
<RegistryTab />}
|
||||
</Suspense>
|
||||
</ErrorBoundary>
|
||||
</WorkspaceShell>
|
||||
);
|
||||
}
|
||||
@@ -1891,15 +1924,17 @@ export default function App() {
|
||||
kicker="Schema Governance"
|
||||
compact
|
||||
>
|
||||
<Suspense fallback={<WorkspaceFallback />}>
|
||||
<OntologyWorkspace
|
||||
onJumpToGraphNode={(nodeId: string) => {
|
||||
setGraphFocusRequest({ nodeId, token: Date.now() });
|
||||
setActiveWorkspace('explore');
|
||||
setExploreView('graph');
|
||||
}}
|
||||
/>
|
||||
</Suspense>
|
||||
<ErrorBoundary key="ontology-hub">
|
||||
<Suspense fallback={<WorkspaceFallback />}>
|
||||
<OntologyWorkspace
|
||||
onJumpToGraphNode={(nodeId: string) => {
|
||||
setGraphFocusRequest({ nodeId, token: Date.now() });
|
||||
setActiveWorkspace('explore');
|
||||
setExploreView('graph');
|
||||
}}
|
||||
/>
|
||||
</Suspense>
|
||||
</ErrorBoundary>
|
||||
</WorkspaceShell>
|
||||
);
|
||||
}
|
||||
@@ -1923,14 +1958,16 @@ export default function App() {
|
||||
</>
|
||||
}
|
||||
>
|
||||
<Suspense fallback={<WorkspaceFallback />}>
|
||||
{manageView === 'lineage' ? <LineageDiagram /> :
|
||||
manageView === 'kg-overview' ? <KGOverviewTab /> :
|
||||
<OntologySummaryTab onOpenVocabularyBrowser={() => {
|
||||
setActiveWorkspace('explore');
|
||||
setExploreView('vocabulary');
|
||||
}} />}
|
||||
</Suspense>
|
||||
<ErrorBoundary key={`manage-${manageView}`}>
|
||||
<Suspense fallback={<WorkspaceFallback />}>
|
||||
{manageView === 'lineage' ? <LineageDiagram /> :
|
||||
manageView === 'kg-overview' ? <KGOverviewTab /> :
|
||||
<OntologySummaryTab onOpenVocabularyBrowser={() => {
|
||||
setActiveWorkspace('explore');
|
||||
setExploreView('vocabulary');
|
||||
}} />}
|
||||
</Suspense>
|
||||
</ErrorBoundary>
|
||||
</WorkspaceShell>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
import { Component, type ErrorInfo, type ReactNode } from 'react';
|
||||
import { AlertCircle } from 'lucide-react';
|
||||
|
||||
interface ErrorBoundaryProps {
|
||||
children: ReactNode;
|
||||
}
|
||||
|
||||
interface ErrorBoundaryState {
|
||||
hasError: boolean;
|
||||
error: Error | null;
|
||||
retryCount: number;
|
||||
}
|
||||
|
||||
const RETRY_SETTLE_MS = 5000;
|
||||
|
||||
export class ErrorBoundary extends Component<ErrorBoundaryProps, ErrorBoundaryState> {
|
||||
private settleTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
constructor(props: ErrorBoundaryProps) {
|
||||
super(props);
|
||||
this.state = { hasError: false, error: null, retryCount: 0 };
|
||||
}
|
||||
|
||||
static getDerivedStateFromError(error: Error): Partial<ErrorBoundaryState> {
|
||||
return { hasError: true, error };
|
||||
}
|
||||
|
||||
componentDidCatch(error: Error, errorInfo: ErrorInfo) {
|
||||
console.error("ErrorBoundary caught an error:", error, errorInfo);
|
||||
this.clearSettleTimer();
|
||||
}
|
||||
|
||||
componentWillUnmount() {
|
||||
this.clearSettleTimer();
|
||||
}
|
||||
|
||||
private clearSettleTimer() {
|
||||
if (this.settleTimer !== null) {
|
||||
clearTimeout(this.settleTimer);
|
||||
this.settleTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
resetErrorBoundary = () => {
|
||||
this.clearSettleTimer();
|
||||
this.setState((prev) => ({
|
||||
hasError: false,
|
||||
error: null,
|
||||
retryCount: prev.retryCount + 1
|
||||
}));
|
||||
// Only clear the retry count once the workspace has stayed error-free for a
|
||||
// sustained period, rather than on the next committed render (which can fire
|
||||
// while Suspense is still showing its fallback) or immediately on retry
|
||||
// (which would allow an unbounded number of clicks on a deterministic crash).
|
||||
this.settleTimer = setTimeout(() => {
|
||||
this.settleTimer = null;
|
||||
this.setState({ retryCount: 0 });
|
||||
}, RETRY_SETTLE_MS);
|
||||
};
|
||||
|
||||
render() {
|
||||
if (this.state.hasError) {
|
||||
const maxRetriesReached = this.state.retryCount >= 3;
|
||||
|
||||
return (
|
||||
<div
|
||||
className="workspace-loading"
|
||||
style={{
|
||||
flexDirection: 'column',
|
||||
gap: 12,
|
||||
color: 'var(--ws-red)'
|
||||
}}
|
||||
>
|
||||
<AlertCircle size={32} style={{ marginBottom: 4, opacity: 0.8 }} />
|
||||
<div style={{ fontWeight: 500, fontSize: '15px' }}>
|
||||
Something went wrong in this view.
|
||||
</div>
|
||||
<div style={{ fontSize: '13px', opacity: 0.7, maxWidth: 450, textAlign: 'center', marginBottom: 8, lineHeight: 1.5 }}>
|
||||
{maxRetriesReached
|
||||
? "This view continues to encounter a critical error. Please switch to another workspace or reload the page to restore functionality."
|
||||
: "An unexpected problem occurred while rendering this workspace. Your data is safe, but this view cannot be displayed."}
|
||||
</div>
|
||||
{!maxRetriesReached ? (
|
||||
<button
|
||||
className="ws-btn ws-btn--ghost"
|
||||
style={{
|
||||
borderColor: 'var(--ws-red-soft)',
|
||||
color: 'var(--ws-red)'
|
||||
}}
|
||||
onClick={this.resetErrorBoundary}
|
||||
>
|
||||
Try Again
|
||||
</button>
|
||||
) : (
|
||||
<button
|
||||
className="ws-btn ws-btn--ghost"
|
||||
style={{
|
||||
borderColor: 'var(--ws-border)',
|
||||
color: 'var(--ws-text)'
|
||||
}}
|
||||
onClick={() => window.location.reload()}
|
||||
>
|
||||
Reload Application
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return this.props.children;
|
||||
}
|
||||
}
|
||||
@@ -92,6 +92,7 @@ export function DecisionWorkspace() {
|
||||
const [chainLoading, setChainLoading] = useState(false);
|
||||
const [listLoading, setListLoading] = useState(true);
|
||||
const [filter, setFilter] = useState("");
|
||||
const [error, setError] = useState("");
|
||||
|
||||
// Tracks the active chain request so stale responses from rapid selections are ignored.
|
||||
const chainCtrlRef = useRef<AbortController | null>(null);
|
||||
@@ -99,14 +100,24 @@ export function DecisionWorkspace() {
|
||||
useEffect(() => {
|
||||
const ctrl = new AbortController();
|
||||
setListLoading(true);
|
||||
setError("");
|
||||
fetch("/api/decisions", { signal: ctrl.signal })
|
||||
.then((r) => r.ok ? r.json() : Promise.reject(r.status))
|
||||
.then(async (r) => {
|
||||
if (!r.ok) throw new Error(`HTTP ${r.status}`);
|
||||
const data = await r.json();
|
||||
if (r.status === 207) setError(data.message || "Warning: Partial success loading decisions.");
|
||||
return data;
|
||||
})
|
||||
.then((data) => {
|
||||
if (ctrl.signal.aborted) return;
|
||||
setDecisions(data);
|
||||
if (data.length > 0) void loadChain(data[0]);
|
||||
})
|
||||
.catch((e) => { if (e?.name !== "AbortError") console.error(e); })
|
||||
.catch((e) => {
|
||||
if (e?.name !== "AbortError") {
|
||||
setError(e instanceof Error ? e.message : "Failed to load decisions.");
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
if (!ctrl.signal.aborted) setListLoading(false);
|
||||
});
|
||||
@@ -125,13 +136,19 @@ export function DecisionWorkspace() {
|
||||
setSelected(d);
|
||||
setChainLoading(true);
|
||||
setChain([]);
|
||||
setError("");
|
||||
try {
|
||||
const res = await fetch(`/api/decisions/${encodeURIComponent(d.decision_id)}/chain`, { signal: ctrl.signal });
|
||||
if (!res.ok) throw new Error(`${res.status}`);
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const data = await res.json();
|
||||
if (res.status === 207 && !ctrl.signal.aborted) {
|
||||
setError(data.message || "Warning: Partial success loading chain.");
|
||||
}
|
||||
if (!ctrl.signal.aborted) setChain(data.chain || []);
|
||||
} catch (e) {
|
||||
if (e instanceof Error && e.name !== "AbortError") console.error(e);
|
||||
if (e instanceof Error && e.name !== "AbortError") {
|
||||
setError(e.message);
|
||||
}
|
||||
} finally {
|
||||
if (!ctrl.signal.aborted) setChainLoading(false);
|
||||
}
|
||||
@@ -213,6 +230,12 @@ export function DecisionWorkspace() {
|
||||
<div style={{ flex: 1, display: "flex", flexDirection: "column", overflow: "hidden", position: "relative" }}>
|
||||
<div style={{ position: "absolute", inset: 0, background: "radial-gradient(ellipse 60% 40% at 70% 20%, rgba(74,163,255,0.04), transparent 55%)", pointerEvents: "none" }} />
|
||||
|
||||
{error ? (
|
||||
<div style={{ padding: 12, borderRadius: 14, color: "#ffb4c2", background: "rgba(255,157,175,0.1)", border: "1px solid rgba(255,157,175,0.18)", margin: "16px 16px 0 16px", zIndex: 2, position: "relative" }}>
|
||||
{error}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{selected ? (
|
||||
<div className="ws-scroll ws-padded ws-animate-in" style={{ position: "relative", zIndex: 1 }}>
|
||||
{/* Decision header */}
|
||||
|
||||
@@ -192,6 +192,7 @@ export function EntityResolutionTab() {
|
||||
}, [threshold]);
|
||||
|
||||
const handleMerge = useCallback(async (primaryId: string, duplicateId: string) => {
|
||||
setScanError("");
|
||||
try {
|
||||
const res = await fetch("/api/enrich/merge", {
|
||||
method: "POST",
|
||||
@@ -200,6 +201,9 @@ export function EntityResolutionTab() {
|
||||
});
|
||||
if (!res.ok) throw new Error(`Merge failed (${res.status})`);
|
||||
const data = await res.json();
|
||||
if (res.status === 207) {
|
||||
setScanError(data.message || "Warning: Partial merge.");
|
||||
}
|
||||
logEvent("merge", `Merged ${duplicateId} → ${primaryId} · ${data.edges_updated ?? 0} edges redirected`, {
|
||||
primary: primaryId,
|
||||
duplicate: duplicateId,
|
||||
@@ -207,7 +211,7 @@ export function EntityResolutionTab() {
|
||||
});
|
||||
setPairs((prev) => prev.filter((p) => !(p.a.id === primaryId && p.b.id === duplicateId)));
|
||||
} catch (err) {
|
||||
console.error("[EntityResolution] merge failed", err);
|
||||
setScanError(err instanceof Error ? err.message : "Merge failed");
|
||||
}
|
||||
}, []);
|
||||
|
||||
|
||||
@@ -162,7 +162,11 @@ const SIGMA_SETTINGS = {
|
||||
hideLabelsOnMove: true,
|
||||
hideEdgesOnMove: true,
|
||||
enableEdgeEvents: true,
|
||||
renderEdgeLabels: false,
|
||||
// #1009: edge labels (the edge `type` — "works_for", "leads", ...) were
|
||||
// hardcoded off, so edge text never rendered regardless of data. The
|
||||
// labelDensity / labelGridCellSize / labelRenderedSizeThreshold settings
|
||||
// below already throttle label density for both nodes and edges.
|
||||
renderEdgeLabels: true,
|
||||
labelDensity: 0.7,
|
||||
labelGridCellSize: 140,
|
||||
zIndex: true,
|
||||
@@ -741,6 +745,12 @@ function buildEffectAvailability(
|
||||
? { enabled: true, available: true, reason: "Panel enabled" }
|
||||
: { enabled: false, available: false, reason: "Disabled by toggle" };
|
||||
|
||||
// #1009: edge labels are immediately available once the graph is loaded —
|
||||
// they have no async analytics or zoom-tier dependency.
|
||||
const edgeLabels = effectsState.edgeLabelsEnabled
|
||||
? { enabled: true, available: true, reason: "Ready" }
|
||||
: { enabled: false, available: false, reason: "Disabled by toggle" };
|
||||
|
||||
const diagnostics = !GRAPH_THEME.effects.diagnostics.enabledInDev
|
||||
? { enabled: false, available: false, reason: "Disabled in production" }
|
||||
: effectsState.diagnosticsEnabled
|
||||
@@ -758,6 +768,7 @@ function buildEffectAvailability(
|
||||
communities,
|
||||
centrality,
|
||||
legend,
|
||||
edgeLabels,
|
||||
diagnostics,
|
||||
};
|
||||
}
|
||||
@@ -1211,6 +1222,12 @@ function applySceneState(
|
||||
size: resolvedStyle.size,
|
||||
zIndex: resolvedStyle.zIndex,
|
||||
curvature: resolvedStyle.curvature,
|
||||
// #1009: Sigma's edge label renderer draws data.label — the graph
|
||||
// stores the relationship type in edgeType, which the renderer never
|
||||
// saw, so enabling renderEdgeLabels alone left edges blank.
|
||||
// Use || rather than ?? so that an empty-string edgeType (possible
|
||||
// when the API returns type: "") does not produce a blank label.
|
||||
label: resolvedStyle.hidden ? undefined : String(attrs.edgeType || data.label || ""),
|
||||
};
|
||||
});
|
||||
|
||||
@@ -1295,6 +1312,9 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
const onEdgeClickRef = useRef(onEdgeClick);
|
||||
const onSceneRuntimeChangeRef = useRef(onSceneRuntimeChange);
|
||||
const onCameraStateChangeRef = useRef(onCameraStateChange);
|
||||
// #1009: tracked as a ref so the Sigma creation effect always reads the
|
||||
// current value without needing effectsState in its dependency array.
|
||||
const effectsStateRef = useRef(effectsState);
|
||||
const [hoveredNodeId, setHoveredNodeId] = useState<string | null>(null);
|
||||
const [zoomTier, setZoomTier] = useState<GraphZoomTier>("overview");
|
||||
const [analyticsSnapshot, setAnalyticsSnapshot] = useState<GraphAnalyticsSnapshot | null>(null);
|
||||
@@ -1323,6 +1343,7 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
onEdgeClickRef.current = onEdgeClick;
|
||||
onSceneRuntimeChangeRef.current = onSceneRuntimeChange;
|
||||
onCameraStateChangeRef.current = onCameraStateChange;
|
||||
effectsStateRef.current = effectsState;
|
||||
|
||||
const behaviors = useMemo<GraphBehavior[]>(
|
||||
() => [
|
||||
@@ -1835,7 +1856,13 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
return;
|
||||
}
|
||||
|
||||
const sigma = new Sigma(displayGraphRef.current, containerRef.current, SIGMA_SETTINGS);
|
||||
const sigma = new Sigma(displayGraphRef.current, containerRef.current, {
|
||||
...SIGMA_SETTINGS,
|
||||
// #1009: initialize with the current toggle value rather than the
|
||||
// static default so that a user who disabled Edge Labels before
|
||||
// graph/Sigma initialization sees the correct state after mount.
|
||||
renderEdgeLabels: effectsStateRef.current.edgeLabelsEnabled,
|
||||
});
|
||||
sigmaRef.current = sigma;
|
||||
appliedGraphVersionRef.current = graphVersionRef.current;
|
||||
|
||||
@@ -1937,6 +1964,17 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
});
|
||||
}, [behaviors, dispatchToBehaviors, getBehaviorContext, graphReady, syncCameraState]);
|
||||
|
||||
// #1009: renderEdgeLabels follows the Effects-panel toggle instead of
|
||||
// staying hardcoded — dense graphs get their label-free edges back.
|
||||
useEffect(() => {
|
||||
const sigma = sigmaRef.current;
|
||||
if (!sigma) {
|
||||
return;
|
||||
}
|
||||
sigma.setSetting("renderEdgeLabels", effectsState.edgeLabelsEnabled);
|
||||
sigma.scheduleRefresh();
|
||||
}, [effectsState.edgeLabelsEnabled]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
const sigma = sigmaRef.current;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useEffect, useRef, useState, type CSSProperties } from "react";
|
||||
import { AlertTriangle, RefreshCw } from "lucide-react";
|
||||
|
||||
import { GRAPH_THEME, withAlpha } from "./graphTheme";
|
||||
import { GRAPH_LOAD_STAGE_SEQUENCE, createGraphLoadProgress, getGraphLoadStageLabel } from "./graphLoading";
|
||||
@@ -121,6 +122,58 @@ const LOADING_OVERLAY_CSS = `
|
||||
0% { transform: translateX(-120%); }
|
||||
100% { transform: translateX(360%); }
|
||||
}
|
||||
.graph-stage-loader-card[data-error="true"] {
|
||||
pointer-events: auto;
|
||||
border-color: rgba(255, 123, 114, 0.32);
|
||||
background:
|
||||
radial-gradient(circle at top left, rgba(255, 123, 114, 0.12), transparent 32%),
|
||||
linear-gradient(145deg, rgba(7, 17, 31, 0.96), rgba(24, 14, 18, 0.86));
|
||||
}
|
||||
.graph-stage-loader-error-mark {
|
||||
width: 38px;
|
||||
height: 38px;
|
||||
flex: 0 0 auto;
|
||||
border-radius: 12px;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
color: #ff9e97;
|
||||
background: rgba(255, 123, 114, 0.12);
|
||||
border: 1px solid rgba(255, 123, 114, 0.28);
|
||||
}
|
||||
.graph-stage-loader-error-detail {
|
||||
padding: 10px 12px;
|
||||
border-radius: 10px;
|
||||
background: rgba(0, 0, 0, 0.32);
|
||||
border: 1px solid rgba(255, 123, 114, 0.18);
|
||||
color: #ffb4ae;
|
||||
font-family: "JetBrains Mono", "Fira Code", Consolas, monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.55;
|
||||
word-break: break-word;
|
||||
}
|
||||
.graph-stage-loader-retry {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 7px;
|
||||
padding: 9px 16px;
|
||||
border-radius: 8px;
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
cursor: pointer;
|
||||
border: 1px solid rgba(127, 208, 255, 0.4);
|
||||
background: linear-gradient(135deg, rgba(74, 163, 255, 0.28), rgba(56, 210, 160, 0.16));
|
||||
color: #e8f6ff;
|
||||
transition: 160ms ease;
|
||||
}
|
||||
.graph-stage-loader-retry:hover {
|
||||
border-color: rgba(127, 208, 255, 0.62);
|
||||
background: linear-gradient(135deg, rgba(74, 163, 255, 0.4), rgba(56, 210, 160, 0.24));
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
.graph-stage-loader-retry:focus-visible {
|
||||
outline: 2px solid #7fd0ff;
|
||||
outline-offset: 2px;
|
||||
}
|
||||
`;
|
||||
|
||||
function formatLayoutSource(source: GraphLoadProgress["layoutSource"]) {
|
||||
@@ -170,10 +223,14 @@ export function GraphLoadingOverlay({
|
||||
progress,
|
||||
visible,
|
||||
showGraphBehind,
|
||||
error = null,
|
||||
onRetry,
|
||||
}: {
|
||||
progress: GraphLoadProgress | null;
|
||||
visible: boolean;
|
||||
showGraphBehind: boolean;
|
||||
error?: string | null;
|
||||
onRetry?: () => void;
|
||||
}) {
|
||||
const [renderVisible, setRenderVisible] = useState(visible);
|
||||
const [exiting, setExiting] = useState(false);
|
||||
@@ -226,6 +283,44 @@ export function GraphLoadingOverlay({
|
||||
return null;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div
|
||||
className="graph-stage-loader"
|
||||
data-exiting={exiting}
|
||||
style={{ background: "linear-gradient(180deg, rgba(1,4,9,0.22), rgba(1,4,9,0.5))" }}
|
||||
>
|
||||
<style>{LOADING_OVERLAY_CSS}</style>
|
||||
<div className="graph-stage-loader-card" data-error="true" role="alert">
|
||||
<div style={{ display: "flex", alignItems: "flex-start", gap: 14, marginBottom: 14 }}>
|
||||
<div className="graph-stage-loader-error-mark" aria-hidden="true">
|
||||
<AlertTriangle size={18} strokeWidth={2.2} />
|
||||
</div>
|
||||
<div style={{ minWidth: 0 }}>
|
||||
<div style={{ color: "#ffffff", fontSize: 20, fontWeight: 700, letterSpacing: "-0.03em", marginBottom: 6 }}>
|
||||
Could not load the graph
|
||||
</div>
|
||||
<div style={{ color: "#8fa8c6", fontSize: 13, lineHeight: 1.5 }}>
|
||||
The Explorer API did not return graph data. Check that the backend is running and reachable, then try again.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="graph-stage-loader-error-detail">{error}</div>
|
||||
|
||||
{onRetry ? (
|
||||
<div style={{ display: "flex", gap: 10, marginTop: 16 }}>
|
||||
<button type="button" className="graph-stage-loader-retry" onClick={onRetry}>
|
||||
<RefreshCw size={14} strokeWidth={2.2} aria-hidden />
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const activeProgress = progress ?? displayProgress;
|
||||
const isLiveStage = activeProgress.phase === "stabilizing_layout" || activeProgress.showGraphBehind || showGraphBehind;
|
||||
const overlayBackground = isLiveStage
|
||||
|
||||
@@ -1,479 +0,0 @@
|
||||
import { forwardRef, useEffect, useImperativeHandle, useMemo, useRef, useState } from "react";
|
||||
|
||||
import { batchMergeEdges, batchMergeNodes, clearGraph, graph, type EdgeAttributes, type NodeAttributes } from "../../store/graphStore";
|
||||
import { SigmaSceneAdapter } from "./SigmaSceneAdapter";
|
||||
import { createGraphLoadProgress } from "./graphLoading";
|
||||
import { resolveDisplayGraph } from "./graphSceneState";
|
||||
import {
|
||||
chooseColorAccessor,
|
||||
colorForNodeKey,
|
||||
computeDegreeMap,
|
||||
computeEdgeSize,
|
||||
computeNodeSize,
|
||||
computePageRank,
|
||||
deterministicPosition,
|
||||
} from "./graphAnalytics";
|
||||
import { GRAPH_THEME } from "./graphConfig";
|
||||
import type { GraphSceneHandle } from "./scene";
|
||||
import type {
|
||||
GraphDataSnapshot,
|
||||
GraphEffectsState,
|
||||
GraphLayoutSource,
|
||||
GraphLayoutStatus,
|
||||
GraphLoadProgress,
|
||||
GraphPath,
|
||||
GraphSelectedNodeState,
|
||||
GraphStageHandle,
|
||||
GraphViewMode,
|
||||
} from "./types";
|
||||
|
||||
const STAGE_EFFECTS_STATE: GraphEffectsState = {
|
||||
pathPulseEnabled: false,
|
||||
pathFlowEnabled: false,
|
||||
lensEnabled: false,
|
||||
temporalEmphasisEnabled: false,
|
||||
semanticRegionsEnabled: false,
|
||||
contoursEnabled: false,
|
||||
pathfindingEnabled: false,
|
||||
communitiesEnabled: false,
|
||||
centralityEnabled: false,
|
||||
legendEnabled: false,
|
||||
diagnosticsEnabled: false,
|
||||
lensMode: "neighborhood",
|
||||
effectQuality: "bounded",
|
||||
};
|
||||
const EMPTY_PATH: string[] = [];
|
||||
|
||||
const socketProtocol = () => (window.location.protocol === "https:" ? "wss:" : "ws:");
|
||||
|
||||
function yieldToMain(): Promise<void> {
|
||||
if ("scheduler" in window && typeof (window as Window & { scheduler?: { yield?: () => Promise<void> } }).scheduler?.yield === "function") {
|
||||
return (window as Window & { scheduler: { yield: () => Promise<void> } }).scheduler.yield();
|
||||
}
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
function buildSelectedNodeState(nodeId: string): GraphSelectedNodeState | null {
|
||||
if (!nodeId || !graph.hasNode(nodeId)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const attributes = graph.getNodeAttributes(nodeId) as NodeAttributes;
|
||||
return {
|
||||
id: nodeId,
|
||||
label: String(attributes.label || nodeId),
|
||||
content: String(attributes.content || attributes.label || nodeId),
|
||||
nodeType: attributes.nodeType || "entity",
|
||||
color: attributes.color,
|
||||
valid_from: attributes.valid_from ?? null,
|
||||
valid_until: attributes.valid_until ?? null,
|
||||
properties: attributes.properties ?? {},
|
||||
neighborCount: graph.neighbors(nodeId).length,
|
||||
visibleNeighborCount: graph.neighbors(nodeId).length,
|
||||
collapsedNeighborCount: 0,
|
||||
isNeighborhoodCollapsed: false,
|
||||
canCollapseNeighborhood: graph.neighbors(nodeId).length > 8,
|
||||
};
|
||||
}
|
||||
|
||||
function hasUsableCoordinate(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
interface GraphRuntimeStageProps {
|
||||
snapshot: GraphDataSnapshot | null | undefined;
|
||||
selectedNodeId: string;
|
||||
activePath: GraphPath;
|
||||
onNodeSelect: (nodeId: string) => void;
|
||||
onSelectedNodeStateChange: (state: GraphSelectedNodeState | null) => void;
|
||||
isLayoutRunning: boolean;
|
||||
onLayoutRunningChange: (running: boolean) => void;
|
||||
viewMode: GraphViewMode;
|
||||
temporalTime: Date | null;
|
||||
onActiveNodeCountChange: (count: number | null) => void;
|
||||
onProgressChange: (progress: GraphLoadProgress | null) => void;
|
||||
onLayoutStatusChange: (status: GraphLayoutStatus) => void;
|
||||
onRuntimeReady: () => void;
|
||||
}
|
||||
|
||||
export const GraphRuntimeStage = forwardRef<GraphStageHandle, GraphRuntimeStageProps>(
|
||||
function GraphRuntimeStage(
|
||||
{
|
||||
snapshot,
|
||||
selectedNodeId,
|
||||
activePath,
|
||||
onNodeSelect,
|
||||
onSelectedNodeStateChange,
|
||||
isLayoutRunning,
|
||||
onLayoutRunningChange,
|
||||
viewMode,
|
||||
temporalTime,
|
||||
onActiveNodeCountChange,
|
||||
onProgressChange,
|
||||
onLayoutStatusChange,
|
||||
onRuntimeReady,
|
||||
},
|
||||
ref,
|
||||
) {
|
||||
const sceneRef = useRef<GraphSceneHandle>(null);
|
||||
const prevActiveIdsRef = useRef<Set<string>>(new Set());
|
||||
const [graphVersion, setGraphVersion] = useState(0);
|
||||
const [runtimeLayoutSource, setRuntimeLayoutSource] = useState<GraphLayoutSource>(snapshot?.summary.layoutSource ?? "runtime");
|
||||
const displayResult = useMemo(
|
||||
() => resolveDisplayGraph(selectedNodeId, activePath, EMPTY_PATH, viewMode, { aggregationEnabled: true }),
|
||||
[activePath, graphVersion, selectedNodeId, viewMode],
|
||||
);
|
||||
|
||||
const stageSignature = useMemo(() => (snapshot ? `${snapshot.fetchedAt}:${snapshot.summary.nodeCount}:${snapshot.summary.edgeCount}` : null), [snapshot]);
|
||||
|
||||
useImperativeHandle(ref, () => ({
|
||||
fitView: () => sceneRef.current?.fitView(),
|
||||
focusNode: (nodeId: string) => sceneRef.current?.focusNode(nodeId),
|
||||
}), []);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
async function hydrateSnapshot() {
|
||||
if (!snapshot) {
|
||||
return;
|
||||
}
|
||||
|
||||
onProgressChange(createGraphLoadProgress({
|
||||
phase: "computing_styling",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: snapshot.summary.nodeCount,
|
||||
nodesTotal: snapshot.summary.nodeCount,
|
||||
edgesLoaded: snapshot.summary.edgeCount,
|
||||
edgesTotal: snapshot.summary.edgeCount,
|
||||
message: "Computing runtime graph styling",
|
||||
showGraphBehind: false,
|
||||
}));
|
||||
|
||||
const degreeByNode = computeDegreeMap(snapshot.nodes, snapshot.edges);
|
||||
const pageRankByNode = computePageRank(snapshot.nodes, snapshot.edges);
|
||||
const nodeIndexById = new Map(snapshot.nodes.map((node, index) => [node.id, index]));
|
||||
const previousPositions = new Map<string, { x: number; y: number }>();
|
||||
|
||||
graph.forEachNode((nodeId, attributes) => {
|
||||
const raw = attributes as Partial<NodeAttributes>;
|
||||
const x = Number(raw.x);
|
||||
const y = Number(raw.y);
|
||||
if (Number.isFinite(x) && Number.isFinite(y)) {
|
||||
previousPositions.set(nodeId, { x, y });
|
||||
}
|
||||
});
|
||||
|
||||
let explicitCoordinateCount = 0;
|
||||
let carriedCoordinateCount = 0;
|
||||
const draftAttributes = snapshot.nodes.map((node) => {
|
||||
const previousPosition = previousPositions.get(node.id);
|
||||
const position = hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)
|
||||
? { x: node.x, y: node.y }
|
||||
: previousPosition
|
||||
? previousPosition
|
||||
: deterministicPosition(node.id, nodeIndexById.get(node.id) ?? 0, snapshot.nodes.length);
|
||||
|
||||
if (hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)) {
|
||||
explicitCoordinateCount += 1;
|
||||
} else if (previousPosition) {
|
||||
carriedCoordinateCount += 1;
|
||||
}
|
||||
|
||||
return {
|
||||
id: node.id,
|
||||
attributes: {
|
||||
label: node.content || node.id,
|
||||
x: position.x,
|
||||
y: position.y,
|
||||
nodeType: node.type,
|
||||
content: node.content,
|
||||
valid_from: node.valid_from,
|
||||
valid_until: node.valid_until,
|
||||
properties: node.properties,
|
||||
} as NodeAttributes,
|
||||
};
|
||||
});
|
||||
|
||||
const layoutSource: GraphLayoutSource = explicitCoordinateCount > 0
|
||||
? "provided"
|
||||
: carriedCoordinateCount > 0
|
||||
? "carried"
|
||||
: "runtime";
|
||||
const hasCoordinates = explicitCoordinateCount > 0 || carriedCoordinateCount > 0;
|
||||
setRuntimeLayoutSource(layoutSource);
|
||||
|
||||
const colorAccessor = chooseColorAccessor(draftAttributes);
|
||||
await yieldToMain();
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
onProgressChange(createGraphLoadProgress({
|
||||
phase: "hydrating_scene",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: snapshot.summary.nodeCount,
|
||||
nodesTotal: snapshot.summary.nodeCount,
|
||||
edgesLoaded: snapshot.summary.edgeCount,
|
||||
edgesTotal: snapshot.summary.edgeCount,
|
||||
message: "Hydrating graph scene and renderer",
|
||||
showGraphBehind: false,
|
||||
}));
|
||||
|
||||
const nodesToMerge = draftAttributes.map(({ id, attributes }) => {
|
||||
const colorKey = colorAccessor(id, attributes);
|
||||
const baseColor = colorForNodeKey(colorKey);
|
||||
const dynamicSize = computeNodeSize(id, degreeByNode, pageRankByNode);
|
||||
return {
|
||||
id,
|
||||
attributes: {
|
||||
...attributes,
|
||||
color: baseColor,
|
||||
baseColor,
|
||||
size: dynamicSize,
|
||||
baseSize: dynamicSize,
|
||||
degree: degreeByNode.get(id) ?? 0,
|
||||
pageRank: pageRankByNode.get(id) ?? 0,
|
||||
glowColor: baseColor,
|
||||
borderColor: GRAPH_THEME.nodes.border,
|
||||
borderSize: 1,
|
||||
} as NodeAttributes,
|
||||
};
|
||||
});
|
||||
|
||||
const edgesToMerge = snapshot.edges.map((edge) => ({
|
||||
id: edge.id,
|
||||
familyId: edge.familyId,
|
||||
source: edge.source,
|
||||
target: edge.target,
|
||||
attributes: {
|
||||
edgeId: edge.id,
|
||||
familyId: edge.familyId,
|
||||
sourceId: edge.source,
|
||||
targetId: edge.target,
|
||||
weight: edge.weight,
|
||||
edgeType: edge.type,
|
||||
properties: edge.properties,
|
||||
size: computeEdgeSize(edge.weight),
|
||||
baseSize: computeEdgeSize(edge.weight),
|
||||
color: GRAPH_THEME.edges.baseColor,
|
||||
baseColor: GRAPH_THEME.edges.baseColor,
|
||||
} as EdgeAttributes,
|
||||
}));
|
||||
|
||||
clearGraph();
|
||||
batchMergeNodes(nodesToMerge);
|
||||
batchMergeEdges(edgesToMerge);
|
||||
prevActiveIdsRef.current = new Set(snapshot.nodes.map((node) => node.id));
|
||||
|
||||
await yieldToMain();
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
onLayoutStatusChange({
|
||||
state: layoutSource === "runtime" ? "bootstrapping" : "interactive",
|
||||
source: layoutSource,
|
||||
hasCoordinates,
|
||||
layoutReady: layoutSource !== "runtime",
|
||||
displacement: null,
|
||||
elapsedMs: 0,
|
||||
stableSamples: 0,
|
||||
});
|
||||
|
||||
onLayoutRunningChange(layoutSource === "runtime");
|
||||
if (selectedNodeId) {
|
||||
sceneRef.current?.focusNode(selectedNodeId);
|
||||
} else {
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
}
|
||||
setGraphVersion((current) => current + 1);
|
||||
if (layoutSource !== "runtime") {
|
||||
onProgressChange(null);
|
||||
} else {
|
||||
onProgressChange(createGraphLoadProgress({
|
||||
phase: "stabilizing_layout",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: snapshot.summary.nodeCount,
|
||||
nodesTotal: snapshot.summary.nodeCount,
|
||||
edgesLoaded: snapshot.summary.edgeCount,
|
||||
edgesTotal: snapshot.summary.edgeCount,
|
||||
message: "Settling runtime layout",
|
||||
showGraphBehind: true,
|
||||
layoutSource,
|
||||
layoutState: "bootstrapping",
|
||||
}));
|
||||
}
|
||||
onRuntimeReady();
|
||||
}
|
||||
|
||||
void hydrateSnapshot();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [onLayoutRunningChange, onLayoutStatusChange, onProgressChange, onRuntimeReady, selectedNodeId, snapshot, stageSignature]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!selectedNodeId) {
|
||||
onSelectedNodeStateChange(null);
|
||||
return;
|
||||
}
|
||||
|
||||
onSelectedNodeStateChange(buildSelectedNodeState(selectedNodeId));
|
||||
}, [graphVersion, onSelectedNodeStateChange, selectedNodeId, viewMode]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!snapshot || !temporalTime) {
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
const applySnapshot = async () => {
|
||||
try {
|
||||
const response = await fetch(`/api/temporal/snapshot?at=${encodeURIComponent(temporalTime.toISOString())}`);
|
||||
if (!response.ok || cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const data: { active_node_ids: string[]; active_node_count: number } = await response.json();
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const nextActiveIds = new Set(data.active_node_ids);
|
||||
requestAnimationFrame(() => {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const previous = prevActiveIdsRef.current;
|
||||
previous.forEach((id) => {
|
||||
if (!nextActiveIds.has(id) && graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", true);
|
||||
}
|
||||
});
|
||||
nextActiveIds.forEach((id) => {
|
||||
if (graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", false);
|
||||
}
|
||||
});
|
||||
|
||||
prevActiveIdsRef.current = nextActiveIds;
|
||||
onActiveNodeCountChange(data.active_node_count);
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
});
|
||||
} catch (error) {
|
||||
if (!cancelled) {
|
||||
console.error("[GraphRuntimeStage] temporal snapshot failed", error);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void applySnapshot();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [onActiveNodeCountChange, snapshot, temporalTime]);
|
||||
|
||||
useEffect(() => {
|
||||
const socket = new WebSocket(`${socketProtocol()}//${window.location.host}/ws/graph-updates`);
|
||||
|
||||
socket.onmessage = (event) => {
|
||||
try {
|
||||
const message = JSON.parse(event.data);
|
||||
if (message.event === "connection_ack" || message.event !== "graph_mutation") {
|
||||
return;
|
||||
}
|
||||
|
||||
const eventType = message.data?.event_type;
|
||||
const payload = message.data?.payload;
|
||||
if (eventType === "ADD_NODE" && payload?.id) {
|
||||
batchMergeNodes([
|
||||
{
|
||||
id: payload.id,
|
||||
attributes: {
|
||||
label: payload.properties?.content || payload.id,
|
||||
x: Number.isFinite(Number(payload.x ?? payload.properties?.x))
|
||||
? Number(payload.x ?? payload.properties?.x)
|
||||
: deterministicPosition(payload.id, graph.order + 1, Math.max(graph.order + 1, 1)).x,
|
||||
y: Number.isFinite(Number(payload.y ?? payload.properties?.y))
|
||||
? Number(payload.y ?? payload.properties?.y)
|
||||
: deterministicPosition(payload.id, graph.order + 1, Math.max(graph.order + 1, 1)).y,
|
||||
nodeType: payload.type,
|
||||
content: payload.properties?.content || payload.id,
|
||||
valid_from: payload.properties?.valid_from ?? null,
|
||||
valid_until: payload.properties?.valid_until ?? null,
|
||||
properties: payload.properties || {},
|
||||
size: 8,
|
||||
color: colorForNodeKey(`${payload.type || "entity"}:${payload.id}`),
|
||||
baseColor: colorForNodeKey(`${payload.type || "entity"}:${payload.id}`),
|
||||
baseSize: 8,
|
||||
glowColor: colorForNodeKey(`${payload.type || "entity"}:${payload.id}`),
|
||||
borderColor: GRAPH_THEME.nodes.border,
|
||||
borderSize: 1,
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
if (eventType === "ADD_EDGE" && payload?.source_id && payload?.target_id) {
|
||||
batchMergeEdges([
|
||||
{
|
||||
id: String(payload.id),
|
||||
familyId: payload.familyId ? String(payload.familyId) : String(payload.id),
|
||||
source: payload.source_id,
|
||||
target: payload.target_id,
|
||||
attributes: {
|
||||
edgeId: String(payload.id),
|
||||
familyId: payload.familyId ? String(payload.familyId) : String(payload.id),
|
||||
sourceId: payload.source_id,
|
||||
targetId: payload.target_id,
|
||||
weight: Number(payload.weight ?? 1),
|
||||
edgeType: payload.type,
|
||||
properties: payload.properties || {},
|
||||
size: computeEdgeSize(Number(payload.weight ?? 1)),
|
||||
baseSize: computeEdgeSize(Number(payload.weight ?? 1)),
|
||||
color: payload.properties?.inferred ? GRAPH_THEME.edges.pathColor : GRAPH_THEME.edges.baseColor,
|
||||
baseColor: GRAPH_THEME.edges.baseColor,
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
setGraphVersion((current) => current + 1);
|
||||
} catch (error) {
|
||||
console.error("[GraphRuntimeStage] websocket update failed", error);
|
||||
}
|
||||
};
|
||||
|
||||
return () => {
|
||||
socket.close();
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<SigmaSceneAdapter
|
||||
ref={sceneRef}
|
||||
onNodeSelect={onNodeSelect}
|
||||
graphVersion={graphVersion}
|
||||
graphReady={Boolean(snapshot)}
|
||||
displayGraph={displayResult.graph}
|
||||
displayMeta={displayResult.meta}
|
||||
displayState={displayResult.state}
|
||||
selectedEdgeId=""
|
||||
selectedNodeId={selectedNodeId}
|
||||
focusedNodeId={viewMode === "focused" ? selectedNodeId : ""}
|
||||
activePath={activePath}
|
||||
activePathEdgeIds={EMPTY_PATH}
|
||||
effectsState={STAGE_EFFECTS_STATE}
|
||||
isLayoutRunning={isLayoutRunning}
|
||||
onLayoutRunningChange={onLayoutRunningChange}
|
||||
layoutSource={runtimeLayoutSource}
|
||||
onLayoutStatusChange={onLayoutStatusChange}
|
||||
viewMode={viewMode}
|
||||
/>
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState, type ComponentType, type ReactNode } from "react";
|
||||
import { useCallback, useEffect, useId, useMemo, useRef, useState, type ComponentType, type ReactNode } from "react";
|
||||
import {
|
||||
Activity,
|
||||
Clock3,
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
RefreshCw,
|
||||
Search,
|
||||
Users,
|
||||
X,
|
||||
ZoomIn,
|
||||
ZoomOut,
|
||||
} from "lucide-react";
|
||||
@@ -38,6 +39,8 @@ import {
|
||||
type GraphPluginPanelDescriptor,
|
||||
type GraphPluginToolbarItem,
|
||||
} from "./plugins";
|
||||
import { explorationEffectsShouldLoad, neighborhoodPanelShouldLoad, temporalOverlayShouldLoad } from "./pluginRegistryPredicates";
|
||||
import { shouldFetchTemporalBounds, shouldFetchTemporalSnapshot } from "./temporalLifecyclePredicates";
|
||||
import type { LinkPrediction, PathResponse } from "./GraphInspectorPanel";
|
||||
import type { GraphSceneHandle, GraphSceneRuntime } from "./scene";
|
||||
import type {
|
||||
@@ -126,7 +129,7 @@ type LazyPluginRegistryEntry = {
|
||||
load: () => Promise<GraphPlugin>;
|
||||
shouldLoad: (context: {
|
||||
panelState: Record<string, boolean>;
|
||||
temporalState: GraphTemporalState | null;
|
||||
temporalState?: GraphTemporalState | null;
|
||||
}) => boolean;
|
||||
};
|
||||
|
||||
@@ -145,6 +148,7 @@ const DEFAULT_EFFECTS_STATE: GraphEffectsState = {
|
||||
communitiesEnabled: false,
|
||||
centralityEnabled: false,
|
||||
legendEnabled: false,
|
||||
edgeLabelsEnabled: true,
|
||||
diagnosticsEnabled: false,
|
||||
lensMode: "neighborhood",
|
||||
effectQuality: "bounded",
|
||||
@@ -281,37 +285,168 @@ function SegmentedModeControl({ items }: { items: GraphToolbarItem[] }) {
|
||||
);
|
||||
}
|
||||
|
||||
const SUGGESTION_DEBOUNCE_MS = 250;
|
||||
const SUGGESTION_LIMIT = 6;
|
||||
|
||||
function SearchCommandBar({
|
||||
value,
|
||||
disabled,
|
||||
onChange,
|
||||
onSubmit,
|
||||
onSelectSuggestion,
|
||||
}: {
|
||||
value: string;
|
||||
disabled: boolean;
|
||||
onChange: (value: string) => void;
|
||||
onSubmit: () => void;
|
||||
onSelectSuggestion: (result: SearchResult) => void;
|
||||
}) {
|
||||
const [suggestions, setSuggestions] = useState<SearchResult[]>([]);
|
||||
const [suggestionsOpen, setSuggestionsOpen] = useState(false);
|
||||
const [highlightedIndex, setHighlightedIndex] = useState(-1);
|
||||
const abortRef = useRef<AbortController | null>(null);
|
||||
const debounceRef = useRef<number | null>(null);
|
||||
const listboxId = useId();
|
||||
|
||||
useEffect(() => {
|
||||
if (debounceRef.current !== null) {
|
||||
window.clearTimeout(debounceRef.current);
|
||||
}
|
||||
|
||||
const query = value.trim();
|
||||
if (disabled || !query) {
|
||||
abortRef.current?.abort();
|
||||
setSuggestions([]);
|
||||
setSuggestionsOpen(false);
|
||||
setHighlightedIndex(-1);
|
||||
return;
|
||||
}
|
||||
|
||||
debounceRef.current = window.setTimeout(() => {
|
||||
abortRef.current?.abort();
|
||||
const controller = new AbortController();
|
||||
abortRef.current = controller;
|
||||
|
||||
fetch("/api/graph/search", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ query, limit: SUGGESTION_LIMIT }),
|
||||
signal: controller.signal,
|
||||
})
|
||||
.then((response) => {
|
||||
if (!response.ok) {
|
||||
throw new Error(`Search failed with status ${response.status}`);
|
||||
}
|
||||
return response.json();
|
||||
})
|
||||
.then((data: { results?: SearchResult[] }) => {
|
||||
setSuggestions(data.results ?? []);
|
||||
setSuggestionsOpen(true);
|
||||
setHighlightedIndex(-1);
|
||||
})
|
||||
.catch((suggestionError: unknown) => {
|
||||
if (suggestionError instanceof DOMException && suggestionError.name === "AbortError") {
|
||||
return;
|
||||
}
|
||||
setSuggestions([]);
|
||||
setSuggestionsOpen(false);
|
||||
setHighlightedIndex(-1);
|
||||
});
|
||||
}, SUGGESTION_DEBOUNCE_MS);
|
||||
|
||||
return () => {
|
||||
if (debounceRef.current !== null) {
|
||||
window.clearTimeout(debounceRef.current);
|
||||
}
|
||||
abortRef.current?.abort();
|
||||
};
|
||||
}, [value, disabled]);
|
||||
|
||||
const closeSuggestions = () => {
|
||||
setSuggestionsOpen(false);
|
||||
setHighlightedIndex(-1);
|
||||
};
|
||||
|
||||
const selectSuggestion = (result: SearchResult) => {
|
||||
setSuggestions([]);
|
||||
closeSuggestions();
|
||||
onSelectSuggestion(result);
|
||||
};
|
||||
|
||||
return (
|
||||
<form
|
||||
className="explore-search-command"
|
||||
role="combobox"
|
||||
aria-expanded={suggestionsOpen && suggestions.length > 0}
|
||||
aria-haspopup="listbox"
|
||||
aria-owns={listboxId}
|
||||
onSubmit={(event) => {
|
||||
event.preventDefault();
|
||||
if (!disabled) {
|
||||
onSubmit();
|
||||
if (disabled) return;
|
||||
if (suggestionsOpen && highlightedIndex >= 0 && suggestions[highlightedIndex]) {
|
||||
selectSuggestion(suggestions[highlightedIndex]);
|
||||
return;
|
||||
}
|
||||
closeSuggestions();
|
||||
onSubmit();
|
||||
}}
|
||||
>
|
||||
<Search size={17} strokeWidth={2.15} aria-hidden />
|
||||
<input
|
||||
value={value}
|
||||
onChange={(event) => onChange(event.target.value)}
|
||||
onFocus={() => {
|
||||
if (suggestions.length > 0) {
|
||||
setSuggestionsOpen(true);
|
||||
}
|
||||
}}
|
||||
onBlur={() => {
|
||||
window.setTimeout(closeSuggestions, 120);
|
||||
}}
|
||||
onKeyDown={(event) => {
|
||||
if (!suggestionsOpen || suggestions.length === 0) return;
|
||||
if (event.key === "ArrowDown") {
|
||||
event.preventDefault();
|
||||
setHighlightedIndex((current) => (current + 1) % suggestions.length);
|
||||
} else if (event.key === "ArrowUp") {
|
||||
event.preventDefault();
|
||||
setHighlightedIndex((current) => (current <= 0 ? suggestions.length - 1 : current - 1));
|
||||
} else if (event.key === "Escape") {
|
||||
event.preventDefault();
|
||||
closeSuggestions();
|
||||
}
|
||||
}}
|
||||
placeholder="Search command, node, or concept"
|
||||
aria-label="Search graph nodes"
|
||||
aria-autocomplete="list"
|
||||
aria-controls={listboxId}
|
||||
aria-activedescendant={highlightedIndex >= 0 ? `${listboxId}-${highlightedIndex}` : undefined}
|
||||
/>
|
||||
<button type="submit" disabled={disabled} aria-label="Search for the current query">
|
||||
Search
|
||||
</button>
|
||||
|
||||
{suggestionsOpen && suggestions.length > 0 ? (
|
||||
<ul id={listboxId} role="listbox" className="explore-search-suggestions" aria-label="Search suggestions">
|
||||
{suggestions.map((result, index) => (
|
||||
<li
|
||||
key={result.node.id}
|
||||
id={`${listboxId}-${index}`}
|
||||
role="option"
|
||||
aria-selected={index === highlightedIndex}
|
||||
data-highlighted={index === highlightedIndex}
|
||||
onMouseDown={(event) => {
|
||||
event.preventDefault();
|
||||
selectSuggestion(result);
|
||||
}}
|
||||
onMouseEnter={() => setHighlightedIndex(index)}
|
||||
>
|
||||
<span className="explore-search-suggestion-label">{result.node.content || result.node.id}</span>
|
||||
<span className="explore-search-suggestion-type">{result.node.type}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : null}
|
||||
</form>
|
||||
);
|
||||
}
|
||||
@@ -576,6 +711,7 @@ const HUD_CSS = `
|
||||
gap: 10px;
|
||||
}
|
||||
.explore-search-command {
|
||||
position: relative;
|
||||
min-width: 0;
|
||||
height: 43px;
|
||||
display: grid;
|
||||
@@ -591,6 +727,50 @@ const HUD_CSS = `
|
||||
color: ${GRAPH_THEME.ui.text.muted};
|
||||
box-shadow: inset 0 1px 0 rgba(255,255,255,0.045), 0 14px 30px rgba(0,0,0,0.16);
|
||||
}
|
||||
.explore-search-suggestions {
|
||||
position: absolute;
|
||||
top: calc(100% + 6px);
|
||||
left: 0;
|
||||
right: 0;
|
||||
z-index: 30;
|
||||
margin: 0;
|
||||
padding: 6px;
|
||||
list-style: none;
|
||||
max-height: 288px;
|
||||
overflow-y: auto;
|
||||
border-radius: 14px;
|
||||
border: 1px solid ${GRAPH_THEME.ui.control.inputBorder};
|
||||
background: ${GRAPH_THEME.ui.surface.cardStrong};
|
||||
box-shadow: 0 18px 40px rgba(0,0,0,0.32), inset 0 1px 0 rgba(255,255,255,0.04);
|
||||
}
|
||||
.explore-search-suggestions li {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
justify-content: space-between;
|
||||
gap: 10px;
|
||||
padding: 8px 10px;
|
||||
border-radius: 10px;
|
||||
cursor: pointer;
|
||||
color: ${GRAPH_THEME.ui.text.body};
|
||||
}
|
||||
.explore-search-suggestions li[data-highlighted="true"] {
|
||||
background: ${GRAPH_THEME.ui.control.hoverBg};
|
||||
}
|
||||
.explore-search-suggestion-label {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
}
|
||||
.explore-search-suggestion-type {
|
||||
flex-shrink: 0;
|
||||
font-size: 11px;
|
||||
color: ${GRAPH_THEME.ui.text.subtle};
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.explore-search-command:focus-within {
|
||||
border-color: ${GRAPH_THEME.ui.control.activeBorder};
|
||||
box-shadow: inset 0 1px 0 rgba(255,255,255,0.06), 0 0 0 1px ${GRAPH_THEME.ui.control.focusRing}, 0 16px 32px rgba(0,0,0,0.18);
|
||||
@@ -1119,6 +1299,18 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
const [activeNodeCount, setActiveNodeCount] = useState<number | null>(null);
|
||||
const [temporalBounds, setTemporalBounds] = useState<TemporalBounds | null>(null);
|
||||
const [scrubberTime, setScrubberTime] = useState<Date | null>(null);
|
||||
// Deduplicates setScrubberTime calls by millisecond value so that React 18
|
||||
// concurrent-mode re-renders with a new Date object for the same timestamp
|
||||
// do not churn temporalState and retrigger the diagnostics effect (issue #830).
|
||||
const lastScrubberMsRef = useRef<number | null>(null);
|
||||
const onTimeChange = useCallback((time: Date) => {
|
||||
const ms = time.getTime();
|
||||
if (ms === lastScrubberMsRef.current) {
|
||||
return;
|
||||
}
|
||||
lastScrubberMsRef.current = ms;
|
||||
setScrubberTime(time);
|
||||
}, []);
|
||||
const [loadingProgress, setLoadingProgress] = useState<GraphLoadProgress | null>(null);
|
||||
const [pluginPanelState, setPluginPanelState] = useState<Record<string, boolean>>({
|
||||
"effects-panel": false,
|
||||
@@ -1129,6 +1321,9 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
const [pluginRuntimeVersion, setPluginRuntimeVersion] = useState(0);
|
||||
const [effectsState, setEffectsState] = useState<GraphEffectsState>(DEFAULT_EFFECTS_STATE);
|
||||
const [graphDiagnosticsState, setGraphDiagnosticsState] = useState<GraphRuntimeDiagnosticsSnapshot | null>(null);
|
||||
// Tracks the last accepted diagnostics outside React's state cycle, allowing
|
||||
// handleDiagnosticsChange to compare synchronously before calling setState.
|
||||
const lastDiagnosticsRef = useRef<GraphRuntimeDiagnosticsSnapshot | null>(null);
|
||||
const [graphAnalyticsState, setGraphAnalyticsState] = useState<GraphAnalyticsSnapshot | null>(null);
|
||||
const [loadedPlugins, setLoadedPlugins] = useState<Record<string, GraphPlugin>>({});
|
||||
|
||||
@@ -1209,12 +1404,28 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
}));
|
||||
}, []);
|
||||
|
||||
const { data: summary, isLoading, isFetching } = useLoadGraph({
|
||||
const {
|
||||
data: summary,
|
||||
isLoading,
|
||||
isFetching,
|
||||
isError: isGraphLoadError,
|
||||
error: graphLoadError,
|
||||
refetch: refetchGraph,
|
||||
} = useLoadGraph({
|
||||
enabled: true,
|
||||
onGraphReady: applyGraphReadySummary,
|
||||
onProgress: handleLoadProgress,
|
||||
});
|
||||
|
||||
const graphLoadErrorMessage = isGraphLoadError
|
||||
? (graphLoadError instanceof Error ? graphLoadError.message : "Unknown error while loading the graph.")
|
||||
: null;
|
||||
|
||||
const handleRetryGraphLoad = useCallback(() => {
|
||||
setLoadingProgress(null);
|
||||
void refetchGraph();
|
||||
}, [refetchGraph]);
|
||||
|
||||
useEffect(() => {
|
||||
if (isLayoutRunning) {
|
||||
return;
|
||||
@@ -1231,7 +1442,18 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
applyGraphReadySummary(summary);
|
||||
}, [applyGraphReadySummary, graphReady, summary]);
|
||||
|
||||
const canFetchTemporalBounds = shouldFetchTemporalBounds(summary);
|
||||
const canFetchTemporalSnapshot = shouldFetchTemporalSnapshot({
|
||||
debouncedTime,
|
||||
isLoading,
|
||||
summary,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (!canFetchTemporalBounds) {
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
const loadBounds = async () => {
|
||||
try {
|
||||
@@ -1251,10 +1473,21 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [summary?.nodeCount, summary?.edgeCount]);
|
||||
}, [
|
||||
canFetchTemporalBounds,
|
||||
summary?.nodeCount,
|
||||
summary?.edgeCount,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!debouncedTime || isLoading) return;
|
||||
if (!canFetchTemporalSnapshot) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!debouncedTime) {
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
const applySnapshot = async () => {
|
||||
@@ -1296,7 +1529,10 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [debouncedTime, isLoading]);
|
||||
}, [
|
||||
canFetchTemporalSnapshot,
|
||||
debouncedTime,
|
||||
]);
|
||||
|
||||
const resolveNodeIdForFocusedMode = useCallback((
|
||||
nodeId: string,
|
||||
@@ -1507,6 +1743,11 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
}
|
||||
}, [searchQuery]);
|
||||
|
||||
const handleClearSearchResults = useCallback(() => {
|
||||
setSearchResults([]);
|
||||
setSearchError("");
|
||||
}, []);
|
||||
|
||||
const handleRunPredictions = useCallback(async () => {
|
||||
if (!inspectableNodeId) return;
|
||||
setIsRunningPredictions(true);
|
||||
@@ -1885,7 +2126,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
viewMode,
|
||||
]);
|
||||
|
||||
const showLoadingOverlay = !graphReady && (isLoading || isFetching || Boolean(loadingProgress));
|
||||
const showLoadingOverlay = !graphReady && (isLoading || isFetching || Boolean(loadingProgress) || isGraphLoadError);
|
||||
const showSettlingStatus = graphReady && loadingProgress?.phase === "stabilizing_layout";
|
||||
const hasGraphContent = Boolean(summary?.nodeCount);
|
||||
const activePath = pathResult?.path ?? EMPTY_PATH;
|
||||
@@ -2056,7 +2297,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
title: "Open exploration effects controls",
|
||||
order: 18,
|
||||
load: loadExplorationEffectsPlugin,
|
||||
shouldLoad: ({ panelState }) => Boolean(panelState["effects-panel"]),
|
||||
shouldLoad: explorationEffectsShouldLoad,
|
||||
},
|
||||
{
|
||||
id: "neighborhood-panel",
|
||||
@@ -2065,7 +2306,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
title: "Toggle neighborhood panel",
|
||||
order: 30,
|
||||
load: loadNeighborhoodPanelPlugin,
|
||||
shouldLoad: ({ panelState }) => Boolean(panelState["neighborhood-panel"]),
|
||||
shouldLoad: neighborhoodPanelShouldLoad,
|
||||
},
|
||||
{
|
||||
id: "temporal-overlay",
|
||||
@@ -2074,7 +2315,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
title: "Toggle temporal context panel",
|
||||
order: 40,
|
||||
load: loadTemporalOverlayPlugin,
|
||||
shouldLoad: ({ panelState, temporalState }) => Boolean(panelState["temporal-panel"] || temporalState?.currentTime),
|
||||
shouldLoad: temporalOverlayShouldLoad,
|
||||
},
|
||||
],
|
||||
[],
|
||||
@@ -2092,7 +2333,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return;
|
||||
}
|
||||
|
||||
if (!entry.shouldLoad({ panelState: pluginPanelState, temporalState })) {
|
||||
if (!entry.shouldLoad({ panelState: pluginPanelState })) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -2111,7 +2352,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [loadedPlugins, pluginPanelState, pluginRegistry, temporalState]);
|
||||
}, [loadedPlugins, pluginPanelState, pluginRegistry]);
|
||||
|
||||
const setEffectToggle = useCallback((effect: GraphEffectToggle, enabled: boolean | ((current: boolean) => boolean)) => {
|
||||
setEffectsState((current) => {
|
||||
@@ -2274,6 +2515,55 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
if (!GRAPH_THEME.effects.diagnostics.enabledInDev) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compare against the last accepted snapshot synchronously before calling
|
||||
// setState. buildEffectAvailability always returns a new object, so an
|
||||
// unconditional setGraphDiagnosticsState on every call created a
|
||||
// render → diagnostics effect → setState → render cycle that exceeded
|
||||
// React's max update depth in dev mode (issue #830).
|
||||
const prev = lastDiagnosticsRef.current;
|
||||
if (prev !== null) {
|
||||
const EFFECT_KEYS = [
|
||||
"pathPulse", "pathFlow", "lens", "temporalEmphasis", "semanticRegions",
|
||||
"contours", "pathfinding", "communities", "centrality", "legend", "diagnostics",
|
||||
] as const;
|
||||
const prevEA = prev.effectAvailability;
|
||||
const nextEA = diagnostics.effectAvailability;
|
||||
const availabilityChanged = EFFECT_KEYS.some((key) => {
|
||||
const p = prevEA[key];
|
||||
const n = nextEA[key];
|
||||
return (
|
||||
p.enabled !== n.enabled ||
|
||||
p.available !== n.available ||
|
||||
p.reason !== n.reason ||
|
||||
p.detail !== n.detail ||
|
||||
p.visibleSegments !== n.visibleSegments ||
|
||||
p.segmentCap !== n.segmentCap
|
||||
);
|
||||
});
|
||||
|
||||
const edgeClassesChanged =
|
||||
prev.edgeClasses?.updatedAt !== diagnostics.edgeClasses?.updatedAt;
|
||||
|
||||
const structureLayerChanged =
|
||||
prev.structureLayer?.cacheKey !== diagnostics.structureLayer?.cacheKey ||
|
||||
prev.structureLayer?.lastDrawAt !== diagnostics.structureLayer?.lastDrawAt ||
|
||||
prev.structureLayer?.enabled !== diagnostics.structureLayer?.enabled ||
|
||||
prev.structureLayer?.disabledReason !== diagnostics.structureLayer?.disabledReason ||
|
||||
prev.structureLayer?.curveCount !== diagnostics.structureLayer?.curveCount ||
|
||||
prev.structureLayer?.bridgeCurveCount !== diagnostics.structureLayer?.bridgeCurveCount ||
|
||||
prev.structureLayer?.backboneCurveCount !== diagnostics.structureLayer?.backboneCurveCount;
|
||||
|
||||
// distanceVisual is compared by reference: GraphCanvas passes the same
|
||||
// object when distances haven't changed.
|
||||
const distanceVisualChanged = prev.distanceVisual !== diagnostics.distanceVisual;
|
||||
|
||||
if (!availabilityChanged && !edgeClassesChanged && !structureLayerChanged && !distanceVisualChanged) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
lastDiagnosticsRef.current = diagnostics;
|
||||
setGraphDiagnosticsState(diagnostics);
|
||||
}, []);
|
||||
|
||||
@@ -2351,16 +2641,17 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
showPluginDock: openDockPanels.length > 0,
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (!openDockPanels.length) {
|
||||
setActiveDockPanelId(null);
|
||||
return;
|
||||
}
|
||||
const openDockPanelIdsString = openDockPanels.map((p) => p.id).join("|");
|
||||
const [prevOpenDockPanelIdsString, setPrevOpenDockPanelIdsString] = useState(openDockPanelIdsString);
|
||||
|
||||
if (!activeDockPanelId || !openDockPanels.some((panel) => panel.id === activeDockPanelId)) {
|
||||
if (openDockPanelIdsString !== prevOpenDockPanelIdsString) {
|
||||
setPrevOpenDockPanelIdsString(openDockPanelIdsString);
|
||||
if (!openDockPanels.length) {
|
||||
if (activeDockPanelId !== null) setActiveDockPanelId(null);
|
||||
} else if (!activeDockPanelId || !openDockPanels.some((panel) => panel.id === activeDockPanelId)) {
|
||||
setActiveDockPanelId(openDockPanels[0].id);
|
||||
}
|
||||
}, [activeDockPanelId, openDockPanels]);
|
||||
}
|
||||
|
||||
const viewModeItems = useMemo<GraphToolbarItem[]>(() => {
|
||||
if (!hasGraphContent) {
|
||||
@@ -2698,6 +2989,10 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
disabled={searchDisabled}
|
||||
onChange={setSearchQuery}
|
||||
onSubmit={() => void handleSearch()}
|
||||
onSelectSuggestion={(result) => {
|
||||
setSearchQuery("");
|
||||
focusNode(result.node.id);
|
||||
}}
|
||||
/>
|
||||
<SegmentedModeControl items={viewModeItems} />
|
||||
<div className="explore-toolbelt">
|
||||
@@ -2773,20 +3068,36 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
{searchError ? <div style={{ color: "#ff7b72", fontSize: 12 }}>{searchError}</div> : null}
|
||||
|
||||
{searchResults.length ? (
|
||||
<div className="explore-search-results hud-scrollbar" style={searchResultsStripStyle}>
|
||||
{searchResults.map((result) => (
|
||||
<button key={result.node.id} style={predictionCardStyle} onClick={() => focusNode(result.node.id)}>
|
||||
<div style={{ display: "flex", justifyContent: "space-between", gap: 12 }}>
|
||||
<div style={{ minWidth: 0 }}>
|
||||
<div style={{ color: "#fff", fontWeight: 600 }}>{result.node.content || result.node.id}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{result.node.type}</div>
|
||||
</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, whiteSpace: "nowrap" }}>
|
||||
{result.score.toFixed(3)}
|
||||
</div>
|
||||
</div>
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
<div style={{ display: "flex", alignItems: "center", justifyContent: "space-between", gap: 12 }}>
|
||||
<span style={{ color: "#8b949e", fontSize: 12 }}>
|
||||
{searchResults.length} result{searchResults.length === 1 ? "" : "s"}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleClearSearchResults}
|
||||
style={{ ...secondaryActionButtonStyle, minHeight: 26, padding: "4px 9px", gap: 5 }}
|
||||
aria-label="Dismiss search results"
|
||||
>
|
||||
<X size={12} strokeWidth={2.4} />
|
||||
Dismiss
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<div className="explore-search-results hud-scrollbar" style={searchResultsStripStyle}>
|
||||
{searchResults.map((result) => (
|
||||
<button key={result.node.id} style={predictionCardStyle} onClick={() => focusNode(result.node.id)}>
|
||||
<div style={{ display: "flex", justifyContent: "space-between", gap: 12 }}>
|
||||
<div style={{ minWidth: 0 }}>
|
||||
<div style={{ color: "#fff", fontWeight: 600 }}>{result.node.content || result.node.id}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{result.node.type}</div>
|
||||
</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, whiteSpace: "nowrap" }}>
|
||||
{Math.round(result.score)}
|
||||
</div>
|
||||
</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
@@ -2880,6 +3191,8 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
progress={loadingProgress}
|
||||
visible={showLoadingOverlay}
|
||||
showGraphBehind={hasGraphContent || Boolean(loadingProgress?.showGraphBehind)}
|
||||
error={graphLoadErrorMessage}
|
||||
onRetry={handleRetryGraphLoad}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
@@ -2921,7 +3234,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
<div className="explore-scene-footer">
|
||||
<Suspense fallback={<div style={timelineFallbackStyle}>Loading timeline…</div>}>
|
||||
<LazyTimelinePanel
|
||||
onTimeChange={setScrubberTime}
|
||||
onTimeChange={onTimeChange}
|
||||
minDate={temporalBounds?.min ?? undefined}
|
||||
maxDate={temporalBounds?.max ?? undefined}
|
||||
/>
|
||||
|
||||
@@ -1,849 +0,0 @@
|
||||
import { lazy, Suspense, useCallback, useEffect, useMemo, useRef, useState, type CSSProperties } from "react";
|
||||
|
||||
import { GraphLoadingOverlay } from "./GraphLoadingOverlay";
|
||||
import { getGraphLoadTitle } from "./graphLoading";
|
||||
import { useGraphData, useReloadGraphData } from "./useGraphData";
|
||||
import type {
|
||||
ApiNode,
|
||||
GraphLayoutStatus,
|
||||
GraphLoadProgress,
|
||||
GraphPath,
|
||||
GraphSelectedNodeState,
|
||||
GraphStageHandle,
|
||||
GraphViewMode,
|
||||
} from "./types";
|
||||
|
||||
type SearchResult = {
|
||||
node: {
|
||||
id: string;
|
||||
type: string;
|
||||
content: string;
|
||||
properties: Record<string, unknown>;
|
||||
};
|
||||
score: number;
|
||||
};
|
||||
|
||||
type LinkPrediction = {
|
||||
target: string;
|
||||
type: string;
|
||||
label?: string;
|
||||
score: number;
|
||||
};
|
||||
|
||||
type PathResponse = {
|
||||
path: GraphPath;
|
||||
total_weight: number;
|
||||
hop_count: number;
|
||||
distance_band: "direct" | "near" | "mid-range" | "distant";
|
||||
};
|
||||
|
||||
type TemporalBounds = {
|
||||
min?: string | null;
|
||||
max?: string | null;
|
||||
};
|
||||
|
||||
const GraphRuntimeStage = lazy(() =>
|
||||
import("./GraphRuntimeStage").then((module) => ({ default: module.GraphRuntimeStage })),
|
||||
);
|
||||
const TimelinePanel = lazy(() =>
|
||||
import("./TimelinePanel").then((module) => ({ default: module.TimelinePanel })),
|
||||
);
|
||||
|
||||
const HUD_CSS = `
|
||||
.palantir-bg {
|
||||
background:
|
||||
radial-gradient(circle at top, rgba(103, 182, 255, 0.1), transparent 24%),
|
||||
linear-gradient(180deg, #07111d 0%, #02060e 100%);
|
||||
}
|
||||
.palantir-grid {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background-image:
|
||||
linear-gradient(rgba(88, 166, 255, 0.04) 1px, transparent 1px),
|
||||
linear-gradient(90deg, rgba(88, 166, 255, 0.04) 1px, transparent 1px);
|
||||
background-size: 44px 44px;
|
||||
pointer-events: none;
|
||||
z-index: 1;
|
||||
opacity: 0.78;
|
||||
}
|
||||
.palantir-vignette {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background: radial-gradient(ellipse at center, transparent 34%, rgba(1, 4, 9, 0.88) 100%);
|
||||
pointer-events: none;
|
||||
z-index: 2;
|
||||
}
|
||||
.hud-scrollbar::-webkit-scrollbar { width: 6px; }
|
||||
.hud-scrollbar::-webkit-scrollbar-track { background: transparent; }
|
||||
.hud-scrollbar::-webkit-scrollbar-thumb { background: rgba(88, 166, 255, 0.25); border-radius: 6px; }
|
||||
.graph-shell-top { position: absolute; top: 18px; left: 18px; right: 18px; z-index: 10; display: flex; justify-content: space-between; align-items: flex-start; gap: 16px; pointer-events: none; }
|
||||
.graph-status-card, .graph-command-card {
|
||||
pointer-events: auto;
|
||||
border: 1px solid rgba(132, 197, 255, 0.12);
|
||||
background: linear-gradient(180deg, rgba(7, 16, 29, 0.86), rgba(10, 22, 39, 0.72)), radial-gradient(circle at top, rgba(103, 182, 255, 0.08), transparent 50%);
|
||||
box-shadow: 0 18px 42px rgba(0, 0, 0, 0.28), inset 0 1px 0 rgba(255,255,255,0.04);
|
||||
backdrop-filter: blur(18px);
|
||||
}
|
||||
.graph-status-card { width: min(420px, 38vw); border-radius: 24px; padding: 16px 18px; }
|
||||
.graph-command-card { width: min(620px, 55vw); border-radius: 24px; padding: 14px; display: flex; flex-direction: column; gap: 12px; }
|
||||
.graph-status-label { display: inline-flex; align-items: center; gap: 8px; color: rgba(160, 191, 223, 0.88); font-size: 11px; font-weight: 800; text-transform: uppercase; letter-spacing: 0.08em; margin-bottom: 10px; }
|
||||
.graph-status-label::before { content: ""; width: 7px; height: 7px; border-radius: 999px; background: linear-gradient(135deg, #8ed3ff, #ffb36a); box-shadow: 0 0 12px rgba(142, 211, 255, 0.5); }
|
||||
.graph-status-title { color: #eef5ff; font-size: 20px; font-weight: 800; letter-spacing: -0.04em; margin-bottom: 6px; }
|
||||
.graph-status-copy { color: #8fa8c6; font-size: 12px; line-height: 1.55; margin-bottom: 14px; max-width: 40ch; }
|
||||
.graph-status-metrics, .graph-command-row, .graph-toggle-cluster, .graph-action-cluster { display: flex; gap: 8px; flex-wrap: wrap; }
|
||||
.graph-command-row { justify-content: space-between; align-items: center; gap: 10px; }
|
||||
.graph-search-shell { flex: 1; min-width: 260px; display: flex; align-items: center; gap: 10px; padding: 8px 10px 8px 14px; border-radius: 18px; border: 1px solid rgba(132, 197, 255, 0.12); background: rgba(0, 0, 0, 0.18); box-shadow: inset 0 1px 0 rgba(255,255,255,0.03); }
|
||||
.graph-search-shell input { flex: 1; min-width: 0; border: none !important; background: transparent !important; padding: 0 !important; margin: 0 !important; }
|
||||
.graph-search-shell input:focus { outline: none; }
|
||||
.graph-search-results { position: absolute; top: 120px; right: 18px; width: min(420px, calc(100vw - 132px)); max-height: 320px; overflow-y: auto; padding: 12px; border-radius: 20px; border: 1px solid rgba(132, 197, 255, 0.14); background: linear-gradient(180deg, rgba(8, 18, 33, 0.94), rgba(10, 21, 38, 0.86)); box-shadow: 0 18px 50px rgba(0,0,0,0.34); backdrop-filter: blur(18px); pointer-events: auto; z-index: 11; }
|
||||
.graph-search-results-label { color: #6f89ab; font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.08em; margin-bottom: 10px; }
|
||||
.graph-search-result-card { width: 100%; text-align: left; padding: 12px 14px; border-radius: 16px; border: 1px solid rgba(132, 197, 255, 0.08); background: rgba(255, 255, 255, 0.025); cursor: pointer; transition: transform 160ms ease, border-color 160ms ease, background 160ms ease; }
|
||||
.graph-search-result-card:hover { transform: translateY(-1px); border-color: rgba(132, 197, 255, 0.18); background: rgba(103, 182, 255, 0.08); }
|
||||
.graph-inspector { pointer-events: auto; position: absolute; right: 18px; top: 154px; bottom: 108px; width: 380px; overflow-y: auto; transition: transform 0.34s cubic-bezier(0.16,1,0.3,1), opacity 0.22s ease; border-radius: 28px; border: 1px solid rgba(132, 197, 255, 0.14); background: linear-gradient(180deg, rgba(8, 18, 33, 0.9), rgba(6, 12, 22, 0.88)), radial-gradient(circle at top, rgba(103, 182, 255, 0.08), transparent 40%); box-shadow: -18px 0 48px rgba(0, 0, 0, 0.32), inset 0 1px 0 rgba(255,255,255,0.04); backdrop-filter: blur(20px); }
|
||||
.graph-inspector[data-open='false'] { transform: translateX(calc(100% + 24px)); opacity: 0; }
|
||||
@keyframes sem-loader-pulse {
|
||||
0%, 100% { transform: translateY(0) scale(0.92); opacity: 0.55; }
|
||||
50% { transform: translateY(-4px) scale(1.08); opacity: 1; }
|
||||
}
|
||||
@media (max-width: 1220px) {
|
||||
.graph-shell-top { flex-direction: column; align-items: stretch; }
|
||||
.graph-status-card, .graph-command-card { width: auto; }
|
||||
.graph-search-results { top: 202px; right: 18px; left: 18px; width: auto; }
|
||||
}
|
||||
`;
|
||||
|
||||
function useDebounce<T>(value: T, delay: number): T {
|
||||
const [debouncedValue, setDebouncedValue] = useState<T>(value);
|
||||
useEffect(() => {
|
||||
const timeout = setTimeout(() => setDebouncedValue(value), delay);
|
||||
return () => clearTimeout(timeout);
|
||||
}, [delay, value]);
|
||||
return debouncedValue;
|
||||
}
|
||||
|
||||
function sourceAttribution(properties: Record<string, unknown>) {
|
||||
const keys = ["source", "source_url", "pmid", "pmids", "evidence", "provenance", "confidence"];
|
||||
return keys
|
||||
.filter((key) => key in properties)
|
||||
.map((key) => ({ key, value: properties[key] }));
|
||||
}
|
||||
|
||||
function toSelectedNodeState(node: ApiNode, neighborCount: number, fallbackColor = "#58a6ff"): GraphSelectedNodeState {
|
||||
return {
|
||||
id: node.id,
|
||||
label: node.content || node.id,
|
||||
content: node.content || node.id,
|
||||
nodeType: node.type,
|
||||
color: fallbackColor,
|
||||
valid_from: node.valid_from ?? null,
|
||||
valid_until: node.valid_until ?? null,
|
||||
properties: node.properties ?? {},
|
||||
neighborCount,
|
||||
visibleNeighborCount: neighborCount,
|
||||
collapsedNeighborCount: 0,
|
||||
isNeighborhoodCollapsed: false,
|
||||
canCollapseNeighborhood: neighborCount > 8,
|
||||
};
|
||||
}
|
||||
|
||||
function TimelineFallback({ min, max }: TemporalBounds) {
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: "100%",
|
||||
height: "90px",
|
||||
borderTop: "1px solid rgba(88, 166, 255, 0.2)",
|
||||
background: "rgba(1, 4, 9, 0.88)",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "space-between",
|
||||
padding: "0 18px",
|
||||
color: "#8fa8c6",
|
||||
fontSize: 12,
|
||||
flexShrink: 0,
|
||||
}}
|
||||
>
|
||||
<span>Temporal scrubber</span>
|
||||
<span>{min || max ? "Preparing timeline runtime..." : "Temporal bounds loading..."}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function NodePanel({
|
||||
node,
|
||||
predictions,
|
||||
predictionType,
|
||||
onPredictionTypeChange,
|
||||
onRunPredictions,
|
||||
pathTargetId,
|
||||
onPathTargetChange,
|
||||
onTracePath,
|
||||
pathResult,
|
||||
onDownloadProvenance,
|
||||
}: {
|
||||
node: GraphSelectedNodeState | null;
|
||||
predictions: LinkPrediction[];
|
||||
predictionType: string;
|
||||
onPredictionTypeChange: (value: string) => void;
|
||||
onRunPredictions: () => void;
|
||||
pathTargetId: string;
|
||||
onPathTargetChange: (value: string) => void;
|
||||
onTracePath: () => void;
|
||||
pathResult: PathResponse | null;
|
||||
onDownloadProvenance: (format: "json" | "markdown") => void;
|
||||
}) {
|
||||
if (!node) {
|
||||
return (
|
||||
<div style={{ padding: 32, textAlign: "center" }}>
|
||||
<p style={{ color: "#8b949e", fontSize: 14, margin: 0 }}>
|
||||
Search for a node or click one in the canvas to inspect its properties.
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const properties = node.properties ?? {};
|
||||
const attribution = sourceAttribution(properties);
|
||||
const accentColor = node.color || "#58a6ff";
|
||||
const propertyEntries = Object.entries(properties).filter(([key]) => !["x", "y", "valid_from", "valid_until", "content", "source", "source_url", "pmid", "pmids", "evidence", "provenance", "confidence"].includes(key));
|
||||
|
||||
return (
|
||||
<aside style={{ padding: 24, display: "flex", flexDirection: "column", gap: 18 }}>
|
||||
<div style={{ borderBottom: "1px solid rgba(88, 166, 255, 0.14)", paddingBottom: 16 }}>
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 10, marginBottom: 8 }}>
|
||||
<span style={{ background: accentColor, boxShadow: `0 0 10px ${accentColor}`, width: 8, height: 8, borderRadius: "50%" }} />
|
||||
<span style={{ color: accentColor, fontSize: 12, fontWeight: 800, textTransform: "uppercase", letterSpacing: "0.08em" }}>{node.nodeType || "Entity"}</span>
|
||||
</div>
|
||||
<h3 style={{ margin: 0, color: "#fff", fontSize: 24, lineHeight: 1, fontWeight: 800, letterSpacing: "-0.04em", wordBreak: "break-word" }}>{node.label}</h3>
|
||||
<div style={{ color: "#8b949e", fontSize: 12, marginTop: 8 }}>{node.id}</div>
|
||||
<div style={{ display: "flex", gap: 8, flexWrap: "wrap", marginTop: 12 }}>
|
||||
{node.valid_from || node.valid_until ? <span style={subtleChipStyle}>temporal</span> : null}
|
||||
<span style={subtleChipStyle}>{node.neighborCount} neighbors</span>
|
||||
{attribution.length ? <span style={subtleChipStyle}>{attribution.length} source fields</span> : null}
|
||||
{predictions.length ? <span style={subtleChipStyle}>{predictions.length} candidate links</span> : null}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<section style={sectionStyle}>
|
||||
<div style={sectionTitleStyle}>Actions</div>
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 10 }}>
|
||||
<button style={{ ...actionButtonStyle, width: "100%", justifyContent: "center" }} onClick={onRunPredictions}>Run Link Prediction</button>
|
||||
<div style={{ display: "flex", gap: 8, flexWrap: "wrap" }}>
|
||||
<button style={secondaryActionButtonStyle} onClick={() => onDownloadProvenance("json")}>Provenance JSON</button>
|
||||
<button style={secondaryActionButtonStyle} onClick={() => onDownloadProvenance("markdown")}>Provenance MD</button>
|
||||
</div>
|
||||
</div>
|
||||
<input value={predictionType} onChange={(event) => onPredictionTypeChange(event.target.value)} placeholder="Optional candidate type filter, e.g. disease" style={inputStyle} />
|
||||
</section>
|
||||
|
||||
<section style={sectionStyle}>
|
||||
<div style={sectionTitleStyle}>Trace Path</div>
|
||||
<input value={pathTargetId} onChange={(event) => onPathTargetChange(event.target.value)} placeholder="Target node ID" style={inputStyle} />
|
||||
<button style={actionButtonStyle} onClick={onTracePath}>Trace Causal Path</button>
|
||||
{pathResult?.path?.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 6, marginTop: 10 }}>
|
||||
{pathResult.path.map((step, index) => (
|
||||
<div key={`${step}-${index}`} style={pathStepStyle}>{index + 1}. {step}</div>
|
||||
))}
|
||||
<div style={{ color: "#79c0ff", fontSize: 12, marginTop: 4 }}>total weight: {pathResult.total_weight.toFixed(3)}</div>
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>Choose a target or click a candidate prediction to prepare a path trace.</div>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<details style={collapseStyle} open={predictions.length > 0}>
|
||||
<summary style={summaryStyle}>Candidate Links</summary>
|
||||
<div style={{ padding: "0 14px 14px" }}>
|
||||
{predictions.length > 0 ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{predictions.map((prediction) => (
|
||||
<button key={`${prediction.target}-${prediction.type}`} style={predictionCardStyle} onClick={() => onPathTargetChange(prediction.target)}>
|
||||
<div style={{ color: "#fff", fontWeight: 600 }}>{prediction.label || prediction.target}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{prediction.type}</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, marginTop: 4 }}>confidence {prediction.score.toFixed(3)}</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>Run link prediction to surface likely next-hop relationships.</div>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details style={collapseStyle}>
|
||||
<summary style={summaryStyle}>Source Attribution</summary>
|
||||
<div style={{ padding: "0 14px 14px" }}>
|
||||
{attribution.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{attribution.map(({ key, value }) => (
|
||||
<div key={key} style={propertyCardStyle}>
|
||||
<div style={{ color: "rgba(88, 166, 255, 0.7)", fontSize: 11, marginBottom: 4 }}>{key}</div>
|
||||
<div style={{ color: "#e6edf3", fontSize: 13, wordBreak: "break-word" }}>{typeof value === "object" ? JSON.stringify(value) : String(value)}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>No explicit attribution metadata was found on this node.</div>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details style={collapseStyle}>
|
||||
<summary style={summaryStyle}>Properties</summary>
|
||||
<div style={{ padding: "0 14px 14px" }}>
|
||||
{propertyEntries.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{propertyEntries.map(([key, value]) => (
|
||||
<div key={key} style={propertyCardStyle}>
|
||||
<div style={{ color: "rgba(88, 166, 255, 0.7)", fontSize: 11, marginBottom: 4 }}>{key}</div>
|
||||
<div style={{ color: "#e6edf3", fontSize: 13, wordBreak: "break-word" }}>{typeof value === "object" ? JSON.stringify(value) : String(value)}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>No additional properties are attached to this node.</div>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
</aside>
|
||||
);
|
||||
}
|
||||
|
||||
export function GraphWorkspaceShell() {
|
||||
const [selectedNodeId, setSelectedNodeId] = useState("");
|
||||
const [selectedNodeState, setSelectedNodeState] = useState<GraphSelectedNodeState | null>(null);
|
||||
const [isLayoutRunning, setIsLayoutRunning] = useState(false);
|
||||
const [viewMode, setViewMode] = useState<GraphViewMode>("full");
|
||||
const [searchQuery, setSearchQuery] = useState("");
|
||||
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
|
||||
const [searchError, setSearchError] = useState("");
|
||||
const [predictionType, setPredictionType] = useState("");
|
||||
const [predictions, setPredictions] = useState<LinkPrediction[]>([]);
|
||||
const [pathTargetId, setPathTargetId] = useState("");
|
||||
const [pathResult, setPathResult] = useState<PathResponse | null>(null);
|
||||
const [activeNodeCount, setActiveNodeCount] = useState<number | null>(null);
|
||||
const [temporalBounds, setTemporalBounds] = useState<TemporalBounds | null>(null);
|
||||
const [scrubberTime, setScrubberTime] = useState<Date | null>(null);
|
||||
const [loadingProgress, setLoadingProgress] = useState<GraphLoadProgress | null>(null);
|
||||
const [isGraphStageReady, setIsGraphStageReady] = useState(false);
|
||||
const [layoutStatus, setLayoutStatus] = useState<GraphLayoutStatus>({
|
||||
state: "idle",
|
||||
source: "runtime",
|
||||
hasCoordinates: false,
|
||||
layoutReady: false,
|
||||
displacement: null,
|
||||
elapsedMs: 0,
|
||||
stableSamples: 0,
|
||||
});
|
||||
|
||||
const debouncedTime = useDebounce(scrubberTime, 150);
|
||||
const stageRef = useRef<GraphStageHandle>(null);
|
||||
const reload = useReloadGraphData();
|
||||
const { data: snapshot, isLoading, isFetching, isError, error } = useGraphData({ enabled: true, onProgress: setLoadingProgress });
|
||||
|
||||
const handleSelectedNodeStateChange = useCallback((state: GraphSelectedNodeState | null) => {
|
||||
setSelectedNodeState(state);
|
||||
}, []);
|
||||
|
||||
const handleLayoutRunningChange = useCallback((running: boolean) => {
|
||||
setIsLayoutRunning(running);
|
||||
}, []);
|
||||
|
||||
const handleActiveNodeCountChange = useCallback((count: number | null) => {
|
||||
setActiveNodeCount(count);
|
||||
}, []);
|
||||
|
||||
const handleProgressChange = useCallback((progress: GraphLoadProgress | null) => {
|
||||
setLoadingProgress(progress);
|
||||
}, []);
|
||||
|
||||
const handleRuntimeReady = useCallback(() => {
|
||||
setIsGraphStageReady(true);
|
||||
}, []);
|
||||
|
||||
const handleLayoutStatusChange = useCallback((status: GraphLayoutStatus) => {
|
||||
setLayoutStatus(status);
|
||||
if (status.layoutReady) {
|
||||
setLoadingProgress(null);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (snapshot) {
|
||||
setIsGraphStageReady(false);
|
||||
setActiveNodeCount(null);
|
||||
setLayoutStatus({
|
||||
state: snapshot.summary.layoutReady ? "interactive" : "idle",
|
||||
source: snapshot.summary.layoutSource ?? "runtime",
|
||||
hasCoordinates: snapshot.summary.hasCoordinates ?? false,
|
||||
layoutReady: snapshot.summary.layoutReady ?? false,
|
||||
displacement: null,
|
||||
elapsedMs: 0,
|
||||
stableSamples: 0,
|
||||
});
|
||||
}
|
||||
}, [snapshot?.fetchedAt]);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
const loadBounds = async () => {
|
||||
try {
|
||||
const response = await fetch("/api/temporal/bounds");
|
||||
if (!response.ok || cancelled) return;
|
||||
const data: TemporalBounds = await response.json();
|
||||
if (!cancelled) setTemporalBounds(data);
|
||||
} catch {
|
||||
if (!cancelled) setTemporalBounds(null);
|
||||
}
|
||||
};
|
||||
void loadBounds();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [snapshot?.summary.nodeCount, snapshot?.summary.edgeCount]);
|
||||
|
||||
const neighborCountMap = useMemo(() => {
|
||||
const map = new Map<string, number>();
|
||||
if (!snapshot) return map;
|
||||
for (const node of snapshot.nodes) map.set(node.id, 0);
|
||||
for (const edge of snapshot.edges) {
|
||||
map.set(edge.source, (map.get(edge.source) ?? 0) + 1);
|
||||
map.set(edge.target, (map.get(edge.target) ?? 0) + 1);
|
||||
}
|
||||
return map;
|
||||
}, [snapshot]);
|
||||
|
||||
const visibleSelectedNode = useMemo(() => {
|
||||
if (!selectedNodeId) return null;
|
||||
if (selectedNodeState?.id === selectedNodeId) return selectedNodeState;
|
||||
const snapshotNode = snapshot?.nodes.find((candidate) => candidate.id === selectedNodeId);
|
||||
if (snapshotNode) return toSelectedNodeState(snapshotNode, neighborCountMap.get(snapshotNode.id) ?? 0);
|
||||
const searchNode = searchResults.find((candidate) => candidate.node.id === selectedNodeId)?.node;
|
||||
return searchNode
|
||||
? {
|
||||
id: searchNode.id,
|
||||
label: searchNode.content || searchNode.id,
|
||||
content: searchNode.content || searchNode.id,
|
||||
nodeType: searchNode.type,
|
||||
color: "#58a6ff",
|
||||
valid_from: null,
|
||||
valid_until: null,
|
||||
properties: searchNode.properties ?? {},
|
||||
neighborCount: 0,
|
||||
visibleNeighborCount: 0,
|
||||
collapsedNeighborCount: 0,
|
||||
isNeighborhoodCollapsed: false,
|
||||
canCollapseNeighborhood: false,
|
||||
}
|
||||
: null;
|
||||
}, [neighborCountMap, searchResults, selectedNodeId, selectedNodeState, snapshot]);
|
||||
|
||||
const focusNode = useCallback((nodeId: string) => {
|
||||
setSelectedNodeId(nodeId);
|
||||
setPathResult(null);
|
||||
|
||||
if (!nodeId) {
|
||||
setSelectedNodeState(null);
|
||||
setPredictions([]);
|
||||
return;
|
||||
}
|
||||
|
||||
setSearchResults([]);
|
||||
setIsLayoutRunning(false);
|
||||
}, []);
|
||||
|
||||
const handleSearch = useCallback(async () => {
|
||||
if (!searchQuery.trim()) {
|
||||
setSearchResults([]);
|
||||
return;
|
||||
}
|
||||
|
||||
setSearchError("");
|
||||
try {
|
||||
const response = await fetch("/api/graph/search", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ query: searchQuery, limit: 8 }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Search failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
setSearchResults(data.results || []);
|
||||
if (data.results?.length) {
|
||||
focusNode(data.results[0].node.id);
|
||||
}
|
||||
} catch (searchFetchError) {
|
||||
setSearchError(searchFetchError instanceof Error ? searchFetchError.message : "Search failed");
|
||||
}
|
||||
}, [focusNode, searchQuery]);
|
||||
|
||||
const handleRunPredictions = useCallback(async () => {
|
||||
if (!selectedNodeId) return;
|
||||
|
||||
try {
|
||||
const response = await fetch("/api/enrich/links", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
node_id: selectedNodeId,
|
||||
top_n: 6,
|
||||
candidate_type: predictionType || undefined,
|
||||
min_score: 0,
|
||||
}),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Link prediction failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
setPredictions(data.predictions || []);
|
||||
} catch (predictionError) {
|
||||
console.error("[GraphWorkspaceShell] prediction failed", predictionError);
|
||||
setPredictions([]);
|
||||
}
|
||||
}, [predictionType, selectedNodeId]);
|
||||
|
||||
const handleTracePath = useCallback(async () => {
|
||||
if (!selectedNodeId || !pathTargetId.trim()) return;
|
||||
|
||||
try {
|
||||
const pathParams = new URLSearchParams({
|
||||
source: selectedNodeId,
|
||||
target: pathTargetId.trim(),
|
||||
algorithm: "dijkstra",
|
||||
});
|
||||
const response = await fetch(
|
||||
`/api/graph/path?${pathParams.toString()}`,
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Path lookup failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const data: PathResponse = await response.json();
|
||||
setPathResult(data);
|
||||
if (data.path?.length) {
|
||||
const lastStep = data.path[data.path.length - 1];
|
||||
stageRef.current?.focusNode(lastStep);
|
||||
}
|
||||
} catch (pathError) {
|
||||
console.error("[GraphWorkspaceShell] path trace failed", pathError);
|
||||
setPathResult(null);
|
||||
}
|
||||
}, [pathTargetId, selectedNodeId]);
|
||||
|
||||
const handleDownloadProvenance = useCallback(async (format: "json" | "markdown") => {
|
||||
if (!selectedNodeId) return;
|
||||
|
||||
const suffix = format === "markdown" ? "markdown" : "json";
|
||||
const response = await fetch(`/api/provenance/report?node_id=${encodeURIComponent(selectedNodeId)}&format=${suffix}`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Provenance report failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
const url = window.URL.createObjectURL(blob);
|
||||
const anchor = document.createElement("a");
|
||||
anchor.href = url;
|
||||
anchor.download = `${selectedNodeId}_provenance.${format === "markdown" ? "md" : "json"}`;
|
||||
document.body.appendChild(anchor);
|
||||
anchor.click();
|
||||
window.URL.revokeObjectURL(url);
|
||||
document.body.removeChild(anchor);
|
||||
}, [selectedNodeId]);
|
||||
|
||||
const searchSummary = useMemo(() => {
|
||||
if (!searchResults.length) return null;
|
||||
return `${searchResults.length} search result${searchResults.length === 1 ? "" : "s"}`;
|
||||
}, [searchResults.length]);
|
||||
|
||||
const focusedSummary = useMemo(() => {
|
||||
if (!visibleSelectedNode) return null;
|
||||
if (viewMode === "focused") {
|
||||
const visibleNeighbors = Math.min(visibleSelectedNode.neighborCount, 16);
|
||||
return `${visibleNeighbors + 1} nodes in focused view`;
|
||||
}
|
||||
return `${visibleSelectedNode.neighborCount} direct neighbors highlighted`;
|
||||
}, [viewMode, visibleSelectedNode]);
|
||||
|
||||
const requestViewMode = useCallback((nextViewMode: GraphViewMode) => {
|
||||
if (nextViewMode === "focused") {
|
||||
if (!selectedNodeId) {
|
||||
return;
|
||||
}
|
||||
setViewMode("focused");
|
||||
setIsLayoutRunning(false);
|
||||
return;
|
||||
}
|
||||
|
||||
setViewMode("full");
|
||||
}, [selectedNodeId]);
|
||||
|
||||
const showLoadingOverlay =
|
||||
isLoading
|
||||
|| isFetching
|
||||
|| !isGraphStageReady
|
||||
|| (layoutStatus.source === "runtime" && !layoutStatus.layoutReady && !selectedNodeId && viewMode === "full");
|
||||
|
||||
const layoutStatusLabel = useMemo(() => {
|
||||
if (layoutStatus.source === "provided" && layoutStatus.layoutReady) return "Persisted layout";
|
||||
if (layoutStatus.source === "carried" && layoutStatus.layoutReady) return "Preserved layout";
|
||||
if (layoutStatus.state === "bootstrapping") return "Bootstrapping layout";
|
||||
if (layoutStatus.state === "running") return "Stabilizing layout";
|
||||
if (layoutStatus.state === "failed") return "Layout timeout fallback";
|
||||
return null;
|
||||
}, [layoutStatus]);
|
||||
|
||||
return (
|
||||
<div className="palantir-bg" style={{ position: "relative", width: "100%", height: "100%", overflow: "hidden", display: "flex", flexDirection: "column" }}>
|
||||
<style>{HUD_CSS}</style>
|
||||
<div className="palantir-grid" />
|
||||
<div className="palantir-vignette" />
|
||||
|
||||
<div style={{ flex: 1, position: "relative", zIndex: 3, minHeight: 0 }}>
|
||||
<Suspense fallback={null}>
|
||||
<GraphRuntimeStage
|
||||
ref={stageRef}
|
||||
snapshot={snapshot}
|
||||
selectedNodeId={selectedNodeId}
|
||||
activePath={pathResult?.path ?? []}
|
||||
onNodeSelect={focusNode}
|
||||
onSelectedNodeStateChange={handleSelectedNodeStateChange}
|
||||
isLayoutRunning={isLayoutRunning}
|
||||
onLayoutRunningChange={handleLayoutRunningChange}
|
||||
viewMode={viewMode}
|
||||
temporalTime={debouncedTime}
|
||||
onActiveNodeCountChange={handleActiveNodeCountChange}
|
||||
onProgressChange={handleProgressChange}
|
||||
onLayoutStatusChange={handleLayoutStatusChange}
|
||||
onRuntimeReady={handleRuntimeReady}
|
||||
/>
|
||||
</Suspense>
|
||||
<GraphLoadingOverlay
|
||||
progress={loadingProgress}
|
||||
visible={showLoadingOverlay}
|
||||
showGraphBehind={Boolean(loadingProgress?.showGraphBehind || isGraphStageReady)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<Suspense fallback={<TimelineFallback min={temporalBounds?.min ?? null} max={temporalBounds?.max ?? null} />}>
|
||||
<TimelinePanel
|
||||
onTimeChange={setScrubberTime}
|
||||
minDate={temporalBounds?.min ?? undefined}
|
||||
maxDate={temporalBounds?.max ?? undefined}
|
||||
/>
|
||||
</Suspense>
|
||||
|
||||
<div style={{ position: "absolute", inset: 0, pointerEvents: "none", zIndex: 10 }}>
|
||||
<div className="graph-shell-top">
|
||||
<section className="graph-status-card">
|
||||
<div className="graph-status-label">Graph Studio</div>
|
||||
<div className="graph-status-title">{visibleSelectedNode ? visibleSelectedNode.label : "Knowledge Explorer"}</div>
|
||||
<div className="graph-status-metrics">
|
||||
{showLoadingOverlay && loadingProgress ? <span style={{ ...metricPillStyle, color: "#a9ddff" }}>{getGraphLoadTitle(loadingProgress.phase)}</span> : null}
|
||||
{layoutStatusLabel ? <span style={{ ...metricPillStyle, color: "#a9ddff" }}>{layoutStatusLabel}</span> : null}
|
||||
{snapshot ? <span style={metricPillStyle}>{snapshot.summary.nodeCount.toLocaleString()} nodes · {snapshot.summary.edgeCount.toLocaleString()} edges</span> : null}
|
||||
{activeNodeCount !== null ? <span style={{ ...metricPillStyle, color: "#4fd49c", borderColor: "rgba(79, 212, 156, 0.22)" }}>{activeNodeCount.toLocaleString()} active</span> : null}
|
||||
{searchSummary ? <span style={metricPillStyle}>{searchSummary}</span> : null}
|
||||
{focusedSummary ? <span style={{ ...metricPillStyle, color: "#f2b66d", borderColor: "rgba(242, 182, 109, 0.24)" }}>{focusedSummary}</span> : null}
|
||||
{isError ? <span style={{ ...metricPillStyle, color: "#ff8f85", borderColor: "rgba(255, 123, 114, 0.22)" }}>{(error as Error).message}</span> : null}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="graph-command-card">
|
||||
<div className="graph-command-row">
|
||||
<div className="graph-toggle-cluster">
|
||||
{selectedNodeId ? (
|
||||
<>
|
||||
<button onClick={() => requestViewMode("focused")} style={{ ...actionButtonStyle, background: viewMode === "focused" ? "rgba(31, 111, 235, 0.38)" : actionButtonStyle.background, borderColor: viewMode === "focused" ? "rgba(127, 208, 255, 0.42)" : "rgba(88, 166, 255, 0.2)" }}>Focused View</button>
|
||||
<button onClick={() => requestViewMode("full")} style={{ ...actionButtonStyle, background: viewMode === "full" ? "rgba(31, 111, 235, 0.38)" : actionButtonStyle.background, borderColor: viewMode === "full" ? "rgba(127, 208, 255, 0.42)" : "rgba(88, 166, 255, 0.2)" }}>Full Graph</button>
|
||||
</>
|
||||
) : (
|
||||
<span style={{ color: "#7f95b3", fontSize: 12 }}>Select a node to switch graph views</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="graph-action-cluster">
|
||||
<button onClick={() => setIsLayoutRunning((value) => !value)} style={secondaryActionButtonStyle} disabled={isLoading || isFetching}>
|
||||
{isLayoutRunning ? "Pause Layout" : "Run Layout"}
|
||||
</button>
|
||||
<button onClick={() => { setIsGraphStageReady(false); reload(); }} style={secondaryActionButtonStyle} disabled={isLoading || isFetching}>
|
||||
Reload
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="graph-command-row">
|
||||
<div className="graph-search-shell">
|
||||
<input
|
||||
value={searchQuery}
|
||||
onChange={(event) => setSearchQuery(event.target.value)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter") {
|
||||
void handleSearch();
|
||||
}
|
||||
}}
|
||||
placeholder="Search a node, e.g. Metformin"
|
||||
style={{ ...inputStyle, minWidth: 260 }}
|
||||
disabled={showLoadingOverlay && !selectedNodeId}
|
||||
/>
|
||||
<button onClick={() => void handleSearch()} style={actionButtonStyle} disabled={showLoadingOverlay && !selectedNodeId}>Search</button>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
{searchError ? <div style={{ position: "absolute", top: 144, right: 34, color: "#ff7b72", fontSize: 12, pointerEvents: "auto" }}>{searchError}</div> : null}
|
||||
{searchResults.length ? (
|
||||
<div className="graph-search-results hud-scrollbar">
|
||||
<div className="graph-search-results-label">Search Results</div>
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{searchResults.map((result) => (
|
||||
<button key={result.node.id} className="graph-search-result-card" onClick={() => focusNode(result.node.id)}>
|
||||
<div style={{ color: "#fff", fontWeight: 700 }}>{result.node.content || result.node.id}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{result.node.type}</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, marginTop: 4 }}>score {result.score.toFixed(3)}</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="graph-inspector hud-scrollbar" data-open={selectedNodeId ? "true" : "false"}>
|
||||
<NodePanel
|
||||
node={visibleSelectedNode}
|
||||
predictions={predictions}
|
||||
predictionType={predictionType}
|
||||
onPredictionTypeChange={setPredictionType}
|
||||
onRunPredictions={() => void handleRunPredictions()}
|
||||
pathTargetId={pathTargetId}
|
||||
onPathTargetChange={setPathTargetId}
|
||||
onTracePath={() => void handleTracePath()}
|
||||
pathResult={pathResult}
|
||||
onDownloadProvenance={(format) => void handleDownloadProvenance(format)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const metricPillStyle: CSSProperties = {
|
||||
background: "rgba(88, 166, 255, 0.08)",
|
||||
color: "#8ed3ff",
|
||||
padding: "6px 11px",
|
||||
borderRadius: 999,
|
||||
fontSize: 12,
|
||||
fontWeight: 700,
|
||||
border: "1px solid rgba(88, 166, 255, 0.14)",
|
||||
};
|
||||
|
||||
const sectionStyle: CSSProperties = {
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
gap: 10,
|
||||
padding: 14,
|
||||
background: "linear-gradient(180deg, rgba(255,255,255,0.025), rgba(255,255,255,0.01))",
|
||||
border: "1px solid rgba(255, 255, 255, 0.06)",
|
||||
borderRadius: 16,
|
||||
};
|
||||
|
||||
const sectionTitleStyle: CSSProperties = {
|
||||
color: "#8fa8c6",
|
||||
fontSize: 11,
|
||||
fontWeight: 800,
|
||||
textTransform: "uppercase",
|
||||
letterSpacing: "0.08em",
|
||||
};
|
||||
|
||||
const inputStyle: CSSProperties = {
|
||||
width: "100%",
|
||||
background: "rgba(0, 0, 0, 0.24)",
|
||||
border: "1px solid rgba(88, 166, 255, 0.14)",
|
||||
color: "#fff",
|
||||
borderRadius: 12,
|
||||
padding: "10px 12px",
|
||||
fontSize: 13,
|
||||
};
|
||||
|
||||
const actionButtonStyle: CSSProperties = {
|
||||
background: "linear-gradient(180deg, rgba(53, 130, 245, 0.28), rgba(25, 88, 185, 0.18))",
|
||||
color: "#fff",
|
||||
border: "1px solid rgba(88, 166, 255, 0.2)",
|
||||
borderRadius: 12,
|
||||
padding: "10px 13px",
|
||||
cursor: "pointer",
|
||||
fontWeight: 700,
|
||||
fontSize: 12,
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
boxShadow: "inset 0 1px 0 rgba(255,255,255,0.05)",
|
||||
};
|
||||
|
||||
const secondaryActionButtonStyle: CSSProperties = {
|
||||
...actionButtonStyle,
|
||||
background: "rgba(255, 255, 255, 0.035)",
|
||||
border: "1px solid rgba(255, 255, 255, 0.06)",
|
||||
color: "#d6e5f8",
|
||||
fontWeight: 500,
|
||||
};
|
||||
|
||||
const predictionCardStyle: CSSProperties = {
|
||||
textAlign: "left",
|
||||
padding: 12,
|
||||
background: "rgba(88, 166, 255, 0.06)",
|
||||
border: "1px solid rgba(88, 166, 255, 0.1)",
|
||||
borderRadius: 14,
|
||||
cursor: "pointer",
|
||||
};
|
||||
|
||||
const pathStepStyle: CSSProperties = {
|
||||
color: "#e6edf3",
|
||||
fontSize: 13,
|
||||
padding: "8px 10px",
|
||||
background: "rgba(255, 255, 255, 0.03)",
|
||||
borderRadius: 8,
|
||||
};
|
||||
|
||||
const propertyCardStyle: CSSProperties = {
|
||||
background: "rgba(0, 0, 0, 0.18)",
|
||||
padding: "10px 12px",
|
||||
borderRadius: 12,
|
||||
border: "1px solid rgba(255, 255, 255, 0.05)",
|
||||
};
|
||||
|
||||
const emptyTextStyle: CSSProperties = {
|
||||
color: "#8b949e",
|
||||
fontSize: 12,
|
||||
lineHeight: 1.5,
|
||||
};
|
||||
|
||||
const subtleChipStyle: CSSProperties = {
|
||||
background: "rgba(255, 255, 255, 0.035)",
|
||||
color: "#9fb6d2",
|
||||
padding: "5px 9px",
|
||||
borderRadius: 999,
|
||||
fontSize: 11,
|
||||
border: "1px solid rgba(255, 255, 255, 0.06)",
|
||||
};
|
||||
|
||||
const collapseStyle: CSSProperties = {
|
||||
border: "1px solid rgba(255, 255, 255, 0.05)",
|
||||
borderRadius: 14,
|
||||
background: "rgba(0, 0, 0, 0.14)",
|
||||
overflow: "hidden",
|
||||
};
|
||||
|
||||
const summaryStyle: CSSProperties = {
|
||||
cursor: "pointer",
|
||||
listStyle: "none",
|
||||
padding: "12px 14px",
|
||||
color: "#c6d4e3",
|
||||
fontSize: 12,
|
||||
fontWeight: 700,
|
||||
letterSpacing: "0.04em",
|
||||
textTransform: "uppercase",
|
||||
};
|
||||
@@ -2099,6 +2099,15 @@ function createCollapsedNeighborhoodGraph(
|
||||
return collapsedGraph;
|
||||
}
|
||||
|
||||
// Normalize an edge relationship type: empty string, null, and undefined all
|
||||
// fall back to the project-wide default used consistently across every
|
||||
// aggregation path. Keep this local — it exists only to guarantee that the
|
||||
// three code paths (single-entry, multi-entry, community-grouped) produce the
|
||||
// same semantics and do not diverge again.
|
||||
function normalizeEdgeType(value: string | null | undefined): string {
|
||||
return value || "related_to";
|
||||
}
|
||||
|
||||
function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAttributes> {
|
||||
const aggregated = new Graph<NodeAttributes, EdgeAttributes>({
|
||||
type: "directed",
|
||||
@@ -2124,10 +2133,13 @@ function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAt
|
||||
const [{ edgeId, attrs }] = entries;
|
||||
aggregated.mergeDirectedEdgeWithKey(edgeId, sourceId, targetId, {
|
||||
...attrs,
|
||||
// #1009: normalize empty/null/undefined edgeType so Sigma's label
|
||||
// renderer never receives a blank string on the single-entry path.
|
||||
edgeType: normalizeEdgeType(attrs.edgeType),
|
||||
dominantEdgeType: normalizeEdgeType(attrs.dominantEdgeType ?? attrs.edgeType),
|
||||
rawEdgeIds: collectRawEdgeIds(attrs, edgeId),
|
||||
isAggregated: isAggregatedEdgeAttributes(attrs),
|
||||
aggregateCount: attrs.aggregateCount ?? collectRawEdgeIds(attrs, edgeId).length,
|
||||
dominantEdgeType: attrs.dominantEdgeType ?? attrs.edgeType,
|
||||
representativeWeight: attrs.representativeWeight ?? Number(attrs.weight ?? 1),
|
||||
});
|
||||
return;
|
||||
@@ -2150,10 +2162,11 @@ function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAt
|
||||
const rawEdgeIds = entries.flatMap(({ edgeId, attrs }) => collectRawEdgeIds(attrs, edgeId));
|
||||
const typeCounts = new Map<string, number>();
|
||||
entries.forEach(({ attrs }) => {
|
||||
const edgeType = String(attrs.edgeType ?? "related_to");
|
||||
const edgeType = normalizeEdgeType(attrs.edgeType);
|
||||
typeCounts.set(edgeType, (typeCounts.get(edgeType) ?? 0) + 1);
|
||||
});
|
||||
const dominantEdgeType = [...typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0] ?? representative.attrs.edgeType ?? "related_to";
|
||||
const dominantEdgeType = [...typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0]
|
||||
?? normalizeEdgeType(representative.attrs.edgeType);
|
||||
const reverseKey = `${targetId}→${sourceId}`;
|
||||
const isBidirectionalBundle = groupedEdges.has(reverseKey);
|
||||
const syntheticEdgeId = `${AGGREGATED_EDGE_PREFIX}${sourceId}::${targetId}`;
|
||||
@@ -2167,10 +2180,10 @@ function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAt
|
||||
rawEdgeIds,
|
||||
isAggregated: true,
|
||||
aggregateCount: rawEdgeIds.length,
|
||||
dominantEdgeType: String(dominantEdgeType),
|
||||
dominantEdgeType: dominantEdgeType,
|
||||
representativeWeight: Number(representative.attrs.weight ?? 1),
|
||||
weight: Number(representative.attrs.weight ?? 1),
|
||||
edgeType: String(representative.attrs.edgeType ?? dominantEdgeType ?? "related_to"),
|
||||
edgeType: representative.attrs.edgeType || dominantEdgeType,
|
||||
parallelCount: rawEdgeIds.length,
|
||||
familySize: rawEdgeIds.length,
|
||||
bundleKind: isBidirectionalBundle ? "bidirectional" : "parallel",
|
||||
@@ -2280,7 +2293,7 @@ function buildCommunityGroupedGraph(): GraphDisplayResult {
|
||||
};
|
||||
bucket.rawEdgeIds.push(String(edgeId));
|
||||
bucket.weight = Math.max(bucket.weight, Number((attrs as EdgeAttributes).weight ?? 1));
|
||||
const edgeType = String((attrs as EdgeAttributes).edgeType ?? "related_to");
|
||||
const edgeType = normalizeEdgeType((attrs as EdgeAttributes).edgeType);
|
||||
bucket.typeCounts.set(edgeType, (bucket.typeCounts.get(edgeType) ?? 0) + 1);
|
||||
groupedEdges.set(key, bucket);
|
||||
});
|
||||
@@ -2396,7 +2409,8 @@ function buildCommunityGroupedGraph(): GraphDisplayResult {
|
||||
if (!visibleGroupedEdgeKeys.has(key)) {
|
||||
return;
|
||||
}
|
||||
const dominantEdgeType = [...bundle.typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0] ?? "related_to";
|
||||
const dominantEdgeType = [...bundle.typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0]
|
||||
?? "related_to";
|
||||
const reverseKey = `${bundle.targetId}→${bundle.sourceId}`;
|
||||
const syntheticEdgeId = `${AGGREGATED_EDGE_PREFIX}${key}`;
|
||||
const aggregateCount = bundle.rawEdgeIds.length;
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
/**
|
||||
* shouldLoad predicates for the GraphWorkspace lazy plugin registry.
|
||||
*
|
||||
* Extracted into a pure module so the predicates can be unit-tested without
|
||||
* importing the full GraphWorkspace React component. Each predicate gates
|
||||
* whether a plugin's module is lazily imported; none reference temporalState
|
||||
* so temporal scrubber updates never retrigger plugin loading (issue #830).
|
||||
*/
|
||||
|
||||
export type PluginShouldLoadContext = {
|
||||
panelState: Record<string, boolean>;
|
||||
};
|
||||
|
||||
export function explorationEffectsShouldLoad({ panelState }: PluginShouldLoadContext): boolean {
|
||||
return Boolean(panelState["effects-panel"]);
|
||||
}
|
||||
|
||||
export function neighborhoodPanelShouldLoad({ panelState }: PluginShouldLoadContext): boolean {
|
||||
return Boolean(panelState["neighborhood-panel"]);
|
||||
}
|
||||
|
||||
export function temporalOverlayShouldLoad({ panelState }: PluginShouldLoadContext): boolean {
|
||||
return Boolean(panelState["temporal-panel"]);
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { CSSProperties } from "react";
|
||||
|
||||
import type {
|
||||
GraphDiagnosticsSnapshot,
|
||||
GraphEffectAvailability,
|
||||
GraphEffectToggle,
|
||||
} from "../types";
|
||||
@@ -30,6 +31,11 @@ const EFFECT_ROWS: EffectRowConfig[] = [
|
||||
label: "Neighborhood Lens",
|
||||
description: "Local emphasis around the hovered or selected node.",
|
||||
},
|
||||
{
|
||||
key: "edgeLabelsEnabled",
|
||||
label: "Edge Labels",
|
||||
description: "Draw the relationship type on graph edges. Off restores label-free edges on dense graphs.",
|
||||
},
|
||||
{
|
||||
key: "legendEnabled",
|
||||
label: "Semantic Legend",
|
||||
@@ -37,6 +43,17 @@ const EFFECT_ROWS: EffectRowConfig[] = [
|
||||
},
|
||||
];
|
||||
|
||||
// Maps the effect toggle keys rendered by this plugin to their corresponding
|
||||
// availability keys in GraphDiagnosticsSnapshot["effectAvailability"]. Kept
|
||||
// local because this plugin only renders a subset of all effects.
|
||||
const EFFECT_AVAILABILITY_KEYS: Partial<Record<GraphEffectToggle, keyof GraphDiagnosticsSnapshot["effectAvailability"]>> = {
|
||||
pathPulseEnabled: "pathPulse",
|
||||
pathFlowEnabled: "pathFlow",
|
||||
lensEnabled: "lens",
|
||||
edgeLabelsEnabled: "edgeLabels",
|
||||
legendEnabled: "legend",
|
||||
};
|
||||
|
||||
function renderAvailabilityText(availability: GraphEffectAvailability) {
|
||||
if (availability.available) {
|
||||
if (typeof availability.visibleSegments === "number" && typeof availability.segmentCap === "number") {
|
||||
@@ -139,15 +156,9 @@ export const explorationEffectsPlugin: GraphPlugin = {
|
||||
description={row.description}
|
||||
checked={effectsState[row.key]}
|
||||
availability={
|
||||
availability?.[
|
||||
row.key === "pathPulseEnabled"
|
||||
? "pathPulse"
|
||||
: row.key === "pathFlowEnabled"
|
||||
? "pathFlow"
|
||||
: row.key === "lensEnabled"
|
||||
? "lens"
|
||||
: "legend"
|
||||
] ?? {
|
||||
(EFFECT_AVAILABILITY_KEYS[row.key] !== undefined
|
||||
? availability?.[EFFECT_AVAILABILITY_KEYS[row.key]!]
|
||||
: undefined) ?? {
|
||||
enabled: effectsState[row.key],
|
||||
available: false,
|
||||
reason: "Waiting for graph runtime",
|
||||
|
||||
@@ -47,6 +47,11 @@ const SCENE_EFFECT_ROWS: EffectRowConfig[] = [
|
||||
label: "Contours",
|
||||
description: "Low-contrast density halos around the strongest visible anchors.",
|
||||
},
|
||||
{
|
||||
key: "edgeLabelsEnabled",
|
||||
label: "Edge Labels",
|
||||
description: "Draw the relationship type on graph edges. Off restores label-free edges on dense graphs.",
|
||||
},
|
||||
{
|
||||
key: "legendEnabled",
|
||||
label: "Regions Summary",
|
||||
@@ -83,6 +88,7 @@ const AVAILABILITY_KEYS: Record<GraphEffectToggle, keyof GraphDiagnosticsSnapsho
|
||||
communitiesEnabled: "communities",
|
||||
centralityEnabled: "centrality",
|
||||
legendEnabled: "legend",
|
||||
edgeLabelsEnabled: "edgeLabels",
|
||||
diagnosticsEnabled: "diagnostics",
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import type { GraphLoadSummary } from "./types";
|
||||
|
||||
/**
|
||||
* Predicates for gating GraphWorkspace temporal API requests.
|
||||
*
|
||||
* Temporal bounds and snapshot requests must strictly not execute until the
|
||||
* initial graph load has succeeded (summary !== undefined). An empty graph
|
||||
* (nodeCount: 0) is still a successful load and must not be rejected.
|
||||
*/
|
||||
|
||||
export function shouldFetchTemporalBounds(
|
||||
summary: GraphLoadSummary | undefined,
|
||||
): boolean {
|
||||
return summary !== undefined;
|
||||
}
|
||||
|
||||
export function shouldFetchTemporalSnapshot({
|
||||
debouncedTime,
|
||||
isLoading,
|
||||
summary,
|
||||
}: {
|
||||
debouncedTime: Date | null;
|
||||
isLoading: boolean;
|
||||
summary: GraphLoadSummary | undefined;
|
||||
}): boolean {
|
||||
return (
|
||||
summary !== undefined &&
|
||||
debouncedTime !== null &&
|
||||
!isLoading
|
||||
);
|
||||
}
|
||||
@@ -103,6 +103,7 @@ export type GraphEffectToggle =
|
||||
| "communitiesEnabled"
|
||||
| "centralityEnabled"
|
||||
| "legendEnabled"
|
||||
| "edgeLabelsEnabled"
|
||||
| "diagnosticsEnabled";
|
||||
|
||||
export interface GraphEffectsState {
|
||||
@@ -113,6 +114,7 @@ export interface GraphEffectsState {
|
||||
semanticRegionsEnabled: boolean;
|
||||
contoursEnabled: boolean;
|
||||
pathfindingEnabled: boolean;
|
||||
edgeLabelsEnabled: boolean;
|
||||
communitiesEnabled: boolean;
|
||||
centralityEnabled: boolean;
|
||||
legendEnabled: boolean;
|
||||
@@ -186,6 +188,7 @@ export interface GraphDiagnosticsSnapshot {
|
||||
communities: GraphEffectAvailability;
|
||||
centrality: GraphEffectAvailability;
|
||||
legend: GraphEffectAvailability;
|
||||
edgeLabels: GraphEffectAvailability;
|
||||
diagnostics: GraphEffectAvailability;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,223 +0,0 @@
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { createGraphLoadProgress } from "./graphLoading";
|
||||
import type { ApiEdge, ApiNode, GraphDataSnapshot, GraphLoadProgress, GraphLayoutSource } from "./types";
|
||||
|
||||
interface NodeListResponse {
|
||||
nodes: ApiNode[];
|
||||
total: number;
|
||||
skip: number;
|
||||
limit: number;
|
||||
next_cursor?: string | null;
|
||||
}
|
||||
|
||||
interface EdgeListResponse {
|
||||
edges: ApiEdge[];
|
||||
total: number;
|
||||
skip: number;
|
||||
limit: number;
|
||||
next_cursor?: string | null;
|
||||
}
|
||||
|
||||
const PAGE_LIMIT = 1000;
|
||||
|
||||
async function fetchAllNodes(
|
||||
signal: AbortSignal,
|
||||
onProgress?: (progress: GraphLoadProgress) => void,
|
||||
): Promise<ApiNode[]> {
|
||||
let cursor: string | null = null;
|
||||
const collected: ApiNode[] = [];
|
||||
let total: number | null = null;
|
||||
|
||||
while (true) {
|
||||
const url = new URL("/api/graph/nodes", window.location.origin);
|
||||
url.searchParams.set("limit", String(PAGE_LIMIT));
|
||||
if (cursor) {
|
||||
url.searchParams.set("cursor", cursor);
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), { signal });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fetch failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const data: NodeListResponse = await response.json();
|
||||
if (!data.nodes?.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
total = data.total ?? total;
|
||||
collected.push(...data.nodes);
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "fetching_nodes",
|
||||
progressKind: total ? "determinate" : "indeterminate",
|
||||
loaded: collected.length,
|
||||
total,
|
||||
nodesLoaded: collected.length,
|
||||
nodesTotal: total,
|
||||
edgesLoaded: 0,
|
||||
edgesTotal: null,
|
||||
message: total
|
||||
? `Loading nodes ${collected.length.toLocaleString()} of ${total.toLocaleString()}`
|
||||
: `Loading nodes ${collected.length.toLocaleString()}`,
|
||||
}));
|
||||
|
||||
if (!data.next_cursor) {
|
||||
break;
|
||||
}
|
||||
cursor = data.next_cursor;
|
||||
await yieldToMain();
|
||||
}
|
||||
|
||||
return collected;
|
||||
}
|
||||
|
||||
async function fetchAllEdges(
|
||||
signal: AbortSignal,
|
||||
nodeIds: Set<string>,
|
||||
nodeProgress: { loaded: number; total: number | null },
|
||||
onProgress?: (progress: GraphLoadProgress) => void,
|
||||
): Promise<ApiEdge[]> {
|
||||
let cursor: string | null = null;
|
||||
const collected: ApiEdge[] = [];
|
||||
const seenEdgeIds = new Set<string>();
|
||||
let total: number | null = null;
|
||||
let warnedOverTotal = false;
|
||||
|
||||
while (true) {
|
||||
const url = new URL("/api/graph/edges", window.location.origin);
|
||||
url.searchParams.set("limit", String(PAGE_LIMIT));
|
||||
if (cursor) {
|
||||
url.searchParams.set("cursor", cursor);
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), { signal });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fetch failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const data: EdgeListResponse = await response.json();
|
||||
if (!data.edges?.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
total = data.total ?? total;
|
||||
const validEdges = data.edges.filter((edge) => {
|
||||
if (!nodeIds.has(edge.source) || !nodeIds.has(edge.target)) {
|
||||
return false;
|
||||
}
|
||||
if (seenEdgeIds.has(edge.id)) {
|
||||
return false;
|
||||
}
|
||||
seenEdgeIds.add(edge.id);
|
||||
return true;
|
||||
});
|
||||
collected.push(...validEdges);
|
||||
const safeLoaded = total ? Math.min(seenEdgeIds.size, total) : seenEdgeIds.size;
|
||||
if (!warnedOverTotal && total !== null && seenEdgeIds.size > total) {
|
||||
warnedOverTotal = true;
|
||||
console.warn("[graph-runtime] edge pagination returned more unique edge ids than total", {
|
||||
uniqueEdgesLoaded: seenEdgeIds.size,
|
||||
total,
|
||||
});
|
||||
}
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "fetching_edges",
|
||||
progressKind: total ? "determinate" : "indeterminate",
|
||||
loaded: safeLoaded,
|
||||
total,
|
||||
nodesLoaded: nodeProgress.loaded,
|
||||
nodesTotal: nodeProgress.total,
|
||||
edgesLoaded: safeLoaded,
|
||||
edgesTotal: total,
|
||||
message: total
|
||||
? `Loading edges ${safeLoaded.toLocaleString()} of ${total.toLocaleString()}`
|
||||
: `Loading edges ${safeLoaded.toLocaleString()}`,
|
||||
}));
|
||||
|
||||
if (!data.next_cursor) {
|
||||
break;
|
||||
}
|
||||
cursor = data.next_cursor;
|
||||
await yieldToMain();
|
||||
}
|
||||
|
||||
return collected;
|
||||
}
|
||||
|
||||
function yieldToMain(): Promise<void> {
|
||||
if ("scheduler" in window && typeof (window as Window & { scheduler?: { yield?: () => Promise<void> } }).scheduler?.yield === "function") {
|
||||
return (window as Window & { scheduler: { yield: () => Promise<void> } }).scheduler.yield();
|
||||
}
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
function hasUsableCoordinate(value: number | null | undefined): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
interface UseGraphDataOptions {
|
||||
enabled?: boolean;
|
||||
onProgress?: (progress: GraphLoadProgress) => void;
|
||||
}
|
||||
|
||||
export function useGraphData(options: UseGraphDataOptions = {}) {
|
||||
const { enabled = true, onProgress } = options;
|
||||
|
||||
return useQuery<GraphDataSnapshot>({
|
||||
queryKey: ["graph", "runtime-snapshot"],
|
||||
enabled,
|
||||
staleTime: Infinity,
|
||||
queryFn: async ({ signal }): Promise<GraphDataSnapshot> => {
|
||||
const startedAt = performance.now();
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "bootstrapping",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: 0,
|
||||
nodesTotal: null,
|
||||
edgesLoaded: 0,
|
||||
edgesTotal: null,
|
||||
message: "Preparing graph session",
|
||||
}));
|
||||
|
||||
const nodes = await fetchAllNodes(signal, onProgress);
|
||||
const nodeIds = new Set(nodes.map((node) => node.id));
|
||||
const edges = await fetchAllEdges(
|
||||
signal,
|
||||
nodeIds,
|
||||
{ loaded: nodes.length, total: nodes.length },
|
||||
onProgress,
|
||||
);
|
||||
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "hydrating_scene",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: nodes.length,
|
||||
nodesTotal: nodes.length,
|
||||
edgesLoaded: edges.length,
|
||||
edgesTotal: edges.length,
|
||||
message: "Preparing graph runtime snapshot",
|
||||
}));
|
||||
|
||||
return {
|
||||
nodes,
|
||||
edges,
|
||||
summary: {
|
||||
nodeCount: nodes.length,
|
||||
edgeCount: edges.length,
|
||||
loadTimeMs: Math.round(performance.now() - startedAt),
|
||||
hasCoordinates: nodes.some((node) => hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)),
|
||||
layoutSource: (nodes.some((node) => hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y))
|
||||
? "provided"
|
||||
: "runtime") as GraphLayoutSource,
|
||||
layoutReady: nodes.some((node) => hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)),
|
||||
},
|
||||
fetchedAt: Date.now(),
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useReloadGraphData() {
|
||||
const queryClient = useQueryClient();
|
||||
return () => queryClient.invalidateQueries({ queryKey: ["graph", "runtime-snapshot"] });
|
||||
}
|
||||
@@ -33,6 +33,7 @@ export function LineageDiagram() {
|
||||
const [edges, setEdges] = useState<any[]>([]);
|
||||
const [searchId, setSearchId] = useState("");
|
||||
const [activeId, setActiveId] = useState("");
|
||||
const [error, setError] = useState("");
|
||||
|
||||
const downloadReport = async (format: "json" | "markdown") => {
|
||||
if (!activeId) return;
|
||||
@@ -51,12 +52,17 @@ export function LineageDiagram() {
|
||||
document.body.removeChild(anchor);
|
||||
};
|
||||
|
||||
const [prevActiveId, setPrevActiveId] = useState(activeId);
|
||||
if (activeId !== prevActiveId) {
|
||||
setPrevActiveId(activeId);
|
||||
setError("");
|
||||
setNodes([]);
|
||||
setEdges([]);
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (!activeId) {
|
||||
setNodes([]);
|
||||
setEdges([]);
|
||||
return;
|
||||
}
|
||||
let ignore = false;
|
||||
if (!activeId) return;
|
||||
|
||||
const xLanes = [
|
||||
{ id: "group_agent", type: "group", position: { x: 50, y: 50 }, style: { width: 800, height: 120 } },
|
||||
@@ -65,22 +71,24 @@ export function LineageDiagram() {
|
||||
];
|
||||
|
||||
const fetchLineage = async () => {
|
||||
setError("");
|
||||
try {
|
||||
const res = await fetch("/api/provenance?node_id=" + encodeURIComponent(activeId));
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
console.error(`HTTP ${res.status}: API Route missing or failed.`, text.substring(0, 100));
|
||||
return;
|
||||
throw new Error(`HTTP ${res.status}: API Route missing or failed. ${text.substring(0, 100)}`);
|
||||
}
|
||||
|
||||
const contentType = res.headers.get("content-type");
|
||||
if (!contentType || !contentType.includes("application/json")) {
|
||||
console.error("Backend returned non-JSON response (likely an HTML fallback). Check FastAPI routing.");
|
||||
return;
|
||||
throw new Error("Backend returned non-JSON response (likely an HTML fallback).");
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
if (res.status === 207) {
|
||||
setError(data.message || "Warning: Partial success loading lineage.");
|
||||
}
|
||||
|
||||
const counters: Record<string, number> = { "group_agent": 0, "group_activity": 0, "group_entity": 0 };
|
||||
|
||||
@@ -89,7 +97,7 @@ export function LineageDiagram() {
|
||||
counters[n.parent_id] = c + 1;
|
||||
return {
|
||||
id: n.id,
|
||||
data: { label: n.label + "\\n(" + n.prov_type + ")" },
|
||||
data: { label: n.label + "\n(" + n.prov_type + ")" },
|
||||
position: { x: 50 + c * 180, y: 30 },
|
||||
parentId: n.parent_id,
|
||||
extent: "parent",
|
||||
@@ -106,13 +114,16 @@ export function LineageDiagram() {
|
||||
style: { stroke: "#58a6ff" }
|
||||
}));
|
||||
|
||||
setNodes([...xLanes, ...mappedNodes]);
|
||||
setEdges(mappedEdges);
|
||||
if (!ignore) {
|
||||
setNodes([...xLanes, ...mappedNodes]);
|
||||
setEdges(mappedEdges);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
setError(err instanceof Error ? err.message : "Failed to load lineage.");
|
||||
}
|
||||
};
|
||||
fetchLineage();
|
||||
void fetchLineage();
|
||||
return () => { ignore = true; };
|
||||
}, [activeId]);
|
||||
|
||||
return (
|
||||
@@ -143,6 +154,12 @@ export function LineageDiagram() {
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{error ? (
|
||||
<div style={{ position: "absolute", top: 60, left: 14, right: 14, zIndex: 10, padding: 12, borderRadius: 14, color: "#ffb4c2", background: "rgba(255,157,175,0.1)", border: "1px solid rgba(255,157,175,0.18)" }}>
|
||||
{error}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{activeId ? (
|
||||
<ReactFlow nodes={nodes} edges={edges} fitView>
|
||||
<Background color="rgba(74,163,255,0.08)" gap={24} />
|
||||
|
||||
@@ -81,15 +81,79 @@ export function KGOverviewTab() {
|
||||
fetch("/api/graph/nodes?limit=500"),
|
||||
]);
|
||||
|
||||
if (statsRes.ok) {
|
||||
const statsData: KGStats = await statsRes.json();
|
||||
setStats(statsData);
|
||||
if (!statsRes.ok) throw new Error(`Stats fetch failed (${statsRes.status})`);
|
||||
if (!nodesRes.ok) throw new Error(`Nodes fetch failed (${nodesRes.status})`);
|
||||
|
||||
const statsData: KGStats = await statsRes.json();
|
||||
setStats(statsData);
|
||||
if (statsRes.status === 207) {
|
||||
setError((statsData as any).message || "Warning: Partial success loading stats.");
|
||||
}
|
||||
|
||||
if (nodesRes.ok) {
|
||||
const nodesData: NodeListResponse = await nodesRes.json();
|
||||
const nodes = nodesData.nodes ?? [];
|
||||
setNodeTypeMap(buildTypeMap(nodes, "type"));
|
||||
if (nodesRes.status === 207) {
|
||||
const nodesMessage = (nodesData as any).message || "Warning: Partial success loading nodes.";
|
||||
setError((prev) => (prev ? `${prev} ${nodesMessage}` : nodesMessage));
|
||||
}
|
||||
|
||||
// Simulate neighbor counts via edges fetch for top-N
|
||||
const edgesRes = await fetch("/api/graph/edges?limit=2000");
|
||||
if (edgesRes.ok) {
|
||||
const edgesData = await edgesRes.json();
|
||||
const edges: { source: string; target: string }[] = edgesData.edges ?? [];
|
||||
const degreeMap: Record<string, number> = {};
|
||||
for (const edge of edges) {
|
||||
degreeMap[edge.source] = (degreeMap[edge.source] ?? 0) + 1;
|
||||
degreeMap[edge.target] = (degreeMap[edge.target] ?? 0) + 1;
|
||||
}
|
||||
const sorted = nodes
|
||||
.map((n) => ({ node: n, neighborCount: degreeMap[n.id] ?? 0 }))
|
||||
.sort((a, b) => b.neighborCount - a.neighborCount)
|
||||
.slice(0, 10);
|
||||
setTopNodes(sorted);
|
||||
}
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : "Failed to load graph overview. Ensure the server is running.");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
let ignore = false;
|
||||
async function fetchInitial() {
|
||||
if (!ignore) {
|
||||
setLoading(true);
|
||||
setError("");
|
||||
}
|
||||
try {
|
||||
const [statsRes, nodesRes] = await Promise.all([
|
||||
fetch("/api/graph/stats"),
|
||||
fetch("/api/graph/nodes?limit=500"),
|
||||
]);
|
||||
|
||||
if (!statsRes.ok) throw new Error(`Stats fetch failed (${statsRes.status})`);
|
||||
if (!nodesRes.ok) throw new Error(`Nodes fetch failed (${nodesRes.status})`);
|
||||
|
||||
const statsData: KGStats = await statsRes.json();
|
||||
if (!ignore) {
|
||||
setStats(statsData);
|
||||
if (statsRes.status === 207) {
|
||||
setError((statsData as { message?: string }).message || "Warning: Partial success loading stats.");
|
||||
}
|
||||
}
|
||||
|
||||
const nodesData: NodeListResponse = await nodesRes.json();
|
||||
const nodes = nodesData.nodes ?? [];
|
||||
setNodeTypeMap(buildTypeMap(nodes, "type"));
|
||||
if (!ignore) {
|
||||
setNodeTypeMap(buildTypeMap(nodes, "type"));
|
||||
if (nodesRes.status === 207) {
|
||||
const nodesMessage = (nodesData as { message?: string }).message || "Warning: Partial success loading nodes.";
|
||||
setError((prev) => (prev ? `${prev} ${nodesMessage}` : nodesMessage));
|
||||
}
|
||||
}
|
||||
|
||||
// Simulate neighbor counts via edges fetch for top-N
|
||||
const edgesRes = await fetch("/api/graph/edges?limit=2000");
|
||||
@@ -105,20 +169,18 @@ export function KGOverviewTab() {
|
||||
.map((n) => ({ node: n, neighborCount: degreeMap[n.id] ?? 0 }))
|
||||
.sort((a, b) => b.neighborCount - a.neighborCount)
|
||||
.slice(0, 10);
|
||||
setTopNodes(sorted);
|
||||
if (!ignore) setTopNodes(sorted);
|
||||
}
|
||||
} catch (err) {
|
||||
if (!ignore) setError(err instanceof Error ? err.message : "Failed to load graph overview. Ensure the server is running.");
|
||||
} finally {
|
||||
if (!ignore) setLoading(false);
|
||||
}
|
||||
} catch {
|
||||
setError("Failed to load graph overview. Ensure the server is running.");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
void fetchInitial();
|
||||
return () => { ignore = true; };
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void fetchOverview();
|
||||
}, [fetchOverview]);
|
||||
|
||||
const nodeTypeEntries = Object.entries(nodeTypeMap).sort((a, b) => b[1] - a[1]);
|
||||
const edgeTypeEntries = stats?.edge_types
|
||||
? Object.entries(stats.edge_types).sort((a, b) => b[1] - a[1])
|
||||
@@ -135,6 +197,12 @@ export function KGOverviewTab() {
|
||||
|
||||
return (
|
||||
<div className="ws-page">
|
||||
{error ? (
|
||||
<div style={{ margin: "16px 22px 0 22px", padding: 12, borderRadius: 14, color: "#ffb4c2", background: "rgba(255,157,175,0.1)", border: "1px solid rgba(255,157,175,0.18)" }}>
|
||||
{error}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{/* Header */}
|
||||
<div style={{ display: "flex", alignItems: "center", justifyContent: "space-between", padding: "16px 22px", borderBottom: "1px solid var(--ws-border)", flexShrink: 0 }}>
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 10 }}>
|
||||
@@ -152,11 +220,7 @@ export function KGOverviewTab() {
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div style={{ margin: "12px 22px", padding: "10px 14px", borderRadius: "var(--ws-radius-sm)", background: "var(--ws-red-soft)", border: "1px solid rgba(255,123,114,0.28)", color: "#fca5a5", fontSize: 13 }}>
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
|
||||
<div className="ws-scroll" style={{ flex: 1, padding: "18px 22px", display: "flex", flexDirection: "column", gap: 16 }}>
|
||||
{/* Stat cards */}
|
||||
|
||||
@@ -54,20 +54,49 @@ export function AlignmentsTab() {
|
||||
loadAlignments(),
|
||||
]);
|
||||
|
||||
const errors: string[] = [];
|
||||
if (registryResult.status === "fulfilled") {
|
||||
setRegistry(registryResult.value);
|
||||
setSourceOntology((current) => current || registryResult.value[0]?.uri || "");
|
||||
setTargetOntology((current) => current || registryResult.value[1]?.uri || registryResult.value[0]?.uri || "");
|
||||
} else {
|
||||
errors.push(registryResult.reason instanceof Error ? registryResult.reason.message : "Failed to load ontology registry.");
|
||||
}
|
||||
if (alignmentResult.status === "fulfilled") {
|
||||
setAlignments(alignmentResult.value);
|
||||
} else {
|
||||
errors.push(alignmentResult.reason instanceof Error ? alignmentResult.reason.message : "Failed to load alignments.");
|
||||
}
|
||||
|
||||
if (errors.length) setError(errors.join(" "));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void reload();
|
||||
}, [reload]);
|
||||
let ignore = false;
|
||||
async function fetchInitial() {
|
||||
const [registryResult, alignmentResult] = await Promise.allSettled([
|
||||
loadOntologyRegistry(),
|
||||
loadAlignments(),
|
||||
]);
|
||||
if (ignore) return;
|
||||
|
||||
const errors: string[] = [];
|
||||
if (registryResult.status === "fulfilled") {
|
||||
setRegistry(registryResult.value);
|
||||
setSourceOntology((current) => current || registryResult.value[0]?.uri || "");
|
||||
setTargetOntology((current) => current || registryResult.value[1]?.uri || registryResult.value[0]?.uri || "");
|
||||
} else {
|
||||
errors.push(registryResult.reason instanceof Error ? registryResult.reason.message : "Failed to load ontology registry.");
|
||||
}
|
||||
if (alignmentResult.status === "fulfilled") {
|
||||
setAlignments(alignmentResult.value);
|
||||
} else {
|
||||
errors.push(alignmentResult.reason instanceof Error ? alignmentResult.reason.message : "Failed to load alignments.");
|
||||
}
|
||||
if (errors.length) setError(errors.join(" "));
|
||||
}
|
||||
void fetchInitial();
|
||||
return () => { ignore = true; };
|
||||
}, []);
|
||||
|
||||
const relationCounts = useMemo(() => {
|
||||
const counts = new Map<string, number>();
|
||||
|
||||
@@ -23,29 +23,40 @@ export function HealthTab({ onFixInEditor }: HealthTabProps) {
|
||||
setRegistry(entries);
|
||||
setSelectedUri((current) => current || entries[0]?.uri || "");
|
||||
})
|
||||
.catch(() => { /* backend unavailable — leave registry empty */ });
|
||||
.catch((err) => {
|
||||
if (cancelled) return;
|
||||
setError(err instanceof Error ? err.message : "Failed to load ontology registry.");
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, []);
|
||||
|
||||
const loadHealth = useCallback(async (uri: string) => {
|
||||
if (!uri) return;
|
||||
setLoading(true);
|
||||
setError("");
|
||||
try {
|
||||
setHealth(await loadOntologyHealth(uri));
|
||||
} catch {
|
||||
// Backend unavailable — show "select an ontology" placeholder, not an error
|
||||
setHealth(null);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
const [prevUri, setPrevUri] = useState(selectedUri);
|
||||
if (selectedUri !== prevUri) {
|
||||
setPrevUri(selectedUri);
|
||||
if (selectedUri) {
|
||||
setLoading(true);
|
||||
setError("");
|
||||
}
|
||||
}, []);
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
void loadHealth(selectedUri);
|
||||
}, [selectedUri, loadHealth]);
|
||||
let ignore = false;
|
||||
async function fetchHealth() {
|
||||
if (!selectedUri) return;
|
||||
try {
|
||||
const data = await loadOntologyHealth(selectedUri);
|
||||
if (!ignore) setHealth(data);
|
||||
} catch {
|
||||
if (!ignore) setHealth(null);
|
||||
} finally {
|
||||
if (!ignore) setLoading(false);
|
||||
}
|
||||
}
|
||||
void fetchHealth();
|
||||
return () => { ignore = true; };
|
||||
}, [selectedUri]);
|
||||
|
||||
const exportReport = useCallback(() => {
|
||||
if (!health) return;
|
||||
|
||||
@@ -248,6 +248,18 @@ export function OntologyManager() {
|
||||
const [rightPanel, setRightPanel] = useState<RightPanel>("none");
|
||||
const [actionMsg, setActionMsg] = useState<{ type: "ok" | "err"; text: string } | null>(null);
|
||||
|
||||
const [prevSearchQ, setPrevSearchQ] = useState(searchQ);
|
||||
if (searchQ !== prevSearchQ) {
|
||||
setPrevSearchQ(searchQ);
|
||||
setLoading(true);
|
||||
setActionMsg(null);
|
||||
}
|
||||
|
||||
const flashMsg = useCallback((type: "ok" | "err", text: string) => {
|
||||
setActionMsg({ type, text });
|
||||
setTimeout(() => setActionMsg(null), 3000);
|
||||
}, []);
|
||||
|
||||
const fetchRegistry = useCallback(async () => {
|
||||
setLoading(true);
|
||||
setActionMsg(null);
|
||||
@@ -255,26 +267,42 @@ export function OntologyManager() {
|
||||
const params = new URLSearchParams();
|
||||
if (searchQ) params.set("q", searchQ);
|
||||
const res = await fetch(`/api/ontology/registry?${params}`);
|
||||
if (res.ok) {
|
||||
setEntries(await res.json());
|
||||
} else {
|
||||
setEntries([]);
|
||||
}
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const data = await res.json();
|
||||
setEntries(data);
|
||||
if (res.status === 207) flashMsg("err", data.message || "Warning: Partial success loading registry.");
|
||||
} catch {
|
||||
setEntries([]);
|
||||
flashMsg("err", "Failed to load ontology registry");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [searchQ, statusFilter]);
|
||||
}, [searchQ, flashMsg]);
|
||||
|
||||
useEffect(() => {
|
||||
fetchRegistry();
|
||||
}, [fetchRegistry]);
|
||||
|
||||
const flashMsg = (type: "ok" | "err", text: string) => {
|
||||
setActionMsg({ type, text });
|
||||
setTimeout(() => setActionMsg(null), 3000);
|
||||
};
|
||||
let ignore = false;
|
||||
async function fetchInitial() {
|
||||
try {
|
||||
const params = new URLSearchParams();
|
||||
if (searchQ) params.set("q", searchQ);
|
||||
const res = await fetch(`/api/ontology/registry?${params}`);
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const data = await res.json();
|
||||
if (ignore) return;
|
||||
setEntries(data);
|
||||
if (res.status === 207) flashMsg("err", data.message || "Warning: Partial success loading registry.");
|
||||
} catch {
|
||||
if (!ignore) {
|
||||
setEntries([]);
|
||||
flashMsg("err", "Failed to load ontology registry");
|
||||
}
|
||||
} finally {
|
||||
if (!ignore) setLoading(false);
|
||||
}
|
||||
}
|
||||
void fetchInitial();
|
||||
return () => { ignore = true; };
|
||||
}, [searchQ, flashMsg]);
|
||||
|
||||
const handleToggle = useCallback(async (uri: string) => {
|
||||
try {
|
||||
|
||||
@@ -178,17 +178,33 @@ function DetailPanel({
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState("");
|
||||
|
||||
useEffect(() => {
|
||||
const [prevUri, setPrevUri] = useState(uri);
|
||||
if (uri !== prevUri) {
|
||||
setPrevUri(uri);
|
||||
setLoading(true);
|
||||
setError("");
|
||||
setDetail(null);
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
let ignore = false;
|
||||
fetch(`/api/ontology/entity/${encodeURIComponent(uri)}`)
|
||||
.then((r) => {
|
||||
.then(async (r) => {
|
||||
if (!r.ok) throw new Error("Not found");
|
||||
return r.json();
|
||||
const data = await r.json();
|
||||
if (r.status === 207) setError(data.message || "Warning: Partial success loading entity.");
|
||||
return data;
|
||||
})
|
||||
.then(setDetail)
|
||||
.catch((e) => setError(e.message))
|
||||
.finally(() => setLoading(false));
|
||||
.then((data) => {
|
||||
if (!ignore) setDetail(data);
|
||||
})
|
||||
.catch((e) => {
|
||||
if (!ignore) setError(e.message);
|
||||
})
|
||||
.finally(() => {
|
||||
if (!ignore) setLoading(false);
|
||||
});
|
||||
return () => { ignore = true; };
|
||||
}, [uri]);
|
||||
|
||||
return (
|
||||
|
||||
@@ -40,19 +40,6 @@ export function ProposalReview({ proposalId }: { proposalId: string }) {
|
||||
const [selectedElement, setSelectedElement] = useState<string | null>(null);
|
||||
const [commentText, setCommentText] = useState("");
|
||||
|
||||
const loadProposal = useCallback(async () => {
|
||||
try {
|
||||
const response = await fetch(`/api/ontology/proposals/${proposalId}`);
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
setProposal(data);
|
||||
generateDiff(data);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to load proposal:", error);
|
||||
}
|
||||
}, [proposalId]);
|
||||
|
||||
const generateDiff = useCallback((prop: Proposal) => {
|
||||
const changes: DiffChange[] = [];
|
||||
|
||||
@@ -75,9 +62,38 @@ export function ProposalReview({ proposalId }: { proposalId: string }) {
|
||||
setDiff(changes);
|
||||
}, []);
|
||||
|
||||
const loadProposal = useCallback(async () => {
|
||||
try {
|
||||
const response = await fetch(`/api/ontology/proposals/${proposalId}`);
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
setProposal(data);
|
||||
generateDiff(data);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to load proposal:", error);
|
||||
}
|
||||
}, [proposalId, generateDiff]);
|
||||
|
||||
useEffect(() => {
|
||||
loadProposal();
|
||||
}, [loadProposal]);
|
||||
let ignore = false;
|
||||
async function fetchInitial() {
|
||||
try {
|
||||
const response = await fetch(`/api/ontology/proposals/${proposalId}`);
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
if (!ignore) {
|
||||
setProposal(data);
|
||||
generateDiff(data);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to load proposal:", error);
|
||||
}
|
||||
}
|
||||
void fetchInitial();
|
||||
return () => { ignore = true; };
|
||||
}, [proposalId, generateDiff]);
|
||||
|
||||
const addComment = useCallback(async () => {
|
||||
if (!selectedElement || !commentText || !proposal) return;
|
||||
|
||||
@@ -77,17 +77,35 @@ function ConceptDetailPanel({
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState("");
|
||||
|
||||
useEffect(() => {
|
||||
const [prevUri, setPrevUri] = useState(uri);
|
||||
if (uri !== prevUri) {
|
||||
setPrevUri(uri);
|
||||
setLoading(true);
|
||||
setError("");
|
||||
setDetail(null);
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
let ignore = false;
|
||||
fetch(`/api/ontology/skos/concept/${encodeURIComponent(uri)}`)
|
||||
.then((r) => {
|
||||
.then(async (r) => {
|
||||
if (!r.ok) throw new Error("Concept not found");
|
||||
return r.json();
|
||||
const data = await r.json();
|
||||
if (r.status === 207) setError(data.message || "Warning: Partial success loading concept.");
|
||||
return data;
|
||||
})
|
||||
.then(setDetail)
|
||||
.catch((e) => setError(e.message))
|
||||
.finally(() => setLoading(false));
|
||||
.then((data) => {
|
||||
if (!ignore) setDetail(data);
|
||||
})
|
||||
.catch((e) => {
|
||||
if (!ignore) setError(e.message);
|
||||
})
|
||||
.finally(() => {
|
||||
if (!ignore) setLoading(false);
|
||||
});
|
||||
return () => {
|
||||
ignore = true;
|
||||
};
|
||||
}, [uri]);
|
||||
|
||||
const renderUriList = (label: string, uris: string[]) => {
|
||||
@@ -322,16 +340,48 @@ function SchemePanel({
|
||||
}) {
|
||||
const [expanded, setExpanded] = useState(true);
|
||||
const [hierarchy, setHierarchy] = useState<ConceptNode[]>([]);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [loading, setLoading] = useState(expanded);
|
||||
const [error, setError] = useState("");
|
||||
|
||||
const [prevExpanded, setPrevExpanded] = useState(expanded);
|
||||
const [prevSchemeUri, setPrevSchemeUri] = useState(scheme.uri);
|
||||
if (expanded !== prevExpanded || scheme.uri !== prevSchemeUri) {
|
||||
setPrevExpanded(expanded);
|
||||
setPrevSchemeUri(scheme.uri);
|
||||
if (expanded) {
|
||||
setLoading(true);
|
||||
setError("");
|
||||
setHierarchy([]);
|
||||
} else {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
let ignore = false;
|
||||
if (!expanded) return;
|
||||
setLoading(true);
|
||||
fetch(`/api/vocabulary/hierarchy?scheme=${encodeURIComponent(scheme.uri)}`)
|
||||
.then((r) => (r.ok ? r.json() : []))
|
||||
.then(setHierarchy)
|
||||
.catch(() => setHierarchy([]))
|
||||
.finally(() => setLoading(false));
|
||||
.then(async (r) => {
|
||||
if (!r.ok) throw new Error(`HTTP ${r.status}`);
|
||||
const data = await r.json();
|
||||
if (r.status === 207 && !ignore) setError(data.message || "Warning: Partial success loading hierarchy.");
|
||||
return data;
|
||||
})
|
||||
.then((data) => {
|
||||
if (!ignore) setHierarchy(data);
|
||||
})
|
||||
.catch((err) => {
|
||||
if (!ignore) {
|
||||
setHierarchy([]);
|
||||
setError(err instanceof Error ? err.message : "Failed to load hierarchy.");
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
if (!ignore) setLoading(false);
|
||||
});
|
||||
return () => {
|
||||
ignore = true;
|
||||
};
|
||||
}, [scheme.uri, expanded]);
|
||||
|
||||
const totalConcepts = countConcepts(hierarchy);
|
||||
@@ -366,6 +416,7 @@ function SchemePanel({
|
||||
|
||||
{expanded && (
|
||||
<div style={{ paddingBottom: 8 }}>
|
||||
{error ? <div style={errorStyle}>{error}</div> : null}
|
||||
{loading ? (
|
||||
<div style={{ padding: "10px 20px", display: "flex", alignItems: "center", gap: 8 }}>
|
||||
<Loader2 size={12} color="#4aa3ff" style={{ animation: "spin 0.8s linear infinite" }} />
|
||||
@@ -410,9 +461,13 @@ export function SKOSVocabularyManager({ schemeUri }: Props) {
|
||||
const [selectedUri, setSelectedUri] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
setLoading(true);
|
||||
fetch("/api/ontology/skos/schemes")
|
||||
.then((r) => (r.ok ? r.json() : []))
|
||||
.then(async (r) => {
|
||||
if (!r.ok) throw new Error(`Failed to load schemes (${r.status})`);
|
||||
const data = await r.json();
|
||||
if (r.status === 207) setError(data.message || "Warning: Partial success loading schemes.");
|
||||
return data;
|
||||
})
|
||||
.then(setSchemes)
|
||||
.catch((e) => setError(e.message))
|
||||
.finally(() => setLoading(false));
|
||||
@@ -424,6 +479,7 @@ export function SKOSVocabularyManager({ schemeUri }: Props) {
|
||||
|
||||
return (
|
||||
<div style={managerShellStyle}>
|
||||
{error ? <div style={errorStyle}>{error}</div> : null}
|
||||
{/* Search bar */}
|
||||
<div style={skosToolbarStyle}>
|
||||
<div style={skosSearchBarStyle}>
|
||||
@@ -628,6 +684,8 @@ const navLinkStyle: React.CSSProperties = {
|
||||
textAlign: "left",
|
||||
};
|
||||
|
||||
const errorStyle: React.CSSProperties = { padding: 12, borderRadius: 14, color: "#ffb4c2", background: "rgba(255,157,175,0.1)", border: "1px solid rgba(255,157,175,0.18)", margin: "0 10px 10px 10px", fontSize: 12 };
|
||||
|
||||
const centerStyle: React.CSSProperties = {
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
|
||||
@@ -33,38 +33,51 @@ export function ShaclStudio({ onJumpToNode }: ShaclStudioProps) {
|
||||
setRegistry(entries);
|
||||
setSelectedUri((current) => current || entries[0]?.uri || "");
|
||||
})
|
||||
.catch(() => { /* backend unavailable — leave registry empty */ });
|
||||
.catch((err) => {
|
||||
if (cancelled) return;
|
||||
setError(err instanceof Error ? err.message : "Failed to load ontology registry.");
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, []);
|
||||
|
||||
const loadShapes = useCallback(async (uri: string) => {
|
||||
if (!uri) return;
|
||||
setLoading(true);
|
||||
setError("");
|
||||
try {
|
||||
const data = await loadShaclShapes(uri);
|
||||
setShapes(data.shapes);
|
||||
const turtle = data.shacl_turtle;
|
||||
setFullShacl(turtle);
|
||||
setShacl((current) => current || turtle);
|
||||
setSelectedShapeId(null);
|
||||
setValidation(null);
|
||||
} catch {
|
||||
// Shapes not yet generated or backend unavailable — show empty shape list
|
||||
setShapes([]);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const [prevUri, setPrevUri] = useState(selectedUri);
|
||||
if (selectedUri !== prevUri) {
|
||||
setPrevUri(selectedUri);
|
||||
setShacl("");
|
||||
setFullShacl("");
|
||||
setSelectedShapeId(null);
|
||||
void loadShapes(selectedUri);
|
||||
}, [selectedUri, loadShapes]);
|
||||
setValidation(null);
|
||||
setLoading(true);
|
||||
setError("");
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
let ignore = false;
|
||||
async function fetchShapes() {
|
||||
if (!selectedUri) return;
|
||||
try {
|
||||
const data = await loadShaclShapes(selectedUri);
|
||||
if (!ignore) {
|
||||
setShapes(data.shapes);
|
||||
const turtle = data.shacl_turtle;
|
||||
setFullShacl(turtle);
|
||||
setShacl((current) => current || turtle);
|
||||
setSelectedShapeId(null);
|
||||
setValidation(null);
|
||||
}
|
||||
} catch {
|
||||
if (!ignore) setShapes([]);
|
||||
} finally {
|
||||
if (!ignore) setLoading(false);
|
||||
}
|
||||
}
|
||||
void fetchShapes();
|
||||
return () => {
|
||||
ignore = true;
|
||||
};
|
||||
}, [selectedUri]);
|
||||
|
||||
const handleGenerate = useCallback(async () => {
|
||||
if (!selectedUri) return;
|
||||
|
||||
@@ -47,86 +47,151 @@ export function VersionsTab() {
|
||||
const [comparePair, setComparePair] = useState<{ v1: string; v2: string } | null>(null);
|
||||
const [compareResult, setCompareResult] = useState<Record<string, any> | null>(null);
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
const [error, setError] = useState("");
|
||||
|
||||
const loadVersions = useCallback(async () => {
|
||||
if (!ontologyUri) return;
|
||||
setError("");
|
||||
try {
|
||||
const response = await fetch(`/api/ontology/versions/${encodeURIComponent(ontologyUri)}`);
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
setVersions(data);
|
||||
if (response.status === 207) setError(data.message || "Warning: Partial success loading versions.");
|
||||
} else {
|
||||
setError(`Failed to load versions (${response.status})`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to load versions:", error);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : "Failed to load versions.");
|
||||
}
|
||||
}, [ontologyUri]);
|
||||
|
||||
const loadProposals = useCallback(async () => {
|
||||
setError("");
|
||||
try {
|
||||
const response = await fetch("/api/ontology/proposals");
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
setProposals(data);
|
||||
if (response.status === 207) setError(data.message || "Warning: Partial success loading proposals.");
|
||||
} else {
|
||||
setError(`Failed to load proposals (${response.status})`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to load proposals:", error);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : "Failed to load proposals.");
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
loadVersions();
|
||||
loadProposals();
|
||||
}, [loadVersions, loadProposals]);
|
||||
let ignore = false;
|
||||
async function fetchInitial() {
|
||||
if (!ignore) setError("");
|
||||
try {
|
||||
const propRes = await fetch("/api/ontology/proposals");
|
||||
if (propRes.ok) {
|
||||
const propData = await propRes.json();
|
||||
if (!ignore) {
|
||||
setProposals(propData);
|
||||
if (propRes.status === 207) setError(propData.message || "Warning: Partial success loading proposals.");
|
||||
}
|
||||
} else if (!ignore) {
|
||||
setError(`Failed to load proposals (${propRes.status})`);
|
||||
}
|
||||
} catch (err) {
|
||||
if (!ignore) setError(err instanceof Error ? err.message : "Failed to load proposals.");
|
||||
}
|
||||
|
||||
if (!ontologyUri) return;
|
||||
try {
|
||||
const verRes = await fetch(`/api/ontology/versions/${encodeURIComponent(ontologyUri)}`);
|
||||
if (verRes.ok) {
|
||||
const verData = await verRes.json();
|
||||
if (!ignore) {
|
||||
setVersions(verData);
|
||||
if (verRes.status === 207) setError(verData.message || "Warning: Partial success loading versions.");
|
||||
}
|
||||
} else if (!ignore) {
|
||||
setError((prev) => prev || `Failed to load versions (${verRes.status})`);
|
||||
}
|
||||
} catch (err) {
|
||||
if (!ignore) setError((prev) => prev || (err instanceof Error ? err.message : "Failed to load versions."));
|
||||
}
|
||||
}
|
||||
void fetchInitial();
|
||||
return () => { ignore = true; };
|
||||
}, [ontologyUri]);
|
||||
|
||||
const approveProposal = useCallback(async (proposalId: string) => {
|
||||
setError("");
|
||||
try {
|
||||
const response = await fetch(`/api/ontology/proposals/${proposalId}/approve`, {
|
||||
method: "POST",
|
||||
});
|
||||
if (response.ok) {
|
||||
alert("Proposal approved");
|
||||
if (response.status === 207) {
|
||||
const data = await response.json().catch(() => ({}));
|
||||
setError(data.message || "Warning: Partial success approving proposal.");
|
||||
} else {
|
||||
alert("Proposal approved");
|
||||
}
|
||||
loadProposals();
|
||||
} else {
|
||||
setError(`Failed to approve proposal (${response.status})`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to approve proposal:", error);
|
||||
alert("Failed to approve proposal");
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : "Failed to approve proposal.");
|
||||
}
|
||||
}, [loadProposals]);
|
||||
|
||||
const rejectProposal = useCallback(async (proposalId: string) => {
|
||||
setError("");
|
||||
try {
|
||||
const response = await fetch(`/api/ontology/proposals/${proposalId}/reject`, {
|
||||
method: "POST",
|
||||
});
|
||||
if (response.ok) {
|
||||
alert("Proposal rejected");
|
||||
if (response.status === 207) {
|
||||
const data = await response.json().catch(() => ({}));
|
||||
setError(data.message || "Warning: Partial success rejecting proposal.");
|
||||
} else {
|
||||
alert("Proposal rejected");
|
||||
}
|
||||
loadProposals();
|
||||
} else {
|
||||
setError(`Failed to reject proposal (${response.status})`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to reject proposal:", error);
|
||||
alert("Failed to reject proposal");
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : "Failed to reject proposal.");
|
||||
}
|
||||
}, [loadProposals]);
|
||||
|
||||
const publishProposal = useCallback(async (proposalId: string) => {
|
||||
setError("");
|
||||
try {
|
||||
const response = await fetch(`/api/ontology/proposals/${proposalId}/publish`, {
|
||||
method: "POST",
|
||||
});
|
||||
if (response.ok) {
|
||||
alert("Proposal published");
|
||||
if (response.status === 207) {
|
||||
const data = await response.json().catch(() => ({}));
|
||||
setError(data.message || "Warning: Partial success publishing proposal.");
|
||||
} else {
|
||||
alert("Proposal published");
|
||||
}
|
||||
loadProposals();
|
||||
loadVersions();
|
||||
} else {
|
||||
setError(`Failed to publish proposal (${response.status})`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to publish proposal:", error);
|
||||
alert("Failed to publish proposal");
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : "Failed to publish proposal.");
|
||||
}
|
||||
}, [loadProposals, loadVersions]);
|
||||
|
||||
const runVersionComparison = useCallback(async () => {
|
||||
if (!comparePair || !ontologyUri) return;
|
||||
setIsLoading(true);
|
||||
setError("");
|
||||
try {
|
||||
const response = await fetch(`/api/ontology/versions/${encodeURIComponent(ontologyUri)}/compare`, {
|
||||
method: "POST",
|
||||
@@ -138,11 +203,13 @@ export function VersionsTab() {
|
||||
});
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
if (response.status === 207) setError(data.message || "Warning: Partial success comparing versions.");
|
||||
setCompareResult(data);
|
||||
} else {
|
||||
setError(`Failed to compare versions (${response.status})`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to compare versions:", error);
|
||||
alert("Failed to compare versions");
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : "Failed to compare versions.");
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
@@ -265,6 +332,8 @@ export function VersionsTab() {
|
||||
marginBottom: "12px",
|
||||
};
|
||||
|
||||
const errorStyle: React.CSSProperties = { padding: "12px", borderRadius: "14px", color: "#ffb4c2", background: "rgba(255,157,175,0.1)", border: "1px solid rgba(255,157,175,0.18)", marginBottom: "16px" };
|
||||
|
||||
return (
|
||||
<div style={containerStyle}>
|
||||
<div style={headerStyle}>
|
||||
@@ -278,6 +347,8 @@ export function VersionsTab() {
|
||||
/>
|
||||
</div>
|
||||
|
||||
{error ? <div style={errorStyle}>{error}</div> : null}
|
||||
|
||||
<div style={sectionStyle}>
|
||||
<h2 style={sectionTitleStyle}>
|
||||
<Layers size={16} />
|
||||
|
||||
@@ -20,7 +20,11 @@ async function parseResponse<T>(response: Response): Promise<T> {
|
||||
}
|
||||
throw new Error(detail);
|
||||
}
|
||||
return response.json() as Promise<T>;
|
||||
const data = await response.json();
|
||||
if (response.status === 207) {
|
||||
console.warn("Partial Success:", data.message || "Warning: 207 Multi-Status");
|
||||
}
|
||||
return data as T;
|
||||
}
|
||||
|
||||
export async function loadOntologyRegistry(): Promise<OntologyEntry[]> {
|
||||
|
||||
@@ -57,6 +57,7 @@ export function ReasoningWorkspace() {
|
||||
});
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.detail || `Status ${response.status}`);
|
||||
if (response.status === 207) setError(data.message || "Warning: Partial success reasoning.");
|
||||
setResult(data);
|
||||
if (data.mutated) queryClient.invalidateQueries({ queryKey: ["graph", "full-load"] });
|
||||
} catch (e) {
|
||||
|
||||
@@ -72,7 +72,15 @@ export function SparqlWorkspace() {
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ query }),
|
||||
});
|
||||
if (!res.headers.get("content-type")?.includes("application/json")) {
|
||||
const text = await res.text();
|
||||
throw new Error(`HTTP ${res.status}: ${text.substring(0, 100)}`);
|
||||
}
|
||||
const data = await res.json();
|
||||
if (res.status === 207) {
|
||||
data.error = data.message || "Warning: Partial success running query.";
|
||||
}
|
||||
|
||||
if (data.error && data.error_line && monaco && editorRef.current) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(monaco as any).editor.setModelMarkers((editorRef.current as any).getModel(), "sparql", [{
|
||||
@@ -81,12 +89,14 @@ export function SparqlWorkspace() {
|
||||
endLineNumber: data.error_line,
|
||||
endColumn: 100,
|
||||
message: data.error,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
severity: (monaco as any).MarkerSeverity.Error,
|
||||
}]);
|
||||
}
|
||||
setResult(data);
|
||||
} catch {
|
||||
setResult({ error: "Network error — could not reach the SPARQL endpoint." });
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : "Network error — could not reach the SPARQL endpoint.";
|
||||
setResult({ error: msg });
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
|
||||
@@ -1061,3 +1061,198 @@ test("checkGroupedViewAvailability returns available when communities exist", ()
|
||||
assert.equal(result.reason, null);
|
||||
});
|
||||
|
||||
|
||||
// ── #1009: edge label data-path regression tests ─────────────────────────────
|
||||
|
||||
test("resolveDisplayGraph parallel-bundle preserves edgeType on aggregated edge", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
batchMergeEdges([
|
||||
{ id: "e1", source: "a", target: "b", attributes: { edgeType: "causes", weight: 1, properties: {} } },
|
||||
{ id: "e2", source: "a", target: "b", attributes: { edgeType: "causes", weight: 2, properties: {} } },
|
||||
]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string; isAggregated?: boolean };
|
||||
assert.equal(attrs.isAggregated, true);
|
||||
// The aggregated representative must carry the relationship text through to
|
||||
// the edgeReducer's label assignment.
|
||||
assert.equal(typeof attrs.edgeType, "string");
|
||||
assert.ok((attrs.edgeType ?? "").length > 0, "aggregated edge must have a non-empty edgeType");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph parallel-bundle picks dominant edgeType across mixed types", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
batchMergeEdges([
|
||||
{ id: "e1", source: "a", target: "b", attributes: { edgeType: "inhibits", weight: 1, properties: {} } },
|
||||
{ id: "e2", source: "a", target: "b", attributes: { edgeType: "inhibits", weight: 1, properties: {} } },
|
||||
{ id: "e3", source: "a", target: "b", attributes: { edgeType: "activates", weight: 1, properties: {} } },
|
||||
]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string; dominantEdgeType?: string };
|
||||
// "inhibits" appears twice so it must be the dominant type.
|
||||
assert.equal(attrs.edgeType, "inhibits");
|
||||
assert.equal(attrs.dominantEdgeType, "inhibits");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph grouped view community edges carry non-empty edgeType", () => {
|
||||
const left = ["g1", "g2", "g3", "g4"];
|
||||
const right = ["h1", "h2", "h3", "h4"];
|
||||
[...left, ...right].forEach((nodeId, index) => addNode(nodeId, index < left.length ? "left" : "right"));
|
||||
|
||||
let edgeIndex = 0;
|
||||
for (let i = 0; i < left.length; i += 1) {
|
||||
for (let j = 0; j < left.length; j += 1) {
|
||||
if (i !== j) {
|
||||
batchMergeEdges([{
|
||||
id: `lg-${edgeIndex++}`,
|
||||
source: left[i],
|
||||
target: left[j],
|
||||
attributes: { edgeType: "co_occurs", weight: 3, properties: {} },
|
||||
}]);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < right.length; i += 1) {
|
||||
for (let j = 0; j < right.length; j += 1) {
|
||||
if (i !== j) {
|
||||
batchMergeEdges([{
|
||||
id: `rg-${edgeIndex++}`,
|
||||
source: right[i],
|
||||
target: right[j],
|
||||
attributes: { edgeType: "co_occurs", weight: 3, properties: {} },
|
||||
}]);
|
||||
}
|
||||
}
|
||||
}
|
||||
batchMergeEdges([{ id: "bridge-g", source: "g1", target: "h1", attributes: { edgeType: "interacts_with", weight: 0.1, properties: {} } }]);
|
||||
|
||||
const { graph: displayGraph, state } = resolveDisplayGraph("", [], [], "grouped", { aggregationEnabled: true });
|
||||
assert.equal(state.groupedViewAvailable, true);
|
||||
|
||||
const communityEdges = displayGraph.edges().filter((edgeId) => {
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { bundleKind?: string };
|
||||
return attrs.bundleKind === "community";
|
||||
});
|
||||
assert.ok(communityEdges.length > 0, "expected at least one community bundle edge");
|
||||
|
||||
for (const edgeId of communityEdges) {
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string };
|
||||
assert.equal(typeof attrs.edgeType, "string");
|
||||
assert.ok((attrs.edgeType ?? "").length > 0, `community edge ${edgeId} must have a non-empty edgeType`);
|
||||
}
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph raw edge preserves exact edgeType string for label rendering", () => {
|
||||
addNode("src");
|
||||
addNode("tgt");
|
||||
batchMergeEdges([{
|
||||
id: "raw-1",
|
||||
source: "src",
|
||||
target: "tgt",
|
||||
attributes: { edgeType: "works_for", weight: 1, properties: {} },
|
||||
}]);
|
||||
|
||||
// In full view without aggregation the edge passes through unchanged.
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: false });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string };
|
||||
assert.equal(attrs.edgeType, "works_for");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph does not produce empty-string edgeType on aggregated edges when source has empty type", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
// Simulate an API response where type is empty string — the aggregation
|
||||
// path must not propagate a blank label.
|
||||
batchMergeEdges([
|
||||
{ id: "e-empty-1", source: "a", target: "b", attributes: { edgeType: "", weight: 1, properties: {} } },
|
||||
{ id: "e-empty-2", source: "a", target: "b", attributes: { edgeType: "", weight: 1, properties: {} } },
|
||||
]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as {
|
||||
edgeType?: string;
|
||||
isAggregated?: boolean;
|
||||
};
|
||||
assert.equal(attrs.isAggregated, true);
|
||||
// The aggregation falls back to "related_to" when all source edgeTypes are
|
||||
// empty, so the rendered label should never be an empty string.
|
||||
assert.equal(attrs.edgeType, "related_to");
|
||||
});
|
||||
|
||||
test("resolveEdgeElementStyle hidden class produces hidden:true for suppressed edges", () => {
|
||||
// Verify the data condition the edgeReducer relies on: hidden-classified
|
||||
// edges must have hidden:true so that the label assignment sets undefined.
|
||||
const style = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"overview",
|
||||
"inactive",
|
||||
{
|
||||
edgeType: "causes",
|
||||
weight: 1,
|
||||
properties: {},
|
||||
edgeVariant: "line",
|
||||
visualPriority: 0.05,
|
||||
baseSize: 0.3,
|
||||
},
|
||||
"source",
|
||||
"target",
|
||||
"full",
|
||||
"inactive-edge",
|
||||
"hidden",
|
||||
);
|
||||
assert.equal(style.hidden, true);
|
||||
});
|
||||
|
||||
// ── #1009 maintainer-blocking regression: single-edge empty edgeType ─────────
|
||||
|
||||
test("resolveDisplayGraph single-edge normalizes empty-string edgeType to related_to", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
// One edge only — exercises the entries.length === 1 path in aggregateDisplayGraph.
|
||||
batchMergeEdges([{
|
||||
id: "e-single-empty",
|
||||
source: "a",
|
||||
target: "b",
|
||||
attributes: { edgeType: "", weight: 1, properties: {} },
|
||||
}]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string; dominantEdgeType?: string };
|
||||
assert.equal(attrs.edgeType, "related_to",
|
||||
"single-edge path must normalize empty edgeType to the canonical fallback");
|
||||
assert.equal(attrs.dominantEdgeType, "related_to",
|
||||
"single-edge dominantEdgeType must also be normalized");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph single-edge preserves a valid non-empty edgeType unchanged", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
batchMergeEdges([{
|
||||
id: "e-single-valid",
|
||||
source: "a",
|
||||
target: "b",
|
||||
attributes: { edgeType: "works_for", weight: 1, properties: {} },
|
||||
}]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string };
|
||||
assert.equal(attrs.edgeType, "works_for",
|
||||
"single-edge path must not alter a valid relationship type");
|
||||
});
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
/**
|
||||
* Regression tests for issue #830: plugin registry shouldLoad predicates.
|
||||
*
|
||||
* Imports the production predicates from pluginRegistryPredicates.ts so that
|
||||
* a regression in GraphWorkspace.tsx is detected here. The key invariant: no
|
||||
* predicate may read temporalState — doing so caused a render loop because
|
||||
* temporalState.currentTime is non-null from startup, which triggered eager
|
||||
* plugin loads on every scrubber update and continuously cancelled in-flight
|
||||
* load() calls before they could register the plugin.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
const {
|
||||
explorationEffectsShouldLoad,
|
||||
neighborhoodPanelShouldLoad,
|
||||
temporalOverlayShouldLoad,
|
||||
} = require("../src/workspaces/GraphWorkspace/pluginRegistryPredicates.ts");
|
||||
|
||||
// ── temporal-overlay ─────────────────────────────────────────────────────────
|
||||
|
||||
test("temporal-overlay shouldLoad: false when panel is closed and no scrubber time", () => {
|
||||
assert.equal(
|
||||
temporalOverlayShouldLoad({ panelState: { "temporal-panel": false } }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal-overlay shouldLoad: false when panel is closed even if scrubber time is set", () => {
|
||||
// Before the fix, a non-null currentTime caused an eager load on every scrubber update.
|
||||
assert.equal(
|
||||
temporalOverlayShouldLoad({
|
||||
panelState: { "temporal-panel": false },
|
||||
temporalState: { currentTime: new Date() },
|
||||
}),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal-overlay shouldLoad: true only when the panel is explicitly opened", () => {
|
||||
assert.equal(
|
||||
temporalOverlayShouldLoad({ panelState: { "temporal-panel": true } }),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal-overlay shouldLoad: true when panel opened even without a scrubber time", () => {
|
||||
assert.equal(
|
||||
temporalOverlayShouldLoad({
|
||||
panelState: { "temporal-panel": true },
|
||||
temporalState: { currentTime: null },
|
||||
}),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
// ── other entries — confirm they also gate only on panelState ─────────────────
|
||||
|
||||
test("exploration-effects shouldLoad: gates only on effects-panel state", () => {
|
||||
assert.equal(explorationEffectsShouldLoad({ panelState: { "effects-panel": false } }), false);
|
||||
assert.equal(explorationEffectsShouldLoad({ panelState: { "effects-panel": true } }), true);
|
||||
});
|
||||
|
||||
test("neighborhood-panel shouldLoad: gates only on neighborhood-panel state", () => {
|
||||
assert.equal(neighborhoodPanelShouldLoad({ panelState: { "neighborhood-panel": false } }), false);
|
||||
assert.equal(neighborhoodPanelShouldLoad({ panelState: { "neighborhood-panel": true } }), true);
|
||||
});
|
||||
|
||||
test("all three shouldLoad conditions are consistent: none reference temporalState", () => {
|
||||
// A regressed predicate reading temporalState?.currentTime would return true
|
||||
// for a closed panel when currentTime is set — detecting the loop bug.
|
||||
const nonNullTemporalState = { currentTime: new Date(), activeNodeCount: 6 };
|
||||
|
||||
assert.equal(
|
||||
temporalOverlayShouldLoad({ panelState: { "temporal-panel": false }, temporalState: nonNullTemporalState }),
|
||||
false,
|
||||
"temporal-overlay must not load when panel is closed, regardless of scrubber time",
|
||||
);
|
||||
assert.equal(
|
||||
explorationEffectsShouldLoad({ panelState: { "effects-panel": false }, temporalState: nonNullTemporalState }),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
neighborhoodPanelShouldLoad({ panelState: { "neighborhood-panel": false }, temporalState: nonNullTemporalState }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,114 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
shouldFetchTemporalBounds,
|
||||
shouldFetchTemporalSnapshot,
|
||||
} from "../src/workspaces/GraphWorkspace/temporalLifecyclePredicates.ts";
|
||||
import type { GraphLoadSummary } from "../src/workspaces/GraphWorkspace/types.ts";
|
||||
|
||||
const sampleSummary: GraphLoadSummary = {
|
||||
nodeCount: 42,
|
||||
edgeCount: 78,
|
||||
loadTimeMs: 120,
|
||||
hasCoordinates: true,
|
||||
layoutSource: "provided",
|
||||
layoutReady: true,
|
||||
};
|
||||
|
||||
const emptyGraphSummary: GraphLoadSummary = {
|
||||
nodeCount: 0,
|
||||
edgeCount: 0,
|
||||
loadTimeMs: 15,
|
||||
hasCoordinates: false,
|
||||
layoutSource: "runtime",
|
||||
layoutReady: false,
|
||||
};
|
||||
|
||||
// ── shouldFetchTemporalBounds ────────────────────────────────────────────────
|
||||
|
||||
test("temporal bounds: false when summary is undefined (initial mount or failed load)", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalBounds(undefined),
|
||||
false,
|
||||
"bounds request must not run before graph load succeeds",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal bounds: true when non-empty summary is present", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalBounds(sampleSummary),
|
||||
true,
|
||||
"bounds request should run when successful graph summary exists",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal bounds: true when successful summary has nodeCount of 0", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalBounds(emptyGraphSummary),
|
||||
true,
|
||||
"an empty graph is still a successful load and must allow bounds fetching",
|
||||
);
|
||||
});
|
||||
|
||||
// ── shouldFetchTemporalSnapshot ──────────────────────────────────────────────
|
||||
|
||||
test("temporal snapshot: false when summary is undefined even if scrubber time is set and isLoading is false", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: false,
|
||||
summary: undefined,
|
||||
}),
|
||||
false,
|
||||
"snapshot request must not run when graph load failed",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: false when graph is currently loading", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: true,
|
||||
summary: sampleSummary,
|
||||
}),
|
||||
false,
|
||||
"snapshot request must not run while graph is loading",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: false when debouncedTime is null", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: null,
|
||||
isLoading: false,
|
||||
summary: sampleSummary,
|
||||
}),
|
||||
false,
|
||||
"snapshot request must not run without a scrubber timestamp",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: true when summary exists, isLoading is false, and time is set", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: false,
|
||||
summary: sampleSummary,
|
||||
}),
|
||||
true,
|
||||
"snapshot request should run after graph load succeeds and time is set",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: true when successful summary has 0 nodes, isLoading is false, and time is set", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: false,
|
||||
summary: emptyGraphSummary,
|
||||
}),
|
||||
true,
|
||||
"empty successful graph must allow snapshot requests once ready",
|
||||
);
|
||||
});
|
||||
@@ -57,6 +57,13 @@ export default defineConfig({
|
||||
},
|
||||
},
|
||||
},
|
||||
optimizeDeps: {
|
||||
// Keep dependency pre-bundling aligned with the production build target.
|
||||
// esbuild >=0.28 no longer lowers destructuring for Vite's default target.
|
||||
esbuildOptions: {
|
||||
target: 'esnext',
|
||||
},
|
||||
},
|
||||
server: {
|
||||
proxy: {
|
||||
'/api': {
|
||||
|
||||
@@ -123,12 +123,10 @@ class AgnoDecisionKit(_ToolkitBase): # type: ignore[misc]
|
||||
tools_to_register.append(self.check_policy)
|
||||
|
||||
for fn in tools_to_register:
|
||||
self._tools.append(fn)
|
||||
if AGNO_AVAILABLE:
|
||||
try:
|
||||
self.register(fn)
|
||||
except Exception:
|
||||
pass
|
||||
self.register(fn)
|
||||
if fn not in self._tools:
|
||||
self._tools.append(fn)
|
||||
|
||||
logger.info("AgnoDecisionKit initialised")
|
||||
|
||||
@@ -312,18 +310,33 @@ class AgnoDecisionKit(_ToolkitBase): # type: ignore[misc]
|
||||
Rules are evaluated inline using simple comparison expressions. This
|
||||
avoids misuse of ``PolicyEngine.check_compliance`` (which requires a
|
||||
stored ``Decision`` + ``policy_id``) and ensures exceptions never
|
||||
silently return ``compliant=True``.
|
||||
silently return ``compliant=True``. A rule that references a field
|
||||
missing from ``decision_data``, a field whose value is JSON ``null``,
|
||||
or a rule that doesn't match the expected ``<field> <op> <value>``
|
||||
format, cannot be evaluated — it is recorded in ``warnings`` (not
|
||||
``violations``) since we don't know whether it would have passed or
|
||||
failed. These are reported as distinct messages (missing key vs.
|
||||
null value) so the warning is actionable.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
decision_data:
|
||||
JSON string describing the decision (must include ``category``,
|
||||
``outcome``, ``confidence`` keys at minimum).
|
||||
``outcome``, ``confidence`` keys at minimum). Must decode to a
|
||||
JSON object — any other shape (list, number, string, bool) is
|
||||
rejected with a single ``violations`` entry, the same as
|
||||
malformed JSON, rather than being passed through to per-rule
|
||||
evaluation where it would produce confusing internal errors.
|
||||
policy_rules:
|
||||
JSON list of rule strings, e.g.
|
||||
``'["confidence >= 0.7", "category != \\"test\\""]'``.
|
||||
Each rule is a simple comparison: ``<field> <op> <value>``
|
||||
where op is one of ``>=``, ``<=``, ``!=``, ``==``, ``>``, ``<``.
|
||||
A JSON-encoded bare string (e.g. ``'"confidence >= 0.7"'``) is
|
||||
treated as a single rule. Any other decoded JSON shape (e.g. a
|
||||
number or object), or a non-string list element, is recorded as
|
||||
one ``warnings`` entry and otherwise ignored rather than being
|
||||
iterated character-by-character.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -341,16 +354,47 @@ class AgnoDecisionKit(_ToolkitBase): # type: ignore[misc]
|
||||
}
|
||||
)
|
||||
|
||||
rules: List[str] = []
|
||||
if policy_rules:
|
||||
try:
|
||||
rules = json.loads(policy_rules)
|
||||
except json.JSONDecodeError:
|
||||
rules = [r.strip() for r in policy_rules.split(",") if r.strip()]
|
||||
if not isinstance(data, dict):
|
||||
return json.dumps(
|
||||
{
|
||||
"compliant": False,
|
||||
"violations": [
|
||||
f"decision_data must decode to a JSON object, "
|
||||
f"got {type(data).__name__}: {data!r}"
|
||||
],
|
||||
"warnings": [],
|
||||
}
|
||||
)
|
||||
|
||||
violations: List[str] = []
|
||||
warnings: List[str] = []
|
||||
|
||||
rules: List[str] = []
|
||||
if policy_rules:
|
||||
try:
|
||||
parsed_rules = json.loads(policy_rules)
|
||||
except json.JSONDecodeError:
|
||||
rules = [r.strip() for r in policy_rules.split(",") if r.strip()]
|
||||
else:
|
||||
if isinstance(parsed_rules, str):
|
||||
# A single rule encoded as a bare JSON string, e.g.
|
||||
# policy_rules='"confidence >= 0.7"'. Treat it as one
|
||||
# rule rather than iterating it character-by-character.
|
||||
rules = [parsed_rules]
|
||||
elif isinstance(parsed_rules, list):
|
||||
for item in parsed_rules:
|
||||
if isinstance(item, str):
|
||||
rules.append(item)
|
||||
else:
|
||||
warnings.append(
|
||||
f"Ignoring non-string policy rule entry: {item!r}"
|
||||
)
|
||||
else:
|
||||
warnings.append(
|
||||
f"policy_rules must decode to a JSON list of rule strings, "
|
||||
f"got {type(parsed_rules).__name__}: {parsed_rules!r}"
|
||||
)
|
||||
|
||||
for rule in rules:
|
||||
try:
|
||||
if not self._eval_rule(rule, data):
|
||||
@@ -369,14 +413,23 @@ class AgnoDecisionKit(_ToolkitBase): # type: ignore[misc]
|
||||
)
|
||||
|
||||
def _eval_rule(self, rule: str, data: Dict[str, Any]) -> bool:
|
||||
"""Evaluate a simple comparison rule (``field op value``) against data."""
|
||||
"""
|
||||
Evaluate a simple comparison rule (``field op value``) against data.
|
||||
|
||||
Raises ``ValueError`` when the rule cannot be evaluated (unrecognised
|
||||
format, the referenced field is absent from ``data``, or the field's
|
||||
value is JSON ``null``) so that ``check_policy`` records it as a
|
||||
``warnings`` entry instead of silently treating it as passed.
|
||||
"""
|
||||
m = re.match(r"(\w+)\s*(>=|<=|!=|==|>|<)\s*(.+)", rule.strip())
|
||||
if not m:
|
||||
return True # unrecognised format — pass through
|
||||
raise ValueError(f"unrecognised rule format: {rule!r}")
|
||||
field, op, val_str = m.group(1), m.group(2), m.group(3).strip().strip("\"'")
|
||||
actual = data.get(field)
|
||||
if field not in data:
|
||||
raise ValueError(f"rule references undefined field {field!r}")
|
||||
actual = data[field]
|
||||
if actual is None:
|
||||
return True # field absent — cannot evaluate
|
||||
raise ValueError(f"field {field!r} is null — cannot evaluate rule")
|
||||
try:
|
||||
val: Any = type(actual)(val_str)
|
||||
except (ValueError, TypeError):
|
||||
|
||||
@@ -122,12 +122,10 @@ class AgnoKGToolkit(_ToolkitBase): # type: ignore[misc]
|
||||
self.export_subgraph,
|
||||
]
|
||||
for fn in tools_to_register:
|
||||
self._tools.append(fn)
|
||||
if AGNO_AVAILABLE:
|
||||
try:
|
||||
self.register(fn)
|
||||
except Exception:
|
||||
pass
|
||||
self.register(fn)
|
||||
if fn not in self._tools:
|
||||
self._tools.append(fn)
|
||||
|
||||
logger.info("AgnoKGToolkit initialised (backend=%s)", graph_store_backend)
|
||||
|
||||
|
||||
@@ -83,7 +83,7 @@ class _AgentScopedStore(AgnoContextStore):
|
||||
try:
|
||||
self._context.store(mem_text, conversation_id=self.session_id)
|
||||
except Exception as exc:
|
||||
logger.warning("[%s] store failed: %s", self._role, exc)
|
||||
logger.warning("[%s] store failed: %s", self._role, exc, exc_info=True)
|
||||
|
||||
if self.decision_tracking:
|
||||
try:
|
||||
@@ -94,8 +94,8 @@ class _AgentScopedStore(AgnoContextStore):
|
||||
outcome="stored",
|
||||
confidence=1.0,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.warning("[%s] record_decision failed: %s", self._role, exc, exc_info=True)
|
||||
|
||||
if hasattr(memory, "id"):
|
||||
memory.id = mem_id
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
# Semantica × CrewAI
|
||||
|
||||
First-class integration between Semantica and [CrewAI](https://github.com/crewAIInc/crewAI) — give your crews a shared semantic knowledge graph, decision intelligence, and graph-based retrieval.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install semantica[crewai]
|
||||
```
|
||||
|
||||
Requires `crewai >= 0.80.0`. If `crewai` is not installed, the integration still imports (classes degrade gracefully), but you can't pass the objects to a `Crew`.
|
||||
|
||||
> **⚠️ Security note:** crewai hard-requires `chromadb~=1.1.0`, which is currently affected by the unpatched pre-authentication code-injection advisory **CVE-2026-45829** (no fixed release — even the latest chromadb 1.5.9 is affected). Installing `semantica[crewai]` pulls that dependency into your environment. The `crewai` extra is intentionally **not** part of `semantica[all]` for this reason — only install it where you actually use CrewAI, and follow chromadb for a patched release.
|
||||
|
||||
## 1. SemanticaKGTool
|
||||
|
||||
A `BaseTool` that lets agents **build and query** a shared `ContextGraph` mid-reasoning:
|
||||
|
||||
- `extract_entities` — extract named entities from `text`
|
||||
- `extract_relations` — extract relationships from `text`
|
||||
- `add_to_graph` — extract entities/relations from `text` and add them to the shared graph
|
||||
- `query_graph` — keyword-search the graph using `query`
|
||||
- `find_related` — find concepts related to `entity` within `hops`
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKGTool
|
||||
|
||||
graph = ContextGraph()
|
||||
|
||||
analyst = Agent(
|
||||
role="Knowledge Analyst",
|
||||
goal="Build and explore a knowledge graph from documents",
|
||||
backstory="You map entities and relationships into a shared graph.",
|
||||
tools=[SemanticaKGTool(graph=graph)],
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[analyst],
|
||||
tasks=[Task(description="Extract and link key entities from the brief", expected_output="JSON", agent=analyst)],
|
||||
)
|
||||
result = crew.kickoff()
|
||||
```
|
||||
|
||||
All actions return JSON, so agents get parseable results.
|
||||
|
||||
## 2. SemanticaDecisionTool
|
||||
|
||||
A `BaseTool` that wraps `AgentContext` and exposes decision intelligence:
|
||||
|
||||
- `record_decision` — record a decision with reasoning and outcome
|
||||
- `find_precedents` — retrieve past decisions similar to a scenario
|
||||
- `trace_causal_chain` — trace the causal chain from a decision
|
||||
- `analyze_impact` — assess downstream influence using graph centrality
|
||||
- `check_policy` — validate a proposed decision against rule-based policies
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from integrations.crewai import SemanticaDecisionTool
|
||||
|
||||
planner = Agent(
|
||||
role="Decision Planner",
|
||||
goal="Make grounded, precedented decisions",
|
||||
backstory="You record decisions and validate them against policy.",
|
||||
tools=[SemanticaDecisionTool()],
|
||||
)
|
||||
|
||||
crew = Crew(agents=[planner], tasks=[...])
|
||||
```
|
||||
|
||||
When no `AgentContext` is passed, one is created in-memory with `decision_tracking=True`.
|
||||
|
||||
## 3. SemanticaKnowledgeSource
|
||||
|
||||
A `BaseKnowledgeSource` that serializes the current state of a `ContextGraph` (nodes, edges, metadata) into CrewAI's knowledge storage, giving **every agent in the crew** retrieval access to the graph:
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKnowledgeSource
|
||||
|
||||
graph = ContextGraph()
|
||||
graph.add_node(node_id="privacy", node_type="policy", content="...")
|
||||
|
||||
researcher = Agent(
|
||||
role="Policy Researcher",
|
||||
goal="Answer questions from the knowledge base",
|
||||
backstory="You retrieve from graph knowledge to answer accurately.",
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[researcher],
|
||||
tasks=[...],
|
||||
knowledge_sources=[SemanticaKnowledgeSource(graph=graph)],
|
||||
)
|
||||
```
|
||||
|
||||
> **Embedder required:** storing chunks goes through CrewAI's knowledge pipeline, which needs an embedder. Set `Crew(embedder=...)` (or provide CrewAI's default credentials, e.g. `OPENAI_API_KEY`). Without a working embedder, storage fails, an ERROR is logged, and agents retrieve nothing — the crew still runs with empty knowledge queries.
|
||||
|
||||
### Compatibility note
|
||||
|
||||
CrewAI's `BaseKnowledgeSource` contract changed between `0.80.x` and current releases (`load_content()` → `validate_content()`/`aadd()`). `SemanticaKnowledgeSource` implements both the legacy and current methods, so it works across `crewai>=0.80.0`.
|
||||
|
||||
### Sharing state & checkpoints
|
||||
|
||||
- Each tool/source holds whatever `graph`/`context` you pass it. When omitted, a fresh in-memory object is created and a warning is logged — instances that auto-create their own state do **not** share knowledge, so pass the same object to every agent that must share.
|
||||
- Live state (`ContextGraph`, `AgentContext`, extractors) is excluded from CrewAI's JSON serialization. After restoring from a checkpoint, re-attach the live graph/context to the restored objects.
|
||||
@@ -0,0 +1,44 @@
|
||||
"""
|
||||
Semantica × CrewAI Integration
|
||||
==============================
|
||||
|
||||
First-class integration between the Semantica semantic intelligence stack and
|
||||
the `CrewAI <https://github.com/crewAIInc/crewAI>`_ agentic framework.
|
||||
|
||||
Public surface
|
||||
--------------
|
||||
SemanticaKGTool — CrewAI ``BaseTool`` exposing KG construction/query actions
|
||||
SemanticaDecisionTool — CrewAI ``BaseTool`` exposing decision-intelligence actions
|
||||
SemanticaKnowledgeSource— CrewAI ``BaseKnowledgeSource`` giving crews graph knowledge
|
||||
|
||||
Quick start
|
||||
-----------
|
||||
pip install semantica[crewai]
|
||||
|
||||
>>> from integrations.crewai import (
|
||||
... SemanticaKGTool,
|
||||
... SemanticaDecisionTool,
|
||||
... SemanticaKnowledgeSource,
|
||||
... )
|
||||
|
||||
Compatibility
|
||||
-------------
|
||||
Requires ``crewai >= 0.80.0``. All three classes degrade gracefully when
|
||||
``crewai`` is not installed — they are still importable and carry the full
|
||||
Semantica API, but cannot be passed to ``Crew`` / ``Agent`` constructors.
|
||||
"""
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE, CREWAI_IMPORT_ERROR
|
||||
from .decision_tool import SemanticaDecisionTool
|
||||
from .kg_tool import SemanticaKGTool
|
||||
from .knowledge_source import SemanticaKnowledgeSource
|
||||
|
||||
__all__ = [
|
||||
"SemanticaKGTool",
|
||||
"SemanticaDecisionTool",
|
||||
"SemanticaKnowledgeSource",
|
||||
"CREWAI_AVAILABLE",
|
||||
"CREWAI_IMPORT_ERROR",
|
||||
]
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,24 @@
|
||||
"""
|
||||
Shared CrewAI availability probe.
|
||||
|
||||
Every integration module needs to know whether the real ``crewai`` package is
|
||||
installed. Probing once here (instead of once per module) guarantees the
|
||||
exported ``CREWAI_AVAILABLE`` flag means the *whole* integration is ready — a
|
||||
caller gating on it will never see tools using CrewAI while a knowledge source
|
||||
silently degrades (or vice versa).
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
CREWAI_AVAILABLE = False
|
||||
CREWAI_IMPORT_ERROR: Optional[str] = None
|
||||
|
||||
try:
|
||||
from crewai.knowledge.source.base_knowledge_source import ( # noqa: F401
|
||||
BaseKnowledgeSource,
|
||||
)
|
||||
from crewai.tools import BaseTool # noqa: F401
|
||||
|
||||
CREWAI_AVAILABLE = True
|
||||
except ImportError as exc:
|
||||
CREWAI_IMPORT_ERROR = str(exc)
|
||||
@@ -0,0 +1,555 @@
|
||||
"""
|
||||
SemanticaDecisionTool — a CrewAI ``BaseTool`` exposing Semantica's decision
|
||||
intelligence (``AgentContext``) to agents.
|
||||
|
||||
Lets agents record decisions with reasoning, retrieve past precedents, trace
|
||||
causal chains, analyse downstream impact, and validate proposed decisions
|
||||
against policy rules.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[crewai]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from integrations.crewai import SemanticaDecisionTool
|
||||
>>> from crewai import Agent, Crew, Task
|
||||
>>> tool = SemanticaDecisionTool()
|
||||
>>> crew = Crew(
|
||||
... agents=[Agent(role="...", goal="...", backstory="...", tools=[tool])],
|
||||
... tasks=[...],
|
||||
... )
|
||||
|
||||
Tools exposed
|
||||
-------------
|
||||
record_decision — Record a decision with reasoning and outcome
|
||||
find_precedents — Search past decisions similar to a scenario
|
||||
trace_causal_chain— Trace the causal chain from a decision node
|
||||
analyze_impact — Assess downstream influence of a decision
|
||||
check_policy — Validate a proposed decision against policy rules
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Literal, Optional, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: CrewAI BaseTool base class
|
||||
# ---------------------------------------------------------------------------
|
||||
_BaseTool: Any = object
|
||||
|
||||
if CREWAI_AVAILABLE:
|
||||
from crewai.tools import BaseTool as _BaseTool # type: ignore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input schema
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaDecisionToolInput(BaseModel):
|
||||
"""
|
||||
Input schema for ``SemanticaDecisionTool``.
|
||||
|
||||
Exactly one action is dispatched per call; the remaining fields are only
|
||||
used by the actions that need them.
|
||||
"""
|
||||
|
||||
action: Literal[
|
||||
"record_decision",
|
||||
"find_precedents",
|
||||
"trace_causal_chain",
|
||||
"analyze_impact",
|
||||
"check_policy",
|
||||
] = Field(
|
||||
...,
|
||||
description=(
|
||||
"Which decision-intelligence operation to run. One of: "
|
||||
"'record_decision', 'find_precedents', 'trace_causal_chain', "
|
||||
"'analyze_impact', 'check_policy'."
|
||||
),
|
||||
)
|
||||
category: Optional[str] = Field(
|
||||
None,
|
||||
description="Domain category, e.g. 'loan_approval'. Used by 'record_decision'.",
|
||||
)
|
||||
scenario: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"Short description of the situation. Used by 'record_decision' and "
|
||||
"'find_precedents'."
|
||||
),
|
||||
)
|
||||
reasoning: Optional[str] = Field(
|
||||
None, description="Why this outcome was chosen. Used by 'record_decision'."
|
||||
)
|
||||
outcome: Optional[str] = Field(
|
||||
None, description="The decision result. Used by 'record_decision'."
|
||||
)
|
||||
confidence: float = Field(
|
||||
0.8,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Confidence score in [0, 1]. Used by 'record_decision'.",
|
||||
)
|
||||
entities: Optional[str] = Field(
|
||||
None,
|
||||
description="Comma-separated entity names. Used by 'record_decision'.",
|
||||
)
|
||||
decision_id: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"Identifier of a decision. Used by 'trace_causal_chain' and "
|
||||
"'analyze_impact'."
|
||||
),
|
||||
)
|
||||
depth: int = Field(
|
||||
3,
|
||||
ge=1,
|
||||
le=20,
|
||||
description="Maximum chain depth. Used by 'trace_causal_chain'.",
|
||||
)
|
||||
decision_data: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"JSON object describing a proposed decision. Used by 'check_policy'."
|
||||
),
|
||||
)
|
||||
policy_rules: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"JSON list of rule strings like 'confidence >= 0.7'. Used by "
|
||||
"'check_policy'."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SemanticaDecisionTool
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaDecisionTool(_BaseTool): # type: ignore[misc]
|
||||
"""
|
||||
CrewAI tool that surfaces Semantica's decision intelligence as agent actions.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
context:
|
||||
A ``semantica.context.AgentContext`` (or compatible object exposing
|
||||
``record_decision``, ``find_precedents_advanced``,
|
||||
``analyze_decision_influence``). A fresh in-memory context is created
|
||||
when ``None``.
|
||||
max_precedents:
|
||||
Default number of precedents returned by ``find_precedents``.
|
||||
causal_depth:
|
||||
Default chain depth used by ``trace_causal_chain``.
|
||||
"""
|
||||
|
||||
name: str = "semantica_decision"
|
||||
description: str = (
|
||||
"Decision intelligence toolkit. Actions: 'record_decision' (record a "
|
||||
"decision with category, scenario, reasoning, outcome, confidence), "
|
||||
"'find_precedents' (search past decisions similar to 'scenario'), "
|
||||
"'trace_causal_chain' (trace the causal chain from 'decision_id'), "
|
||||
"'analyze_impact' (assess downstream influence of 'decision_id'), "
|
||||
"'check_policy' (validate 'decision_data' JSON against 'policy_rules' "
|
||||
"rules like 'confidence >= 0.7'). Returns JSON."
|
||||
)
|
||||
args_schema: Type[BaseModel] = SemanticaDecisionToolInput
|
||||
context: Any = Field(default=None, exclude=True)
|
||||
max_precedents: int = 5
|
||||
causal_depth: int = 3
|
||||
had_live_state: bool = False
|
||||
reconstructed_state: bool = Field(default=False, exclude=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
context: Any = None,
|
||||
max_precedents: int = 5,
|
||||
causal_depth: int = 3,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if CREWAI_AVAILABLE:
|
||||
super().__init__(
|
||||
context=context,
|
||||
max_precedents=max_precedents,
|
||||
causal_depth=causal_depth,
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
super().__init__()
|
||||
self.context = context
|
||||
self.max_precedents = max_precedents
|
||||
self.causal_depth = causal_depth
|
||||
# Degraded mode is a plain class — no model_post_init lifecycle.
|
||||
self._ensure_defaults()
|
||||
|
||||
logger.info("SemanticaDecisionTool initialised (crewai=%s)", CREWAI_AVAILABLE)
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
"""Re-create default state after validation/deserialisation.
|
||||
|
||||
``context`` is excluded from JSON serialisation (CrewAI checkpoints
|
||||
serialise every tool via ``model_dump(mode="json")``), so a tool
|
||||
restored from a checkpoint has ``None`` state until this runs.
|
||||
"""
|
||||
self._ensure_defaults()
|
||||
super().model_post_init(__context)
|
||||
|
||||
def _ensure_defaults(self) -> None:
|
||||
"""Lazy-import and build a real AgentContext when none is wired."""
|
||||
if self.context is None:
|
||||
from semantica.context import AgentContext, ContextGraph
|
||||
from semantica.vector_store import VectorStore
|
||||
|
||||
self.context = AgentContext(
|
||||
vector_store=VectorStore(backend="faiss"),
|
||||
decision_tracking=True,
|
||||
knowledge_graph=ContextGraph(),
|
||||
)
|
||||
if self.had_live_state:
|
||||
self.reconstructed_state = True
|
||||
logger.warning(
|
||||
"SemanticaDecisionTool: the live decision context was lost "
|
||||
"during serialization/checkpoint restore — an EMPTY "
|
||||
"context was reconstructed; re-attach the original context "
|
||||
"before continuing"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"SemanticaDecisionTool created a fresh in-memory "
|
||||
"AgentContext — agents sharing decision state must be "
|
||||
"wired to the same context"
|
||||
)
|
||||
self.had_live_state = True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CrewAI entry points
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run(
|
||||
self,
|
||||
action: str,
|
||||
category: Optional[str] = None,
|
||||
scenario: Optional[str] = None,
|
||||
reasoning: Optional[str] = None,
|
||||
outcome: Optional[str] = None,
|
||||
confidence: float = 0.8,
|
||||
entities: Optional[str] = None,
|
||||
decision_id: Optional[str] = None,
|
||||
depth: int = 3,
|
||||
decision_data: Optional[str] = None,
|
||||
policy_rules: Optional[str] = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
valid = {
|
||||
"record_decision",
|
||||
"find_precedents",
|
||||
"trace_causal_chain",
|
||||
"analyze_impact",
|
||||
"check_policy",
|
||||
}
|
||||
if action not in valid:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": f"Unknown action '{action}'. Valid actions: "
|
||||
+ ", ".join(sorted(valid))
|
||||
}
|
||||
)
|
||||
|
||||
if action == "record_decision":
|
||||
return self._record_decision(
|
||||
category=category or "general",
|
||||
scenario=scenario or "decision recorded",
|
||||
reasoning=reasoning or "agent decision",
|
||||
outcome=outcome or "recorded",
|
||||
confidence=confidence,
|
||||
entities=entities,
|
||||
)
|
||||
if action == "find_precedents":
|
||||
return self._find_precedents(scenario=scenario or "", category=category)
|
||||
if action == "trace_causal_chain":
|
||||
return self._trace_causal_chain(decision_id or "", depth=depth)
|
||||
if action == "analyze_impact":
|
||||
return self._analyze_impact(decision_id or "")
|
||||
return self._check_policy(decision_data or "", policy_rules)
|
||||
|
||||
async def _arun(self, action: str, **kwargs: Any) -> str:
|
||||
"""Async variant of ``_run`` for CrewAI's async tool path."""
|
||||
return self._run(action=action, **kwargs)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Actions
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _record_decision(
|
||||
self,
|
||||
category: str,
|
||||
scenario: str,
|
||||
reasoning: str,
|
||||
outcome: str,
|
||||
confidence: float = 0.8,
|
||||
entities: Optional[str] = None,
|
||||
) -> str:
|
||||
entity_list: Optional[List[str]] = None
|
||||
if entities:
|
||||
entity_list = [e.strip() for e in entities.split(",") if e.strip()]
|
||||
|
||||
try:
|
||||
decision_id = self.context.record_decision(
|
||||
category=category,
|
||||
scenario=scenario,
|
||||
reasoning=reasoning,
|
||||
outcome=outcome,
|
||||
confidence=float(confidence),
|
||||
entities=entity_list,
|
||||
)
|
||||
result = {"decision_id": str(decision_id), "status": "recorded"}
|
||||
logger.info("record_decision → %s", decision_id)
|
||||
except Exception as exc:
|
||||
result = {"error": str(exc), "status": "failed"}
|
||||
logger.warning("record_decision failed: %s", exc)
|
||||
|
||||
return json.dumps(result)
|
||||
|
||||
def _find_precedents(
|
||||
self,
|
||||
scenario: str,
|
||||
category: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> str:
|
||||
k = limit if limit is not None else self.max_precedents
|
||||
try:
|
||||
precedents = self.context.find_precedents_advanced(
|
||||
scenario=scenario,
|
||||
category=category,
|
||||
limit=k,
|
||||
)
|
||||
out: List[Dict[str, Any]] = []
|
||||
for p in (precedents or [])[:k]:
|
||||
if isinstance(p, dict):
|
||||
out.append(p)
|
||||
else:
|
||||
out.append(
|
||||
{
|
||||
"scenario": getattr(p, "scenario", str(p)),
|
||||
"outcome": getattr(p, "outcome", ""),
|
||||
"confidence": getattr(p, "confidence", 0.0),
|
||||
"category": getattr(p, "category", ""),
|
||||
}
|
||||
)
|
||||
logger.info("find_precedents('%s') → %d results", scenario, len(out))
|
||||
return json.dumps({"precedents": out, "count": len(out)})
|
||||
except Exception as exc:
|
||||
logger.warning("find_precedents failed: %s", exc)
|
||||
return json.dumps({"precedents": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _trace_causal_chain(self, decision_id: str, depth: Optional[int] = None) -> str:
|
||||
if not decision_id:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": "decision_id is required for trace_causal_chain",
|
||||
"causal_chain": [],
|
||||
"decision_id": "",
|
||||
}
|
||||
)
|
||||
max_depth = depth or self.causal_depth
|
||||
try:
|
||||
graph = getattr(self.context, "knowledge_graph", None)
|
||||
if graph is None:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": (
|
||||
"causal tracing is not available on this knowledge "
|
||||
"graph (the decision context has no knowledge_graph)"
|
||||
),
|
||||
"causal_chain": [],
|
||||
"decision_id": decision_id,
|
||||
}
|
||||
)
|
||||
trace = getattr(graph, "trace_decision_causality", None)
|
||||
if trace is None:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": (
|
||||
"causal tracing is not available on this knowledge graph "
|
||||
"(graph.trace_decision_causality is not implemented)"
|
||||
),
|
||||
"causal_chain": [],
|
||||
"decision_id": decision_id,
|
||||
}
|
||||
)
|
||||
chain = trace(decision_id, max_depth=max_depth)
|
||||
return json.dumps({"causal_chain": chain, "decision_id": decision_id})
|
||||
except Exception as exc:
|
||||
logger.warning("trace_causal_chain failed: %s", exc)
|
||||
return json.dumps(
|
||||
{"error": str(exc), "causal_chain": [], "decision_id": decision_id}
|
||||
)
|
||||
|
||||
def _analyze_impact(self, decision_id: str) -> str:
|
||||
try:
|
||||
influence = self.context.analyze_decision_influence(decision_id)
|
||||
if not isinstance(influence, dict):
|
||||
influence = {"influence": str(influence)}
|
||||
influence["decision_id"] = decision_id
|
||||
return json.dumps(influence)
|
||||
except Exception as exc:
|
||||
logger.warning("analyze_impact failed: %s", exc)
|
||||
return json.dumps({"error": str(exc), "decision_id": decision_id})
|
||||
|
||||
def _check_policy(
|
||||
self,
|
||||
decision_data: str,
|
||||
policy_rules: Optional[str] = None,
|
||||
) -> str:
|
||||
try:
|
||||
data = (
|
||||
json.loads(decision_data)
|
||||
if isinstance(decision_data, str)
|
||||
else decision_data
|
||||
)
|
||||
except json.JSONDecodeError as exc:
|
||||
return json.dumps(
|
||||
{
|
||||
"compliant": False,
|
||||
"violations": [f"Invalid decision_data JSON: {exc}"],
|
||||
"warnings": [],
|
||||
}
|
||||
)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return json.dumps(
|
||||
{
|
||||
"compliant": False,
|
||||
"violations": [
|
||||
f"decision_data must decode to a JSON object, "
|
||||
f"got {type(data).__name__}: {data!r}"
|
||||
],
|
||||
"warnings": [],
|
||||
}
|
||||
)
|
||||
|
||||
violations: List[str] = []
|
||||
warnings: List[str] = []
|
||||
|
||||
rules: List[str] = []
|
||||
if policy_rules:
|
||||
try:
|
||||
parsed_rules = json.loads(policy_rules)
|
||||
except json.JSONDecodeError:
|
||||
rules = [r.strip() for r in policy_rules.split(",") if r.strip()]
|
||||
else:
|
||||
if isinstance(parsed_rules, str):
|
||||
rules = [parsed_rules]
|
||||
elif isinstance(parsed_rules, list):
|
||||
for item in parsed_rules:
|
||||
if isinstance(item, str):
|
||||
rules.append(item)
|
||||
else:
|
||||
warnings.append(
|
||||
f"Ignoring non-string policy rule entry: {item!r}"
|
||||
)
|
||||
else:
|
||||
warnings.append(
|
||||
f"policy_rules must decode to a JSON list of rule strings, "
|
||||
f"got {type(parsed_rules).__name__}: {parsed_rules!r}"
|
||||
)
|
||||
|
||||
for rule in rules:
|
||||
try:
|
||||
if not self._eval_rule(rule, data):
|
||||
violations.append(f"Rule violated: {rule}")
|
||||
except Exception as exc:
|
||||
warnings.append(f"Could not evaluate rule '{rule}': {exc}")
|
||||
|
||||
compliant = len(violations) == 0
|
||||
logger.debug(
|
||||
"check_policy: compliant=%s, violations=%d", compliant, len(violations)
|
||||
)
|
||||
return json.dumps(
|
||||
{
|
||||
"compliant": compliant,
|
||||
"violations": violations,
|
||||
"warnings": warnings,
|
||||
}
|
||||
)
|
||||
|
||||
def _eval_rule(self, rule: str, data: Dict[str, Any]) -> bool:
|
||||
"""Evaluate a simple comparison rule (``field op value``) against data.
|
||||
|
||||
This is a small standalone evaluator for the tool's ``check_policy``
|
||||
action — it is intentionally independent of Semantica's policy engine
|
||||
so agents get a bounded, side-effect-free rule check. Rules are
|
||||
``<field> <op> <value>`` comparisons only; there is no expression
|
||||
evaluation (no ``eval``), so untrusted rule strings are safe to pass.
|
||||
|
||||
Values are coerced type-aware: ``true``/``false`` (and ``1``/``0``)
|
||||
become booleans, numeric literals become numbers, and string values
|
||||
that parse as numbers are compared numerically, so ``score == 0.9``
|
||||
holds for ``score: "0.90"`` and ``enabled == false`` holds for
|
||||
``enabled: false``. Field names may contain hyphens, dots and spaces
|
||||
(e.g. ``risk-score >= 0.9``); they are matched against ``data`` keys
|
||||
as-is.
|
||||
"""
|
||||
m = re.match(r"(.+?)\s*(>=|<=|!=|==|>|<)\s*(.+)$", rule.strip())
|
||||
if not m:
|
||||
raise ValueError(f"unrecognised rule format: {rule!r}")
|
||||
field, op, val_str = m.group(1), m.group(2), m.group(3).strip().strip("\"'")
|
||||
if field not in data:
|
||||
raise ValueError(f"rule references undefined field {field!r}")
|
||||
actual = data[field]
|
||||
if actual is None:
|
||||
raise ValueError(f"field {field!r} is null — cannot evaluate rule")
|
||||
val = self._coerce_value(val_str)
|
||||
if isinstance(actual, str):
|
||||
actual = self._coerce_value(actual)
|
||||
ops = {
|
||||
">=": lambda a, b: a >= b,
|
||||
"<=": lambda a, b: a <= b,
|
||||
"!=": lambda a, b: a != b,
|
||||
"==": lambda a, b: a == b,
|
||||
">": lambda a, b: a > b,
|
||||
"<": lambda a, b: a < b,
|
||||
}
|
||||
return ops[op](actual, val)
|
||||
|
||||
@staticmethod
|
||||
def _coerce_value(value: str) -> Any:
|
||||
"""Parse a rule literal into its most specific Python type."""
|
||||
text = value.strip()
|
||||
lowered = text.lower()
|
||||
if lowered in ("true", "1"):
|
||||
return True
|
||||
if lowered in ("false", "0"):
|
||||
return False
|
||||
try:
|
||||
return int(text)
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
return float(text)
|
||||
except ValueError:
|
||||
pass
|
||||
return text
|
||||
|
||||
# When crewai is absent there is no BaseTool to provide the public
|
||||
# ``run``/``arun`` entry points, so expose them directly. With crewai
|
||||
# installed these are left untouched so crewai's own implementations
|
||||
# (usage tracking, ``result_as_answer``) win.
|
||||
if not CREWAI_AVAILABLE:
|
||||
|
||||
def run(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool synchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
|
||||
async def arun(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool asynchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
@@ -0,0 +1,573 @@
|
||||
"""
|
||||
SemanticaKGTool — a CrewAI ``BaseTool`` exposing Semantica's knowledge-graph
|
||||
pipeline (``NERExtractor``, ``RelationExtractor``, ``ContextGraph``) to agents.
|
||||
|
||||
Lets agents build and query a shared ``ContextGraph`` as part of their
|
||||
reasoning loop.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[crewai]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from integrations.crewai import SemanticaKGTool
|
||||
>>> from semantica.context import ContextGraph
|
||||
>>> from crewai import Agent, Crew, Task
|
||||
>>> graph = ContextGraph()
|
||||
>>> tool = SemanticaKGTool(graph=graph)
|
||||
>>> crew = Crew(
|
||||
... agents=[Agent(role="...", goal="...", backstory="...", tools=[tool])],
|
||||
... tasks=[...],
|
||||
... )
|
||||
|
||||
Tools exposed
|
||||
-------------
|
||||
extract_entities — Extract named entities from text
|
||||
extract_relations — Extract relationships between entities
|
||||
add_to_graph — Extract entities/relations from text and add them to the graph
|
||||
query_graph — Query the graph by keyword
|
||||
find_related — Find concepts related to a given entity within ``hops``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import weakref
|
||||
from typing import Any, Dict, List, Literal, Optional, Sequence, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE, CREWAI_IMPORT_ERROR # noqa: F401
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: CrewAI BaseTool base class
|
||||
# ---------------------------------------------------------------------------
|
||||
_BaseTool: Any = object
|
||||
|
||||
if CREWAI_AVAILABLE:
|
||||
from crewai.tools import BaseTool as _BaseTool # type: ignore
|
||||
|
||||
# One re-entrant lock per graph so concurrent tool invocations sharing a graph
|
||||
# cannot double-count duplicate adds (check-then-act is not atomic), while
|
||||
# independent graphs are never serialised against each other. An RLock also
|
||||
# means an extractor callback that re-enters add_to_graph on the same graph
|
||||
# cannot deadlock.
|
||||
_graph_locks_guard = threading.Lock()
|
||||
_graph_locks: "weakref.WeakKeyDictionary[Any, threading.RLock]" = (
|
||||
weakref.WeakKeyDictionary()
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input schema
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaKGToolInput(BaseModel):
|
||||
"""
|
||||
Input schema for ``SemanticaKGTool``.
|
||||
|
||||
Exactly one action is dispatched per call; the remaining fields are only
|
||||
used by the actions that need them.
|
||||
"""
|
||||
|
||||
action: Literal[
|
||||
"extract_entities",
|
||||
"extract_relations",
|
||||
"add_to_graph",
|
||||
"query_graph",
|
||||
"find_related",
|
||||
] = Field(
|
||||
...,
|
||||
description=(
|
||||
"Which graph operation to run. One of: 'extract_entities', "
|
||||
"'extract_relations', 'add_to_graph', 'query_graph', 'find_related'."
|
||||
),
|
||||
)
|
||||
text: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"Input text. Used by 'extract_entities', 'extract_relations' and "
|
||||
"'add_to_graph'."
|
||||
),
|
||||
)
|
||||
query: Optional[str] = Field(
|
||||
None, description="Search query. Used by 'query_graph'."
|
||||
)
|
||||
entity: Optional[str] = Field(
|
||||
None,
|
||||
description="Root entity name. Used by 'find_related'.",
|
||||
)
|
||||
hops: int = Field(
|
||||
1,
|
||||
ge=1,
|
||||
le=10,
|
||||
description="Maximum relationship hops. Used by 'find_related'.",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SemanticaKGTool
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaKGTool(_BaseTool): # type: ignore[misc]
|
||||
"""
|
||||
CrewAI tool that surfaces Semantica's KG pipeline as agent actions.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph:
|
||||
A ``semantica.context.ContextGraph`` to read/write. A fresh in-memory
|
||||
graph is used when ``None``.
|
||||
ner_extractor:
|
||||
A ``semantica.semantic_extract.NERExtractor`` instance; auto-created
|
||||
when ``None``.
|
||||
relation_extractor:
|
||||
A ``semantica.semantic_extract.RelationExtractor`` instance; auto-
|
||||
created when ``None``.
|
||||
"""
|
||||
|
||||
name: str = "semantica_knowledge_graph"
|
||||
description: str = (
|
||||
"Build and query a semantic knowledge graph. Actions: "
|
||||
"'extract_entities' (extract named entities from 'text'), "
|
||||
"'extract_relations' (extract relationships from 'text'), "
|
||||
"'add_to_graph' (extract entities/relations from 'text' and add them "
|
||||
"to the shared graph), 'query_graph' (keyword search using 'query'), "
|
||||
"'find_related' (find concepts related to 'entity' within 'hops' "
|
||||
"hops). Returns JSON."
|
||||
)
|
||||
args_schema: Type[BaseModel] = SemanticaKGToolInput
|
||||
graph: Any = Field(default=None, exclude=True)
|
||||
ner_extractor: Any = Field(default=None, exclude=True)
|
||||
relation_extractor: Any = Field(default=None, exclude=True)
|
||||
had_live_state: bool = False
|
||||
reconstructed_state: bool = Field(default=False, exclude=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
graph: Any = None,
|
||||
ner_extractor: Any = None,
|
||||
relation_extractor: Any = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if CREWAI_AVAILABLE:
|
||||
super().__init__(
|
||||
graph=graph,
|
||||
ner_extractor=ner_extractor,
|
||||
relation_extractor=relation_extractor,
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
super().__init__()
|
||||
self.graph = graph
|
||||
self.ner_extractor = ner_extractor
|
||||
self.relation_extractor = relation_extractor
|
||||
# Degraded mode is a plain class — no model_post_init lifecycle.
|
||||
self._ensure_defaults()
|
||||
|
||||
logger.info("SemanticaKGTool initialised (crewai=%s)", CREWAI_AVAILABLE)
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
"""Re-create default state after validation/deserialisation.
|
||||
|
||||
``graph``/extractors are excluded from JSON serialisation (CrewAI
|
||||
checkpoints serialise every tool via ``model_dump(mode="json")``), so a
|
||||
tool restored from a checkpoint has ``None`` state until this runs.
|
||||
"""
|
||||
self._ensure_defaults()
|
||||
super().model_post_init(__context)
|
||||
|
||||
def _ensure_defaults(self) -> None:
|
||||
"""Lazy-import and build defaults for any missing shared state."""
|
||||
# Lazy imports keep the module importable without heavy deps
|
||||
if self.graph is None:
|
||||
from semantica.context import ContextGraph
|
||||
|
||||
self.graph = ContextGraph()
|
||||
if self.had_live_state:
|
||||
self.reconstructed_state = True
|
||||
logger.warning(
|
||||
"SemanticaKGTool: the live graph was lost during "
|
||||
"serialization/checkpoint restore — an EMPTY graph was "
|
||||
"reconstructed; re-attach the original graph before "
|
||||
"continuing"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"SemanticaKGTool created a fresh in-memory ContextGraph — "
|
||||
"agents sharing this tool's graph must be wired explicitly"
|
||||
)
|
||||
self.had_live_state = True
|
||||
if self.ner_extractor is None:
|
||||
from semantica.semantic_extract import NERExtractor
|
||||
|
||||
self.ner_extractor = NERExtractor()
|
||||
if self.relation_extractor is None:
|
||||
from semantica.semantic_extract import RelationExtractor
|
||||
|
||||
self.relation_extractor = RelationExtractor()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CrewAI entry points
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run(
|
||||
self,
|
||||
action: str,
|
||||
text: Optional[str] = None,
|
||||
query: Optional[str] = None,
|
||||
entity: Optional[str] = None,
|
||||
hops: int = 1,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
"""
|
||||
Dispatch a graph action. Always returns a JSON string so the agent
|
||||
receives a structured, parseable result.
|
||||
"""
|
||||
valid = {
|
||||
"extract_entities",
|
||||
"extract_relations",
|
||||
"add_to_graph",
|
||||
"query_graph",
|
||||
"find_related",
|
||||
}
|
||||
if action not in valid:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": f"Unknown action '{action}'. Valid actions: "
|
||||
+ ", ".join(sorted(valid))
|
||||
}
|
||||
)
|
||||
|
||||
if action == "extract_entities":
|
||||
return self._extract_entities(text or "")
|
||||
if action == "extract_relations":
|
||||
return self._extract_relations(text or "")
|
||||
if action == "add_to_graph":
|
||||
return self._add_from_text(text or "")
|
||||
if action == "query_graph":
|
||||
return self._query_graph(query or "")
|
||||
return self._find_related(entity or "", hops=hops)
|
||||
|
||||
async def _arun(
|
||||
self,
|
||||
action: str,
|
||||
text: Optional[str] = None,
|
||||
query: Optional[str] = None,
|
||||
entity: Optional[str] = None,
|
||||
hops: int = 1,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
"""
|
||||
Async variant of ``_run`` for CrewAI's async tool path.
|
||||
"""
|
||||
return self._run(
|
||||
action=action, text=text, query=query, entity=entity, hops=hops, **kwargs
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Entity/relation field access (handles both Semantica dataclasses and
|
||||
# third-party shapes like MagicMock/plain dicts in stubs)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _first_str(obj: Any, attrs: Sequence[str]) -> str:
|
||||
"""Return the first attribute value that is a non-empty string."""
|
||||
for attr in attrs:
|
||||
value = getattr(obj, attr, None)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
if isinstance(obj, dict):
|
||||
for key in attrs:
|
||||
value = obj.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return ""
|
||||
|
||||
@classmethod
|
||||
def _entity_name(cls, e: Any) -> str:
|
||||
"""Best-effort name for an entity-like object."""
|
||||
return cls._first_str(e, ("name", "text", "label", "node_id", "id"))
|
||||
|
||||
@classmethod
|
||||
def _entity_type(cls, e: Any) -> str:
|
||||
"""Best-effort type/label for an entity-like object."""
|
||||
return cls._first_str(e, ("type", "label")) or "Entity"
|
||||
|
||||
@classmethod
|
||||
def _relation_source(cls, r: Any) -> str:
|
||||
"""Best-effort source of a relation-like object."""
|
||||
src = cls._first_str(r, ("source",))
|
||||
if not src:
|
||||
src = cls._entity_name(getattr(r, "subject", None))
|
||||
return src
|
||||
|
||||
@classmethod
|
||||
def _relation_target(cls, r: Any) -> str:
|
||||
"""Best-effort target of a relation-like object."""
|
||||
tgt = cls._first_str(r, ("target",))
|
||||
if not tgt:
|
||||
tgt = cls._entity_name(getattr(r, "object", None))
|
||||
return tgt
|
||||
|
||||
@classmethod
|
||||
def _relation_type(cls, r: Any) -> str:
|
||||
"""Best-effort relation type of a relation-like object."""
|
||||
rtype = cls._first_str(r, ("type", "relation", "predicate"))
|
||||
return rtype or "related_to"
|
||||
|
||||
@classmethod
|
||||
def _confidence(cls, e: Any) -> float:
|
||||
"""Normalise an entity/relation confidence value to a float."""
|
||||
try:
|
||||
val = getattr(e, "confidence", None)
|
||||
if val is None:
|
||||
return 1.0
|
||||
return round(float(val), 4)
|
||||
except (TypeError, ValueError):
|
||||
return 1.0
|
||||
|
||||
@classmethod
|
||||
def _graph_lock(cls, graph: Any) -> threading.RLock:
|
||||
"""Return the re-entrant lock guarding a specific graph."""
|
||||
with _graph_locks_guard:
|
||||
lock = _graph_locks.get(graph)
|
||||
if lock is None:
|
||||
lock = threading.RLock()
|
||||
_graph_locks[graph] = lock
|
||||
return lock
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Actions
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _extract_entities(self, text: str) -> str:
|
||||
"""Extract named entities from ``text``."""
|
||||
try:
|
||||
raw = self.ner_extractor.extract_entities(text) or []
|
||||
entities = [
|
||||
{
|
||||
"name": self._entity_name(e),
|
||||
"type": self._entity_type(e),
|
||||
"confidence": self._confidence(e),
|
||||
}
|
||||
for e in raw
|
||||
if self._entity_name(e)
|
||||
]
|
||||
logger.debug("extract_entities → %d entities", len(entities))
|
||||
return json.dumps({"entities": entities, "count": len(entities)})
|
||||
except Exception as exc:
|
||||
logger.warning("extract_entities failed: %s", exc)
|
||||
return json.dumps({"entities": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _extract_relations(self, text: str) -> str:
|
||||
"""Extract relationships between entities in ``text``."""
|
||||
try:
|
||||
raw = self.relation_extractor.extract_relations(text) or []
|
||||
relations = [
|
||||
{
|
||||
"source": self._relation_source(r),
|
||||
"relation": self._relation_type(r),
|
||||
"target": self._relation_target(r),
|
||||
"confidence": self._confidence(r),
|
||||
}
|
||||
for r in raw
|
||||
]
|
||||
logger.debug("extract_relations → %d relations", len(relations))
|
||||
return json.dumps({"relations": relations, "count": len(relations)})
|
||||
except Exception as exc:
|
||||
logger.warning("extract_relations failed: %s", exc)
|
||||
return json.dumps({"relations": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _add_from_text(self, text: str) -> str:
|
||||
"""
|
||||
Extract entities and relations from ``text`` and add them to the graph.
|
||||
|
||||
Duplicate nodes/edges (same id, or same source/type/target) are
|
||||
skipped so repeated calls are idempotent. Returns JSON with the
|
||||
number of nodes/edges added.
|
||||
"""
|
||||
nodes_added = 0
|
||||
edges_added = 0
|
||||
try:
|
||||
with self._graph_lock(self.graph):
|
||||
existing_nodes = {
|
||||
n.get("id") or n.get("node_id")
|
||||
for n in (
|
||||
self.graph.find_nodes() or [] # type: ignore[attr-defined]
|
||||
)
|
||||
if n.get("id") or n.get("node_id")
|
||||
}
|
||||
existing_edges = {
|
||||
(e.get("source"), e.get("type") or "related_to", e.get("target"))
|
||||
for e in (
|
||||
self.graph.find_edges() or [] # type: ignore[attr-defined]
|
||||
)
|
||||
if e.get("source") and e.get("target")
|
||||
}
|
||||
|
||||
raw_entities = self.ner_extractor.extract_entities(text) or []
|
||||
entities: List[Any] = []
|
||||
seen: set = set()
|
||||
for e in raw_entities:
|
||||
name = self._entity_name(e)
|
||||
ntype = self._entity_type(e)
|
||||
if not name or name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
entities.append(e)
|
||||
if name in existing_nodes:
|
||||
continue
|
||||
try:
|
||||
if self.graph.add_node(node_id=name, node_type=ntype):
|
||||
nodes_added += 1
|
||||
existing_nodes.add(name)
|
||||
except Exception as exc:
|
||||
logger.debug("add_node(%r) failed: %s", name, exc)
|
||||
|
||||
raw_relations = (
|
||||
self.relation_extractor.extract_relations(text, entities=entities)
|
||||
or []
|
||||
)
|
||||
for r in raw_relations:
|
||||
src = self._relation_source(r)
|
||||
tgt = self._relation_target(r)
|
||||
rtype = self._relation_type(r)
|
||||
if not src or not tgt:
|
||||
continue
|
||||
key = (src, rtype, tgt)
|
||||
if key in existing_edges:
|
||||
continue
|
||||
try:
|
||||
if self.graph.add_edge(
|
||||
source_id=src, target_id=tgt, edge_type=rtype
|
||||
):
|
||||
edges_added += 1
|
||||
existing_edges.add(key)
|
||||
except Exception as exc:
|
||||
logger.debug("add_edge(%r) failed: %s", key, exc)
|
||||
logger.debug("add_to_graph: +%d nodes, +%d edges", nodes_added, edges_added)
|
||||
return json.dumps({"nodes_added": nodes_added, "edges_added": edges_added})
|
||||
except Exception as exc:
|
||||
logger.warning("add_to_graph failed: %s", exc)
|
||||
return json.dumps({"nodes_added": 0, "edges_added": 0, "error": str(exc)})
|
||||
|
||||
def _query_graph(self, query: str) -> str:
|
||||
"""Keyword-search graph nodes by id, type and content."""
|
||||
try:
|
||||
q = (query or "").strip().lower()
|
||||
out: List[dict] = []
|
||||
seen: set = set()
|
||||
|
||||
query_method = getattr(self.graph, "query", None)
|
||||
if query_method is not None:
|
||||
for match in query_method(query) or []:
|
||||
node = match.get("node") or {}
|
||||
nid = node.get("id", "") or node.get("node_id", "")
|
||||
if not nid or nid in seen:
|
||||
continue
|
||||
seen.add(nid)
|
||||
content = match.get("content") or node.get("content", "")
|
||||
out.append(
|
||||
{
|
||||
"id": nid,
|
||||
"type": node.get("type", "") or node.get("node_type", ""),
|
||||
"label": nid,
|
||||
"content": str(content)[:500],
|
||||
"score": round(float(match.get("score") or 0.0), 4),
|
||||
}
|
||||
)
|
||||
|
||||
if q:
|
||||
for n in self.graph.find_nodes() or []: # type: ignore[attr-defined]
|
||||
if isinstance(n, dict):
|
||||
nid = n.get("id", "") or n.get("node_id", "")
|
||||
ntype = n.get("type", "") or n.get("node_type", "")
|
||||
content = str(
|
||||
n.get("content")
|
||||
or (n.get("properties") or {}).get("content", "")
|
||||
or ""
|
||||
)
|
||||
else:
|
||||
nid = getattr(n, "id", getattr(n, "label", ""))
|
||||
ntype = getattr(n, "node_type", "")
|
||||
content = str(getattr(n, "content", "") or "")
|
||||
if not nid or nid in seen:
|
||||
continue
|
||||
if q in str(nid).lower() or q in str(ntype).lower():
|
||||
seen.add(nid)
|
||||
out.append(
|
||||
{
|
||||
"id": nid,
|
||||
"type": ntype,
|
||||
"label": nid,
|
||||
"content": content[:500],
|
||||
"score": 1.0,
|
||||
}
|
||||
)
|
||||
return json.dumps({"results": out, "count": len(out)})
|
||||
except Exception as exc:
|
||||
logger.warning("query_graph failed: %s", exc)
|
||||
return json.dumps({"results": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _find_related(self, entity: str, hops: int = 1) -> str:
|
||||
"""Find concepts related to ``entity`` within ``hops`` graph hops.
|
||||
|
||||
Traversal is undirected — an edge counts as related regardless of
|
||||
direction, so both outgoing and incoming edges are honored.
|
||||
"""
|
||||
try:
|
||||
adjacency: Dict[str, List[str]] = {}
|
||||
for edge in self.graph.find_edges() or []: # type: ignore[attr-defined]
|
||||
if isinstance(edge, dict):
|
||||
src = edge.get("source")
|
||||
tgt = edge.get("target")
|
||||
else:
|
||||
src = getattr(edge, "source", None)
|
||||
tgt = getattr(edge, "target", None)
|
||||
if not src or not tgt:
|
||||
continue
|
||||
adjacency.setdefault(src, []).append(tgt)
|
||||
adjacency.setdefault(tgt, []).append(src)
|
||||
|
||||
related: List[str] = []
|
||||
frontier = [entity]
|
||||
visited = {entity}
|
||||
for _ in range(max(1, hops)):
|
||||
next_frontier: List[str] = []
|
||||
for e in frontier:
|
||||
for n in adjacency.get(e, []):
|
||||
if n in visited:
|
||||
continue
|
||||
visited.add(n)
|
||||
next_frontier.append(n)
|
||||
related.append(n)
|
||||
frontier = next_frontier
|
||||
|
||||
logger.debug("find_related('%s', hops=%d) → %d", entity, hops, len(related))
|
||||
return json.dumps(
|
||||
{"entity": entity, "related": related, "count": len(related)}
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("find_related failed: %s", exc)
|
||||
return json.dumps(
|
||||
{"entity": entity, "related": [], "count": 0, "error": str(exc)}
|
||||
)
|
||||
|
||||
# When crewai is absent there is no BaseTool to provide the public
|
||||
# ``run``/``arun`` entry points, so expose them directly. With crewai
|
||||
# installed these are left untouched so crewai's own implementations
|
||||
# (usage tracking, ``result_as_answer``) win.
|
||||
if not CREWAI_AVAILABLE:
|
||||
|
||||
def run(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool synchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
|
||||
async def arun(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool asynchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
@@ -0,0 +1,331 @@
|
||||
"""
|
||||
SemanticaKnowledgeSource — expose a Semantica ``ContextGraph`` as a CrewAI
|
||||
knowledge source.
|
||||
|
||||
Lets a ``Crew`` load the current state of a knowledge graph (nodes, edges,
|
||||
metadata) into its knowledge storage, so every agent gets retrieval access to
|
||||
graph knowledge during the kickoff.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[crewai]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from integrations.crewai import SemanticaKnowledgeSource
|
||||
>>> from semantica.context import ContextGraph
|
||||
>>> from crewai import Agent, Crew, Task
|
||||
>>> graph = ContextGraph()
|
||||
>>> graph.add_node(node_id="privacy", node_type="policy")
|
||||
>>> crew = Crew(
|
||||
... agents=[...],
|
||||
... tasks=[...],
|
||||
... knowledge_sources=[SemanticaKnowledgeSource(graph=graph)],
|
||||
... )
|
||||
|
||||
Compatibility
|
||||
-------------
|
||||
Works with ``crewai >= 0.80.0``. The ``BaseKnowledgeSource`` contract changed
|
||||
between versions (``load_content`` → ``validate_content``/``aadd``), so this
|
||||
source implements both legacy and current methods. It degrades gracefully
|
||||
when ``crewai`` is not installed: the class is still importable and carries the
|
||||
full Semantica API, but cannot be passed to a ``Crew``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: CrewAI BaseKnowledgeSource base class
|
||||
# ---------------------------------------------------------------------------
|
||||
_BaseKnowledgeSource: Any = object
|
||||
|
||||
if CREWAI_AVAILABLE:
|
||||
from crewai.knowledge.source.base_knowledge_source import (
|
||||
BaseKnowledgeSource as _BaseKnowledgeSource, # type: ignore
|
||||
)
|
||||
|
||||
|
||||
def _chunk_text_manual(text: str, chunk_size: int, chunk_overlap: int) -> List[str]:
|
||||
"""Fallback plain-text chunker for when CrewAI helpers are unavailable."""
|
||||
if not text:
|
||||
return []
|
||||
if int(chunk_size) <= 0:
|
||||
return [text]
|
||||
size = max(1, int(chunk_size))
|
||||
overlap = max(0, int(chunk_overlap))
|
||||
if len(text) <= size:
|
||||
return [text]
|
||||
step = max(1, size - overlap)
|
||||
return [text[i : i + size] for i in range(0, len(text), step)]
|
||||
|
||||
|
||||
class SemanticaKnowledgeSource(_BaseKnowledgeSource): # type: ignore[misc]
|
||||
"""
|
||||
CrewAI knowledge source backed by a Semantica ``ContextGraph``.
|
||||
|
||||
On ``add()`` the graph's nodes and edges are serialised into readable text
|
||||
and pushed through the standard CrewAI chunking / storage pipeline, making
|
||||
graph knowledge retrievable by every agent in the crew.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph:
|
||||
A ``semantica.context.ContextGraph`` to expose. A fresh in-memory
|
||||
graph is created when ``None``.
|
||||
name:
|
||||
Source name. Defaults to ``"semantica_knowledge_graph"``.
|
||||
chunk_size:
|
||||
Max characters per chunk (default 4000).
|
||||
chunk_overlap:
|
||||
Character overlap between adjacent chunks (default 200).
|
||||
"""
|
||||
|
||||
name: str = "semantica_knowledge_graph"
|
||||
graph: Any = Field(default=None, exclude=True)
|
||||
chunk_size: int = 4000
|
||||
chunk_overlap: int = 200
|
||||
had_live_state: bool = False
|
||||
reconstructed_state: bool = Field(default=False, exclude=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
graph: Any = None,
|
||||
name: Optional[str] = None,
|
||||
chunk_size: int = 4000,
|
||||
chunk_overlap: int = 200,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if CREWAI_AVAILABLE:
|
||||
# Do NOT eagerly build a graph here: pydantic calls this ``__init__``
|
||||
# during ``model_validate`` (checkpoint restore), and the eager
|
||||
# build would hide that a live graph was lost. ``model_post_init``
|
||||
# rebuilds defaults and flags ``reconstructed_state`` instead.
|
||||
super().__init__(
|
||||
graph=graph,
|
||||
name=name or "semantica_knowledge_graph",
|
||||
chunk_size=int(chunk_size),
|
||||
chunk_overlap=int(chunk_overlap),
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
if graph is None:
|
||||
from semantica.context import ContextGraph
|
||||
|
||||
graph = ContextGraph()
|
||||
super().__init__()
|
||||
self.graph = graph
|
||||
self.name = name or "semantica_knowledge_graph"
|
||||
self.chunk_size = int(chunk_size)
|
||||
self.chunk_overlap = int(chunk_overlap)
|
||||
|
||||
logger.info(
|
||||
"SemanticaKnowledgeSource initialised (crewai=%s, chunk_size=%d)",
|
||||
CREWAI_AVAILABLE,
|
||||
self.chunk_size,
|
||||
)
|
||||
self.had_live_state = True
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
"""Re-create default state after validation/deserialisation.
|
||||
|
||||
``graph`` is excluded from JSON serialisation (CrewAI checkpoints
|
||||
serialise their models via ``model_dump(mode="json")``), so a source
|
||||
restored from a checkpoint has ``None`` state until this runs.
|
||||
"""
|
||||
if self.graph is None:
|
||||
from semantica.context import ContextGraph
|
||||
|
||||
self.graph = ContextGraph()
|
||||
if self.had_live_state:
|
||||
self.reconstructed_state = True
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource: the live graph was lost during "
|
||||
"serialization/checkpoint restore — an EMPTY graph was "
|
||||
"reconstructed; re-attach the original graph before "
|
||||
"continuing"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource created a fresh in-memory "
|
||||
"ContextGraph — sources sharing knowledge must be wired to "
|
||||
"the same graph explicitly"
|
||||
)
|
||||
self.had_live_state = True
|
||||
super().model_post_init(__context)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Content extraction
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def load_content(self) -> Dict[str, str]:
|
||||
"""
|
||||
Serialise the graph into ``{id: readable_text}`` pairs.
|
||||
|
||||
Nodes are rendered with their type/content/metadata, edges with their
|
||||
source, relation type and target. This satisfies the legacy CrewAI
|
||||
``BaseKnowledgeSource.load_content`` contract.
|
||||
"""
|
||||
content: Dict[str, str] = {}
|
||||
graph = self.graph
|
||||
if graph is None:
|
||||
return content
|
||||
|
||||
try:
|
||||
for node in graph.find_nodes() or []: # type: ignore[attr-defined]
|
||||
nid = node.get("id") or node.get("node_id") or ""
|
||||
if not nid:
|
||||
continue
|
||||
parts = [
|
||||
"Entity",
|
||||
str(nid),
|
||||
"type: " + str(node.get("type", "entity")),
|
||||
]
|
||||
if node.get("content"):
|
||||
parts.append("content: " + str(node["content"]))
|
||||
if node.get("metadata"):
|
||||
try:
|
||||
import json
|
||||
|
||||
parts.append("metadata: " + json.dumps(node["metadata"]))
|
||||
except Exception:
|
||||
parts.append("metadata: " + str(node["metadata"]))
|
||||
content[str(nid)] = " | ".join(parts)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource.load_content (nodes) failed: %s", exc
|
||||
)
|
||||
|
||||
try:
|
||||
for idx, edge in enumerate(
|
||||
graph.find_edges() or [] # type: ignore[attr-defined]
|
||||
):
|
||||
src = edge.get("source")
|
||||
tgt = edge.get("target")
|
||||
if not src or not tgt:
|
||||
continue
|
||||
rel = edge.get("type") or edge.get("edge_type") or "related_to"
|
||||
weight = edge.get("weight")
|
||||
text = f"{src} -[{rel}]-> {tgt}"
|
||||
if weight is not None:
|
||||
text += f" (weight: {weight})"
|
||||
content[f"edge-{idx}"] = text
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource.load_content (edges) failed: %s", exc
|
||||
)
|
||||
|
||||
return content
|
||||
|
||||
def validate_content(self) -> Any:
|
||||
"""
|
||||
Validate that a readable graph is attached.
|
||||
|
||||
Satisfies the current CrewAI ``BaseKnowledgeSource.validate_content``
|
||||
contract.
|
||||
"""
|
||||
if self.graph is None:
|
||||
raise ValueError("SemanticaKnowledgeSource requires a ContextGraph.")
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Chunking + storage (abstract in both CrewAI generations)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _chunk(self, text: str) -> List[str]:
|
||||
"""Chunk ``text`` using CrewAI's helper when available, else manual."""
|
||||
helper = getattr(self, "_chunk_text", None)
|
||||
if helper is not None:
|
||||
try:
|
||||
return list(helper(text) or [])
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"SemanticaKnowledgeSource._chunk_text failed, falling back: %s", exc
|
||||
)
|
||||
return _chunk_text_manual(text, self.chunk_size, self.chunk_overlap)
|
||||
|
||||
def add(self) -> None:
|
||||
"""
|
||||
Process the graph into chunks and store them via CrewAI storage.
|
||||
|
||||
Sets both ``chunks`` (current CrewAI) and ``_chunks`` (legacy CrewAI)
|
||||
so either ``_save_documents`` implementation picks them up. If no
|
||||
storage has been wired (e.g. not yet attached to a ``Crew``), chunks
|
||||
are kept in memory.
|
||||
"""
|
||||
content = self.load_content()
|
||||
if not content:
|
||||
logger.debug("SemanticaKnowledgeSource.add: empty graph — nothing to store")
|
||||
return
|
||||
|
||||
chunks: List[str] = []
|
||||
for _, text in content.items():
|
||||
if text:
|
||||
chunks.extend(self._chunk(text))
|
||||
|
||||
self.chunks = chunks
|
||||
self._chunks = chunks
|
||||
|
||||
save = getattr(self, "_save_documents", None)
|
||||
if save is not None:
|
||||
if getattr(self, "storage", None) is None:
|
||||
logger.debug(
|
||||
"SemanticaKnowledgeSource.add: storage not wired — "
|
||||
"keeping chunks in memory"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
save()
|
||||
logger.info(
|
||||
"SemanticaKnowledgeSource.add: stored %d chunks", len(chunks)
|
||||
)
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"SemanticaKnowledgeSource.add: storage save FAILED (%s) — "
|
||||
"chunks are only kept in memory and agents will retrieve "
|
||||
"nothing. Configure the Crew embedder (e.g. an OpenAI "
|
||||
"embedder with OPENAI_API_KEY, or a local embedder) before "
|
||||
"running the crew.",
|
||||
exc,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"SemanticaKnowledgeSource.add: %d chunks ready in memory", len(chunks)
|
||||
)
|
||||
|
||||
async def aadd(self) -> None:
|
||||
"""
|
||||
Asynchronous variant of ``add()`` (current CrewAI contract).
|
||||
|
||||
The graph serialisation is CPU-bound, so it runs in a thread pool to
|
||||
avoid blocking the event loop.
|
||||
"""
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(None, self.add)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Inspection helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_content_summary(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Summarise what the source exposes (helpful for debugging / testing).
|
||||
"""
|
||||
content = self.load_content()
|
||||
return {
|
||||
"name": self.name,
|
||||
"source_count": len(content),
|
||||
"chunks": len(getattr(self, "chunks", []) or []),
|
||||
"crewai_available": CREWAI_AVAILABLE,
|
||||
}
|
||||
+6
-2
@@ -21,7 +21,11 @@ Configure in Claude Desktop, Windsurf, Cline, Continue, VS Code:
|
||||
}
|
||||
"""
|
||||
|
||||
# `semantica.__version__` is the authoritative package version — see
|
||||
# semantica/mcp_server/__init__.py for why it is used directly rather than
|
||||
# importlib.metadata.version("semantica").
|
||||
from semantica import __version__
|
||||
|
||||
from .server import SemanticaMCPServer, main
|
||||
|
||||
__all__ = ["SemanticaMCPServer", "main"]
|
||||
__version__ = "0.4.0"
|
||||
__all__ = ["SemanticaMCPServer", "main", "__version__"]
|
||||
|
||||
@@ -10,6 +10,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
|
||||
from mcp import __version__
|
||||
from mcp.session import get_graph
|
||||
|
||||
log = logging.getLogger("semantica.mcp.resources")
|
||||
@@ -60,7 +61,7 @@ def _read_decisions_list(uri: str) -> dict:
|
||||
|
||||
def _read_schema_info(uri: str) -> dict:
|
||||
info = {
|
||||
"version": "0.4.0",
|
||||
"version": __version__,
|
||||
"node_types": [
|
||||
"Entity", "decision", "Decision", "Event", "Concept",
|
||||
"Person", "Organisation", "Location",
|
||||
|
||||
+14
-3
@@ -17,6 +17,7 @@ import logging
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from mcp import __version__
|
||||
from mcp.resources import RESOURCE_DEFINITIONS, handle_resource_read
|
||||
from mcp.tools import TOOL_DEFINITIONS
|
||||
|
||||
@@ -63,7 +64,7 @@ def _handle_initialize(req_id: Any, params: dict) -> dict:
|
||||
},
|
||||
"serverInfo": {
|
||||
"name": "semantica-mcp",
|
||||
"version": "0.4.0",
|
||||
"version": __version__,
|
||||
},
|
||||
})
|
||||
|
||||
@@ -92,7 +93,14 @@ def _handle_tools_call(req_id: Any, params: dict) -> dict:
|
||||
result = tool["_handler"](args)
|
||||
except Exception as exc:
|
||||
log.exception("Tool %s raised an exception", name)
|
||||
return _err(req_id, _INTERNAL_ERROR, str(exc))
|
||||
# The exception's class name (e.g. "ValidationError", "TimeoutError")
|
||||
# is safe to surface — unlike str(exc), it never carries paths,
|
||||
# connection strings, or other internal detail — and lets the
|
||||
# client distinguish failure kinds without a full message.
|
||||
return _err(
|
||||
req_id, _INTERNAL_ERROR,
|
||||
f"Tool '{name}' failed ({type(exc).__name__}). See server logs for details.",
|
||||
)
|
||||
|
||||
# MCP spec: content must be a list of content items
|
||||
return _ok(req_id, {
|
||||
@@ -170,7 +178,10 @@ class SemanticaMCPServer:
|
||||
log.exception("Unhandled error in method %s", method)
|
||||
if req_id is None:
|
||||
return None
|
||||
return _err(req_id, _INTERNAL_ERROR, str(exc))
|
||||
return _err(
|
||||
req_id, _INTERNAL_ERROR,
|
||||
f"Method '{method}' failed ({type(exc).__name__}). See server logs for details.",
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def run(self) -> None:
|
||||
|
||||
+80
-4
@@ -91,11 +91,19 @@ def handle_find_precedents(args: dict) -> dict:
|
||||
|
||||
def handle_get_causal_chain(args: dict) -> dict:
|
||||
"""Trace the upstream or downstream causal chain from a decision."""
|
||||
decision_id = args.get("decision_id", "").strip()
|
||||
if not isinstance(args, dict):
|
||||
return {"error": "args must be a dictionary", "chain": []}
|
||||
decision_id = str(args.get("decision_id") or "").strip()
|
||||
if not decision_id:
|
||||
return {"error": "decision_id is required", "chain": []}
|
||||
direction = args.get("direction", "downstream")
|
||||
max_depth = int(args.get("max_depth", 5))
|
||||
direction = str(args.get("direction") or "downstream").strip()
|
||||
try:
|
||||
max_depth = int(args.get("max_depth", 5))
|
||||
if max_depth <= 0:
|
||||
max_depth = 5
|
||||
max_depth = min(max_depth, 100)
|
||||
except (ValueError, TypeError):
|
||||
max_depth = 5
|
||||
try:
|
||||
graph = get_graph()
|
||||
try:
|
||||
@@ -105,7 +113,75 @@ def handle_get_causal_chain(args: dict) -> dict:
|
||||
decision_id, direction=direction, max_depth=max_depth
|
||||
)
|
||||
except (ImportError, AttributeError):
|
||||
chain = graph.get_causal_chain(decision_id) if hasattr(graph, "get_causal_chain") else []
|
||||
if hasattr(graph, "get_causal_chain"):
|
||||
import inspect
|
||||
# Introspect the signature in its own try/except: only
|
||||
# failure to introspect (ValueError/TypeError from
|
||||
# inspect.signature itself, e.g. a C-extension callable)
|
||||
# should fall through to the trial-and-error cascade below.
|
||||
# A call made after a *successful* introspection must not be
|
||||
# wrapped in that cascade's except block — otherwise a
|
||||
# genuine bug inside get_causal_chain (raising an unrelated
|
||||
# TypeError) gets misread as "wrong signature" and the
|
||||
# backend is invoked a second time with identical arguments.
|
||||
try:
|
||||
params = inspect.signature(graph.get_causal_chain).parameters
|
||||
except (ValueError, TypeError):
|
||||
params = None
|
||||
|
||||
if params is not None:
|
||||
has_var_kwargs = any(
|
||||
p.kind == inspect.Parameter.VAR_KEYWORD
|
||||
for p in params.values()
|
||||
)
|
||||
if has_var_kwargs or (
|
||||
"direction" in params and "max_depth" in params
|
||||
):
|
||||
chain = graph.get_causal_chain(
|
||||
decision_id,
|
||||
direction=direction,
|
||||
max_depth=max_depth,
|
||||
)
|
||||
elif "depth" in params:
|
||||
chain = graph.get_causal_chain(
|
||||
decision_id,
|
||||
depth=max_depth,
|
||||
)
|
||||
else:
|
||||
chain = graph.get_causal_chain(decision_id)
|
||||
else:
|
||||
try:
|
||||
chain = graph.get_causal_chain(
|
||||
decision_id,
|
||||
direction=direction,
|
||||
max_depth=max_depth,
|
||||
)
|
||||
except TypeError as exc:
|
||||
if "unexpected keyword argument" in str(
|
||||
exc
|
||||
) or "positional" in str(exc):
|
||||
try:
|
||||
chain = graph.get_causal_chain(
|
||||
decision_id,
|
||||
depth=max_depth,
|
||||
)
|
||||
except TypeError as exc2:
|
||||
if "unexpected keyword argument" in str(
|
||||
exc2
|
||||
) or "positional" in str(exc2):
|
||||
chain = graph.get_causal_chain(decision_id)
|
||||
else:
|
||||
raise
|
||||
else:
|
||||
raise
|
||||
else:
|
||||
return {
|
||||
"error": (
|
||||
"Causal chain analysis is not supported on this graph"
|
||||
" backend"
|
||||
),
|
||||
"chain": [],
|
||||
}
|
||||
result = chain if isinstance(chain, list) else list(chain)
|
||||
return {"chain": result, "count": len(result), "direction": direction}
|
||||
except Exception as exc:
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user