mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
da642f12fa | ||
|
|
5376f046ca | ||
|
|
56d9e9a857 |
@@ -41,6 +41,7 @@ import {
|
||||
} from "./plugins";
|
||||
import { explorationEffectsShouldLoad, neighborhoodPanelShouldLoad, temporalOverlayShouldLoad } from "./pluginRegistryPredicates";
|
||||
import { shouldFetchTemporalBounds, shouldFetchTemporalSnapshot } from "./temporalLifecyclePredicates";
|
||||
import { createTemporalSnapshotGuards, type TemporalSnapshotResponse } from "./temporalSnapshotGuards";
|
||||
import type { LinkPrediction, PathResponse } from "./GraphInspectorPanel";
|
||||
import type { GraphSceneHandle, GraphSceneRuntime } from "./scene";
|
||||
import type {
|
||||
@@ -1479,6 +1480,23 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
summary?.edgeCount,
|
||||
]);
|
||||
|
||||
// Guards the snapshot lifecycle: at most one in-flight request per scrubber
|
||||
// position (identical-`at` polls are deduplicated, breaking the idle/play
|
||||
// polling loop), applied snapshots are cached and re-applied on revisit, and
|
||||
// a response applies only while the scrubber is still on its position
|
||||
// (out-of-order responses cannot clobber the active-node count).
|
||||
const temporalSnapshotGuardsRef = useRef<ReturnType<typeof createTemporalSnapshotGuards> | null>(null);
|
||||
if (temporalSnapshotGuardsRef.current === null) {
|
||||
temporalSnapshotGuardsRef.current = createTemporalSnapshotGuards();
|
||||
}
|
||||
const temporalSnapshotGuards = temporalSnapshotGuardsRef.current;
|
||||
|
||||
// A new graph summary means the graph data was replaced (reload/retry);
|
||||
// snapshots cached against the previous graph are stale, so reset all state.
|
||||
useEffect(() => {
|
||||
temporalSnapshotGuards.reset();
|
||||
}, [summary]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!canFetchTemporalSnapshot) {
|
||||
return;
|
||||
@@ -1488,37 +1506,67 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return;
|
||||
}
|
||||
|
||||
const atMs = debouncedTime.getTime();
|
||||
const { seq, cached } = temporalSnapshotGuards.begin(atMs);
|
||||
if (seq === null) {
|
||||
// An identical request is already in flight: one request per position.
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
const applyData = (data: TemporalSnapshotResponse) => {
|
||||
const nextActiveIds = new Set(data.active_node_ids);
|
||||
requestAnimationFrame(() => {
|
||||
if (cancelled) return;
|
||||
if (!temporalSnapshotGuards.shouldApply(atMs, seq)) {
|
||||
// The scrubber moved on (or this request was superseded): release the
|
||||
// position so a return to it refetches instead of stalling.
|
||||
temporalSnapshotGuards.finish(atMs, seq);
|
||||
return;
|
||||
}
|
||||
const previous = prevActiveIdsRef.current;
|
||||
previous.forEach((id) => {
|
||||
if (!nextActiveIds.has(id) && graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", true);
|
||||
}
|
||||
});
|
||||
nextActiveIds.forEach((id) => {
|
||||
if (graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", false);
|
||||
}
|
||||
});
|
||||
prevActiveIdsRef.current = nextActiveIds;
|
||||
setActiveNodeCount(data.active_node_count);
|
||||
setGraphVersion((current) => current + 1);
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
temporalSnapshotGuards.apply(atMs, seq, data);
|
||||
});
|
||||
};
|
||||
|
||||
if (cached) {
|
||||
// Returning to a position whose snapshot was already applied: re-apply
|
||||
// the cached result without a network request.
|
||||
applyData(cached);
|
||||
return;
|
||||
}
|
||||
|
||||
const applySnapshot = async () => {
|
||||
try {
|
||||
const at = debouncedTime.toISOString();
|
||||
const response = await fetch(`/api/temporal/snapshot?at=${encodeURIComponent(at)}`);
|
||||
if (!response.ok || cancelled) return;
|
||||
|
||||
const data: { active_node_ids: string[]; active_node_count: number } = await response.json();
|
||||
if (!response.ok) {
|
||||
// A failed request must be retryable if the scrubber returns.
|
||||
if (!cancelled) temporalSnapshotGuards.finish(atMs, seq);
|
||||
return;
|
||||
}
|
||||
if (cancelled) return;
|
||||
|
||||
const nextActiveIds = new Set(data.active_node_ids);
|
||||
requestAnimationFrame(() => {
|
||||
if (cancelled) return;
|
||||
const previous = prevActiveIdsRef.current;
|
||||
previous.forEach((id) => {
|
||||
if (!nextActiveIds.has(id) && graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", true);
|
||||
}
|
||||
});
|
||||
nextActiveIds.forEach((id) => {
|
||||
if (graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", false);
|
||||
}
|
||||
});
|
||||
prevActiveIdsRef.current = nextActiveIds;
|
||||
setActiveNodeCount(data.active_node_count);
|
||||
setGraphVersion((current) => current + 1);
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
});
|
||||
const data: TemporalSnapshotResponse = await response.json();
|
||||
if (cancelled) return;
|
||||
applyData(data);
|
||||
} catch (fetchError) {
|
||||
temporalSnapshotGuards.finish(atMs, seq);
|
||||
if (!cancelled) {
|
||||
console.error("[Temporal] Snapshot fetch failed", fetchError);
|
||||
}
|
||||
@@ -1528,6 +1576,8 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
applySnapshot();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
// A cancelled request must be retryable when its position is revisited.
|
||||
temporalSnapshotGuards.finish(atMs, seq);
|
||||
};
|
||||
}, [
|
||||
canFetchTemporalSnapshot,
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* Guards for the temporal snapshot fetch/apply lifecycle.
|
||||
*
|
||||
* The snapshot effect previously fetched /api/temporal/snapshot with no
|
||||
* idempotency or ordering protection. Upstream churn (timeline recreation
|
||||
* while bounds settle, play ticks resetting the playhead, drag events) could
|
||||
* re-request the same `at` repeatedly, and responses could arrive after the
|
||||
* scrubber had moved on.
|
||||
*
|
||||
* The guards enforce:
|
||||
* - at most one in-flight request per scrubber position (identical `at`
|
||||
* values are deduplicated while a request is pending, breaking the
|
||||
* idle/play polling loop);
|
||||
* - successful snapshots are cached per position and re-applied when the
|
||||
* scrubber returns (play wrap-around, back-scrubbing) without a refetch;
|
||||
* - a response is applied only while the scrubber is still on its position,
|
||||
* so out-of-order responses cannot clobber a newer position's count;
|
||||
* - failed, cancelled, or superseded requests release their position so it
|
||||
* can be fetched again on the next visit;
|
||||
* - `reset()` drops all state when the underlying graph data is replaced
|
||||
* (reload/retry), because cached snapshots describe the previous graph.
|
||||
*
|
||||
* `createTemporalSnapshotGuards()` is stateful by design.
|
||||
*/
|
||||
|
||||
export interface TemporalSnapshotResponse {
|
||||
active_node_ids: string[];
|
||||
active_node_count: number;
|
||||
}
|
||||
|
||||
export interface TemporalSnapshotRequest {
|
||||
/** null when the request was deduplicated because one is already in flight. */
|
||||
seq: number | null;
|
||||
/** The snapshot previously applied for this position, when revisiting it. */
|
||||
cached: TemporalSnapshotResponse | null;
|
||||
}
|
||||
|
||||
export interface TemporalSnapshotGuards {
|
||||
/** Begin (or dedupe) a request for `atMs`; marks it as the current position. */
|
||||
begin(atMs: number): TemporalSnapshotRequest;
|
||||
/** True when the response for `atMs`/`seq` may be applied (scrubber still on `atMs`). */
|
||||
shouldApply(atMs: number, seq: number): boolean;
|
||||
/** Record a successful application and cache its snapshot for revisits. */
|
||||
apply(atMs: number, seq: number, data: TemporalSnapshotResponse): void;
|
||||
/** Release a position whose request failed, was cancelled, or was superseded. */
|
||||
finish(atMs: number, seq: number): void;
|
||||
/** Drop all state; call when the underlying graph data is replaced (reload). */
|
||||
reset(): void;
|
||||
}
|
||||
|
||||
interface SnapshotEntry {
|
||||
seq: number;
|
||||
/** null while the request is in flight (or before the first success). */
|
||||
data: TemporalSnapshotResponse | null;
|
||||
}
|
||||
|
||||
/** Upper bound on cached positions so long scrubbing sessions stay bounded. */
|
||||
const MAX_CACHED_POSITIONS = 256;
|
||||
|
||||
export function createTemporalSnapshotGuards(): TemporalSnapshotGuards {
|
||||
const entries = new Map<number, SnapshotEntry>();
|
||||
let latestRequestSeq = 0;
|
||||
let currentAtMs: number | null = null;
|
||||
|
||||
const evictOldest = () => {
|
||||
while (entries.size > MAX_CACHED_POSITIONS) {
|
||||
const oldestAtMs = entries.keys().next().value;
|
||||
if (oldestAtMs === undefined) return;
|
||||
entries.delete(oldestAtMs);
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
begin(atMs) {
|
||||
const existing = entries.get(atMs);
|
||||
if (existing && existing.data === null) {
|
||||
// Identical request already in flight: dedupe, but the scrubber is here now.
|
||||
currentAtMs = atMs;
|
||||
return { seq: null, cached: null };
|
||||
}
|
||||
latestRequestSeq += 1;
|
||||
const seq = latestRequestSeq;
|
||||
entries.set(atMs, { seq, data: existing?.data ?? null });
|
||||
currentAtMs = atMs;
|
||||
evictOldest();
|
||||
return { seq, cached: existing?.data ?? null };
|
||||
},
|
||||
|
||||
shouldApply(atMs, seq) {
|
||||
return atMs === currentAtMs && entries.get(atMs)?.seq === seq;
|
||||
},
|
||||
|
||||
apply(atMs, seq, data) {
|
||||
const entry = entries.get(atMs);
|
||||
if (entry && entry.seq === seq) {
|
||||
entry.data = data;
|
||||
}
|
||||
},
|
||||
|
||||
finish(atMs, seq) {
|
||||
const entry = entries.get(atMs);
|
||||
if (entry && entry.seq === seq && entry.data === null) {
|
||||
entries.delete(atMs);
|
||||
}
|
||||
},
|
||||
|
||||
reset() {
|
||||
entries.clear();
|
||||
latestRequestSeq = 0;
|
||||
currentAtMs = null;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { createTemporalSnapshotGuards } from "../src/workspaces/GraphWorkspace/temporalSnapshotGuards.ts";
|
||||
|
||||
const POSITION_1 = new Date("2023-07-02T00:00:00Z").getTime();
|
||||
const POSITION_2 = new Date("2024-01-02T00:00:00Z").getTime();
|
||||
const POSITION_3 = new Date("2024-07-02T00:00:00Z").getTime();
|
||||
|
||||
const SNAPSHOT = { active_node_ids: ["n1", "n2"], active_node_count: 2 };
|
||||
|
||||
// ── begin: one request per scrubber position ─────────────────────────────────
|
||||
|
||||
test("begin: a new position returns a fresh request sequence", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
assert.deepEqual(guards.begin(POSITION_1), { seq: 1, cached: null });
|
||||
});
|
||||
|
||||
test("begin: an identical in-flight request is deduplicated (no duplicate fetch)", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
guards.begin(POSITION_1);
|
||||
assert.deepEqual(guards.begin(POSITION_1), { seq: null, cached: null });
|
||||
});
|
||||
|
||||
test("begin: distinct positions request independently", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
assert.equal(guards.begin(POSITION_1).seq, 1);
|
||||
assert.equal(guards.begin(POSITION_2).seq, 2);
|
||||
});
|
||||
|
||||
test("begin: revisiting an applied position returns its cached snapshot", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
const revisit = guards.begin(POSITION_1);
|
||||
assert.equal(revisit.seq, 2);
|
||||
assert.deepEqual(revisit.cached, SNAPSHOT);
|
||||
});
|
||||
|
||||
test("begin: a failed position (finished) can be requested again", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.finish(POSITION_1, seq);
|
||||
const retry = guards.begin(POSITION_1);
|
||||
assert.equal(retry.seq, 2);
|
||||
assert.equal(retry.cached, null);
|
||||
});
|
||||
|
||||
test("finish: does not clear a position whose snapshot was already applied", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.finish(POSITION_1, seq);
|
||||
assert.deepEqual(guards.begin(POSITION_1).cached, SNAPSHOT);
|
||||
});
|
||||
|
||||
test("finish: a stale sequence cannot release a newer request's position", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const first = guards.begin(POSITION_1);
|
||||
guards.finish(POSITION_1, first.seq);
|
||||
guards.begin(POSITION_1); // seq 2, in flight again
|
||||
guards.finish(POSITION_1, first.seq); // stale seq: must not release seq 2
|
||||
assert.deepEqual(guards.begin(POSITION_1), { seq: null, cached: null });
|
||||
});
|
||||
|
||||
// ── shouldApply: applied only while the scrubber is on that position ─────────
|
||||
|
||||
test("shouldApply: the current position's response is applied", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq), true);
|
||||
});
|
||||
|
||||
test("shouldApply: a response for a position the scrubber left is discarded", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq: seq1 } = guards.begin(POSITION_1);
|
||||
guards.begin(POSITION_2);
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq1), false);
|
||||
assert.equal(guards.shouldApply(POSITION_2, 2), true);
|
||||
});
|
||||
|
||||
test("shouldApply: a late response for the position the scrubber returned to is applied", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq: seq1 } = guards.begin(POSITION_1);
|
||||
const { seq: seq2 } = guards.begin(POSITION_2);
|
||||
guards.begin(POSITION_1); // back to 1: deduplicated, no new request
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq1), true);
|
||||
assert.equal(guards.shouldApply(POSITION_2, seq2), false);
|
||||
});
|
||||
|
||||
test("shouldApply: an unknown sequence is discarded", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
guards.begin(POSITION_1);
|
||||
assert.equal(guards.shouldApply(POSITION_1, 99), false);
|
||||
});
|
||||
|
||||
test("shouldApply: after a reset no pre-reset response applies", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.reset();
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq), false);
|
||||
});
|
||||
|
||||
// ── apply: caching for revisits ─────────────────────────────────────────────
|
||||
|
||||
test("apply: stores the snapshot so a revisit re-applies it without a request", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.begin(POSITION_2);
|
||||
assert.deepEqual(guards.begin(POSITION_1).cached, SNAPSHOT);
|
||||
});
|
||||
|
||||
test("apply: play wrap-around re-applies the wrapped-to position's snapshot", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.begin(POSITION_2);
|
||||
guards.begin(POSITION_3);
|
||||
const wrap = guards.begin(POSITION_1);
|
||||
assert.deepEqual(wrap.cached, SNAPSHOT);
|
||||
assert.equal(guards.shouldApply(POSITION_1, wrap.seq), true);
|
||||
});
|
||||
|
||||
// ── reset: graph reload ─────────────────────────────────────────────────────
|
||||
|
||||
test("reset: clears requested and cached state so positions refetch", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.reset();
|
||||
const fresh = guards.begin(POSITION_1);
|
||||
assert.equal(fresh.seq, 1);
|
||||
assert.equal(fresh.cached, null);
|
||||
});
|
||||
|
||||
// ── cache bound ─────────────────────────────────────────────────────────────
|
||||
|
||||
test("cache: oldest positions are evicted when the cache is full", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const count = 300;
|
||||
for (let i = 0; i < count; i++) {
|
||||
const { seq } = guards.begin(POSITION_1 + i * 1000);
|
||||
guards.apply(POSITION_1 + i * 1000, seq, SNAPSHOT);
|
||||
}
|
||||
const oldest = guards.begin(POSITION_1);
|
||||
assert.equal(oldest.cached, null); // evicted: must refetch on revisit
|
||||
const newest = guards.begin(POSITION_1 + (count - 1) * 1000);
|
||||
assert.deepEqual(newest.cached, SNAPSHOT); // still cached
|
||||
});
|
||||
@@ -306,10 +306,10 @@ class JSONExporter:
|
||||
|
||||
self.logger.debug(f"Exporting {len(entities)} entity(ies) to JSON")
|
||||
|
||||
# Build JSON data with JSON-LD context
|
||||
# Build JSON data with JSON-LD context. No @vocab: it would expand
|
||||
# every bare key in the caller's entity dicts into ns# (#1146).
|
||||
json_data = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"entities": {"@id": "semantica:entities", "@container": "@list"},
|
||||
},
|
||||
@@ -339,7 +339,6 @@ class JSONExporter:
|
||||
"""
|
||||
json_data = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"relationships": {
|
||||
"@id": "semantica:relationships",
|
||||
@@ -434,11 +433,14 @@ class JSONExporter:
|
||||
Returns:
|
||||
Dictionary in JSON-LD format with @context, @graph/@value, and metadata
|
||||
"""
|
||||
# Initialize JSON-LD structure with context
|
||||
# Initialize JSON-LD structure with context. No @vocab: for a generic
|
||||
# payload it turned whatever bare keys the caller happened to use into
|
||||
# ns# terms (#1146). Undeclared terms now simply expand to nothing,
|
||||
# which is standard JSON-LD behaviour for a context that does not
|
||||
# know them; the raw payload is still in the document.
|
||||
jsonld = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": "https://semantica.dev/ns#",
|
||||
"semantica": SEMANTICA_NS,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -598,13 +600,21 @@ class JSONExporter:
|
||||
Returns:
|
||||
Dictionary in JSON-LD format with @context, @id, @type, and graph data
|
||||
"""
|
||||
# Initialize JSON-LD structure with RDF context
|
||||
# Initialize JSON-LD structure with RDF context. No @vocab: it applied
|
||||
# to every bare term in caller data, so an extracted type like "ORG"
|
||||
# became ns#ORG and a metadata key like "source" collided with the
|
||||
# real sem:source object property (#1146). Only explicit semantica:
|
||||
# terms resolve now, and the caller's metadata dict is typed @json so
|
||||
# it survives as one rdf:JSON literal instead of expanding its keys.
|
||||
jsonld = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": "https://semantica.dev/ns#",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
||||
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
||||
"semantica:metadata": {
|
||||
"@id": "semantica:metadata",
|
||||
"@type": "@json",
|
||||
},
|
||||
},
|
||||
# Minted from the graph's own content rather than the wall clock
|
||||
# (#1147): re-exporting an unchanged graph must produce the same
|
||||
@@ -664,14 +674,23 @@ class JSONExporter:
|
||||
entity_text = entity.get("text") or entity.get("label", "unknown")
|
||||
entity_id = entity.get("id") or mint_entity_iri(entity_text)
|
||||
|
||||
# The caller's type label is data, not a class we define: minting it
|
||||
# into @type expanded it through @vocab into ns#ORG and friends, terms
|
||||
# that look official but do not exist (#1146). The node is always a
|
||||
# semantica:Entity and the label travels as semantica:type, exactly
|
||||
# how _relationship_to_jsonld has always carried the relationship type.
|
||||
jsonld = {
|
||||
"@id": entity_id,
|
||||
"@type": entity.get("type") or "semantica:Entity",
|
||||
"@type": "semantica:Entity",
|
||||
"semantica:text": entity.get("text") or entity.get("label", ""),
|
||||
"semantica:confidence": entity.get("confidence", 1.0),
|
||||
}
|
||||
entity_type = entity.get("type")
|
||||
if entity_type:
|
||||
jsonld["semantica:type"] = entity_type
|
||||
|
||||
# Add metadata if present
|
||||
# Add metadata if present. The @json term definition on
|
||||
# semantica:metadata keeps the whole dict one rdf:JSON literal.
|
||||
if "metadata" in entity:
|
||||
jsonld["semantica:metadata"] = entity["metadata"]
|
||||
|
||||
|
||||
@@ -1226,11 +1226,14 @@ class RDFSerializer:
|
||||
metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None))
|
||||
graph_uri: Optional[str] = options.pop("graph_uri", None)
|
||||
|
||||
# Initialize JSON-LD structure with context
|
||||
# Initialize JSON-LD structure with context. No @vocab: it applied to
|
||||
# every bare term in caller data, so an extracted type like "ORG"
|
||||
# became ns#ORG and a metadata key like "source" collided with the
|
||||
# real sem:source object property (#1146). Only explicit semantica:
|
||||
# terms resolve now.
|
||||
jsonld = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": "https://semantica.dev/ns#",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
||||
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
||||
},
|
||||
@@ -1252,11 +1255,20 @@ class RDFSerializer:
|
||||
# and was dropped in full by a JSON-LD parser, silently.
|
||||
entity_id = entity.get("id") or mint_entity_iri(entity.get("text", ""))
|
||||
|
||||
# The caller's type label is data, not a class we define: minting
|
||||
# it into @type expanded it through @vocab into ns#ORG and
|
||||
# friends, terms that look official but do not exist (#1146).
|
||||
# The node is always a semantica:Entity and the label travels as
|
||||
# semantica:type, matching the relationship node below and
|
||||
# JSONExporter._entity_to_jsonld.
|
||||
node = {
|
||||
"@id": entity_id,
|
||||
"@type": entity.get("type", "semantica:Entity"),
|
||||
"@type": "semantica:Entity",
|
||||
"semantica:text": entity.get("text") or entity.get("label", ""),
|
||||
}
|
||||
entity_type = entity.get("type")
|
||||
if entity_type:
|
||||
node["semantica:type"] = entity_type
|
||||
confidence = normalize_confidence(entity.get("confidence", 1.0))
|
||||
if confidence is None:
|
||||
self.logger.warning(
|
||||
|
||||
@@ -117,6 +117,8 @@ class PropertyGenerator:
|
||||
data_properties = self._infer_data_properties(entities, classes, **options)
|
||||
properties.extend(data_properties)
|
||||
|
||||
properties = self._coalesce_normalized_properties(properties)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
@@ -193,6 +195,67 @@ class PropertyGenerator:
|
||||
|
||||
return properties
|
||||
|
||||
def _coalesce_normalized_properties(
|
||||
self, properties: List[Dict[str, Any]]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Merge same-kind properties that normalize to the same name."""
|
||||
property_kinds = defaultdict(set)
|
||||
for prop in properties:
|
||||
property_kinds[prop["name"]].add(prop.get("type"))
|
||||
|
||||
collisions = {
|
||||
name: sorted(kind for kind in kinds if kind is not None)
|
||||
for name, kinds in property_kinds.items()
|
||||
if len({kind for kind in kinds if kind is not None}) > 1
|
||||
}
|
||||
if collisions:
|
||||
raise ValidationError(
|
||||
"Normalized property names cannot be shared by object and "
|
||||
"data properties.",
|
||||
validation_context={"property_kind_collisions": collisions},
|
||||
)
|
||||
|
||||
merged: Dict[tuple, Dict[str, Any]] = {}
|
||||
result = []
|
||||
for prop in properties:
|
||||
key = (prop.get("type"), prop["name"])
|
||||
existing = merged.get(key)
|
||||
if existing is None:
|
||||
merged[key] = prop
|
||||
result.append(prop)
|
||||
continue
|
||||
|
||||
existing["domain"] = self._merge_property_values(
|
||||
existing.get("domain", []), prop.get("domain", [])
|
||||
)
|
||||
if prop.get("type") == "object":
|
||||
existing["range"] = self._merge_property_values(
|
||||
existing.get("range", []), prop.get("range", [])
|
||||
)
|
||||
existing_metadata = existing.setdefault("metadata", {})
|
||||
existing_metadata["occurrence_count"] = (
|
||||
existing_metadata.get("occurrence_count", 0)
|
||||
+ prop.get("metadata", {}).get("occurrence_count", 0)
|
||||
)
|
||||
elif existing.get("range") != prop.get("range"):
|
||||
existing["range"] = self._get_more_general_type(
|
||||
existing["range"], prop["range"]
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _merge_property_values(current: Any, incoming: Any) -> List[Any]:
|
||||
"""Merge scalar-or-list property values while preserving input order."""
|
||||
values = list(current) if isinstance(current, list) else [current]
|
||||
incoming_values = (
|
||||
incoming if isinstance(incoming, list) else [incoming]
|
||||
)
|
||||
for value in incoming_values:
|
||||
if value not in values:
|
||||
values.append(value)
|
||||
return [value for value in values if value is not None]
|
||||
|
||||
def _infer_data_properties(
|
||||
self, entities: List[Dict[str, Any]], classes: List[Dict[str, Any]], **options
|
||||
) -> List[Dict[str, Any]]:
|
||||
|
||||
@@ -70,7 +70,8 @@ sem:metadata a owl:AnnotationProperty ;
|
||||
rdfs:label "metadata" ;
|
||||
rdfs:comment """Free-form metadata carried through from extraction. An
|
||||
annotation property because its value is an arbitrary structure rather than a
|
||||
modelled one.""" ;
|
||||
modelled one; in the JSON-LD export the whole mapping is written as one
|
||||
rdf:JSON literal so caller keys never expand into this namespace (#1146).""" ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
# ── Relationship terms (JSON-LD export) ──────────────────────────────────────
|
||||
@@ -96,10 +97,10 @@ sem:target a owl:ObjectProperty ;
|
||||
|
||||
sem:type a owl:DatatypeProperty ;
|
||||
rdfs:label "type" ;
|
||||
rdfs:comment """The relationship type as a label, as emitted in the JSON-LD
|
||||
export. Distinct from rdf:type, which relates a node to a class rather than to
|
||||
a string.""" ;
|
||||
rdfs:domain sem:Relationship ;
|
||||
rdfs:comment """The entity or relationship type as a label, as emitted in
|
||||
the JSON-LD export. Distinct from rdf:type, which relates a node to a class
|
||||
rather than to a string. Emitted for both entities and relationships, so the
|
||||
domain is left open rather than tied to sem:Relationship.""" ;
|
||||
rdfs:range xsd:string ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
|
||||
@@ -67,7 +67,7 @@ def test_merging_repeated_exports_yields_one_graph_node(tmp_path):
|
||||
assert len(kg_nodes) == 1
|
||||
|
||||
entity_nodes = set(
|
||||
merged.subjects(RDF.type, URIRef("https://semantica.dev/vocab/ORG"))
|
||||
merged.subjects(RDF.type, URIRef("https://semantica.dev/ns#Entity"))
|
||||
)
|
||||
assert len(entity_nodes) == 1
|
||||
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Caller data must never expand into the Semantica namespace (#1146).
|
||||
|
||||
``@vocab`` used to sit in every JSON-LD context pointing at ``ns#``, so every
|
||||
bare term in caller data expanded into it: an extracted type like ``"ORG"``
|
||||
became ``ns#ORG`` (a term the vocabulary does not define), and a metadata key
|
||||
like ``"source"`` collided with the real ``sem:source`` object property,
|
||||
attaching a plain string to a property whose range is a resource. The fix
|
||||
removes ``@vocab`` outright: only explicit ``semantica:``-prefixed terms
|
||||
resolve, caller type labels travel as ``semantica:type`` strings, and caller
|
||||
metadata survives as one ``rdf:JSON`` literal.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from rdflib import RDF, Graph, Literal, URIRef
|
||||
|
||||
from semantica.export.json_exporter import JSONExporter
|
||||
from semantica.export.rdf_exporter import RDFExporter, SEMANTICA_NS
|
||||
|
||||
NS = SEMANTICA_NS
|
||||
E1 = "https://example.org/e1"
|
||||
|
||||
KG = {
|
||||
"entities": [
|
||||
{
|
||||
"id": E1,
|
||||
"text": "Acme",
|
||||
"type": "ORG",
|
||||
"metadata": {"source": "crm_export_2024"},
|
||||
}
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"source_id": E1,
|
||||
"target_id": "https://example.org/e2",
|
||||
"type": "employs",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _jsonld_file(exporter, kind, tmp_path, name):
|
||||
path = tmp_path / name
|
||||
if kind == "knowledge_graph":
|
||||
exporter.export_knowledge_graph(KG, path, format="json-ld")
|
||||
elif kind == "entities":
|
||||
exporter.export_entities(KG["entities"], path, format="json-ld")
|
||||
elif kind == "relationships":
|
||||
exporter.export_relationships(KG["relationships"], path, format="json-ld")
|
||||
elif kind == "generic":
|
||||
exporter.export({"note": "plain payload, no @id"}, path, format="json-ld")
|
||||
else:
|
||||
raise AssertionError(kind)
|
||||
return json.loads(path.read_text())
|
||||
|
||||
|
||||
def test_no_jsonld_context_declares_a_vocab(tmp_path):
|
||||
exporter = JSONExporter()
|
||||
for kind in ("knowledge_graph", "entities", "relationships", "generic"):
|
||||
context = _jsonld_file(exporter, kind, tmp_path, f"{kind}.jsonld")[
|
||||
"@context"
|
||||
]
|
||||
assert "@vocab" not in context, f"{kind}: @vocab expands caller data"
|
||||
|
||||
context = json.loads(RDFExporter().export_to_rdf(KG, format="jsonld"))[
|
||||
"@context"
|
||||
]
|
||||
assert "@vocab" not in context
|
||||
|
||||
|
||||
def test_extracted_type_labels_stay_out_of_the_namespace(tmp_path):
|
||||
path = tmp_path / "kg.jsonld"
|
||||
JSONExporter().export_knowledge_graph(KG, path, format="json-ld")
|
||||
|
||||
graph = Graph()
|
||||
graph.parse(str(path), format="json-ld")
|
||||
|
||||
assert (None, RDF.type, URIRef(NS + "ORG")) not in graph, (
|
||||
"the caller's type label was minted as a class in ns#"
|
||||
)
|
||||
assert (URIRef(E1), RDF.type, URIRef(NS + "Entity")) in graph
|
||||
assert (URIRef(E1), URIRef(NS + "type"), Literal("ORG")) in graph, (
|
||||
"the label itself must survive, as a string"
|
||||
)
|
||||
|
||||
|
||||
def test_metadata_keys_stay_out_of_the_namespace(tmp_path):
|
||||
path = tmp_path / "kg.jsonld"
|
||||
JSONExporter().export_knowledge_graph(KG, path, format="json-ld")
|
||||
|
||||
graph = Graph()
|
||||
graph.parse(str(path), format="json-ld")
|
||||
|
||||
assert (None, URIRef(NS + "source"), Literal("crm_export_2024")) not in (
|
||||
graph
|
||||
), "caller metadata value attached to the real sem:source object property"
|
||||
for _, _, o in graph.triples((None, URIRef(NS + "source"), None)):
|
||||
assert not isinstance(o, Literal), (
|
||||
"sem:source has a resource range but received a plain literal"
|
||||
)
|
||||
|
||||
literals = [
|
||||
o
|
||||
for o in graph.objects(None, URIRef(NS + "metadata"))
|
||||
if isinstance(o, Literal)
|
||||
]
|
||||
assert literals, "the metadata dict was dropped instead of preserved"
|
||||
assert literals[0].datatype == RDF.JSON
|
||||
assert json.loads(str(literals[0])) == {"source": "crm_export_2024"}
|
||||
|
||||
|
||||
def test_rdf_exporter_jsonld_keeps_type_labels_out_of_the_namespace():
|
||||
graph = Graph()
|
||||
graph.parse(
|
||||
data=RDFExporter().export_to_rdf(KG, format="jsonld"), format="json-ld"
|
||||
)
|
||||
|
||||
assert (None, RDF.type, URIRef(NS + "ORG")) not in graph
|
||||
assert (URIRef(E1), RDF.type, URIRef(NS + "Entity")) in graph
|
||||
assert (URIRef(E1), URIRef(NS + "type"), Literal("ORG")) in graph
|
||||
@@ -0,0 +1,51 @@
|
||||
import pytest
|
||||
|
||||
from semantica.ontology.class_inferrer import ClassInferrer
|
||||
from semantica.ontology.property_generator import PropertyGenerator
|
||||
from semantica.utils.exceptions import ValidationError
|
||||
|
||||
|
||||
def test_same_kind_normalized_object_properties_are_merged():
|
||||
entities = [
|
||||
{"id": "p1", "type": "Person", "name": "Alice"},
|
||||
{"id": "o1", "type": "Organization", "name": "Acme"},
|
||||
]
|
||||
classes = ClassInferrer(min_occurrences=1).infer_classes(entities)
|
||||
relationships = [
|
||||
{
|
||||
"source_type": "Person",
|
||||
"target_type": "Organization",
|
||||
"type": "works_for",
|
||||
},
|
||||
{
|
||||
"source_type": "Person",
|
||||
"target_type": "Organization",
|
||||
"type": "worksFor",
|
||||
},
|
||||
]
|
||||
|
||||
properties = PropertyGenerator().infer_properties(
|
||||
entities, relationships, classes, min_occurrences=1
|
||||
)
|
||||
|
||||
works_for = [prop for prop in properties if prop["name"] == "worksFor"]
|
||||
assert len(works_for) == 1
|
||||
assert works_for[0]["domain"] == ["Person"]
|
||||
assert works_for[0]["range"] == ["Organization"]
|
||||
|
||||
|
||||
def test_normalized_name_cannot_be_both_object_and_data_property():
|
||||
entities = [
|
||||
{"id": "p1", "type": "Person", "value": "Alice"},
|
||||
{"id": "p2", "type": "Person", "value": "Bob"},
|
||||
]
|
||||
classes = ClassInferrer(min_occurrences=1).infer_classes(entities)
|
||||
relationships = [
|
||||
{"source_type": "Person", "target_type": "Person", "type": "value"},
|
||||
{"source_type": "Person", "target_type": "Person", "type": "value"},
|
||||
]
|
||||
|
||||
with pytest.raises(ValidationError, match="object and data"):
|
||||
PropertyGenerator().infer_properties(
|
||||
entities, relationships, classes, min_occurrences=1
|
||||
)
|
||||
Reference in New Issue
Block a user