mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Merge branch 'main' into dependabot/pip/pypickle-2.0.2
This commit is contained in:
@@ -1,8 +1,9 @@
|
|||||||
import { useState, useRef, useEffect, type CSSProperties } from "react";
|
import { useState, useRef, useEffect, useMemo, type CSSProperties } from "react";
|
||||||
import ReactMarkdown from "react-markdown";
|
import ReactMarkdown, { type Components } from "react-markdown";
|
||||||
import remarkGfm from "remark-gfm";
|
import remarkGfm from "remark-gfm";
|
||||||
import { Check, Copy, Code2, Eye, ExternalLink, Image as ImageIcon } from "lucide-react";
|
import { Check, Copy, Code2, Eye, ExternalLink, Image as ImageIcon } from "lucide-react";
|
||||||
import { GRAPH_THEME } from "./graphTheme";
|
import { GRAPH_THEME } from "./graphTheme";
|
||||||
|
import { isSafeUrl } from "./markdownUrlSafety";
|
||||||
|
|
||||||
export interface MarkdownContentViewerProps {
|
export interface MarkdownContentViewerProps {
|
||||||
content?: string | null;
|
content?: string | null;
|
||||||
@@ -10,25 +11,6 @@ export interface MarkdownContentViewerProps {
|
|||||||
defaultMode?: "preview" | "source";
|
defaultMode?: "preview" | "source";
|
||||||
}
|
}
|
||||||
|
|
||||||
export function isSafeUrl(url?: string): boolean {
|
|
||||||
if (!url) return false;
|
|
||||||
const trimmed = url.trim();
|
|
||||||
// Reject whitespace-only strings — new URL("", base) would resolve to the base
|
|
||||||
// protocol and produce a false positive. This guards direct callers of the exported
|
|
||||||
// function; markdown parsers normalise whitespace-only destinations to "" which
|
|
||||||
// already fails the !url check above.
|
|
||||||
if (!trimmed) return false;
|
|
||||||
if (trimmed.startsWith("//")) return false;
|
|
||||||
if (trimmed.startsWith("#")) return true;
|
|
||||||
if (trimmed.startsWith("/")) return true;
|
|
||||||
try {
|
|
||||||
const parsed = new URL(trimmed, "http://localhost");
|
|
||||||
return ["http:", "https:", "mailto:"].includes(parsed.protocol);
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export function MarkdownContentViewer({
|
export function MarkdownContentViewer({
|
||||||
content,
|
content,
|
||||||
className,
|
className,
|
||||||
@@ -65,6 +47,20 @@ export function MarkdownContentViewer({
|
|||||||
const rawContent = typeof content === "string" ? content : "";
|
const rawContent = typeof content === "string" ? content : "";
|
||||||
const hasContent = rawContent.trim().length > 0;
|
const hasContent = rawContent.trim().length > 0;
|
||||||
|
|
||||||
|
// react-markdown runs the whole remark pipeline synchronously inside its own
|
||||||
|
// render, so without this memo every unrelated re-render of this component --
|
||||||
|
// clicking Copy, toggling Preview/Source -- re-parses the entire document.
|
||||||
|
// Measured at ~364ms per re-render for a 1000-row GFM table (issue #1118).
|
||||||
|
// Keyed on rawContent so a genuine node change still re-parses exactly once.
|
||||||
|
const renderedMarkdown = useMemo(
|
||||||
|
() => (
|
||||||
|
<ReactMarkdown remarkPlugins={REMARK_PLUGINS} components={MARKDOWN_COMPONENTS}>
|
||||||
|
{rawContent}
|
||||||
|
</ReactMarkdown>
|
||||||
|
),
|
||||||
|
[rawContent],
|
||||||
|
);
|
||||||
|
|
||||||
const handleCopy = async () => {
|
const handleCopy = async () => {
|
||||||
if (!hasContent) return;
|
if (!hasContent) return;
|
||||||
try {
|
try {
|
||||||
@@ -130,10 +126,24 @@ export function MarkdownContentViewer({
|
|||||||
<code style={sourceCodeStyle}>{rawContent}</code>
|
<code style={sourceCodeStyle}>{rawContent}</code>
|
||||||
</pre>
|
</pre>
|
||||||
) : (
|
) : (
|
||||||
<div style={previewStyle}>
|
<div style={previewStyle}>{renderedMarkdown}</div>
|
||||||
<ReactMarkdown
|
)}
|
||||||
remarkPlugins={[remarkGfm]}
|
</div>
|
||||||
components={{
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ─── Markdown rendering config ───────────────────────────────────── */
|
||||||
|
|
||||||
|
// Both props are hoisted to module scope so they keep a stable identity across
|
||||||
|
// renders. As inline literals they allocated a fresh plugin array and ~20 fresh
|
||||||
|
// arrow components on every render, which made React treat every mapped tag as a
|
||||||
|
// new element type and remount the entire rendered subtree instead of updating
|
||||||
|
// it (issue #1118). The arrow bodies only read the style constants below at call
|
||||||
|
// time, so declaring the map before them is safe.
|
||||||
|
const REMARK_PLUGINS = [remarkGfm];
|
||||||
|
|
||||||
|
const MARKDOWN_COMPONENTS: Components = {
|
||||||
// C-1: react-markdown passes a HAST `node` prop (the raw AST
|
// C-1: react-markdown passes a HAST `node` prop (the raw AST
|
||||||
// Element) to every custom component override via passNode:true.
|
// Element) to every custom component override via passNode:true.
|
||||||
// In React 19 any unknown prop spreads onto a native element are
|
// In React 19 any unknown prop spreads onto a native element are
|
||||||
@@ -211,16 +221,7 @@ export function MarkdownContentViewer({
|
|||||||
</code>
|
</code>
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
}}
|
};
|
||||||
>
|
|
||||||
{rawContent}
|
|
||||||
</ReactMarkdown>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ─── Styles ──────────────────────────────────────────────────────── */
|
/* ─── Styles ──────────────────────────────────────────────────────── */
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
/**
|
||||||
|
* URL-safety predicate for the Markdown content viewer.
|
||||||
|
*
|
||||||
|
* Extracted into a pure module so the check can be unit-tested without
|
||||||
|
* importing the MarkdownContentViewer React component, and so the component
|
||||||
|
* module exports only components (react-refresh/only-export-components,
|
||||||
|
* issue #1119). The behaviour is unchanged from the original in-component
|
||||||
|
* implementation: only http, https, mailto, in-document fragments, and
|
||||||
|
* root-relative paths are permitted.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export function isSafeUrl(url?: string): boolean {
|
||||||
|
if (!url) return false;
|
||||||
|
const trimmed = url.trim();
|
||||||
|
// Reject whitespace-only strings — new URL("", base) would resolve to the base
|
||||||
|
// protocol and produce a false positive. This guards direct callers of the exported
|
||||||
|
// function; markdown parsers normalise whitespace-only destinations to "" which
|
||||||
|
// already fails the !url check above.
|
||||||
|
if (!trimmed) return false;
|
||||||
|
if (trimmed.startsWith("//")) return false;
|
||||||
|
if (trimmed.startsWith("#")) return true;
|
||||||
|
if (trimmed.startsWith("/")) return true;
|
||||||
|
try {
|
||||||
|
const parsed = new URL(trimmed, "http://localhost");
|
||||||
|
return ["http:", "https:", "mailto:"].includes(parsed.protocol);
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,7 +5,8 @@ import { renderToString } from "react-dom/server";
|
|||||||
|
|
||||||
(globalThis as any).React = React;
|
(globalThis as any).React = React;
|
||||||
|
|
||||||
import { isSafeUrl, MarkdownContentViewer } from "../src/workspaces/GraphWorkspace/MarkdownContentViewer.tsx";
|
import { MarkdownContentViewer } from "../src/workspaces/GraphWorkspace/MarkdownContentViewer.tsx";
|
||||||
|
import { isSafeUrl } from "../src/workspaces/GraphWorkspace/markdownUrlSafety.ts";
|
||||||
|
|
||||||
test("isSafeUrl permits safe http, https, and mailto URLs and relative paths", () => {
|
test("isSafeUrl permits safe http, https, and mailto URLs and relative paths", () => {
|
||||||
assert.equal(isSafeUrl("https://example.com"), true);
|
assert.equal(isSafeUrl("https://example.com"), true);
|
||||||
|
|||||||
@@ -277,25 +277,22 @@ class AgnoKnowledgeGraph(_KnowledgeBase): # type: ignore[misc]
|
|||||||
def load_urls(self, urls: List[str]) -> None:
|
def load_urls(self, urls: List[str]) -> None:
|
||||||
"""Fetch each URL and ingest the response body.
|
"""Fetch each URL and ingest the response body.
|
||||||
|
|
||||||
Only ``http`` and ``https`` schemes are permitted to prevent SSRF.
|
Uses the shared SSRF guard so that ``http`` and ``https`` are the only
|
||||||
|
permitted schemes, private/loopback/link-local/cloud-metadata addresses
|
||||||
|
are blocked by default, DNS resolution is validated, and every redirect
|
||||||
|
hop is re-checked before being followed.
|
||||||
"""
|
"""
|
||||||
import urllib.request
|
from semantica.ingest.ssrf import request_with_ssrf_guard
|
||||||
from urllib.parse import urlparse
|
from semantica.utils.exceptions import ValidationError
|
||||||
|
|
||||||
for url in urls:
|
for url in urls:
|
||||||
parsed = urlparse(url)
|
|
||||||
if parsed.scheme not in ("http", "https"):
|
|
||||||
logger.warning(
|
|
||||||
"Skipping URL with disallowed scheme '%s': %s",
|
|
||||||
parsed.scheme,
|
|
||||||
url,
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(url, timeout=10) as resp: # noqa: S310
|
response = request_with_ssrf_guard("GET", url, timeout=10)
|
||||||
text = resp.read().decode("utf-8", errors="replace")
|
text = response.text
|
||||||
self._ingest_text(text, source=url)
|
self._ingest_text(text, source=url)
|
||||||
logger.info("Loaded URL: %s", url)
|
logger.info("Loaded URL: %s", url)
|
||||||
|
except ValidationError as exc:
|
||||||
|
logger.warning("Skipping URL (SSRF check failed) %s: %s", url, exc)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.warning("Failed to fetch %s: %s", url, exc)
|
logger.warning("Failed to fetch %s: %s", url, exc)
|
||||||
|
|
||||||
|
|||||||
@@ -116,7 +116,41 @@ class OpenClawKGTool:
|
|||||||
)
|
)
|
||||||
|
|
||||||
def __init__(self, base_url: str = "http://localhost:8000", timeout: int = 30) -> None:
|
def __init__(self, base_url: str = "http://localhost:8000", timeout: int = 30) -> None:
|
||||||
self.base_url = base_url.rstrip("/")
|
# Validate base_url at construction time so callers get an immediate,
|
||||||
|
# actionable error rather than a cryptic failure on the first request.
|
||||||
|
# allow_private_ips=True because the documented default (localhost:8000)
|
||||||
|
# is intentionally a local Semantica server; the scheme check and
|
||||||
|
# URL-structure check still apply unconditionally.
|
||||||
|
try:
|
||||||
|
from semantica.ingest.ssrf import validate_url_for_request
|
||||||
|
validate_url_for_request(base_url, allow_private_ips=True)
|
||||||
|
except ImportError:
|
||||||
|
# semantica.ingest not installed in minimal openclaw-only environments;
|
||||||
|
# mirror the structural checks that validate_url_for_request performs
|
||||||
|
# unconditionally (before allow_private_ips is consulted), so the
|
||||||
|
# guarantee in the comment above — "scheme check and URL-structure check
|
||||||
|
# still apply unconditionally" — holds in this path too.
|
||||||
|
from urllib.parse import urlparse as _urlparse
|
||||||
|
if not isinstance(base_url, str) or not base_url.strip():
|
||||||
|
raise ValueError("OpenClawKGTool base_url must be a non-empty string.")
|
||||||
|
_parsed = _urlparse(base_url.strip())
|
||||||
|
_scheme = (_parsed.scheme or "").lower()
|
||||||
|
if _scheme not in ("http", "https"):
|
||||||
|
raise ValueError(
|
||||||
|
f"OpenClawKGTool base_url scheme '{_parsed.scheme}' is not permitted. "
|
||||||
|
"Only http and https are allowed."
|
||||||
|
)
|
||||||
|
if not _parsed.netloc:
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid OpenClawKGTool base_url '{base_url}': "
|
||||||
|
"URL must include a netloc (domain or host)."
|
||||||
|
)
|
||||||
|
if not _parsed.hostname:
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid OpenClawKGTool base_url '{base_url}': "
|
||||||
|
"URL must include a hostname."
|
||||||
|
)
|
||||||
|
self.base_url = base_url.strip().rstrip("/")
|
||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
self._session: Any = None
|
self._session: Any = None
|
||||||
|
|
||||||
|
|||||||
@@ -21,6 +21,17 @@ Configure in Claude Desktop, Windsurf, Cline, Continue, VS Code:
|
|||||||
}
|
}
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
# MCP stdio framing IS stdout: any progress bar or console renderer that writes
|
||||||
|
# to stdout would interleave with the JSON-RPC stream and corrupt framing for
|
||||||
|
# every client. This package is always used as an MCP stdio server, so force
|
||||||
|
# progress tracking off for the entire process. Set before importing server /
|
||||||
|
# tools so the Semantica progress-tracker singleton is never created with
|
||||||
|
# output enabled (the singleton reads this variable at construction time and
|
||||||
|
# the enabled.setter re-checks it, so later re-enable attempts are also blocked).
|
||||||
|
os.environ["SEMANTICA_DISABLE_PROGRESS"] = "1"
|
||||||
|
|
||||||
# `semantica.__version__` is the authoritative package version — see
|
# `semantica.__version__` is the authoritative package version — see
|
||||||
# semantica/mcp_server/__init__.py for why it is used directly rather than
|
# semantica/mcp_server/__init__.py for why it is used directly rather than
|
||||||
# importlib.metadata.version("semantica").
|
# importlib.metadata.version("semantica").
|
||||||
|
|||||||
+6
-1
@@ -80,7 +80,12 @@ def handle_export_graph(args: dict) -> dict:
|
|||||||
if rdf_fmt:
|
if rdf_fmt:
|
||||||
try:
|
try:
|
||||||
from semantica.export import RDFExporter
|
from semantica.export import RDFExporter
|
||||||
rdf_str = RDFExporter().export_to_rdf(graph, format=rdf_fmt)
|
# RDFExporter.export_to_rdf() expects the canonical kg dict
|
||||||
|
# {"entities": [...], "relationships": [...]}, not a ContextGraph
|
||||||
|
# object. Convert before handing off; passing the raw graph
|
||||||
|
# caused AttributeError: 'ContextGraph' object has no attribute
|
||||||
|
# 'get' on every RDF format.
|
||||||
|
rdf_str = RDFExporter().export_to_rdf(graph.to_kg_dict(), format=rdf_fmt)
|
||||||
return {"format": rdf_fmt, "data": rdf_str}
|
return {"format": rdf_fmt, "data": rdf_str}
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return {"error": f"RDF export failed: {exc}"}
|
return {"error": f"RDF export failed: {exc}"}
|
||||||
|
|||||||
@@ -187,15 +187,6 @@ def poc_vuln3():
|
|||||||
})
|
})
|
||||||
return nodes
|
return nodes
|
||||||
|
|
||||||
# Simulate the CSV parser — mirrors export_import.py lines 131-133
|
|
||||||
def parse_import_csv_row(row: dict) -> dict:
|
|
||||||
"""Mirrors export_import.py CSV node ID extraction (no sanitization)."""
|
|
||||||
node_id = row.get("id") or row.get("node_id") or row.get(":ID") or row.get("_id")
|
|
||||||
return {
|
|
||||||
"id": str(node_id), # ← UNSANITIZED
|
|
||||||
"type": row.get("type", "entity"),
|
|
||||||
}
|
|
||||||
|
|
||||||
# Attack payloads
|
# Attack payloads
|
||||||
payloads = [
|
payloads = [
|
||||||
# Header injection payload (chained with VULN-1)
|
# Header injection payload (chained with VULN-1)
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ import hashlib
|
|||||||
import json
|
import json
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import threading
|
import threading
|
||||||
|
import warnings
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -62,6 +63,12 @@ def create_graph_snapshot_record(
|
|||||||
"""
|
"""
|
||||||
Creates a standardized snapshot metadata record for a named graph.
|
Creates a standardized snapshot metadata record for a named graph.
|
||||||
|
|
||||||
|
.. deprecated::
|
||||||
|
``create_graph_snapshot_record()`` is deprecated and will be removed in
|
||||||
|
a future major version. It has no callers inside Semantica; build the
|
||||||
|
record inline and checksum it with
|
||||||
|
:func:`semantica.change_management.compute_checksum` instead.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
version_id: Unique identifier for this snapshot
|
version_id: Unique identifier for this snapshot
|
||||||
graph_uri: The underlying named graph URI in the triplet store
|
graph_uri: The underlying named graph URI in the triplet store
|
||||||
@@ -69,6 +76,13 @@ def create_graph_snapshot_record(
|
|||||||
description: Purpose or context of the snapshot
|
description: Purpose or context of the snapshot
|
||||||
metadata: Additional tags or pipeline context
|
metadata: Additional tags or pipeline context
|
||||||
"""
|
"""
|
||||||
|
warnings.warn(
|
||||||
|
"create_graph_snapshot_record() is deprecated and will be removed in a "
|
||||||
|
"future major version. Build the snapshot record inline and use "
|
||||||
|
"semantica.change_management.compute_checksum() instead.",
|
||||||
|
DeprecationWarning,
|
||||||
|
stacklevel=2,
|
||||||
|
)
|
||||||
|
|
||||||
record = {
|
record = {
|
||||||
"label": version_id,
|
"label": version_id,
|
||||||
|
|||||||
@@ -899,6 +899,11 @@ class ContextGraph:
|
|||||||
return
|
return
|
||||||
node.properties.update(attributes)
|
node.properties.update(attributes)
|
||||||
node.metadata.update(attributes)
|
node.metadata.update(attributes)
|
||||||
|
# Keep derived decision indexes consistent when a decision node is
|
||||||
|
# mutated so that category / entity / temporal lookups reflect the
|
||||||
|
# new property values without requiring a full graph reload.
|
||||||
|
if (getattr(node, "node_type", None) or "").lower() == "decision":
|
||||||
|
self._sync_decision_from_node(node_id)
|
||||||
|
|
||||||
if getattr(self, "mutation_callback", None) and not getattr(
|
if getattr(self, "mutation_callback", None) and not getattr(
|
||||||
self, "_suspend_mutation_callback", False
|
self, "_suspend_mutation_callback", False
|
||||||
@@ -1291,6 +1296,14 @@ class ContextGraph:
|
|||||||
if link_id:
|
if link_id:
|
||||||
self._unresolved_links[link_id] = link_meta
|
self._unresolved_links[link_id] = link_meta
|
||||||
|
|
||||||
|
# Rebuild all derived decision indexes from the freshly-loaded
|
||||||
|
# nodes so that find_precedents_by_scenario, find_similar_decisions,
|
||||||
|
# and all decision analytics work correctly after a reload.
|
||||||
|
# _rebuild_decision_indexes() unconditionally clears the old indexes
|
||||||
|
# first, so repeated load_from_file calls never accumulate stale
|
||||||
|
# entries from a previous file.
|
||||||
|
self._rebuild_decision_indexes()
|
||||||
|
|
||||||
self.logger.info(f"Loaded context graph from {path}")
|
self.logger.info(f"Loaded context graph from {path}")
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -1633,6 +1646,8 @@ class ContextGraph:
|
|||||||
self._analytics_cache.clear()
|
self._analytics_cache.clear()
|
||||||
self._retractions.clear()
|
self._retractions.clear()
|
||||||
self._tombstones.clear()
|
self._tombstones.clear()
|
||||||
|
# Rebuild derived decision indexes from the freshly-loaded nodes.
|
||||||
|
self._rebuild_decision_indexes()
|
||||||
|
|
||||||
if self.mutation_callback and not self._suspend_mutation_callback:
|
if self.mutation_callback and not self._suspend_mutation_callback:
|
||||||
mutation_events = [
|
mutation_events = [
|
||||||
@@ -2825,6 +2840,12 @@ class ContextGraph:
|
|||||||
self._unresolved_links.clear()
|
self._unresolved_links.clear()
|
||||||
self._retractions.clear()
|
self._retractions.clear()
|
||||||
self._tombstones.clear()
|
self._tombstones.clear()
|
||||||
|
# Reset derived decision indexes so that decision queries against
|
||||||
|
# a cleared graph return empty results rather than stale data.
|
||||||
|
self._decisions = {}
|
||||||
|
self._decision_index = defaultdict(set)
|
||||||
|
self._entity_index = defaultdict(set)
|
||||||
|
self._temporal_index = []
|
||||||
self.logger.debug("Graph state fully cleared.")
|
self.logger.debug("Graph state fully cleared.")
|
||||||
|
|
||||||
# --- Internal Helpers ---
|
# --- Internal Helpers ---
|
||||||
@@ -3482,6 +3503,9 @@ class ContextGraph:
|
|||||||
)
|
)
|
||||||
self._add_internal_edge(edge)
|
self._add_internal_edge(edge)
|
||||||
|
|
||||||
|
# Rebuild derived decision indexes from the now-populated node store.
|
||||||
|
self._rebuild_decision_indexes()
|
||||||
|
|
||||||
def state_at(self, timestamp: Union[str, int, float, datetime]) -> Dict[str, Any]:
|
def state_at(self, timestamp: Union[str, int, float, datetime]) -> Dict[str, Any]:
|
||||||
"""Return a serializable snapshot of graph state valid at the given time."""
|
"""Return a serializable snapshot of graph state valid at the given time."""
|
||||||
at_time = self._normalize_timestamp(timestamp)
|
at_time = self._normalize_timestamp(timestamp)
|
||||||
@@ -4707,6 +4731,7 @@ class ContextGraph:
|
|||||||
scenario=decision["scenario"],
|
scenario=decision["scenario"],
|
||||||
decision_maker=decision.get("decision_maker", ""),
|
decision_maker=decision.get("decision_maker", ""),
|
||||||
reasoning=decision["reasoning"],
|
reasoning=decision["reasoning"],
|
||||||
|
recorded_at=decision.get("recorded_at", ""),
|
||||||
**safe_metadata,
|
**safe_metadata,
|
||||||
**extra_properties,
|
**extra_properties,
|
||||||
)
|
)
|
||||||
@@ -4788,20 +4813,288 @@ class ContextGraph:
|
|||||||
return False
|
return False
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def _calculate_decision_content_similarity(self, scenario: str, decision: Dict[str, Any]) -> float:
|
# ── decision-index helpers ────────────────────────────────────────────────
|
||||||
"""Calculate content similarity between scenario and decision."""
|
|
||||||
|
# Protected set of node properties whose values are *core* decision fields
|
||||||
|
# so that we can distinguish them from user-supplied metadata when
|
||||||
|
# rebuilding the in-memory indexes from a persisted node.
|
||||||
|
_DECISION_CORE_FIELDS: frozenset = frozenset({
|
||||||
|
"id", "category", "scenario", "reasoning", "outcome", "confidence",
|
||||||
|
"entities", "decision_maker", "timestamp", "recorded_at",
|
||||||
|
"valid_from", "valid_until", "content",
|
||||||
|
})
|
||||||
|
|
||||||
|
def _rebuild_decision_indexes(self) -> None:
|
||||||
|
"""Rebuild all derived decision indexes from the current node store.
|
||||||
|
|
||||||
|
This method is the single authoritative rebuild path. It must be
|
||||||
|
called (under the graph lock) after any operation that wholesale
|
||||||
|
replaces ``self.nodes`` — namely ``load_from_file`` (JSON and Markdown
|
||||||
|
paths) and ``from_dict``.
|
||||||
|
|
||||||
|
Contract:
|
||||||
|
- Unconditionally clears ``_decisions``, ``_decision_index``,
|
||||||
|
``_entity_index``, and ``_temporal_index`` before rebuilding so that
|
||||||
|
repeated calls never accumulate stale entries.
|
||||||
|
- Derives ``_decisions[node_id]["metadata"]`` from the full set of
|
||||||
|
node properties, excluding the protected core fields, so that
|
||||||
|
user-supplied metadata survives the round-trip.
|
||||||
|
- Runs under ``self._lock`` when called from load paths; callers that
|
||||||
|
already hold the lock must invoke ``_rebuild_decision_indexes``
|
||||||
|
inside the lock block.
|
||||||
|
"""
|
||||||
|
# Always start fresh so repeated loads don't accumulate stale entries.
|
||||||
|
self._decisions: Dict[str, Any] = {}
|
||||||
|
self._decision_index: Dict[str, set] = defaultdict(set)
|
||||||
|
self._entity_index: Dict[str, set] = defaultdict(set)
|
||||||
|
self._temporal_index: List[Tuple[str, float]] = []
|
||||||
|
|
||||||
|
for node in self.nodes.values():
|
||||||
|
if (getattr(node, "node_type", None) or "").lower() != "decision":
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Merge metadata and properties; properties win on collision.
|
||||||
|
meta: Dict[str, Any] = {}
|
||||||
|
meta.update(getattr(node, "metadata", {}) or {})
|
||||||
|
meta.update(getattr(node, "properties", {}) or {})
|
||||||
|
|
||||||
|
# Timestamp: keep whatever was stored (float epoch or ISO string).
|
||||||
|
# The temporal index uses it for sorting; downstream code handles
|
||||||
|
# both types via _normalize_timestamp.
|
||||||
|
raw_ts = meta.get("timestamp", 0.0)
|
||||||
try:
|
try:
|
||||||
# Simple word-based similarity
|
sort_ts = float(raw_ts)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
sort_ts = 0.0
|
||||||
|
|
||||||
|
# Entities may be stored as a list in meta or inferred from
|
||||||
|
# outgoing "involves" edges if the list field is absent/empty.
|
||||||
|
# _add_decision_to_graph creates entity nodes connected via
|
||||||
|
# "involves" edges; it does NOT store the list as a node property.
|
||||||
|
entities = meta.get("entities") or []
|
||||||
|
if not isinstance(entities, list):
|
||||||
|
entities = []
|
||||||
|
if not entities:
|
||||||
|
# Recover entity list from "involves" edges on this decision node
|
||||||
|
for edge in self._adjacency.get(node.node_id, []):
|
||||||
|
if edge.edge_type == "involves":
|
||||||
|
entities.append(edge.target_id)
|
||||||
|
|
||||||
|
# Everything that isn't a core field is user-supplied metadata.
|
||||||
|
extra_meta = {
|
||||||
|
k: v
|
||||||
|
for k, v in meta.items()
|
||||||
|
if k not in self._DECISION_CORE_FIELDS
|
||||||
|
}
|
||||||
|
|
||||||
|
decision: Dict[str, Any] = {
|
||||||
|
"id": node.node_id,
|
||||||
|
"category": meta.get("category", ""),
|
||||||
|
"scenario": meta.get("scenario", getattr(node, "content", "") or ""),
|
||||||
|
"reasoning": meta.get("reasoning", ""),
|
||||||
|
"outcome": meta.get("outcome", ""),
|
||||||
|
"confidence": float(meta.get("confidence", 0.0) or 0.0),
|
||||||
|
"entities": entities,
|
||||||
|
"decision_maker": meta.get("decision_maker"),
|
||||||
|
"timestamp": raw_ts,
|
||||||
|
"recorded_at": meta.get("recorded_at", ""),
|
||||||
|
"valid_from": getattr(node, "valid_from", None),
|
||||||
|
"valid_until": getattr(node, "valid_until", None),
|
||||||
|
# Preserve all non-core node properties as decision metadata so
|
||||||
|
# that user-supplied fields survive a save → load round-trip.
|
||||||
|
"metadata": extra_meta,
|
||||||
|
}
|
||||||
|
|
||||||
|
self._decisions[node.node_id] = decision
|
||||||
|
|
||||||
|
category = decision["category"]
|
||||||
|
if category:
|
||||||
|
self._decision_index[category].add(node.node_id)
|
||||||
|
|
||||||
|
for entity in entities:
|
||||||
|
self._entity_index[entity].add(node.node_id)
|
||||||
|
|
||||||
|
self._temporal_index.append((node.node_id, sort_ts))
|
||||||
|
|
||||||
|
self._temporal_index.sort(key=lambda x: x[1], reverse=True)
|
||||||
|
|
||||||
|
def _sync_decision_from_node(self, node_id: str) -> None:
|
||||||
|
"""Synchronise a single decision index entry from the node store.
|
||||||
|
|
||||||
|
Called after ``add_node_attribute`` mutates a decision node so that
|
||||||
|
``_decisions`` and the derived indexes stay consistent without
|
||||||
|
requiring a full rebuild of all decisions.
|
||||||
|
"""
|
||||||
|
node = self.nodes.get(node_id)
|
||||||
|
if node is None:
|
||||||
|
return
|
||||||
|
if (getattr(node, "node_type", None) or "").lower() != "decision":
|
||||||
|
return
|
||||||
|
|
||||||
|
if not hasattr(self, "_decisions"):
|
||||||
|
# Indexes don't exist yet — a full rebuild is safer.
|
||||||
|
self._rebuild_decision_indexes()
|
||||||
|
return
|
||||||
|
|
||||||
|
# Remove stale index entries for this decision ID.
|
||||||
|
old = self._decisions.get(node_id)
|
||||||
|
if old:
|
||||||
|
old_cat = old.get("category", "")
|
||||||
|
if old_cat and node_id in self._decision_index.get(old_cat, set()):
|
||||||
|
self._decision_index[old_cat].discard(node_id)
|
||||||
|
for ent in old.get("entities", []):
|
||||||
|
self._entity_index[ent].discard(node_id)
|
||||||
|
self._temporal_index = [
|
||||||
|
(nid, ts) for nid, ts in self._temporal_index if nid != node_id
|
||||||
|
]
|
||||||
|
|
||||||
|
# Rebuild the entry for this node and re-insert index entries.
|
||||||
|
meta: Dict[str, Any] = {}
|
||||||
|
meta.update(getattr(node, "metadata", {}) or {})
|
||||||
|
meta.update(getattr(node, "properties", {}) or {})
|
||||||
|
|
||||||
|
raw_ts = meta.get("timestamp", 0.0)
|
||||||
|
try:
|
||||||
|
sort_ts = float(raw_ts)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
sort_ts = 0.0
|
||||||
|
|
||||||
|
entities = meta.get("entities") or []
|
||||||
|
if not isinstance(entities, list):
|
||||||
|
entities = []
|
||||||
|
if not entities:
|
||||||
|
# Recover entity list from "involves" edges
|
||||||
|
for edge in self._adjacency.get(node_id, []):
|
||||||
|
if edge.edge_type == "involves":
|
||||||
|
entities.append(edge.target_id)
|
||||||
|
|
||||||
|
extra_meta = {
|
||||||
|
k: v for k, v in meta.items() if k not in self._DECISION_CORE_FIELDS
|
||||||
|
}
|
||||||
|
|
||||||
|
decision: Dict[str, Any] = {
|
||||||
|
"id": node_id,
|
||||||
|
"category": meta.get("category", ""),
|
||||||
|
"scenario": meta.get("scenario", getattr(node, "content", "") or ""),
|
||||||
|
"reasoning": meta.get("reasoning", ""),
|
||||||
|
"outcome": meta.get("outcome", ""),
|
||||||
|
"confidence": float(meta.get("confidence", 0.0) or 0.0),
|
||||||
|
"entities": entities,
|
||||||
|
"decision_maker": meta.get("decision_maker"),
|
||||||
|
"timestamp": raw_ts,
|
||||||
|
"recorded_at": meta.get("recorded_at", ""),
|
||||||
|
"valid_from": getattr(node, "valid_from", None),
|
||||||
|
"valid_until": getattr(node, "valid_until", None),
|
||||||
|
"metadata": extra_meta,
|
||||||
|
}
|
||||||
|
|
||||||
|
self._decisions[node_id] = decision
|
||||||
|
if decision["category"]:
|
||||||
|
self._decision_index[decision["category"]].add(node_id)
|
||||||
|
for ent in entities:
|
||||||
|
self._entity_index[ent].add(node_id)
|
||||||
|
self._temporal_index.append((node_id, sort_ts))
|
||||||
|
self._temporal_index.sort(key=lambda x: x[1], reverse=True)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _char_bigrams(text: str) -> set:
|
||||||
|
"""Character bigrams over whitespace-stripped text (CJK fallback).
|
||||||
|
|
||||||
|
Strips whitespace so CJK characters without word-separating spaces are
|
||||||
|
treated as a contiguous character sequence rather than a single token.
|
||||||
|
"""
|
||||||
|
chars = "".join(text.lower().split())
|
||||||
|
return {chars[i:i + 2] for i in range(len(chars) - 1)}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _looks_cjk(text: str) -> bool:
|
||||||
|
"""True if text contains CJK/Japanese/Korean script characters.
|
||||||
|
|
||||||
|
Used to gate the character-bigram similarity fallback so it only
|
||||||
|
activates for scripts where whitespace tokenisation doesn't work.
|
||||||
|
"""
|
||||||
|
for ch in text:
|
||||||
|
code = ord(ch)
|
||||||
|
if (
|
||||||
|
0x4E00 <= code <= 0x9FFF # CJK Unified Ideographs
|
||||||
|
or 0x3400 <= code <= 0x4DBF # CJK Extension A
|
||||||
|
or 0x3040 <= code <= 0x30FF # Hiragana + Katakana
|
||||||
|
or 0xAC00 <= code <= 0xD7A3 # Hangul Syllables
|
||||||
|
or 0x1100 <= code <= 0x11FF # Hangul Jamo
|
||||||
|
):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _calculate_decision_content_similarity(self, scenario: str, decision: Dict[str, Any]) -> float:
|
||||||
|
"""Calculate content similarity between scenario and decision.
|
||||||
|
|
||||||
|
Uses word-level Jaccard for space-separated languages. For text where
|
||||||
|
whitespace tokenisation is unreliable (CJK/Japanese/Korean scripts, or
|
||||||
|
a query with no whitespace at all) a character-bigram Jaccard is
|
||||||
|
computed over the *stripped* character sequences instead.
|
||||||
|
|
||||||
|
The bigram fallback only activates when whitespace tokenisation would
|
||||||
|
not help — i.e. the query is CJK-like or has at most one whitespace
|
||||||
|
token — so it never contributes for ordinary multi-word English
|
||||||
|
queries, where incidental bigram overlap between unrelated sentences
|
||||||
|
would otherwise inflate scores.
|
||||||
|
|
||||||
|
The bigram side uses *Jaccard* (|A∩B|/|A∪B|), not the overlap
|
||||||
|
coefficient, so a 2-character query whose single bigram happens to
|
||||||
|
appear anywhere in a long document does not silently receive a score of
|
||||||
|
1.0. A minimum bigram set size of 3 is required before the bigram
|
||||||
|
signal contributes; this prevents 1- and 2-character English queries
|
||||||
|
from polluting results while still allowing 3-character CJK phrases (2
|
||||||
|
bigrams) to match.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
decision_text = (
|
||||||
|
f"{decision['scenario']} {decision['reasoning']} "
|
||||||
|
f"{' '.join(decision['entities'])}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- word-level Jaccard (primary metric for Latin/space-delimited) ---
|
||||||
scenario_words = set(scenario.lower().split())
|
scenario_words = set(scenario.lower().split())
|
||||||
decision_text = f"{decision['scenario']} {decision['reasoning']} {' '.join(decision['entities'])}"
|
|
||||||
decision_words = set(decision_text.lower().split())
|
decision_words = set(decision_text.lower().split())
|
||||||
|
word_union = scenario_words | decision_words
|
||||||
|
word_sim = (
|
||||||
|
len(scenario_words & decision_words) / len(word_union)
|
||||||
|
if word_union
|
||||||
|
else 0.0
|
||||||
|
)
|
||||||
|
|
||||||
intersection = scenario_words.intersection(decision_words)
|
# --- character-bigram Jaccard (CJK / very-short-query fallback) ---
|
||||||
union = scenario_words.union(decision_words)
|
# Only used when whitespace tokenisation can't do the job: CJK-like
|
||||||
|
# scripts, or a query that is a single whitespace token (no spaces
|
||||||
|
# to split on). Ordinary multi-word English queries rely on
|
||||||
|
# word_sim alone, so incidental bigram overlap between unrelated
|
||||||
|
# sentences can never inflate their score.
|
||||||
|
bigram_sim = 0.0
|
||||||
|
needs_bigram_fallback = (
|
||||||
|
self._looks_cjk(scenario) or len(scenario.split()) <= 1
|
||||||
|
)
|
||||||
|
if needs_bigram_fallback:
|
||||||
|
scenario_bigrams = self._char_bigrams(scenario)
|
||||||
|
decision_bigrams = self._char_bigrams(decision_text)
|
||||||
|
|
||||||
return len(intersection) / len(union) if union else 0.0
|
# Require at least 3 bigrams in the query before the bigram
|
||||||
|
# signal is used. A 2-char query produces only 1 bigram; that
|
||||||
|
# single bigram is far too likely to appear as a substring of
|
||||||
|
# any English word and would produce a spuriously high overlap
|
||||||
|
# coefficient. 3 bigrams correspond to a 4-char stripped query
|
||||||
|
# (e.g. two CJK characters produce 1 bigram each → need ≥3
|
||||||
|
# chars stripped).
|
||||||
|
if len(scenario_bigrams) >= 3 and decision_bigrams:
|
||||||
|
bigram_union = scenario_bigrams | decision_bigrams
|
||||||
|
bigram_sim = (
|
||||||
|
len(scenario_bigrams & decision_bigrams) / len(bigram_union)
|
||||||
|
if bigram_union
|
||||||
|
else 0.0
|
||||||
|
)
|
||||||
|
|
||||||
except Exception as e:
|
return max(word_sim, bigram_sim)
|
||||||
|
|
||||||
|
except Exception:
|
||||||
self.logger.exception("Content similarity calculation failed")
|
self.logger.exception("Content similarity calculation failed")
|
||||||
return 0.0
|
return 0.0
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ including node labels, relationship types, and indexes for graph databases.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import warnings
|
||||||
from typing import Dict, Any, List
|
from typing import Dict, Any, List
|
||||||
|
|
||||||
from ..graph_store import GraphStore
|
from ..graph_store import GraphStore
|
||||||
@@ -460,11 +461,25 @@ def drop_decision_schema(graph_store: GraphStore) -> None:
|
|||||||
"""
|
"""
|
||||||
Drop decision tracking schema (for cleanup/testing).
|
Drop decision tracking schema (for cleanup/testing).
|
||||||
|
|
||||||
|
.. deprecated::
|
||||||
|
``drop_decision_schema()`` is deprecated and will be removed in a future
|
||||||
|
major version. It has no callers inside Semantica; issue the DROP
|
||||||
|
CONSTRAINT / DROP INDEX / DETACH DELETE statements directly against your
|
||||||
|
:class:`~semantica.graph_store.GraphStore` instead.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
graph_store: Graph database instance
|
graph_store: Graph database instance
|
||||||
"""
|
"""
|
||||||
logger = get_logger(__name__)
|
logger = get_logger(__name__)
|
||||||
|
|
||||||
|
warnings.warn(
|
||||||
|
"drop_decision_schema() is deprecated and will be removed in a future "
|
||||||
|
"major version. Issue the DROP CONSTRAINT / DROP INDEX / DETACH DELETE "
|
||||||
|
"statements directly against your GraphStore instead.",
|
||||||
|
DeprecationWarning,
|
||||||
|
stacklevel=2,
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Drop constraints
|
# Drop constraints
|
||||||
constraints = [
|
constraints = [
|
||||||
|
|||||||
@@ -2,8 +2,9 @@
|
|||||||
Semantica Explorer : FastAPI Dependencies
|
Semantica Explorer : FastAPI Dependencies
|
||||||
|
|
||||||
Provides ``Depends()``-compatible callables for injecting the
|
Provides ``Depends()``-compatible callables for injecting the
|
||||||
current ``GraphSession`` and ``ConnectionManager`` into route handlers,
|
current ``GraphSession`` into route handlers, and for enforcing API-key
|
||||||
and for enforcing API-key authentication on protected routes.
|
authentication on protected routes. WebSocket manager access is handled
|
||||||
|
directly via ``app.state.ws_manager``.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import hmac
|
import hmac
|
||||||
@@ -14,7 +15,6 @@ from fastapi import Request, HTTPException, Security, status
|
|||||||
from fastapi.security.api_key import APIKeyHeader
|
from fastapi.security.api_key import APIKeyHeader
|
||||||
|
|
||||||
from .session import GraphSession
|
from .session import GraphSession
|
||||||
from .ws import ConnectionManager
|
|
||||||
|
|
||||||
_api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False)
|
_api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False)
|
||||||
|
|
||||||
@@ -80,13 +80,3 @@ def get_session(request: Request) -> GraphSession:
|
|||||||
detail="GraphSession not initialized."
|
detail="GraphSession not initialized."
|
||||||
)
|
)
|
||||||
return request.app.state.session
|
return request.app.state.session
|
||||||
|
|
||||||
|
|
||||||
def get_ws_manager(request: Request) -> ConnectionManager:
|
|
||||||
"""Retrieve the ConnectionManager stored on ``app.state``."""
|
|
||||||
if not hasattr(request.app.state, "ws_manager") or request.app.state.ws_manager is None:
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
|
||||||
detail="WebSocket manager not initialized.",
|
|
||||||
)
|
|
||||||
return request.app.state.ws_manager
|
|
||||||
|
|||||||
@@ -78,66 +78,6 @@ def _parse_bbox(raw_bbox: Optional[str]) -> Optional[tuple[float, float, float,
|
|||||||
return min_x, min_y, max_x, max_y
|
return min_x, min_y, max_x, max_y
|
||||||
|
|
||||||
|
|
||||||
def _coerce_embedding_vector(value: object) -> Optional[List[float]]:
|
|
||||||
if isinstance(value, dict):
|
|
||||||
# Probe keys in priority order: generic first, then framework-specific.
|
|
||||||
# Must stay aligned with the top-level keys in _extract_node_embeddings.
|
|
||||||
for key in ("embedding", "embeddings", "vector", "values", "node2vec", "semantic"):
|
|
||||||
nested = _coerce_embedding_vector(value.get(key))
|
|
||||||
if nested is not None:
|
|
||||||
return nested
|
|
||||||
return None
|
|
||||||
|
|
||||||
if not isinstance(value, (list, tuple)):
|
|
||||||
return None
|
|
||||||
|
|
||||||
vector: List[float] = []
|
|
||||||
for item in value:
|
|
||||||
try:
|
|
||||||
vector.append(float(item))
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
return None
|
|
||||||
|
|
||||||
return vector if vector else None
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_node_embeddings(graph_dict: dict) -> dict[str, List[float]]:
|
|
||||||
"""Extract embeddings from graph dictionary."""
|
|
||||||
# Top-level keys to probe on each entity (and its metadata/properties dicts).
|
|
||||||
# Priority: generic names first, then KG-extras-specific names.
|
|
||||||
# Must stay aligned with the inner probe list in _coerce_embedding_vector.
|
|
||||||
embedding_keys = (
|
|
||||||
"embedding",
|
|
||||||
"embeddings",
|
|
||||||
"vector",
|
|
||||||
"node_embedding",
|
|
||||||
"node2vec_embedding",
|
|
||||||
"semantic_embedding",
|
|
||||||
"reasoning_embedding",
|
|
||||||
)
|
|
||||||
|
|
||||||
embeddings: dict[str, List[float]] = {}
|
|
||||||
for entity in graph_dict.get("entities") or graph_dict.get("nodes") or []:
|
|
||||||
if not isinstance(entity, dict):
|
|
||||||
continue
|
|
||||||
node_id = entity.get("id") or entity.get("node_id")
|
|
||||||
if not node_id:
|
|
||||||
continue
|
|
||||||
|
|
||||||
metadata = entity.get("metadata") if isinstance(entity.get("metadata"), dict) else {}
|
|
||||||
properties = entity.get("properties") if isinstance(entity.get("properties"), dict) else {}
|
|
||||||
|
|
||||||
for key in embedding_keys:
|
|
||||||
vector = _coerce_embedding_vector(
|
|
||||||
entity.get(key, metadata.get(key, properties.get(key)))
|
|
||||||
)
|
|
||||||
if vector is not None:
|
|
||||||
embeddings[str(node_id)] = vector
|
|
||||||
break
|
|
||||||
|
|
||||||
return embeddings
|
|
||||||
|
|
||||||
|
|
||||||
def _get_cached_embeddings(session: GraphSession) -> dict[str, List[float]]:
|
def _get_cached_embeddings(session: GraphSession) -> dict[str, List[float]]:
|
||||||
"""Get embeddings from session cache for optimal performance."""
|
"""Get embeddings from session cache for optimal performance."""
|
||||||
return session.get_cached_embeddings()
|
return session.get_cached_embeddings()
|
||||||
|
|||||||
@@ -456,10 +456,6 @@ class DraftResponse(BaseModel):
|
|||||||
updated_at: str
|
updated_at: str
|
||||||
|
|
||||||
|
|
||||||
class ProposalState(BaseModel):
|
|
||||||
state: Literal["draft", "proposed", "approved", "published", "rejected"]
|
|
||||||
|
|
||||||
|
|
||||||
class ProposalRequest(BaseModel):
|
class ProposalRequest(BaseModel):
|
||||||
draft_id: str
|
draft_id: str
|
||||||
ontology_uri: str
|
ontology_uri: str
|
||||||
|
|||||||
@@ -8,11 +8,6 @@ from typing import Any, Dict, List, Literal, Optional, Tuple
|
|||||||
from pydantic import BaseModel, Field, field_validator
|
from pydantic import BaseModel, Field, field_validator
|
||||||
|
|
||||||
|
|
||||||
class ErrorResponse(BaseModel):
|
|
||||||
detail: str
|
|
||||||
status_code: int = 500
|
|
||||||
|
|
||||||
|
|
||||||
class NodeResponse(BaseModel):
|
class NodeResponse(BaseModel):
|
||||||
id: str
|
id: str
|
||||||
type: str
|
type: str
|
||||||
@@ -187,12 +182,6 @@ class ComplianceResponse(BaseModel):
|
|||||||
violations: List[Dict[str, Any]] = Field(default_factory=list)
|
violations: List[Dict[str, Any]] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
class TemporalSnapshotResponse(BaseModel):
|
|
||||||
timestamp: str
|
|
||||||
active_nodes: List[NodeResponse]
|
|
||||||
active_node_count: int
|
|
||||||
|
|
||||||
|
|
||||||
class TemporalDiffResponse(BaseModel):
|
class TemporalDiffResponse(BaseModel):
|
||||||
from_time: str
|
from_time: str
|
||||||
to_time: str
|
to_time: str
|
||||||
@@ -256,13 +245,6 @@ class ExportRequest(BaseModel):
|
|||||||
node_ids: Optional[List[str]] = None
|
node_ids: Optional[List[str]] = None
|
||||||
|
|
||||||
|
|
||||||
class ExportResponse(BaseModel):
|
|
||||||
format: str
|
|
||||||
content_type: str
|
|
||||||
filename: str
|
|
||||||
size_bytes: int = 0
|
|
||||||
|
|
||||||
|
|
||||||
class ImportResponse(BaseModel):
|
class ImportResponse(BaseModel):
|
||||||
status: str = "success"
|
status: str = "success"
|
||||||
message: str = "Import successful"
|
message: str = "Import successful"
|
||||||
@@ -272,11 +254,6 @@ class ImportResponse(BaseModel):
|
|||||||
edges_imported: Optional[int] = None
|
edges_imported: Optional[int] = None
|
||||||
|
|
||||||
|
|
||||||
class StandardMessageResponse(BaseModel):
|
|
||||||
status: str
|
|
||||||
message: str
|
|
||||||
|
|
||||||
|
|
||||||
class AnnotationCreate(BaseModel):
|
class AnnotationCreate(BaseModel):
|
||||||
node_id: str
|
node_id: str
|
||||||
content: str
|
content: str
|
||||||
|
|||||||
@@ -738,6 +738,22 @@ class RDFSerializer:
|
|||||||
# node to signal that valid_until is OPEN/unbounded. This keeps the
|
# node to signal that valid_until is OPEN/unbounded. This keeps the
|
||||||
# interval well-formed while remaining human- and machine-readable.
|
# interval well-formed while remaining human- and machine-readable.
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _escape_turtle_literal(value: str) -> str:
|
||||||
|
"""Escape a string value for safe embedding in a Turtle string literal.
|
||||||
|
|
||||||
|
Backslash must be escaped first, then the double quote and the
|
||||||
|
recognized control characters (newline, carriage return, tab), per the
|
||||||
|
RDF 1.1 Turtle grammar for STRING_LITERAL_QUOTE.
|
||||||
|
"""
|
||||||
|
return (
|
||||||
|
value.replace("\\", "\\\\")
|
||||||
|
.replace('"', '\\"')
|
||||||
|
.replace("\n", "\\n")
|
||||||
|
.replace("\r", "\\r")
|
||||||
|
.replace("\t", "\\t")
|
||||||
|
)
|
||||||
|
|
||||||
def serialize_to_turtle(self, rdf_data: Dict[str, Any], **options) -> str:
|
def serialize_to_turtle(self, rdf_data: Dict[str, Any], **options) -> str:
|
||||||
"""
|
"""
|
||||||
Serialize RDF to Turtle format.
|
Serialize RDF to Turtle format.
|
||||||
@@ -807,7 +823,7 @@ class RDFSerializer:
|
|||||||
|
|
||||||
clauses = [
|
clauses = [
|
||||||
f"a <{self._as_turtle_iri(entity_type, merged_namespaces)}>",
|
f"a <{self._as_turtle_iri(entity_type, merged_namespaces)}>",
|
||||||
f'semantica:text "{text}"',
|
f'semantica:text "{self._escape_turtle_literal(text)}"',
|
||||||
]
|
]
|
||||||
if confidence is None:
|
if confidence is None:
|
||||||
self.logger.warning(
|
self.logger.warning(
|
||||||
@@ -999,7 +1015,7 @@ class RDFSerializer:
|
|||||||
lines.append(f" time:hasEnd <{end_id}> .")
|
lines.append(f" time:hasEnd <{end_id}> .")
|
||||||
lines.append(f"<{end_id}> a time:Instant ;")
|
lines.append(f"<{end_id}> a time:Instant ;")
|
||||||
lines.append(
|
lines.append(
|
||||||
f' time:inXSDDateTimeStamp "{until_val}"^^xsd:dateTimeStamp .'
|
f' time:inXSDDateTimeStamp "{self._escape_turtle_literal(until_val)}"^^xsd:dateTimeStamp .'
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
lines[-1] = (
|
lines[-1] = (
|
||||||
@@ -1008,7 +1024,7 @@ class RDFSerializer:
|
|||||||
|
|
||||||
lines.append(f"<{begin_id}> a time:Instant ;")
|
lines.append(f"<{begin_id}> a time:Instant ;")
|
||||||
lines.append(
|
lines.append(
|
||||||
f' time:inXSDDateTimeStamp "{from_val}"^^xsd:dateTimeStamp .'
|
f' time:inXSDDateTimeStamp "{self._escape_turtle_literal(from_val)}"^^xsd:dateTimeStamp .'
|
||||||
)
|
)
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
@@ -1305,7 +1321,7 @@ class RDFSerializer:
|
|||||||
# Text property
|
# Text property
|
||||||
text = entity.get("text") or entity.get("label", "")
|
text = entity.get("text") or entity.get("label", "")
|
||||||
if text:
|
if text:
|
||||||
safe_text = text.replace('"', '\\"').replace("\n", "\\n")
|
safe_text = self._escape_turtle_literal(text)
|
||||||
lines.append(
|
lines.append(
|
||||||
f'{subject} {expand_uri("semantica:text")} "{safe_text}" .'
|
f'{subject} {expand_uri("semantica:text")} "{safe_text}" .'
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ from pathlib import Path
|
|||||||
from typing import Any, Dict, List, Optional, Union
|
from typing import Any, Dict, List, Optional, Union
|
||||||
|
|
||||||
import rdflib
|
import rdflib
|
||||||
from rdflib import RDF, RDFS, OWL, Graph
|
from rdflib import RDF, RDFS, OWL, Dataset, Graph
|
||||||
|
|
||||||
from ..utils.exceptions import ProcessingError, ValidationError
|
from ..utils.exceptions import ProcessingError, ValidationError
|
||||||
from ..utils.logging import get_logger
|
from ..utils.logging import get_logger
|
||||||
@@ -106,7 +106,16 @@ class OntologyIngestor:
|
|||||||
raise ValidationError(f"File not found: {file_path}")
|
raise ValidationError(f"File not found: {file_path}")
|
||||||
|
|
||||||
self.progress.update_tracking(tracking_id, message="Parsing RDF graph...")
|
self.progress.update_tracking(tracking_id, message="Parsing RDF graph...")
|
||||||
g = Graph()
|
# `Dataset`, not `Graph`: a JSON-LD document with a top-level `@id` *and*
|
||||||
|
# `@graph` places its terms in a NAMED graph. `Graph.parse()` loads only the
|
||||||
|
# default graph and discards the rest without an error, so every class and
|
||||||
|
# property in such a document was dropped while the load reported success.
|
||||||
|
# Same migration #757 made for JenaStore; the ingest path was not covered by it.
|
||||||
|
# `default_union=True` makes the Dataset itself present triples from every
|
||||||
|
# graph as one merged view (it is an rdflib.Graph subclass, so it satisfies
|
||||||
|
# _convert_to_dict()'s Graph-typed contract directly) instead of copying every
|
||||||
|
# quad into a second in-memory Graph.
|
||||||
|
ds = Dataset(default_union=True)
|
||||||
|
|
||||||
# Use provided format or let rdflib guess based on extension
|
# Use provided format or let rdflib guess based on extension
|
||||||
parse_kwargs = kwargs.copy()
|
parse_kwargs = kwargs.copy()
|
||||||
@@ -114,7 +123,7 @@ class OntologyIngestor:
|
|||||||
parse_kwargs['format'] = format
|
parse_kwargs['format'] = format
|
||||||
|
|
||||||
try:
|
try:
|
||||||
g.parse(file_path, **parse_kwargs)
|
ds.parse(file_path, **parse_kwargs)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# Fallback: try to guess format from extension if not provided and initial parse failed
|
# Fallback: try to guess format from extension if not provided and initial parse failed
|
||||||
if not format:
|
if not format:
|
||||||
@@ -130,12 +139,14 @@ class OntologyIngestor:
|
|||||||
guessed_fmt = fmt_map.get(ext)
|
guessed_fmt = fmt_map.get(ext)
|
||||||
if guessed_fmt:
|
if guessed_fmt:
|
||||||
self.logger.info(f"Retrying with guessed format: {guessed_fmt}")
|
self.logger.info(f"Retrying with guessed format: {guessed_fmt}")
|
||||||
g.parse(file_path, format=guessed_fmt, **kwargs)
|
ds.parse(file_path, format=guessed_fmt, **kwargs)
|
||||||
else:
|
else:
|
||||||
raise e
|
raise e
|
||||||
else:
|
else:
|
||||||
raise e
|
raise e
|
||||||
|
|
||||||
|
g = ds
|
||||||
|
|
||||||
self.progress.update_tracking(tracking_id, message="Converting to internal format...")
|
self.progress.update_tracking(tracking_id, message="Converting to internal format...")
|
||||||
|
|
||||||
# Determine format for metadata
|
# Determine format for metadata
|
||||||
|
|||||||
@@ -62,6 +62,13 @@ logging.basicConfig(stream=sys.stderr, level=_log_level,
|
|||||||
format="%(asctime)s [semantica-mcp] %(levelname)s %(message)s")
|
format="%(asctime)s [semantica-mcp] %(levelname)s %(message)s")
|
||||||
log = logging.getLogger("semantica.mcp_server")
|
log = logging.getLogger("semantica.mcp_server")
|
||||||
|
|
||||||
|
# MCP stdio framing IS stdout: a progress bar or other console renderer writing
|
||||||
|
# to stdout would interleave with the JSON-RPC stream and hang every client
|
||||||
|
# (observed 2026-08-20: export_graph over MCP timed out at 300s while the same
|
||||||
|
# call returned in <1s directly). Force the progress trackers off for this
|
||||||
|
# process — stdout is not a console here.
|
||||||
|
os.environ["SEMANTICA_DISABLE_PROGRESS"] = "1"
|
||||||
|
|
||||||
# ── lazy graph session ──────────────────────────────────────────────────────
|
# ── lazy graph session ──────────────────────────────────────────────────────
|
||||||
_graph: Any = None
|
_graph: Any = None
|
||||||
|
|
||||||
@@ -74,7 +81,7 @@ def _get_graph():
|
|||||||
kg_path = os.environ.get("SEMANTICA_KG_PATH")
|
kg_path = os.environ.get("SEMANTICA_KG_PATH")
|
||||||
if kg_path and os.path.exists(kg_path):
|
if kg_path and os.path.exists(kg_path):
|
||||||
try:
|
try:
|
||||||
_graph.load(kg_path)
|
_graph.load_from_file(kg_path)
|
||||||
log.info("Loaded graph from %s", kg_path)
|
log.info("Loaded graph from %s", kg_path)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log.warning("Could not load graph from %s: %s", kg_path, exc)
|
log.warning("Could not load graph from %s: %s", kg_path, exc)
|
||||||
@@ -86,30 +93,57 @@ def _get_graph():
|
|||||||
# ══════════════════════════════════════════════════════════════════════════════
|
# ══════════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
def _tool_extract_entities(args: dict) -> dict:
|
def _tool_extract_entities(args: dict) -> dict:
|
||||||
"""Extract named entities from text."""
|
"""Extract named entities from text.
|
||||||
|
|
||||||
|
Optional ``model`` (spaCy pipeline, e.g. ``zh_core_web_sm`` for Chinese)
|
||||||
|
and ``language`` allow non-English NER; defaults to the Semantica English
|
||||||
|
pipeline when omitted. ``method`` defaults to ``ml`` (spaCy); other
|
||||||
|
options are ``huggingface``, ``llm``, ``pattern``.
|
||||||
|
"""
|
||||||
text = args.get("text", "")
|
text = args.get("text", "")
|
||||||
if not text:
|
if not text:
|
||||||
return {"error": "text is required"}
|
return {"error": "text is required"}
|
||||||
from semantica.semantic_extract import NamedEntityRecognizer
|
from semantica.semantic_extract import NamedEntityRecognizer
|
||||||
entities = NamedEntityRecognizer().extract_entities(text)
|
init_kwargs = {}
|
||||||
|
for k in ("model", "language", "confidence_threshold"):
|
||||||
|
if args.get(k) is not None:
|
||||||
|
init_kwargs[k] = args[k]
|
||||||
|
method = args.get("method", "ml")
|
||||||
|
ner = NamedEntityRecognizer(methods=[method], **init_kwargs)
|
||||||
|
entities = ner.extract_entities(text)
|
||||||
return {
|
return {
|
||||||
"entities": [
|
"entities": [
|
||||||
{"label": getattr(e, "label", str(e)),
|
{"text": getattr(e, "text", ""),
|
||||||
"type": getattr(e, "type", None),
|
"label": getattr(e, "label", ""),
|
||||||
"start": getattr(e, "start", None),
|
"type": getattr(e, "label", None),
|
||||||
"end": getattr(e, "end", None)}
|
"start": getattr(e, "start_char", getattr(e, "start", None)),
|
||||||
|
"end": getattr(e, "end_char", getattr(e, "end", None)),
|
||||||
|
"confidence": getattr(e, "confidence", 1.0)}
|
||||||
for e in (entities or [])
|
for e in (entities or [])
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _tool_extract_relations(args: dict) -> dict:
|
def _tool_extract_relations(args: dict) -> dict:
|
||||||
"""Extract relations and triplets from text."""
|
"""Extract relations and triplets from text.
|
||||||
|
|
||||||
|
Optional ``model``/``language`` enable non-English extraction.
|
||||||
|
``method`` defaults to ``pattern``; ``dependency`` uses spaCy syntactic
|
||||||
|
parsing (requires a spaCy model, e.g. ``zh_core_web_sm``).
|
||||||
|
"""
|
||||||
text = args.get("text", "")
|
text = args.get("text", "")
|
||||||
if not text:
|
if not text:
|
||||||
return {"error": "text is required"}
|
return {"error": "text is required"}
|
||||||
from semantica.semantic_extract import RelationExtractor, TripletExtractor
|
from semantica.semantic_extract import NamedEntityRecognizer, RelationExtractor, TripletExtractor
|
||||||
relations = RelationExtractor().extract_relations(text)
|
rel_kwargs = {}
|
||||||
|
ner_kwargs = {}
|
||||||
|
for k in ("model", "language"):
|
||||||
|
if args.get(k) is not None:
|
||||||
|
rel_kwargs[k] = args[k]
|
||||||
|
ner_kwargs[k] = args[k]
|
||||||
|
method = args.get("method", "pattern")
|
||||||
|
entities = NamedEntityRecognizer(methods=["ml"], **ner_kwargs).extract_entities(text) or []
|
||||||
|
relations = RelationExtractor(method=method, **rel_kwargs).extract_relations(text, entities)
|
||||||
triplets = TripletExtractor().extract_triplets(text)
|
triplets = TripletExtractor().extract_triplets(text)
|
||||||
return {
|
return {
|
||||||
"relations": [
|
"relations": [
|
||||||
@@ -156,10 +190,12 @@ def _tool_query_decisions(args: dict) -> dict:
|
|||||||
graph = _get_graph()
|
graph = _get_graph()
|
||||||
try:
|
try:
|
||||||
if query:
|
if query:
|
||||||
results = graph.find_similar_decisions(query, max_results=limit)
|
results = graph.find_similar_decisions(query, max_results=limit, min_similarity=0.05)
|
||||||
elif category:
|
elif category:
|
||||||
nodes = graph.find_nodes(node_type="decision")
|
nodes = graph.find_nodes(node_type="decision")
|
||||||
results = [n for n in nodes if n.get("category") == category][:limit]
|
results = [n for n in nodes
|
||||||
|
if n.get("category") == category
|
||||||
|
or n.get("metadata", {}).get("category") == category][:limit]
|
||||||
else:
|
else:
|
||||||
results = graph.find_nodes(node_type="decision")[:limit]
|
results = graph.find_nodes(node_type="decision")[:limit]
|
||||||
return {"decisions": results if isinstance(results, list) else list(results)}
|
return {"decisions": results if isinstance(results, list) else list(results)}
|
||||||
@@ -175,7 +211,9 @@ def _tool_find_precedents(args: dict) -> dict:
|
|||||||
max_results = int(args.get("max_results", 5))
|
max_results = int(args.get("max_results", 5))
|
||||||
graph = _get_graph()
|
graph = _get_graph()
|
||||||
try:
|
try:
|
||||||
precedents = graph.find_similar_decisions(scenario, max_results=max_results)
|
min_similarity = float(args.get("min_similarity", 0.05))
|
||||||
|
precedents = graph.find_similar_decisions(
|
||||||
|
scenario, max_results=max_results, min_similarity=min_similarity)
|
||||||
return {"precedents": precedents if isinstance(precedents, list) else list(precedents)}
|
return {"precedents": precedents if isinstance(precedents, list) else list(precedents)}
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return {"error": str(exc), "precedents": []}
|
return {"error": str(exc), "precedents": []}
|
||||||
@@ -260,16 +298,28 @@ def _tool_get_graph_analytics(args: dict) -> dict:
|
|||||||
return {"error": str(exc)}
|
return {"error": str(exc)}
|
||||||
|
|
||||||
|
|
||||||
|
_EXPORT_GRAPH_FORMATS = ("turtle", "ttl", "nt", "xml", "json-ld", "json")
|
||||||
|
|
||||||
|
|
||||||
def _tool_export_graph(args: dict) -> dict:
|
def _tool_export_graph(args: dict) -> dict:
|
||||||
"""Export the current knowledge graph to a serialised format."""
|
"""Export the current knowledge graph to a serialised format."""
|
||||||
fmt = args.get("format", "json-ld")
|
fmt = args.get("format", "json-ld")
|
||||||
|
if fmt not in _EXPORT_GRAPH_FORMATS:
|
||||||
|
return {
|
||||||
|
"error": f"Unsupported format '{fmt}'. Supported: {', '.join(_EXPORT_GRAPH_FORMATS)}"
|
||||||
|
}
|
||||||
graph = _get_graph()
|
graph = _get_graph()
|
||||||
try:
|
try:
|
||||||
from semantica.export import RDFExporter, JSONExporter
|
from semantica.export import RDFExporter
|
||||||
|
# The exporters consume the canonical kg dict, not the ContextGraph
|
||||||
|
# object (regression: the old code passed the object straight through,
|
||||||
|
# so every branch failed — JSONExporter.export() with no file_path on
|
||||||
|
# the json branch, AttributeError on the RDF branches).
|
||||||
|
kg = graph.to_kg_dict()
|
||||||
if fmt in ("turtle", "ttl", "nt", "xml", "json-ld"):
|
if fmt in ("turtle", "ttl", "nt", "xml", "json-ld"):
|
||||||
result = RDFExporter().export_to_rdf(graph, format=fmt)
|
result = RDFExporter().export_to_rdf(kg, format=fmt)
|
||||||
else:
|
else:
|
||||||
result = JSONExporter().export(graph)
|
result = json.dumps(kg, indent=2, ensure_ascii=False)
|
||||||
return {"format": fmt, "data": result}
|
return {"format": fmt, "data": result}
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return {"error": str(exc)}
|
return {"error": str(exc)}
|
||||||
@@ -290,6 +340,160 @@ def _tool_get_graph_summary(args: dict) -> dict:
|
|||||||
return {"error": str(exc), "graph_ready": False}
|
return {"error": str(exc), "graph_ready": False}
|
||||||
|
|
||||||
|
|
||||||
|
def _tool_update_node(args: dict) -> dict:
|
||||||
|
"""Update properties of an existing node and persist to SEMANTICA_KG_PATH.
|
||||||
|
|
||||||
|
Common use: mark an action node's status (todo/doing/done) with an
|
||||||
|
optional note. The graph is mutated in-memory then saved back to the
|
||||||
|
file it was loaded from, so changes survive server restarts.
|
||||||
|
"""
|
||||||
|
node_id = args.get("node_id", "")
|
||||||
|
if not node_id:
|
||||||
|
return {"error": "node_id is required"}
|
||||||
|
properties = args.get("properties", {})
|
||||||
|
if not isinstance(properties, dict) or not properties:
|
||||||
|
return {"error": "properties (non-empty object) is required"}
|
||||||
|
graph = _get_graph()
|
||||||
|
try:
|
||||||
|
if not graph.find_node(node_id):
|
||||||
|
return {"error": f"node '{node_id}' not found"}
|
||||||
|
graph.add_node_attribute(node_id, properties)
|
||||||
|
# Persist back to disk so the change survives restarts
|
||||||
|
kg_path = os.environ.get("SEMANTICA_KG_PATH")
|
||||||
|
if kg_path:
|
||||||
|
graph.save_to_file(kg_path)
|
||||||
|
persisted = True
|
||||||
|
else:
|
||||||
|
persisted = False
|
||||||
|
updated = graph.find_node(node_id)
|
||||||
|
return {
|
||||||
|
"status": "updated",
|
||||||
|
"node_id": node_id,
|
||||||
|
"properties": {k: (updated.get("metadata") or {}).get(k) for k in properties},
|
||||||
|
"persisted": persisted,
|
||||||
|
}
|
||||||
|
except Exception as exc:
|
||||||
|
return {"error": str(exc)}
|
||||||
|
|
||||||
|
|
||||||
|
def _tool_delete_node(args: dict) -> dict:
|
||||||
|
"""Archive a node (soft delete) and persist to SEMANTICA_KG_PATH.
|
||||||
|
|
||||||
|
The node is kept in the graph for history but marked status='archived'.
|
||||||
|
Use to retire an action you no longer actively track.
|
||||||
|
"""
|
||||||
|
node_id = args.get("node_id", "")
|
||||||
|
if not node_id:
|
||||||
|
return {"error": "node_id is required"}
|
||||||
|
graph = _get_graph()
|
||||||
|
try:
|
||||||
|
if not graph.find_node(node_id):
|
||||||
|
return {"error": f"node '{node_id}' not found"}
|
||||||
|
graph.add_node_attribute(node_id, {"status": "archived"})
|
||||||
|
kg_path = os.environ.get("SEMANTICA_KG_PATH")
|
||||||
|
if kg_path:
|
||||||
|
graph.save_to_file(kg_path)
|
||||||
|
return {"status": "archived", "node_id": node_id, "persisted": bool(kg_path)}
|
||||||
|
except Exception as exc:
|
||||||
|
return {"error": str(exc)}
|
||||||
|
|
||||||
|
|
||||||
|
def _tool_query_graph(args: dict) -> dict:
|
||||||
|
"""Query the live knowledge graph: node detail, neighbours, or keyword search.
|
||||||
|
|
||||||
|
mode:
|
||||||
|
- "node" : get one node by id (needs node_id)
|
||||||
|
- "neighbors": traverse up to `depth` hops from node_id (default depth=1)
|
||||||
|
- "search" : keyword search over node id+content (needs query)
|
||||||
|
"""
|
||||||
|
graph = _get_graph()
|
||||||
|
mode = args.get("mode", "neighbors")
|
||||||
|
try:
|
||||||
|
if mode == "node":
|
||||||
|
node_id = args.get("node_id", "")
|
||||||
|
if not node_id:
|
||||||
|
return {"error": "node_id is required"}
|
||||||
|
node = graph.find_node(node_id)
|
||||||
|
return {"node": node}
|
||||||
|
|
||||||
|
if mode == "neighbors":
|
||||||
|
node_id = args.get("node_id", "")
|
||||||
|
if not node_id:
|
||||||
|
return {"error": "node_id is required"}
|
||||||
|
depth = int(args.get("depth", 1))
|
||||||
|
rel_types = args.get("relationship_types")
|
||||||
|
if isinstance(rel_types, str):
|
||||||
|
rel_types = [rel_types]
|
||||||
|
rel_set = set(rel_types) if rel_types else None
|
||||||
|
limit = args.get("limit")
|
||||||
|
limit = int(limit) if limit is not None else None
|
||||||
|
depth = min(max(depth, 1), 5)
|
||||||
|
# Out-edges (multi-hop) via get_neighbors
|
||||||
|
nb = graph.get_neighbors(
|
||||||
|
node_id, hops=depth, relationship_types=rel_types, limit=limit,
|
||||||
|
)
|
||||||
|
out = [
|
||||||
|
{"id": n.get("id"), "type": n.get("type"),
|
||||||
|
"content": n.get("content"),
|
||||||
|
"relationship": n.get("relationship"),
|
||||||
|
"direction": "out", "hop": n.get("hop", 1)}
|
||||||
|
for n in (nb or [])
|
||||||
|
]
|
||||||
|
# In-edges (1-hop): scan edges whose target == node_id.
|
||||||
|
# Deduplicate by source node so that multiple edges between the
|
||||||
|
# same pair of nodes (different edge types) produce one entry.
|
||||||
|
# Stop early once we have already collected `limit` inbound results
|
||||||
|
# (if a limit is set) to avoid scanning the full edge list.
|
||||||
|
inb = []
|
||||||
|
seen_inbound = set()
|
||||||
|
for e in graph.find_edges():
|
||||||
|
if e.get("target") != node_id:
|
||||||
|
continue
|
||||||
|
if rel_set is not None and e.get("type") not in rel_set:
|
||||||
|
continue
|
||||||
|
src_id = e.get("source")
|
||||||
|
if src_id in seen_inbound:
|
||||||
|
continue
|
||||||
|
seen_inbound.add(src_id)
|
||||||
|
src = graph.find_node(src_id) or {}
|
||||||
|
inb.append({"id": src_id, "type": src.get("type"),
|
||||||
|
"content": src.get("content"),
|
||||||
|
"relationship": e.get("type"),
|
||||||
|
"direction": "in", "hop": 1})
|
||||||
|
# Early-exit: we already have `limit` inbound results; the
|
||||||
|
# combined list will be truncated to `limit` anyway.
|
||||||
|
if limit is not None and len(inb) >= limit:
|
||||||
|
break
|
||||||
|
neighbors = out + inb
|
||||||
|
# Apply final limit. Use ``is not None`` so limit=0 (zero results)
|
||||||
|
# is honoured correctly; ``if limit:`` would treat 0 as falsy.
|
||||||
|
if limit is not None:
|
||||||
|
neighbors = neighbors[:limit]
|
||||||
|
return {"node_id": node_id, "depth": depth, "neighbors": neighbors}
|
||||||
|
|
||||||
|
if mode == "search":
|
||||||
|
q = (args.get("query") or "").lower()
|
||||||
|
if not q:
|
||||||
|
return {"error": "query is required"}
|
||||||
|
node_type = args.get("node_type")
|
||||||
|
limit = int(args.get("limit", 50))
|
||||||
|
nodes = graph.find_nodes(node_type=node_type) if node_type else graph.find_nodes()
|
||||||
|
hits = []
|
||||||
|
for n in nodes:
|
||||||
|
# Check limit BEFORE appending so limit=0 returns empty.
|
||||||
|
if len(hits) >= limit:
|
||||||
|
break
|
||||||
|
blob = f"{n.get('id','')} {n.get('content','')}".lower()
|
||||||
|
if q in blob:
|
||||||
|
hits.append({"id": n.get("id"), "type": n.get("type"),
|
||||||
|
"content": n.get("content")})
|
||||||
|
return {"query": q, "results": hits, "total": len(hits)}
|
||||||
|
|
||||||
|
return {"error": f"unknown mode '{mode}': use node|neighbors|search"}
|
||||||
|
except Exception as exc:
|
||||||
|
return {"error": str(exc)}
|
||||||
|
|
||||||
|
|
||||||
# ══════════════════════════════════════════════════════════════════════════════
|
# ══════════════════════════════════════════════════════════════════════════════
|
||||||
# MCP protocol tables
|
# MCP protocol tables
|
||||||
# ══════════════════════════════════════════════════════════════════════════════
|
# ══════════════════════════════════════════════════════════════════════════════
|
||||||
@@ -301,7 +505,11 @@ TOOLS = [
|
|||||||
"inputSchema": {
|
"inputSchema": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
"text": {"type": "string", "description": "Input text to extract entities from"}
|
"text": {"type": "string", "description": "Input text to extract entities from"},
|
||||||
|
"model": {"type": "string", "description": "spaCy model name, e.g. 'zh_core_web_sm' for Chinese, 'en_core_web_sm' for English. Defaults to English pipeline."},
|
||||||
|
"language": {"type": "string", "description": "Language code, e.g. 'zh', 'en'."},
|
||||||
|
"method": {"type": "string", "description": "Extraction method: 'ml' (spaCy, default), 'huggingface', 'llm', 'pattern'."},
|
||||||
|
"confidence_threshold": {"type": "number", "description": "Minimum confidence 0-1 (default 0.5)."}
|
||||||
},
|
},
|
||||||
"required": ["text"],
|
"required": ["text"],
|
||||||
},
|
},
|
||||||
@@ -313,7 +521,10 @@ TOOLS = [
|
|||||||
"inputSchema": {
|
"inputSchema": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
"text": {"type": "string", "description": "Input text to extract relations from"}
|
"text": {"type": "string", "description": "Input text to extract relations from"},
|
||||||
|
"model": {"type": "string", "description": "spaCy model name for dependency parsing, e.g. 'zh_core_web_sm'."},
|
||||||
|
"language": {"type": "string", "description": "Language code, e.g. 'zh'."},
|
||||||
|
"method": {"type": "string", "description": "Extraction method: 'pattern' (default), 'dependency', 'cooccurrence', 'huggingface', 'llm'."}
|
||||||
},
|
},
|
||||||
"required": ["text"],
|
"required": ["text"],
|
||||||
},
|
},
|
||||||
@@ -441,7 +652,7 @@ TOOLS = [
|
|||||||
"properties": {
|
"properties": {
|
||||||
"format": {
|
"format": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"enum": ["turtle", "ttl", "nt", "xml", "json-ld", "json"],
|
"enum": list(_EXPORT_GRAPH_FORMATS),
|
||||||
"description": "Export format (default: json-ld)",
|
"description": "Export format (default: json-ld)",
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -454,6 +665,48 @@ TOOLS = [
|
|||||||
"inputSchema": {"type": "object", "properties": {}},
|
"inputSchema": {"type": "object", "properties": {}},
|
||||||
"_handler": _tool_get_graph_summary,
|
"_handler": _tool_get_graph_summary,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "query_graph",
|
||||||
|
"description": "Query the live knowledge graph: get a node, traverse its neighbours (up to 5 hops), or keyword-search nodes by id+content.",
|
||||||
|
"inputSchema": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"mode": {"type": "string", "description": "node | neighbors | search (default: neighbors)"},
|
||||||
|
"node_id": {"type": "string", "description": "Node id (required for node/neighbors mode)"},
|
||||||
|
"depth": {"type": "integer", "description": "Hop depth for neighbors (1-5, default 1)"},
|
||||||
|
"relationship_types": {"type": "array", "items": {"type": "string"}, "description": "Optional filter by edge type(s)"},
|
||||||
|
"query": {"type": "string", "description": "Keyword for search mode (matched against node id+content)"},
|
||||||
|
"node_type": {"type": "string", "description": "Optional node_type filter for search mode"},
|
||||||
|
"limit": {"type": "integer", "description": "Max results for neighbors/search"}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"_handler": _tool_query_graph,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "update_node",
|
||||||
|
"description": "Update properties of an existing node (e.g. mark an action todo/doing/done with a note) and persist to SEMANTICA_KG_PATH.",
|
||||||
|
"inputSchema": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"node_id": {"type": "string", "description": "Node id to update"},
|
||||||
|
"properties": {"type": "object", "description": "Property key-values to merge onto the node, e.g. {\"status\":\"done\",\"updated_at\":\"2026-08-13\",\"note\":\"...\"}"}
|
||||||
|
},
|
||||||
|
"required": ["node_id", "properties"],
|
||||||
|
},
|
||||||
|
"_handler": _tool_update_node,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "delete_node",
|
||||||
|
"description": "Archive a node (soft delete: marks status='archived', keeps it for history) and persist to SEMANTICA_KG_PATH. Use to retire an action you no longer track.",
|
||||||
|
"inputSchema": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"node_id": {"type": "string", "description": "Node id to delete"}
|
||||||
|
},
|
||||||
|
"required": ["node_id"],
|
||||||
|
},
|
||||||
|
"_handler": _tool_delete_node,
|
||||||
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
RESOURCES = [
|
RESOURCES = [
|
||||||
|
|||||||
@@ -272,7 +272,19 @@ class PipelineBuilder:
|
|||||||
step_name = step_config.get("name")
|
step_name = step_config.get("name")
|
||||||
step_type = step_config.get("type")
|
step_type = step_config.get("type")
|
||||||
if step_name and step_type:
|
if step_name and step_type:
|
||||||
self.add_step(step_name, step_type, **step_config.get("config", {}))
|
step = self.add_step(
|
||||||
|
step_name, step_type, **step_config.get("config", {})
|
||||||
|
)
|
||||||
|
step.dependencies = list(
|
||||||
|
step_config.get("dependencies", step.dependencies)
|
||||||
|
)
|
||||||
|
step.delta_mode = step_config.get("delta_mode", step.delta_mode)
|
||||||
|
step.base_version_id = step_config.get(
|
||||||
|
"base_version_id", step.base_version_id
|
||||||
|
)
|
||||||
|
step.target_version_id = step_config.get(
|
||||||
|
"target_version_id", step.target_version_id
|
||||||
|
)
|
||||||
|
|
||||||
# Set parallelism if specified
|
# Set parallelism if specified
|
||||||
if "parallelism" in pipeline_config:
|
if "parallelism" in pipeline_config:
|
||||||
@@ -398,14 +410,29 @@ class PipelineSerializer:
|
|||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Serialized pipeline
|
Serialized pipeline
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
Step handlers are runtime callables and are intentionally omitted from
|
||||||
|
the serialized representation. They must be rebound after deserialization.
|
||||||
"""
|
"""
|
||||||
|
reserved_config_keys = {
|
||||||
|
"handler",
|
||||||
|
"dependencies",
|
||||||
|
"delta_mode",
|
||||||
|
"base_version_id",
|
||||||
|
"target_version_id",
|
||||||
|
}
|
||||||
pipeline_data = {
|
pipeline_data = {
|
||||||
"name": pipeline.name,
|
"name": pipeline.name,
|
||||||
"steps": [
|
"steps": [
|
||||||
{
|
{
|
||||||
"name": step.name,
|
"name": step.name,
|
||||||
"type": step.step_type,
|
"type": step.step_type,
|
||||||
"config": step.config,
|
"config": {
|
||||||
|
key: value
|
||||||
|
for key, value in step.config.items()
|
||||||
|
if key not in reserved_config_keys
|
||||||
|
},
|
||||||
"dependencies": step.dependencies,
|
"dependencies": step.dependencies,
|
||||||
"delta_mode": getattr(step, "delta_mode", False),
|
"delta_mode": getattr(step, "delta_mode", False),
|
||||||
"base_version_id": getattr(step, "base_version_id", None),
|
"base_version_id": getattr(step, "base_version_id", None),
|
||||||
@@ -445,6 +472,18 @@ class PipelineSerializer:
|
|||||||
else:
|
else:
|
||||||
pipeline_data = serialized_pipeline
|
pipeline_data = serialized_pipeline
|
||||||
|
|
||||||
|
# Runtime handlers are process-local and cannot be reconstructed safely
|
||||||
|
# from serialized data. Copy before sanitizing so dict inputs are not mutated.
|
||||||
|
pipeline_data = dict(pipeline_data)
|
||||||
|
sanitized_steps = []
|
||||||
|
for step_data in pipeline_data.get("steps", []):
|
||||||
|
sanitized_step = dict(step_data)
|
||||||
|
step_config = dict(sanitized_step.get("config", {}))
|
||||||
|
step_config.pop("handler", None)
|
||||||
|
sanitized_step["config"] = step_config
|
||||||
|
sanitized_steps.append(sanitized_step)
|
||||||
|
pipeline_data["steps"] = sanitized_steps
|
||||||
|
|
||||||
# Reconstruct pipeline
|
# Reconstruct pipeline
|
||||||
builder = PipelineBuilder(**self.config)
|
builder = PipelineBuilder(**self.config)
|
||||||
pipeline = builder.build_pipeline(pipeline_data, **options)
|
pipeline = builder.build_pipeline(pipeline_data, **options)
|
||||||
|
|||||||
@@ -482,8 +482,8 @@ class SeedDataManager:
|
|||||||
List of loaded data records as dictionaries
|
List of loaded data records as dictionaries
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
ProcessingError: If API request fails, response parsing fails, or
|
ProcessingError: If the API request fails (connection error,
|
||||||
requests library is not available
|
timeout, non-2xx status) or the response cannot be parsed
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
>>> records = manager.load_from_api(
|
>>> records = manager.load_from_api(
|
||||||
@@ -559,10 +559,6 @@ class SeedDataManager:
|
|||||||
self.logger.info(f"Loaded {len(records)} records from API: {full_url}")
|
self.logger.info(f"Loaded {len(records)} records from API: {full_url}")
|
||||||
return records
|
return records
|
||||||
|
|
||||||
except (ImportError, OSError):
|
|
||||||
raise ProcessingError(
|
|
||||||
"requests library not available. Install with: pip install requests"
|
|
||||||
)
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise ProcessingError(f"Failed to load from API: {e}") from e
|
raise ProcessingError(f"Failed to load from API: {e}") from e
|
||||||
|
|
||||||
|
|||||||
@@ -140,6 +140,47 @@ _result_cache = ExtractionCache(
|
|||||||
if not config.get("cache_enabled", True):
|
if not config.get("cache_enabled", True):
|
||||||
_result_cache.enabled = False
|
_result_cache.enabled = False
|
||||||
|
|
||||||
|
# Generation kwargs that affect provider output and must therefore be part of
|
||||||
|
# the cache key. This is the union of every generation-affecting parameter
|
||||||
|
# read across providers.py, including params picked up outside _add_if_set
|
||||||
|
# (e.g. AnthropicProvider's manual pass-through loop). Sensitive values
|
||||||
|
# (api_key, token, etc.) are already filtered out by
|
||||||
|
# ExtractionCache._generate_key, so they need not be excluded here.
|
||||||
|
_GENERATION_CACHE_KEYS = frozenset({
|
||||||
|
"max_tokens",
|
||||||
|
"max_completion_tokens",
|
||||||
|
"temperature",
|
||||||
|
"top_p",
|
||||||
|
"top_k",
|
||||||
|
"seed",
|
||||||
|
"frequency_penalty",
|
||||||
|
"presence_penalty",
|
||||||
|
"stop",
|
||||||
|
"stop_sequences", # Anthropic/Gemini spelling of "stop"
|
||||||
|
"logit_bias",
|
||||||
|
"user",
|
||||||
|
"system", # Anthropic system prompt
|
||||||
|
"metadata", # Anthropic request metadata
|
||||||
|
"candidate_count", # Gemini
|
||||||
|
"repeat_penalty", # Ollama
|
||||||
|
"num_ctx", # Ollama
|
||||||
|
"context_window", # Ollama alias for num_ctx
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def _generation_cache_params(kwargs: dict) -> dict:
|
||||||
|
"""Return the subset of *kwargs* that affects generation output.
|
||||||
|
|
||||||
|
Only keys listed in ``_GENERATION_CACHE_KEYS`` are included so that
|
||||||
|
irrelevant or sensitive caller kwargs do not pollute the cache key.
|
||||||
|
Values that are ``None`` are omitted; a caller passing
|
||||||
|
``temperature=None`` is equivalent to not passing it at all.
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
k: v for k, v in kwargs.items()
|
||||||
|
if k in _GENERATION_CACHE_KEYS and v is not None
|
||||||
|
}
|
||||||
|
|
||||||
# Try to import spaCy
|
# Try to import spaCy
|
||||||
from ..utils.helpers import safe_import
|
from ..utils.helpers import safe_import
|
||||||
|
|
||||||
@@ -957,6 +998,7 @@ def extract_entities_llm(
|
|||||||
"max_text_length": max_text_length,
|
"max_text_length": max_text_length,
|
||||||
"structured_output_mode": structured_output_mode,
|
"structured_output_mode": structured_output_mode,
|
||||||
"entity_types": kwargs.get("entity_types"),
|
"entity_types": kwargs.get("entity_types"),
|
||||||
|
**_generation_cache_params(kwargs),
|
||||||
}
|
}
|
||||||
cached_result = _result_cache.get("entities", text, **cache_params)
|
cached_result = _result_cache.get("entities", text, **cache_params)
|
||||||
if cached_result is not None:
|
if cached_result is not None:
|
||||||
@@ -1124,47 +1166,6 @@ Text to extract from:
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
def _parse_entity_result(result: Any, provider: str, model: Optional[str]) -> List[Entity]:
|
|
||||||
"""Helper to parse raw LLM result into Entity objects."""
|
|
||||||
entities = []
|
|
||||||
items = []
|
|
||||||
|
|
||||||
if isinstance(result, list):
|
|
||||||
items = result
|
|
||||||
elif isinstance(result, dict):
|
|
||||||
# Handle cases where LLM wraps the list in a key
|
|
||||||
for key in ["entities", "data", "results"]:
|
|
||||||
if key in result and isinstance(result[key], list):
|
|
||||||
items = result[key]
|
|
||||||
break
|
|
||||||
if not items and "text" in result: # Single object instead of list
|
|
||||||
items = [result]
|
|
||||||
|
|
||||||
for item in items:
|
|
||||||
if not isinstance(item, dict):
|
|
||||||
continue
|
|
||||||
|
|
||||||
text = item.get("text", "")
|
|
||||||
if not text:
|
|
||||||
continue
|
|
||||||
|
|
||||||
entities.append(
|
|
||||||
Entity(
|
|
||||||
text=text,
|
|
||||||
label=item.get("label", "UNKNOWN"),
|
|
||||||
start_char=item.get("start", 0),
|
|
||||||
end_char=item.get("end", 0),
|
|
||||||
confidence=item.get("confidence", 0.9),
|
|
||||||
metadata={
|
|
||||||
"provider": provider,
|
|
||||||
"model": model,
|
|
||||||
"extraction_method": "llm",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return entities
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_entities_chunked(
|
def _extract_entities_chunked(
|
||||||
text: str,
|
text: str,
|
||||||
provider: str,
|
provider: str,
|
||||||
@@ -1747,7 +1748,8 @@ def extract_relations_llm(
|
|||||||
"relation_types": kwargs.get("relation_types"),
|
"relation_types": kwargs.get("relation_types"),
|
||||||
"extract_temporal_bounds": extract_temporal_bounds,
|
"extract_temporal_bounds": extract_temporal_bounds,
|
||||||
# Include entities hash/str in cache key implicitly via **cache_params
|
# Include entities hash/str in cache key implicitly via **cache_params
|
||||||
"entities_hash": hash(tuple(sorted([e.text for e in entities]))) if entities else 0
|
"entities_hash": hash(tuple(sorted([e.text for e in entities]))) if entities else 0,
|
||||||
|
**_generation_cache_params(kwargs),
|
||||||
}
|
}
|
||||||
cached_result = _result_cache.get("relations", text, **cache_params)
|
cached_result = _result_cache.get("relations", text, **cache_params)
|
||||||
if cached_result is not None:
|
if cached_result is not None:
|
||||||
@@ -1947,13 +1949,10 @@ Entities found in text: {entities_str}"""
|
|||||||
"[methods.extract_relations_llm] Calling llm.generate_typed (%s/%s)...",
|
"[methods.extract_relations_llm] Calling llm.generate_typed (%s/%s)...",
|
||||||
provider, model,
|
provider, model,
|
||||||
)
|
)
|
||||||
# Only forward minimal, safe parameters to provider calls
|
# Forward all caller-supplied generation kwargs so they reach
|
||||||
call_kwargs = {}
|
# generate_typed and the underlying provider API. max_retries is
|
||||||
if "temperature" in kwargs:
|
# always set from the explicit parameter.
|
||||||
call_kwargs["temperature"] = kwargs["temperature"]
|
call_kwargs = kwargs.copy()
|
||||||
if "verbose" in kwargs:
|
|
||||||
call_kwargs["verbose"] = kwargs["verbose"]
|
|
||||||
|
|
||||||
call_kwargs["max_retries"] = max_retries
|
call_kwargs["max_retries"] = max_retries
|
||||||
|
|
||||||
# Select schema based on whether temporal extraction is requested
|
# Select schema based on whether temporal extraction is requested
|
||||||
@@ -2405,7 +2404,8 @@ def extract_triplets_llm(
|
|||||||
"triplet_types": kwargs.get("triplet_types"),
|
"triplet_types": kwargs.get("triplet_types"),
|
||||||
# Include entities/relations hash in cache key implicitly via **cache_params
|
# Include entities/relations hash in cache key implicitly via **cache_params
|
||||||
"entities_hash": hash(tuple(sorted([e.text for e in entities]))) if entities else 0,
|
"entities_hash": hash(tuple(sorted([e.text for e in entities]))) if entities else 0,
|
||||||
"relations_hash": hash(tuple(sorted([str(r) for r in relations]))) if relations else 0
|
"relations_hash": hash(tuple(sorted([str(r) for r in relations]))) if relations else 0,
|
||||||
|
**_generation_cache_params(kwargs),
|
||||||
}
|
}
|
||||||
cached_result = _result_cache.get("triplets", text, **cache_params)
|
cached_result = _result_cache.get("triplets", text, **cache_params)
|
||||||
if cached_result is not None:
|
if cached_result is not None:
|
||||||
@@ -2559,48 +2559,6 @@ Text to extract from:
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
def _parse_triplet_result(result: Any, provider: str, model: Optional[str]) -> List[Triplet]:
|
|
||||||
"""Helper to parse raw LLM result into Triplet objects."""
|
|
||||||
triplets = []
|
|
||||||
items = []
|
|
||||||
|
|
||||||
if isinstance(result, list):
|
|
||||||
items = result
|
|
||||||
elif isinstance(result, dict):
|
|
||||||
for key in ["triplets", "data", "results"]:
|
|
||||||
if key in result and isinstance(result[key], list):
|
|
||||||
items = result[key]
|
|
||||||
break
|
|
||||||
if not items and "subject" in result:
|
|
||||||
items = [result]
|
|
||||||
|
|
||||||
for item in items:
|
|
||||||
if not isinstance(item, dict):
|
|
||||||
continue
|
|
||||||
|
|
||||||
subject = item.get("subject", "")
|
|
||||||
predicate = item.get("predicate", "")
|
|
||||||
obj = item.get("object", "")
|
|
||||||
|
|
||||||
if not subject or not predicate or not obj:
|
|
||||||
continue
|
|
||||||
|
|
||||||
triplets.append(
|
|
||||||
Triplet(
|
|
||||||
subject=str(subject),
|
|
||||||
predicate=str(predicate),
|
|
||||||
object=str(obj),
|
|
||||||
confidence=item.get("confidence", 0.9),
|
|
||||||
metadata={
|
|
||||||
"provider": provider,
|
|
||||||
"model": model,
|
|
||||||
"extraction_method": "llm",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return triplets
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_triplets_chunked(
|
def _extract_triplets_chunked(
|
||||||
text: str,
|
text: str,
|
||||||
provider: str,
|
provider: str,
|
||||||
|
|||||||
@@ -101,7 +101,6 @@ from .triplet_store import TripletStore
|
|||||||
# Global store registry
|
# Global store registry
|
||||||
_global_stores: Dict[str, TripletStore] = {}
|
_global_stores: Dict[str, TripletStore] = {}
|
||||||
_default_store_id: Optional[str] = None
|
_default_store_id: Optional[str] = None
|
||||||
_global_query_engine: Optional[QueryEngine] = None
|
|
||||||
_global_bulk_loader: Optional[BulkLoader] = None
|
_global_bulk_loader: Optional[BulkLoader] = None
|
||||||
|
|
||||||
|
|
||||||
@@ -131,25 +130,6 @@ def _get_store(store_id: Optional[str] = None) -> TripletStore:
|
|||||||
return _global_stores[target_id]
|
return _global_stores[target_id]
|
||||||
|
|
||||||
|
|
||||||
def _get_query_engine() -> QueryEngine:
|
|
||||||
"""Get or create global QueryEngine instance."""
|
|
||||||
global _global_query_engine
|
|
||||||
if _global_query_engine is None:
|
|
||||||
# We need a store backend for the engine, but QueryEngine in this module
|
|
||||||
# seems to be initialized with config in the old code.
|
|
||||||
# In the new code, TripletStore has its own query_engine.
|
|
||||||
# If we use this standalone function, we might need to rely on the store's engine.
|
|
||||||
# But let's keep a standalone one if needed, or better, delegate to store.
|
|
||||||
config = triplet_store_config.get_all()
|
|
||||||
# QueryEngine now expects a backend, but we can initialize it without one
|
|
||||||
# if we pass the backend at execution time?
|
|
||||||
# Checking QueryEngine implementation... it takes `store_backend` in __init__.
|
|
||||||
# So we can't easily have a global one without a store.
|
|
||||||
# We'll rely on the store's engine.
|
|
||||||
pass
|
|
||||||
return None # Deprecated use of global engine
|
|
||||||
|
|
||||||
|
|
||||||
def _get_bulk_loader() -> BulkLoader:
|
def _get_bulk_loader() -> BulkLoader:
|
||||||
"""Get or create global BulkLoader instance."""
|
"""Get or create global BulkLoader instance."""
|
||||||
global _global_bulk_loader
|
global _global_bulk_loader
|
||||||
|
|||||||
@@ -42,6 +42,10 @@ class OxigraphStore:
|
|||||||
ProcessingError: If the store cannot be opened.
|
ProcessingError: If the store cannot be opened.
|
||||||
"""
|
"""
|
||||||
self.logger = get_logger("oxigraph_store")
|
self.logger = get_logger("oxigraph_store")
|
||||||
|
# Accept storage_path as an alias for path (matches the convention used
|
||||||
|
# by other Semantica stores). Pop it so it isn't left in self.config.
|
||||||
|
if path is None and "storage_path" in config:
|
||||||
|
path = config.pop("storage_path")
|
||||||
self.config = config
|
self.config = config
|
||||||
self.path = path if path is not None else config.get("path")
|
self.path = path if path is not None else config.get("path")
|
||||||
|
|
||||||
@@ -74,24 +78,65 @@ class OxigraphStore:
|
|||||||
) from exc
|
) from exc
|
||||||
|
|
||||||
def add_triplet(self, triplet: Triplet, **options) -> Dict[str, Any]:
|
def add_triplet(self, triplet: Triplet, **options) -> Dict[str, Any]:
|
||||||
"""Add one triplet to the default graph or ``options['graph']``."""
|
"""Add one triplet to the default graph or ``options['graph']``.
|
||||||
return self.add_triplets([triplet], **options)
|
|
||||||
|
|
||||||
def add_triplets(self, triplets: List[Triplet], **options) -> Dict[str, Any]:
|
The write is committed to the store's in-memory state immediately.
|
||||||
"""Add triplets in one native Oxigraph batch."""
|
pyoxigraph's background threads will persist it to disk shortly
|
||||||
|
afterward; call :meth:`flush` explicitly if you need a synchronous
|
||||||
|
durability guarantee before reopening or crashing.
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
graph_name = self._graph_name(options.get("graph"))
|
graph_name = self._graph_name(options.get("graph"))
|
||||||
quads = [self._to_quad(triplet, graph_name) for triplet in triplets]
|
self.store.extend([self._to_quad(triplet, graph_name)])
|
||||||
self.store.extend(quads)
|
|
||||||
return {
|
return {
|
||||||
"success": True,
|
"success": True,
|
||||||
"triplets_loaded": len(triplets),
|
"triplets_loaded": 1,
|
||||||
"graph": options.get("graph"),
|
"graph": options.get("graph"),
|
||||||
}
|
}
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self.logger.error(f"Oxigraph load failed: {exc}")
|
self.logger.error(f"Oxigraph load failed: {exc}")
|
||||||
raise ProcessingError(f"Oxigraph load failed: {exc}") from exc
|
raise ProcessingError(f"Oxigraph load failed: {exc}") from exc
|
||||||
|
|
||||||
|
def add_triplets(self, triplets: List[Triplet], **options) -> Dict[str, Any]:
|
||||||
|
"""Add triplets in one native Oxigraph batch.
|
||||||
|
|
||||||
|
The batch is written transactionally and then explicitly flushed to
|
||||||
|
disk before returning. This makes the full batch durable without
|
||||||
|
requiring a separate :meth:`flush` call. In-memory stores skip the
|
||||||
|
flush (there is nothing to sync).
|
||||||
|
|
||||||
|
For high-volume imports the :class:`~.bulk_loader.BulkLoader` splits
|
||||||
|
work into chunks and calls this method once per chunk, so each chunk
|
||||||
|
lands as one atomic, durable unit.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
graph_name = self._graph_name(options.get("graph"))
|
||||||
|
quads = [self._to_quad(triplet, graph_name) for triplet in triplets]
|
||||||
|
self.store.extend(quads)
|
||||||
|
except Exception as exc:
|
||||||
|
self.logger.error(f"Oxigraph load failed: {exc}")
|
||||||
|
raise ProcessingError(f"Oxigraph load failed: {exc}") from exc
|
||||||
|
|
||||||
|
# Flush is kept outside the write try/except so that a flush I/O error
|
||||||
|
# does not produce a misleading "load failed" message when extend()
|
||||||
|
# already committed the batch successfully.
|
||||||
|
if self.path is not None:
|
||||||
|
try:
|
||||||
|
self.flush()
|
||||||
|
except OSError as exc:
|
||||||
|
self.logger.warning(
|
||||||
|
f"Oxigraph flush failed after successful write: {exc}"
|
||||||
|
)
|
||||||
|
raise ProcessingError(
|
||||||
|
f"Oxigraph flush failed after successful write: {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"triplets_loaded": len(triplets),
|
||||||
|
"graph": options.get("graph"),
|
||||||
|
}
|
||||||
|
|
||||||
def bulk_load(self, triplets: List[Triplet], **options) -> Dict[str, Any]:
|
def bulk_load(self, triplets: List[Triplet], **options) -> Dict[str, Any]:
|
||||||
"""Load a batch of triplets using Oxigraph's native bulk operation."""
|
"""Load a batch of triplets using Oxigraph's native bulk operation."""
|
||||||
return self.add_triplets(triplets, **options)
|
return self.add_triplets(triplets, **options)
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,237 @@
|
|||||||
|
"""Regression tests for the /api/decisions 422 bug.
|
||||||
|
|
||||||
|
``ContextGraph.record_decision()`` stores ``timestamp`` as a POSIX float
|
||||||
|
(``datetime.now().timestamp()``). ``DecisionResponse.timestamp`` is typed
|
||||||
|
``Optional[str]``. Without the ``_normalize_timestamp`` field-validator on
|
||||||
|
``DecisionResponse`` the raw float fails Pydantic validation and every decision
|
||||||
|
endpoint returns 422.
|
||||||
|
|
||||||
|
The validator lives on ``DecisionResponse`` in ``semantica/explorer/schemas.py``
|
||||||
|
and converts float/int epochs to ISO-8601 strings via
|
||||||
|
``datetime.fromtimestamp(value, tz=timezone.utc).isoformat()``.
|
||||||
|
|
||||||
|
``_node_to_decision()`` must pass the raw stored value through unchanged so the
|
||||||
|
validator can do its job. A route-level ``str()`` cast would pre-empt the
|
||||||
|
validator and produce raw numeric strings instead of ISO-8601, breaking the API
|
||||||
|
contract and all callers that call ``datetime.fromisoformat()`` on the result.
|
||||||
|
|
||||||
|
Each test below is written so that it *fails* when the route-level ``str()``
|
||||||
|
cast is present (i.e. it would have caught the regression introduced by the
|
||||||
|
incorrect fix).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import math
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from pydantic import ValidationError
|
||||||
|
|
||||||
|
from semantica.context.context_graph import ContextGraph
|
||||||
|
from semantica.explorer.app import create_app
|
||||||
|
from semantica.explorer.routes.decisions import _node_to_decision
|
||||||
|
from semantica.explorer.schemas import DecisionResponse
|
||||||
|
from semantica.explorer.session import GraphSession
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _decision_node(timestamp):
|
||||||
|
"""Minimal node dict as returned by the graph session layer."""
|
||||||
|
return {
|
||||||
|
"id": "d-test",
|
||||||
|
"type": "decision",
|
||||||
|
"properties": {
|
||||||
|
"category": "loan_underwriting",
|
||||||
|
"scenario": "A-7291 review",
|
||||||
|
"reasoning": "DTI within policy",
|
||||||
|
"outcome": "approved",
|
||||||
|
"confidence": 0.94,
|
||||||
|
"timestamp": timestamp,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _recorded_client():
|
||||||
|
"""TestClient backed by a graph built with real record_decision() calls.
|
||||||
|
|
||||||
|
This is the path that was broken in production: record_decision() stores
|
||||||
|
timestamp as a float epoch, which must come out the other side as an
|
||||||
|
ISO-8601 string, not a raw numeric string.
|
||||||
|
"""
|
||||||
|
graph = ContextGraph(advanced_analytics=False)
|
||||||
|
graph.record_decision(
|
||||||
|
category="credit_application",
|
||||||
|
scenario="Personal loan, $85k income, 31% DTI",
|
||||||
|
reasoning="Income meets threshold; employment stable",
|
||||||
|
outcome="proceed_to_underwriting",
|
||||||
|
confidence=0.88,
|
||||||
|
entities=["applicant_A7291"],
|
||||||
|
)
|
||||||
|
return TestClient(create_app(session=GraphSession(graph)))
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Unit tests: _node_to_decision() → DecisionResponse
|
||||||
|
#
|
||||||
|
# Each assertion must fail when the route contains the incorrect str() cast:
|
||||||
|
# timestamp=None if ... is None else str(properties.get("timestamp"))
|
||||||
|
# because that cast turns floats into numeric strings such as "1786513069.69"
|
||||||
|
# rather than ISO-8601 strings such as "2026-08-12T05:37:49+00:00".
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_float_timestamp_becomes_iso8601():
|
||||||
|
"""A POSIX-float epoch must be normalised to an ISO-8601 string.
|
||||||
|
|
||||||
|
Fails with the str() cast because str(1786513069.694965) ==
|
||||||
|
'1786513069.694965', which is not a valid isoformat string.
|
||||||
|
"""
|
||||||
|
decision = _node_to_decision(_decision_node(timestamp=1786513069.694965))
|
||||||
|
|
||||||
|
assert isinstance(decision.timestamp, str)
|
||||||
|
# Must parse as a valid ISO-8601 datetime — this is the key assertion that
|
||||||
|
# the incorrect str() cast breaks.
|
||||||
|
parsed = datetime.fromisoformat(decision.timestamp)
|
||||||
|
# Round-trip: parsed timestamp must be within 1 s of the original epoch.
|
||||||
|
assert abs(parsed.timestamp() - 1786513069.694965) < 1.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_int_timestamp_becomes_iso8601():
|
||||||
|
"""An integer epoch (no sub-second component) must also become ISO-8601.
|
||||||
|
|
||||||
|
Fails with the str() cast because str(1786513069) == '1786513069'.
|
||||||
|
"""
|
||||||
|
decision = _node_to_decision(_decision_node(timestamp=1786513069))
|
||||||
|
|
||||||
|
assert isinstance(decision.timestamp, str)
|
||||||
|
parsed = datetime.fromisoformat(decision.timestamp)
|
||||||
|
assert abs(parsed.timestamp() - 1786513069) < 1.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_none_timestamp_stays_none():
|
||||||
|
"""A stored None must remain None, not become the string 'None'."""
|
||||||
|
decision = _node_to_decision(_decision_node(timestamp=None))
|
||||||
|
|
||||||
|
assert decision.timestamp is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_timestamp_key_stays_none():
|
||||||
|
"""A node without a timestamp key at all must not raise and must be None."""
|
||||||
|
node = {
|
||||||
|
"id": "d-no-ts",
|
||||||
|
"type": "decision",
|
||||||
|
"properties": {"category": "x", "outcome": "y"},
|
||||||
|
}
|
||||||
|
|
||||||
|
decision = _node_to_decision(node)
|
||||||
|
|
||||||
|
assert decision.timestamp is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_iso_string_passes_through_unchanged():
|
||||||
|
"""An already-ISO-8601 string must be returned verbatim."""
|
||||||
|
iso = "2026-08-12T10:04:20+00:00"
|
||||||
|
decision = _node_to_decision(_decision_node(timestamp=iso))
|
||||||
|
|
||||||
|
assert decision.timestamp == iso
|
||||||
|
|
||||||
|
|
||||||
|
def test_nan_timestamp_raises_validation_error():
|
||||||
|
"""NaN must be rejected by the validator, not silently accepted.
|
||||||
|
|
||||||
|
With the str() cast, str(nan) == 'nan' bypasses the validator's finiteness
|
||||||
|
check and is silently accepted — this test would pass the incorrect version
|
||||||
|
of the code if it expected 'nan', but it correctly expects a ValidationError.
|
||||||
|
"""
|
||||||
|
with pytest.raises(ValidationError):
|
||||||
|
_node_to_decision(_decision_node(timestamp=float("nan")))
|
||||||
|
|
||||||
|
|
||||||
|
def test_inf_timestamp_raises_validation_error():
|
||||||
|
"""Positive infinity must be rejected, not silently accepted as 'inf'."""
|
||||||
|
with pytest.raises(ValidationError):
|
||||||
|
_node_to_decision(_decision_node(timestamp=float("inf")))
|
||||||
|
|
||||||
|
|
||||||
|
def test_negative_inf_timestamp_raises_validation_error():
|
||||||
|
"""Negative infinity must be rejected, not silently accepted as '-inf'."""
|
||||||
|
with pytest.raises(ValidationError):
|
||||||
|
_node_to_decision(_decision_node(timestamp=float("-inf")))
|
||||||
|
|
||||||
|
|
||||||
|
def test_out_of_range_epoch_raises_validation_error():
|
||||||
|
"""A millisecond epoch accidentally passed as seconds must be rejected.
|
||||||
|
|
||||||
|
With the str() cast, str(1723600000000) is silently accepted as a string.
|
||||||
|
The validator correctly raises ValidationError for out-of-range epochs.
|
||||||
|
"""
|
||||||
|
with pytest.raises(ValidationError):
|
||||||
|
_node_to_decision(_decision_node(timestamp=1723600000000))
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Integration tests: full HTTP path through TestClient
|
||||||
|
#
|
||||||
|
# These exercise the complete production path:
|
||||||
|
# record_decision() → float stored in graph → HTTP GET → JSON response
|
||||||
|
#
|
||||||
|
# They are the definitive check: if the route emits numeric strings instead of
|
||||||
|
# ISO-8601 the fromisoformat() assertion below fails immediately.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_list_decisions_float_timestamp_serialised_as_iso8601():
|
||||||
|
"""GET /api/decisions must return ISO-8601 timestamps for all decisions.
|
||||||
|
|
||||||
|
This is the exact production failure path. record_decision() stores
|
||||||
|
timestamp as a float; the endpoint must return an ISO-8601 string, not a
|
||||||
|
raw numeric string like '1786513069.69'.
|
||||||
|
"""
|
||||||
|
with _recorded_client() as client:
|
||||||
|
response = client.get("/api/decisions")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
payload = response.json()
|
||||||
|
assert len(payload) >= 1
|
||||||
|
|
||||||
|
for item in payload:
|
||||||
|
ts = item["timestamp"]
|
||||||
|
assert isinstance(ts, str), f"timestamp must be str, got {type(ts)}"
|
||||||
|
# This is the line that fails when the str() cast is present:
|
||||||
|
datetime.fromisoformat(ts)
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_decision_float_timestamp_serialised_as_iso8601():
|
||||||
|
"""GET /api/decisions/{id} must return an ISO-8601 timestamp."""
|
||||||
|
with _recorded_client() as client:
|
||||||
|
decision_id = client.get("/api/decisions").json()[0]["decision_id"]
|
||||||
|
response = client.get(f"/api/decisions/{decision_id}")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
ts = response.json()["timestamp"]
|
||||||
|
assert isinstance(ts, str)
|
||||||
|
datetime.fromisoformat(ts)
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_precedents_float_timestamp_serialised_as_iso8601():
|
||||||
|
"""GET /api/decisions/{id}/precedents must return ISO-8601 timestamps."""
|
||||||
|
graph = ContextGraph(advanced_analytics=False)
|
||||||
|
for i in range(3):
|
||||||
|
graph.record_decision(
|
||||||
|
category="risk",
|
||||||
|
scenario=f"loan assessment scenario {i}",
|
||||||
|
reasoning="standard criteria",
|
||||||
|
outcome="approved",
|
||||||
|
confidence=0.9,
|
||||||
|
)
|
||||||
|
|
||||||
|
with TestClient(create_app(session=GraphSession(graph))) as client:
|
||||||
|
decision_id = client.get("/api/decisions").json()[0]["decision_id"]
|
||||||
|
response = client.get(f"/api/decisions/{decision_id}/precedents")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
for item in response.json():
|
||||||
|
ts = item["timestamp"]
|
||||||
|
assert isinstance(ts, str)
|
||||||
|
datetime.fromisoformat(ts)
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
"""Regression tests for #1098: Turtle/N-Triples literal escaping."""
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from semantica.export.rdf_exporter import RDFExporter, RDFSerializer
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def serializer():
|
||||||
|
return RDFSerializer()
|
||||||
|
|
||||||
|
|
||||||
|
class TestTurtleLiteralEscaping:
|
||||||
|
def test_quote_in_text_is_escaped(self, serializer):
|
||||||
|
kg = {
|
||||||
|
"entities": [{"id": "e1", "text": 'He said "hello"', "type": "ORG"}],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
turtle = serializer.serialize_to_turtle(kg)
|
||||||
|
assert '"He said \\"hello\\""' in turtle
|
||||||
|
|
||||||
|
def test_backslash_in_text_is_escaped(self, serializer):
|
||||||
|
kg = {
|
||||||
|
"entities": [{"id": "e1", "text": r"path\to\file", "type": "ORG"}],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
turtle = serializer.serialize_to_turtle(kg)
|
||||||
|
assert r"path\\to\\file" in turtle
|
||||||
|
|
||||||
|
def test_newline_in_text_is_escaped(self, serializer):
|
||||||
|
kg = {
|
||||||
|
"entities": [{"id": "e1", "text": "line1\nline2", "type": "ORG"}],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
turtle = serializer.serialize_to_turtle(kg)
|
||||||
|
assert "line1\\nline2" in turtle
|
||||||
|
|
||||||
|
def test_tab_in_text_is_escaped(self, serializer):
|
||||||
|
kg = {
|
||||||
|
"entities": [{"id": "e1", "text": "a\tb", "type": "ORG"}],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
turtle = serializer.serialize_to_turtle(kg)
|
||||||
|
assert "a\\tb" in turtle
|
||||||
|
|
||||||
|
def test_plain_text_unchanged(self, serializer):
|
||||||
|
kg = {
|
||||||
|
"entities": [{"id": "e1", "text": "Apple Inc.", "type": "ORG"}],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
turtle = serializer.serialize_to_turtle(kg)
|
||||||
|
assert 'semantica:text "Apple Inc."' in turtle
|
||||||
|
|
||||||
|
|
||||||
|
class TestNTriplesLiteralEscaping:
|
||||||
|
def test_quote_in_text_is_escaped(self, serializer):
|
||||||
|
kg = {
|
||||||
|
"entities": [{"id": "e1", "text": 'He said "hello"', "type": "ORG"}],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
ntriples = serializer.serialize_to_ntriples(kg)
|
||||||
|
assert '\\"hello\\"' in ntriples
|
||||||
|
|
||||||
|
def test_backslash_in_text_is_escaped(self, serializer):
|
||||||
|
kg = {
|
||||||
|
"entities": [{"id": "e1", "text": r"path\to\file", "type": "ORG"}],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
ntriples = serializer.serialize_to_ntriples(kg)
|
||||||
|
assert r"path\\to\\file" in ntriples
|
||||||
|
|
||||||
|
def test_newline_in_text_is_escaped(self, serializer):
|
||||||
|
kg = {
|
||||||
|
"entities": [{"id": "e1", "text": "line1\nline2", "type": "ORG"}],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
ntriples = serializer.serialize_to_ntriples(kg)
|
||||||
|
assert "line1\\nline2" in ntriples
|
||||||
|
|
||||||
|
def test_tab_in_text_is_escaped(self, serializer):
|
||||||
|
kg = {
|
||||||
|
"entities": [{"id": "e1", "text": "a\tb", "type": "ORG"}],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
ntriples = serializer.serialize_to_ntriples(kg)
|
||||||
|
assert "a\\tb" in ntriples
|
||||||
|
|
||||||
|
|
||||||
|
class TestOWLTimeLiteralEscaping:
|
||||||
|
"""Timestamp literals in OWL-Time turtle output must also be escaped."""
|
||||||
|
|
||||||
|
def test_owl_time_timestamps_are_escaped(self):
|
||||||
|
exporter = RDFExporter()
|
||||||
|
kg = {
|
||||||
|
"entities": [],
|
||||||
|
"relationships": [
|
||||||
|
{
|
||||||
|
"id": "r1",
|
||||||
|
"source_id": "a",
|
||||||
|
"target_id": "b",
|
||||||
|
"type": "works_for",
|
||||||
|
"valid_from": "2020-01-01T00:00:00Z",
|
||||||
|
"valid_until": None,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
turtle = exporter.export_to_rdf(kg, format="turtle", include_temporal=True)
|
||||||
|
assert 'time:inXSDDateTimeStamp "2020-01-01T00:00:00Z"' in turtle
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
"""A JSON-LD ontology whose terms live in a named graph must not be silently dropped.
|
||||||
|
|
||||||
|
A JSON-LD document with a top-level ``@id`` *and* ``@graph`` places its terms in a NAMED
|
||||||
|
graph. ``rdflib.Graph.parse()`` loads only the default graph and discards the rest without
|
||||||
|
raising, so every class and property in such a document disappeared while the load reported
|
||||||
|
success — see issue #1129 for the reproduction through the public API.
|
||||||
|
|
||||||
|
This is the same ``Graph`` -> ``Dataset`` migration #757 made for ``JenaStore`` (#756); the
|
||||||
|
ingest path was not covered by it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from semantica.ingest.ontology_ingestor import OntologyIngestor
|
||||||
|
|
||||||
|
NAMED_GRAPH_ONTOLOGY = {
|
||||||
|
"@context": {
|
||||||
|
"ex": "https://example.org/ns#",
|
||||||
|
"owl": "http://www.w3.org/2002/07/owl#",
|
||||||
|
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
||||||
|
},
|
||||||
|
"@id": "https://example.org/ns",
|
||||||
|
"@type": "owl:Ontology",
|
||||||
|
"@graph": [
|
||||||
|
{"@id": "ex:Thing", "@type": "owl:Class", "rdfs:label": "Thing"},
|
||||||
|
{"@id": "ex:Other", "@type": "owl:Class", "rdfs:label": "Other"},
|
||||||
|
{
|
||||||
|
"@id": "ex:relatesTo",
|
||||||
|
"@type": "owl:ObjectProperty",
|
||||||
|
"rdfs:domain": {"@id": "ex:Thing"},
|
||||||
|
"rdfs:range": {"@id": "ex:Other"},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
DEFAULT_GRAPH_ONTOLOGY = {
|
||||||
|
"@context": NAMED_GRAPH_ONTOLOGY["@context"],
|
||||||
|
"@graph": [
|
||||||
|
{"@id": "ex:Thing", "@type": "owl:Class", "rdfs:label": "Thing"},
|
||||||
|
{"@id": "ex:Other", "@type": "owl:Class", "rdfs:label": "Other"},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _write(tmp_path, name, document):
|
||||||
|
path = tmp_path / name
|
||||||
|
path.write_text(json.dumps(document), encoding="utf-8")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def test_terms_in_a_named_graph_are_ingested(tmp_path):
|
||||||
|
"""The regression: two classes and one object property, all inside the named graph."""
|
||||||
|
path = _write(tmp_path, "named.jsonld", NAMED_GRAPH_ONTOLOGY)
|
||||||
|
|
||||||
|
data = OntologyIngestor().ingest_ontology(path).data
|
||||||
|
|
||||||
|
assert len(data["classes"]) == 2, (
|
||||||
|
"classes inside a JSON-LD named graph were dropped; the ingestor is reading only "
|
||||||
|
"the default graph"
|
||||||
|
)
|
||||||
|
assert len(data["properties"]) == 1
|
||||||
|
assert {c["uri"] for c in data["classes"]} == {
|
||||||
|
"https://example.org/ns#Thing",
|
||||||
|
"https://example.org/ns#Other",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_terms_in_the_default_graph_still_work(tmp_path):
|
||||||
|
"""Canary for the test above: a document *without* a top-level ``@id`` keeps its terms
|
||||||
|
in the default graph and always parsed correctly. If this stopped passing, the fix would
|
||||||
|
have traded one blind spot for another."""
|
||||||
|
path = _write(tmp_path, "default.jsonld", DEFAULT_GRAPH_ONTOLOGY)
|
||||||
|
|
||||||
|
data = OntologyIngestor().ingest_ontology(path).data
|
||||||
|
|
||||||
|
assert len(data["classes"]) == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("document", [NAMED_GRAPH_ONTOLOGY, DEFAULT_GRAPH_ONTOLOGY])
|
||||||
|
def test_metadata_reports_what_was_actually_read(tmp_path, document):
|
||||||
|
"""Whatever the shape of the document, the counts reported have to match the terms
|
||||||
|
returned — a load that says it succeeded while returning nothing is what made #1129
|
||||||
|
cost an afternoon to find."""
|
||||||
|
path = _write(tmp_path, "any.jsonld", document)
|
||||||
|
|
||||||
|
result = OntologyIngestor().ingest_ontology(path)
|
||||||
|
|
||||||
|
assert result.data["classes"], "reported success with zero classes"
|
||||||
@@ -0,0 +1,278 @@
|
|||||||
|
"""SSRF regression tests for AgnoKnowledgeGraph.load_urls().
|
||||||
|
|
||||||
|
Prior to the fix, load_urls() used urllib.request.urlopen with only a
|
||||||
|
scheme check — private/loopback/link-local/metadata IPs were not blocked
|
||||||
|
and redirects were followed without re-validation.
|
||||||
|
|
||||||
|
These tests exercise the real SSRF guard (no mock of request_with_ssrf_guard
|
||||||
|
itself) by patching at the socket.getaddrinfo level, confirming that
|
||||||
|
blocked addresses never reach the network layer.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import socket
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
# conftest.py installs the full agno stub before this file is collected.
|
||||||
|
from integrations.agno.knowledge_graph import AgnoKnowledgeGraph
|
||||||
|
|
||||||
|
from semantica.utils.exceptions import ValidationError
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Minimal fakes so AgnoKnowledgeGraph.__init__ succeeds without real imports.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
class _FakeNER:
|
||||||
|
def extract_entities(self, text):
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeRelExtractor:
|
||||||
|
def extract_relations(self, text, entities=None):
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeGraphBuilder:
|
||||||
|
def build(self, sources):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeContextGraph:
|
||||||
|
def find_nodes(self, label=None):
|
||||||
|
return []
|
||||||
|
|
||||||
|
def get_neighbors(self, node_id=None, hops=1):
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _make_kg() -> AgnoKnowledgeGraph:
|
||||||
|
return AgnoKnowledgeGraph(
|
||||||
|
graph_builder=_FakeGraphBuilder(),
|
||||||
|
ner_extractor=_FakeNER(),
|
||||||
|
relation_extractor=_FakeRelExtractor(),
|
||||||
|
context_graph=_FakeContextGraph(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _public_getaddrinfo(host, *args, **kwargs):
|
||||||
|
"""Stub that makes every hostname resolve to a public IP."""
|
||||||
|
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0))]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests: blocked addresses must never be fetched
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestLoadUrlsBlockedAddresses:
|
||||||
|
"""load_urls() must silently skip (warn) any URL that fails the SSRF guard."""
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("url", [
|
||||||
|
"http://127.0.0.1/secret",
|
||||||
|
"http://127.0.0.1:9200/", # common internal service port
|
||||||
|
"http://0.0.0.0/",
|
||||||
|
"http://169.254.169.254/latest/meta-data/",
|
||||||
|
"http://169.254.169.254/computeMetadata/v1/",
|
||||||
|
"http://10.0.0.1/internal",
|
||||||
|
"http://10.255.255.255/",
|
||||||
|
"http://172.16.0.1/",
|
||||||
|
"http://172.31.255.255/",
|
||||||
|
"http://192.168.0.1/admin",
|
||||||
|
"http://192.168.100.200/",
|
||||||
|
"http://[::1]/ipv6-loopback",
|
||||||
|
"http://[fc00::1]/ipv6-ula",
|
||||||
|
"http://[fe80::1]/ipv6-link-local",
|
||||||
|
])
|
||||||
|
def test_blocked_ip_never_reaches_network(self, url):
|
||||||
|
"""Blocked addresses must raise ValidationError inside the guard,
|
||||||
|
which load_urls() catches and logs — _ingest_text must NOT be called."""
|
||||||
|
kg = _make_kg()
|
||||||
|
with patch.object(kg, "_ingest_text") as mock_ingest:
|
||||||
|
kg.load_urls([url])
|
||||||
|
mock_ingest.assert_not_called()
|
||||||
|
|
||||||
|
def test_localhost_hostname_blocked(self):
|
||||||
|
kg = _make_kg()
|
||||||
|
with patch.object(kg, "_ingest_text") as mock_ingest:
|
||||||
|
kg.load_urls(["http://localhost/admin"])
|
||||||
|
mock_ingest.assert_not_called()
|
||||||
|
|
||||||
|
def test_localhost_subdomain_blocked(self):
|
||||||
|
kg = _make_kg()
|
||||||
|
with patch.object(kg, "_ingest_text") as mock_ingest:
|
||||||
|
kg.load_urls(["http://foo.localhost/"])
|
||||||
|
mock_ingest.assert_not_called()
|
||||||
|
|
||||||
|
def test_hostname_resolving_to_private_ip_blocked(self):
|
||||||
|
"""A hostname that resolves to a private IP must be blocked even though
|
||||||
|
the URL string itself looks like a normal hostname."""
|
||||||
|
def _internal_getaddrinfo(host, *a, **kw):
|
||||||
|
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.5", 0))]
|
||||||
|
|
||||||
|
kg = _make_kg()
|
||||||
|
with patch("semantica.ingest.ssrf.socket.getaddrinfo", side_effect=_internal_getaddrinfo):
|
||||||
|
with patch.object(kg, "_ingest_text") as mock_ingest:
|
||||||
|
kg.load_urls(["http://internal.corp/secret"])
|
||||||
|
mock_ingest.assert_not_called()
|
||||||
|
|
||||||
|
def test_hostname_resolving_to_metadata_ip_blocked(self):
|
||||||
|
def _meta_getaddrinfo(host, *a, **kw):
|
||||||
|
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("169.254.169.254", 0))]
|
||||||
|
|
||||||
|
kg = _make_kg()
|
||||||
|
with patch("semantica.ingest.ssrf.socket.getaddrinfo", side_effect=_meta_getaddrinfo):
|
||||||
|
with patch.object(kg, "_ingest_text") as mock_ingest:
|
||||||
|
kg.load_urls(["http://metadata.internal/v1/token"])
|
||||||
|
mock_ingest.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoadUrlsNonHttpSchemes:
|
||||||
|
"""Non-HTTP(S) schemes must be rejected."""
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("url", [
|
||||||
|
"file:///etc/passwd",
|
||||||
|
"file://localhost/etc/shadow",
|
||||||
|
"ftp://example.com/file.txt",
|
||||||
|
"gopher://example.com/1",
|
||||||
|
"dict://example.com/",
|
||||||
|
"sftp://example.com/data",
|
||||||
|
])
|
||||||
|
def test_non_http_scheme_blocked(self, url):
|
||||||
|
kg = _make_kg()
|
||||||
|
with patch.object(kg, "_ingest_text") as mock_ingest:
|
||||||
|
kg.load_urls([url])
|
||||||
|
mock_ingest.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoadUrlsRedirects:
|
||||||
|
"""Redirects to private/blocked addresses must be rejected."""
|
||||||
|
|
||||||
|
def test_redirect_to_loopback_blocked(self):
|
||||||
|
"""A public first hop that redirects to loopback must be blocked."""
|
||||||
|
redirect = MagicMock()
|
||||||
|
redirect.status_code = 302
|
||||||
|
redirect.headers = {"Location": "http://127.0.0.1/secret"}
|
||||||
|
redirect.close = MagicMock()
|
||||||
|
|
||||||
|
kg = _make_kg()
|
||||||
|
with patch(
|
||||||
|
"semantica.ingest.ssrf.socket.getaddrinfo",
|
||||||
|
return_value=[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0))],
|
||||||
|
):
|
||||||
|
# Patch requests.Session so the first hop returns our redirect mock.
|
||||||
|
# The guard sees the 302, then validates the Location — 127.0.0.1 is
|
||||||
|
# blocked without a second network call.
|
||||||
|
with patch("semantica.ingest.ssrf.requests.Session") as MockSession:
|
||||||
|
mock_session = MockSession.return_value
|
||||||
|
mock_session.adapters = {}
|
||||||
|
mock_session.headers = {}
|
||||||
|
mock_session.auth = None
|
||||||
|
mock_session.trust_env = True
|
||||||
|
mock_session.request.return_value = redirect
|
||||||
|
|
||||||
|
with patch.object(kg, "_ingest_text") as mock_ingest:
|
||||||
|
kg.load_urls(["https://example.com/start"])
|
||||||
|
mock_ingest.assert_not_called()
|
||||||
|
|
||||||
|
def test_redirect_to_metadata_ip_blocked(self):
|
||||||
|
"""Redirect to cloud metadata endpoint must be blocked."""
|
||||||
|
redirect = MagicMock()
|
||||||
|
redirect.status_code = 301
|
||||||
|
redirect.headers = {"Location": "http://169.254.169.254/latest/meta-data/"}
|
||||||
|
redirect.close = MagicMock()
|
||||||
|
|
||||||
|
kg = _make_kg()
|
||||||
|
with patch(
|
||||||
|
"semantica.ingest.ssrf.socket.getaddrinfo",
|
||||||
|
return_value=[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0))],
|
||||||
|
):
|
||||||
|
with patch("semantica.ingest.ssrf.requests.Session") as MockSession:
|
||||||
|
mock_session = MockSession.return_value
|
||||||
|
mock_session.adapters = {}
|
||||||
|
mock_session.headers = {}
|
||||||
|
mock_session.auth = None
|
||||||
|
mock_session.trust_env = True
|
||||||
|
mock_session.request.return_value = redirect
|
||||||
|
|
||||||
|
with patch.object(kg, "_ingest_text") as mock_ingest:
|
||||||
|
kg.load_urls(["https://example.com/redirect-me"])
|
||||||
|
mock_ingest.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoadUrlsValidUrls:
|
||||||
|
"""Valid public URLs must succeed and call _ingest_text."""
|
||||||
|
|
||||||
|
def test_valid_public_url_ingested(self):
|
||||||
|
"""A URL resolving to a public IP must be fetched and ingested."""
|
||||||
|
ok_response = MagicMock()
|
||||||
|
ok_response.status_code = 200
|
||||||
|
ok_response.headers = {}
|
||||||
|
ok_response.text = "This is the document content."
|
||||||
|
|
||||||
|
kg = _make_kg()
|
||||||
|
with patch("semantica.ingest.ssrf.socket.getaddrinfo", side_effect=_public_getaddrinfo):
|
||||||
|
with patch("semantica.ingest.ssrf.requests.Session") as MockSession:
|
||||||
|
mock_session = MockSession.return_value
|
||||||
|
mock_session.adapters = {}
|
||||||
|
mock_session.headers = {}
|
||||||
|
mock_session.auth = None
|
||||||
|
mock_session.trust_env = True
|
||||||
|
mock_session.request.return_value = ok_response
|
||||||
|
|
||||||
|
with patch.object(kg, "_ingest_text") as mock_ingest:
|
||||||
|
kg.load_urls(["https://example.com/doc.txt"])
|
||||||
|
|
||||||
|
mock_ingest.assert_called_once_with(
|
||||||
|
"This is the document content.", source="https://example.com/doc.txt"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_multiple_urls_each_independently_validated(self):
|
||||||
|
"""Each URL in the list is independently validated; one blocked URL
|
||||||
|
must not prevent valid subsequent URLs from being ingested."""
|
||||||
|
ok_response = MagicMock()
|
||||||
|
ok_response.status_code = 200
|
||||||
|
ok_response.headers = {}
|
||||||
|
ok_response.text = "Valid content."
|
||||||
|
|
||||||
|
def _selective_getaddrinfo(host, *a, **kw):
|
||||||
|
if host == "internal.corp":
|
||||||
|
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.5", 0))]
|
||||||
|
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0))]
|
||||||
|
|
||||||
|
kg = _make_kg()
|
||||||
|
with patch("semantica.ingest.ssrf.socket.getaddrinfo", side_effect=_selective_getaddrinfo):
|
||||||
|
with patch("semantica.ingest.ssrf.requests.Session") as MockSession:
|
||||||
|
mock_session = MockSession.return_value
|
||||||
|
mock_session.adapters = {}
|
||||||
|
mock_session.headers = {}
|
||||||
|
mock_session.auth = None
|
||||||
|
mock_session.trust_env = True
|
||||||
|
mock_session.request.return_value = ok_response
|
||||||
|
|
||||||
|
with patch.object(kg, "_ingest_text") as mock_ingest:
|
||||||
|
kg.load_urls([
|
||||||
|
"http://internal.corp/secret", # blocked
|
||||||
|
"https://example.com/public.txt", # allowed
|
||||||
|
])
|
||||||
|
|
||||||
|
# Only the valid URL triggers ingestion
|
||||||
|
mock_ingest.assert_called_once_with("Valid content.", source="https://example.com/public.txt")
|
||||||
|
|
||||||
|
def test_failed_fetch_does_not_raise(self):
|
||||||
|
"""A network failure on a valid URL must log a warning, not raise."""
|
||||||
|
import requests as _requests
|
||||||
|
|
||||||
|
kg = _make_kg()
|
||||||
|
with patch("semantica.ingest.ssrf.socket.getaddrinfo", side_effect=_public_getaddrinfo):
|
||||||
|
with patch("semantica.ingest.ssrf.requests.Session") as MockSession:
|
||||||
|
mock_session = MockSession.return_value
|
||||||
|
mock_session.adapters = {}
|
||||||
|
mock_session.headers = {}
|
||||||
|
mock_session.auth = None
|
||||||
|
mock_session.trust_env = True
|
||||||
|
mock_session.request.side_effect = _requests.exceptions.ConnectionError("refused")
|
||||||
|
|
||||||
|
# Must not raise; failure is logged and skipped
|
||||||
|
kg.load_urls(["https://example.com/unreachable"])
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
# tests/integrations/openclaw package
|
||||||
@@ -0,0 +1,290 @@
|
|||||||
|
"""SSRF hardening tests for OpenClawKGTool.
|
||||||
|
|
||||||
|
OpenClawKGTool is designed to speak to a locally-running Semantica REST server
|
||||||
|
(default: http://localhost:8000). The fix validates base_url at construction
|
||||||
|
time so that obviously wrong schemes (file://, ftp://, gopher://, etc.) and
|
||||||
|
malformed URLs are rejected immediately, while localhost and other private
|
||||||
|
addresses remain valid because allow_private_ips=True is the correct posture
|
||||||
|
for this tool's intended use case.
|
||||||
|
|
||||||
|
These are construction-time tests; per-request SSRF guarding is not the
|
||||||
|
contract of this tool (its threat model is operator-configured base_url, not
|
||||||
|
untrusted per-call URLs).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from integrations.openclaw.mcp_tool import OpenClawKGTool
|
||||||
|
from semantica.utils.exceptions import ValidationError
|
||||||
|
|
||||||
|
|
||||||
|
class TestOpenClawKGToolBaseUrlValidation:
|
||||||
|
"""base_url is validated at __init__ time."""
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Valid base_urls — all must construct without raising
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("url", [
|
||||||
|
"http://localhost:8000",
|
||||||
|
"http://localhost",
|
||||||
|
"http://127.0.0.1:8000",
|
||||||
|
"http://127.0.0.1",
|
||||||
|
"https://localhost:8443",
|
||||||
|
"http://0.0.0.0:8000",
|
||||||
|
"http://192.168.1.10:8000", # LAN Semantica server
|
||||||
|
"http://10.0.0.5:8000", # corporate intranet deployment
|
||||||
|
"https://semantica.internal/api",
|
||||||
|
"https://semantica.example.com",
|
||||||
|
])
|
||||||
|
def test_valid_base_url_accepted(self, url):
|
||||||
|
"""All reasonable operator-configured base_urls must be accepted."""
|
||||||
|
tool = OpenClawKGTool(base_url=url)
|
||||||
|
assert tool.base_url == url.rstrip("/")
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Invalid schemes — must raise at construction
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("url", [
|
||||||
|
"file:///etc/passwd",
|
||||||
|
"file://localhost/etc/shadow",
|
||||||
|
"ftp://example.com/",
|
||||||
|
"gopher://example.com/1",
|
||||||
|
"dict://example.com/",
|
||||||
|
"sftp://example.com/",
|
||||||
|
"ldap://example.com/",
|
||||||
|
"javascript:alert(1)",
|
||||||
|
])
|
||||||
|
def test_invalid_scheme_rejected(self, url):
|
||||||
|
"""Non-HTTP(S) schemes must be rejected at construction time."""
|
||||||
|
with pytest.raises((ValidationError, ValueError)):
|
||||||
|
OpenClawKGTool(base_url=url)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Malformed URLs
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_empty_string_rejected(self):
|
||||||
|
with pytest.raises((ValidationError, ValueError)):
|
||||||
|
OpenClawKGTool(base_url="")
|
||||||
|
|
||||||
|
def test_no_scheme_rejected(self):
|
||||||
|
"""A bare hostname without a scheme must be rejected."""
|
||||||
|
with pytest.raises((ValidationError, ValueError)):
|
||||||
|
OpenClawKGTool(base_url="localhost:8000")
|
||||||
|
|
||||||
|
def test_whitespace_only_rejected(self):
|
||||||
|
with pytest.raises((ValidationError, ValueError)):
|
||||||
|
OpenClawKGTool(base_url=" ")
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Default is the documented localhost value
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_default_base_url_is_localhost(self):
|
||||||
|
"""The default must remain http://localhost:8000 for backward compat."""
|
||||||
|
tool = OpenClawKGTool()
|
||||||
|
assert tool.base_url == "http://localhost:8000"
|
||||||
|
|
||||||
|
def test_trailing_slash_stripped(self):
|
||||||
|
"""base_url trailing slash must be stripped so paths concatenate cleanly."""
|
||||||
|
tool = OpenClawKGTool(base_url="http://localhost:8000/")
|
||||||
|
assert tool.base_url == "http://localhost:8000"
|
||||||
|
|
||||||
|
def test_multiple_trailing_slashes_stripped(self):
|
||||||
|
tool = OpenClawKGTool(base_url="http://localhost:8000///")
|
||||||
|
assert tool.base_url == "http://localhost:8000"
|
||||||
|
|
||||||
|
def test_leading_and_trailing_whitespace_stripped(self):
|
||||||
|
"""Whitespace around a valid URL must be stripped before storage so
|
||||||
|
_post/_get don't build requests with space-padded URLs like
|
||||||
|
' http://localhost:8000 /extract'."""
|
||||||
|
tool = OpenClawKGTool(base_url=" http://localhost:8000 ")
|
||||||
|
assert tool.base_url == "http://localhost:8000"
|
||||||
|
|
||||||
|
def test_whitespace_plus_trailing_slash_both_stripped(self):
|
||||||
|
tool = OpenClawKGTool(base_url=" http://localhost:8000/ ")
|
||||||
|
assert tool.base_url == "http://localhost:8000"
|
||||||
|
|
||||||
|
|
||||||
|
class TestOpenClawKGToolFallbackValidation:
|
||||||
|
"""When semantica.ingest.ssrf is unavailable (ImportError path), the fallback
|
||||||
|
must perform the same structural checks as validate_url_for_request:
|
||||||
|
non-empty string, http/https scheme, netloc present, hostname present.
|
||||||
|
|
||||||
|
The fallback is exercised by temporarily hiding semantica.ingest.ssrf
|
||||||
|
from sys.modules so the import inside __init__ raises ImportError.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _hide_ssrf(monkeypatch):
|
||||||
|
"""Return a context in which semantica.ingest.ssrf appears unimportable."""
|
||||||
|
import sys
|
||||||
|
monkeypatch.setitem(sys.modules, "semantica.ingest.ssrf", None)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Valid URLs must still be accepted in the fallback path
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("url", [
|
||||||
|
"http://localhost:8000",
|
||||||
|
"http://127.0.0.1:8000",
|
||||||
|
"https://semantica.example.com",
|
||||||
|
])
|
||||||
|
def test_fallback_valid_url_accepted(self, url, monkeypatch):
|
||||||
|
self._hide_ssrf(monkeypatch)
|
||||||
|
tool = OpenClawKGTool(base_url=url)
|
||||||
|
assert tool.base_url == url.rstrip("/")
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Malformed URLs that the fallback previously let through
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("url", [
|
||||||
|
"http://", # scheme only, no netloc or hostname
|
||||||
|
"https://", # same
|
||||||
|
"http:///path", # empty hostname (netloc is present but hostname is None)
|
||||||
|
])
|
||||||
|
def test_fallback_no_netloc_rejected(self, url, monkeypatch):
|
||||||
|
"""URLs with a valid scheme but missing netloc/hostname must be rejected
|
||||||
|
in the fallback path, matching validate_url_for_request's behaviour."""
|
||||||
|
self._hide_ssrf(monkeypatch)
|
||||||
|
with pytest.raises((ValidationError, ValueError)):
|
||||||
|
OpenClawKGTool(base_url=url)
|
||||||
|
|
||||||
|
def test_fallback_empty_string_rejected(self, monkeypatch):
|
||||||
|
self._hide_ssrf(monkeypatch)
|
||||||
|
with pytest.raises((ValidationError, ValueError)):
|
||||||
|
OpenClawKGTool(base_url="")
|
||||||
|
|
||||||
|
def test_fallback_whitespace_only_rejected(self, monkeypatch):
|
||||||
|
self._hide_ssrf(monkeypatch)
|
||||||
|
with pytest.raises((ValidationError, ValueError)):
|
||||||
|
OpenClawKGTool(base_url=" ")
|
||||||
|
|
||||||
|
def test_fallback_invalid_scheme_rejected(self, monkeypatch):
|
||||||
|
self._hide_ssrf(monkeypatch)
|
||||||
|
with pytest.raises((ValidationError, ValueError)):
|
||||||
|
OpenClawKGTool(base_url="file:///etc/passwd")
|
||||||
|
|
||||||
|
def test_fallback_no_scheme_rejected(self, monkeypatch):
|
||||||
|
self._hide_ssrf(monkeypatch)
|
||||||
|
with pytest.raises((ValidationError, ValueError)):
|
||||||
|
OpenClawKGTool(base_url="localhost:8000")
|
||||||
|
|
||||||
|
def test_fallback_whitespace_padded_valid_url_stored_clean(self, monkeypatch):
|
||||||
|
"""Whitespace around a valid URL must be stripped before storage in the
|
||||||
|
fallback path too — same guarantee as the normal path."""
|
||||||
|
self._hide_ssrf(monkeypatch)
|
||||||
|
tool = OpenClawKGTool(base_url=" http://localhost:8000 ")
|
||||||
|
assert tool.base_url == "http://localhost:8000"
|
||||||
|
|
||||||
|
|
||||||
|
class TestOpenClawKGToolEndpointConstruction:
|
||||||
|
"""Verify that per-method URLs are assembled from base_url + hardcoded paths.
|
||||||
|
|
||||||
|
The endpoint strings are always literals defined in the class body —
|
||||||
|
they are not caller-supplied — so these tests confirm the URL assembly
|
||||||
|
logic is correct rather than testing SSRF guards on the endpoints.
|
||||||
|
|
||||||
|
All HTTP calls are mocked so no real network connection is made.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _mock_session(self, status: int = 200, body: bytes = b"{}") -> "MagicMock":
|
||||||
|
"""Return a mock session whose post/get return a minimal JSON response."""
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
mock_resp = MagicMock()
|
||||||
|
mock_resp.status_code = status
|
||||||
|
mock_resp.raise_for_status = MagicMock()
|
||||||
|
mock_resp.json.return_value = {}
|
||||||
|
session = MagicMock()
|
||||||
|
session.post.return_value = mock_resp
|
||||||
|
session.get.return_value = mock_resp
|
||||||
|
return session
|
||||||
|
|
||||||
|
def test_post_url_constructed_from_base_url(self):
|
||||||
|
"""_post must call session.post with the exact URL base_url+endpoint,
|
||||||
|
the supplied payload as json=, and the tool timeout. No real connection."""
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
tool = OpenClawKGTool(base_url="http://localhost:8000")
|
||||||
|
mock_session = self._mock_session()
|
||||||
|
|
||||||
|
with patch.object(tool, "_get_session", return_value=mock_session):
|
||||||
|
tool._post("/extract", {"text": "hello"})
|
||||||
|
|
||||||
|
mock_session.post.assert_called_once_with(
|
||||||
|
"http://localhost:8000/extract",
|
||||||
|
json={"text": "hello"},
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_post_url_with_custom_base_url(self):
|
||||||
|
"""base_url is reflected correctly in the outbound URL for _post."""
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
tool = OpenClawKGTool(base_url="http://192.168.1.10:9000")
|
||||||
|
mock_session = self._mock_session()
|
||||||
|
|
||||||
|
with patch.object(tool, "_get_session", return_value=mock_session):
|
||||||
|
tool._post("/decisions", {"decision": "deploy"})
|
||||||
|
|
||||||
|
mock_session.post.assert_called_once_with(
|
||||||
|
"http://192.168.1.10:9000/decisions",
|
||||||
|
json={"decision": "deploy"},
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_get_url_constructed_from_base_url(self):
|
||||||
|
"""_get must call session.get with the exact URL base_url+endpoint,
|
||||||
|
params={} when none are supplied, and the tool timeout."""
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
tool = OpenClawKGTool(base_url="http://localhost:8000")
|
||||||
|
mock_session = self._mock_session()
|
||||||
|
|
||||||
|
with patch.object(tool, "_get_session", return_value=mock_session):
|
||||||
|
tool._get("/analytics")
|
||||||
|
|
||||||
|
mock_session.get.assert_called_once_with(
|
||||||
|
"http://localhost:8000/analytics",
|
||||||
|
params={},
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_get_url_with_params(self):
|
||||||
|
"""_get must forward supplied params to session.get."""
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
tool = OpenClawKGTool(base_url="http://localhost:8000")
|
||||||
|
mock_session = self._mock_session()
|
||||||
|
|
||||||
|
with patch.object(tool, "_get_session", return_value=mock_session):
|
||||||
|
tool._get("/decisions/search", {"q": "deploy", "limit": 5})
|
||||||
|
|
||||||
|
mock_session.get.assert_called_once_with(
|
||||||
|
"http://localhost:8000/decisions/search",
|
||||||
|
params={"q": "deploy", "limit": 5},
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_custom_timeout_forwarded(self):
|
||||||
|
"""A non-default timeout must reach session.post and session.get."""
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
tool = OpenClawKGTool(base_url="http://localhost:8000", timeout=60)
|
||||||
|
mock_session = self._mock_session()
|
||||||
|
|
||||||
|
with patch.object(tool, "_get_session", return_value=mock_session):
|
||||||
|
tool._post("/extract", {"text": "x"})
|
||||||
|
tool._get("/analytics")
|
||||||
|
|
||||||
|
assert mock_session.post.call_args.kwargs["timeout"] == 60
|
||||||
|
assert mock_session.get.call_args.kwargs["timeout"] == 60
|
||||||
|
|
||||||
|
def test_repr_includes_base_url(self):
|
||||||
|
tool = OpenClawKGTool(base_url="http://localhost:9000")
|
||||||
|
assert "http://localhost:9000" in repr(tool)
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
import copy
|
||||||
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from semantica.pipeline.pipeline_builder import PipelineBuilder, PipelineSerializer
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("serialization_format", ["dict", "json"])
|
||||||
|
def test_roundtrip_preserves_dependencies_and_delta_metadata(serialization_format):
|
||||||
|
builder = PipelineBuilder()
|
||||||
|
builder.add_step("extract", "source")
|
||||||
|
builder.add_step(
|
||||||
|
"index",
|
||||||
|
"sink",
|
||||||
|
delta_mode=True,
|
||||||
|
base_version_id="v1",
|
||||||
|
target_version_id="v2",
|
||||||
|
)
|
||||||
|
builder.connect_steps("extract", "index")
|
||||||
|
pipeline = builder.build("incremental-index")
|
||||||
|
|
||||||
|
serializer = PipelineSerializer()
|
||||||
|
serialized = serializer.serialize_pipeline(pipeline, format=serialization_format)
|
||||||
|
restored = serializer.deserialize_pipeline(serialized)
|
||||||
|
|
||||||
|
index_step = next(step for step in restored.steps if step.name == "index")
|
||||||
|
assert index_step.dependencies == ["extract"]
|
||||||
|
assert index_step.delta_mode is True
|
||||||
|
assert index_step.base_version_id == "v1"
|
||||||
|
assert index_step.target_version_id == "v2"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("serialization_format", ["dict", "json"])
|
||||||
|
def test_serialization_omits_runtime_handlers(serialization_format):
|
||||||
|
def handler(data, **config):
|
||||||
|
return data
|
||||||
|
|
||||||
|
builder = PipelineBuilder()
|
||||||
|
builder.add_step("extract", "source", handler=handler, batch_size=10)
|
||||||
|
pipeline = builder.build("handler-pipeline")
|
||||||
|
|
||||||
|
serializer = PipelineSerializer()
|
||||||
|
serialized = serializer.serialize_pipeline(pipeline, format=serialization_format)
|
||||||
|
serialized_data = (
|
||||||
|
json.loads(serialized) if isinstance(serialized, str) else serialized
|
||||||
|
)
|
||||||
|
|
||||||
|
assert serialized_data["steps"][0]["config"] == {"batch_size": 10}
|
||||||
|
|
||||||
|
restored = serializer.deserialize_pipeline(serialized)
|
||||||
|
assert restored.steps[0].handler is None
|
||||||
|
assert restored.steps[0].config == {"batch_size": 10}
|
||||||
|
|
||||||
|
|
||||||
|
def test_deserialization_ignores_legacy_stringified_handler():
|
||||||
|
serialized = json.dumps(
|
||||||
|
{
|
||||||
|
"name": "legacy-handler-pipeline",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"name": "extract",
|
||||||
|
"type": "source",
|
||||||
|
"config": {
|
||||||
|
"handler": "<function extract at 0x1234>",
|
||||||
|
"batch_size": 10,
|
||||||
|
},
|
||||||
|
"dependencies": [],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
restored = PipelineSerializer().deserialize_pipeline(serialized)
|
||||||
|
|
||||||
|
assert restored.steps[0].handler is None
|
||||||
|
assert restored.steps[0].config == {"batch_size": 10}
|
||||||
|
|
||||||
|
|
||||||
|
def test_deserialization_does_not_mutate_caller_owned_dict():
|
||||||
|
payload = {
|
||||||
|
"name": "legacy-handler-pipeline",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"name": "extract",
|
||||||
|
"type": "source",
|
||||||
|
"config": {
|
||||||
|
"handler": "<function extract at 0x1234>",
|
||||||
|
"batch_size": 10,
|
||||||
|
},
|
||||||
|
"dependencies": [],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
snapshot = copy.deepcopy(payload)
|
||||||
|
|
||||||
|
restored = PipelineSerializer().deserialize_pipeline(payload)
|
||||||
|
|
||||||
|
assert payload == snapshot
|
||||||
|
assert "handler" in payload["steps"][0]["config"]
|
||||||
|
assert payload is not snapshot
|
||||||
|
assert restored.steps[0].handler is None
|
||||||
|
assert restored.steps[0].config == {"batch_size": 10}
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
|
|
||||||
import unittest
|
|
||||||
from unittest.mock import MagicMock, patch
|
|
||||||
from semantica.semantic_extract.methods import extract_relations_llm, extract_entities_llm, extract_triplets_llm
|
|
||||||
from semantica.semantic_extract.ner_extractor import Entity
|
|
||||||
|
|
||||||
class TestMaxTokensPropagation(unittest.TestCase):
|
|
||||||
@patch("semantica.semantic_extract.methods.create_provider")
|
|
||||||
def test_max_tokens_propagation_relations(self, mock_create_provider):
|
|
||||||
"""Test that max_tokens is passed to generate_typed in extract_relations_llm."""
|
|
||||||
# Setup mock
|
|
||||||
mock_llm = MagicMock()
|
|
||||||
mock_create_provider.return_value = mock_llm
|
|
||||||
mock_llm.is_available.return_value = True
|
|
||||||
|
|
||||||
# Setup return value to avoid pydantic validation errors
|
|
||||||
mock_response = MagicMock()
|
|
||||||
mock_response.relations = []
|
|
||||||
mock_llm.generate_typed.return_value = mock_response
|
|
||||||
|
|
||||||
# Create dummy entities
|
|
||||||
entities = [Entity(text="Foo", label="ORG", start_char=0, end_char=3)]
|
|
||||||
|
|
||||||
# Call the function with max_tokens
|
|
||||||
extract_relations_llm(
|
|
||||||
text="some text",
|
|
||||||
entities=entities,
|
|
||||||
provider="openai",
|
|
||||||
model="gpt-4",
|
|
||||||
max_tokens=128000
|
|
||||||
)
|
|
||||||
|
|
||||||
# Check if generate_typed was called with max_tokens
|
|
||||||
args, kwargs = mock_llm.generate_typed.call_args
|
|
||||||
|
|
||||||
print(f"Relations Call kwargs: {kwargs}")
|
|
||||||
|
|
||||||
self.assertIn("max_tokens", kwargs)
|
|
||||||
self.assertEqual(kwargs["max_tokens"], 128000)
|
|
||||||
|
|
||||||
@patch("semantica.semantic_extract.methods.create_provider")
|
|
||||||
def test_max_tokens_propagation_entities(self, mock_create_provider):
|
|
||||||
"""Test that max_tokens is passed to generate_typed in extract_entities_llm."""
|
|
||||||
# Setup mock
|
|
||||||
mock_llm = MagicMock()
|
|
||||||
mock_create_provider.return_value = mock_llm
|
|
||||||
mock_llm.is_available.return_value = True
|
|
||||||
|
|
||||||
# Setup return value to avoid pydantic validation errors
|
|
||||||
mock_response = MagicMock()
|
|
||||||
mock_response.entities = []
|
|
||||||
mock_llm.generate_typed.return_value = mock_response
|
|
||||||
|
|
||||||
# Call the function with max_tokens
|
|
||||||
extract_entities_llm(
|
|
||||||
text="some text",
|
|
||||||
provider="openai",
|
|
||||||
model="gpt-4",
|
|
||||||
max_tokens=128000
|
|
||||||
)
|
|
||||||
|
|
||||||
# Check if generate_typed was called with max_tokens
|
|
||||||
args, kwargs = mock_llm.generate_typed.call_args
|
|
||||||
|
|
||||||
print(f"Entities Call kwargs: {kwargs}")
|
|
||||||
|
|
||||||
self.assertIn("max_tokens", kwargs)
|
|
||||||
self.assertEqual(kwargs["max_tokens"], 128000)
|
|
||||||
|
|
||||||
@patch("semantica.semantic_extract.methods.create_provider")
|
|
||||||
def test_max_tokens_propagation_triplets(self, mock_create_provider):
|
|
||||||
"""Test that max_tokens is passed to generate_typed in extract_triplets_llm."""
|
|
||||||
# Setup mock
|
|
||||||
mock_llm = MagicMock()
|
|
||||||
mock_create_provider.return_value = mock_llm
|
|
||||||
mock_llm.is_available.return_value = True
|
|
||||||
|
|
||||||
# Setup return value to avoid pydantic validation errors
|
|
||||||
mock_response = MagicMock()
|
|
||||||
mock_response.triplets = []
|
|
||||||
mock_llm.generate_typed.return_value = mock_response
|
|
||||||
|
|
||||||
# Call the function with max_tokens
|
|
||||||
extract_triplets_llm(
|
|
||||||
text="some text",
|
|
||||||
provider="openai",
|
|
||||||
model="gpt-4",
|
|
||||||
max_tokens=128000
|
|
||||||
)
|
|
||||||
|
|
||||||
# Check if generate_typed was called with max_tokens
|
|
||||||
args, kwargs = mock_llm.generate_typed.call_args
|
|
||||||
|
|
||||||
print(f"Triplets Call kwargs: {kwargs}")
|
|
||||||
|
|
||||||
self.assertIn("max_tokens", kwargs)
|
|
||||||
self.assertEqual(kwargs["max_tokens"], 128000)
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@@ -0,0 +1,232 @@
|
|||||||
|
"""Regression tests for the standalone mcp/ package export_graph tool.
|
||||||
|
|
||||||
|
The mcp/ server (python -m mcp / python -m mcp.server) had two failures on
|
||||||
|
every RDF export format:
|
||||||
|
|
||||||
|
1. AttributeError: 'ContextGraph' object has no attribute 'get'
|
||||||
|
handle_export_graph() in mcp/tools/export.py called
|
||||||
|
RDFExporter().export_to_rdf(graph, ...) passing the raw ContextGraph
|
||||||
|
object instead of the canonical kg dict expected by the exporter.
|
||||||
|
|
||||||
|
2. stdout progress corruption
|
||||||
|
RDFExporter.__init__ instantiated the Semantica progress-tracker
|
||||||
|
singleton, which wrote a progress bar to sys.stdout before the
|
||||||
|
AttributeError was raised. stdout is the MCP stdio JSON-RPC transport,
|
||||||
|
so this interleaved non-JSON bytes corrupted framing for every client.
|
||||||
|
|
||||||
|
Fixes applied:
|
||||||
|
- mcp/tools/export.py: convert with graph.to_kg_dict() before export_to_rdf()
|
||||||
|
- mcp/__init__.py: os.environ["SEMANTICA_DISABLE_PROGRESS"] = "1" at
|
||||||
|
package initialisation, before any tool handler can instantiate
|
||||||
|
RDFExporter and therefore before the tracker singleton is created.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
import semantica.utils.progress_tracker as _progress_module
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _make_graph():
|
||||||
|
"""Return a ContextGraph with two entities and one relationship."""
|
||||||
|
from semantica.context.context_graph import ContextGraph
|
||||||
|
g = ContextGraph()
|
||||||
|
g.add_node("n1", node_type="entity")
|
||||||
|
g.add_node("n2", node_type="entity")
|
||||||
|
g.add_edge("n1", "n2", "related_to")
|
||||||
|
return g
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_progress_singleton():
|
||||||
|
"""Destroy any cached progress-tracker singleton so the next call
|
||||||
|
to get_progress_tracker() reads the current environment variable."""
|
||||||
|
_progress_module.ProgressTracker._instance = None
|
||||||
|
_progress_module._global_tracker = None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# RDF export correctness
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestMCPPackageExportGraphRDF(unittest.TestCase):
|
||||||
|
"""handle_export_graph() must return a non-empty RDF string for every
|
||||||
|
supported RDF format, not an error dict."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
# Inject a known graph into the mcp/ session so handlers don't try to
|
||||||
|
# build a full ContextGraph (which requires heavy ML dependencies).
|
||||||
|
import mcp.session as _session
|
||||||
|
self._orig_graph = _session._graph
|
||||||
|
_session._graph = _make_graph()
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
import mcp.session as _session
|
||||||
|
_session._graph = self._orig_graph
|
||||||
|
|
||||||
|
def test_turtle_returns_non_empty_string(self):
|
||||||
|
from mcp.tools.export import handle_export_graph
|
||||||
|
result = handle_export_graph({"format": "turtle"})
|
||||||
|
self.assertNotIn("error", result, result)
|
||||||
|
self.assertIsInstance(result["data"], str)
|
||||||
|
self.assertGreater(len(result["data"]), 0)
|
||||||
|
# Turtle output must carry prefix declarations
|
||||||
|
self.assertIn("@prefix", result["data"])
|
||||||
|
|
||||||
|
def test_ttl_alias_returns_non_empty_string(self):
|
||||||
|
from mcp.tools.export import handle_export_graph
|
||||||
|
result = handle_export_graph({"format": "ttl"})
|
||||||
|
self.assertNotIn("error", result, result)
|
||||||
|
self.assertIsInstance(result["data"], str)
|
||||||
|
self.assertGreater(len(result["data"]), 0)
|
||||||
|
|
||||||
|
def test_nt_returns_non_empty_string(self):
|
||||||
|
from mcp.tools.export import handle_export_graph
|
||||||
|
result = handle_export_graph({"format": "nt"})
|
||||||
|
self.assertNotIn("error", result, result)
|
||||||
|
self.assertIsInstance(result["data"], str)
|
||||||
|
self.assertGreater(len(result["data"]), 0)
|
||||||
|
|
||||||
|
def test_xml_returns_non_empty_string(self):
|
||||||
|
from mcp.tools.export import handle_export_graph
|
||||||
|
result = handle_export_graph({"format": "xml"})
|
||||||
|
self.assertNotIn("error", result, result)
|
||||||
|
self.assertIsInstance(result["data"], str)
|
||||||
|
self.assertGreater(len(result["data"]), 0)
|
||||||
|
|
||||||
|
def test_jsonld_returns_non_empty_string(self):
|
||||||
|
from mcp.tools.export import handle_export_graph
|
||||||
|
result = handle_export_graph({"format": "json-ld"})
|
||||||
|
self.assertNotIn("error", result, result)
|
||||||
|
self.assertIsInstance(result["data"], str)
|
||||||
|
self.assertGreater(len(result["data"]), 0)
|
||||||
|
|
||||||
|
def test_all_rdf_formats_succeed(self):
|
||||||
|
from mcp.tools.export import handle_export_graph
|
||||||
|
for fmt in ("turtle", "ttl", "nt", "xml", "json-ld"):
|
||||||
|
with self.subTest(fmt=fmt):
|
||||||
|
result = handle_export_graph({"format": fmt})
|
||||||
|
self.assertNotIn("error", result, f"format={fmt}: {result}")
|
||||||
|
self.assertIsInstance(result["data"], str)
|
||||||
|
self.assertGreater(len(result["data"]), 0)
|
||||||
|
|
||||||
|
def test_rdf_branch_does_not_raise_context_graph_attribute_error(self):
|
||||||
|
"""The pre-fix code passed ContextGraph directly to export_to_rdf(),
|
||||||
|
causing AttributeError: 'ContextGraph' object has no attribute 'get'.
|
||||||
|
Verify that error does not appear in the result."""
|
||||||
|
from mcp.tools.export import handle_export_graph
|
||||||
|
result = handle_export_graph({"format": "turtle"})
|
||||||
|
if "error" in result:
|
||||||
|
self.assertNotIn("'ContextGraph' object has no attribute 'get'",
|
||||||
|
result["error"])
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# stdout protection — subprocess-based to avoid process-state cross-contamination
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestMCPPackageStdoutProtection(unittest.TestCase):
|
||||||
|
"""The standalone mcp/ server must not write any progress bytes to stdout.
|
||||||
|
stdout is the MCP JSON-RPC transport channel.
|
||||||
|
|
||||||
|
These tests use a subprocess to get a clean process state where
|
||||||
|
SEMANTICA_DISABLE_PROGRESS has not yet been set, so we can verify that
|
||||||
|
importing mcp and running an export produces no progress bytes on stdout.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _run_in_subprocess(self, code: str, timeout: int = 30) -> subprocess.CompletedProcess:
|
||||||
|
"""Run a Python snippet in a clean subprocess with the repo on sys.path."""
|
||||||
|
repo_root = os.path.abspath(
|
||||||
|
os.path.join(os.path.dirname(__file__), "..")
|
||||||
|
)
|
||||||
|
env = os.environ.copy()
|
||||||
|
env["PYTHONPATH"] = repo_root
|
||||||
|
# Start with a clean slate — no pre-set disable flag
|
||||||
|
env.pop("SEMANTICA_DISABLE_PROGRESS", None)
|
||||||
|
return subprocess.run(
|
||||||
|
[sys.executable, "-c", code],
|
||||||
|
cwd=repo_root,
|
||||||
|
env=env,
|
||||||
|
text=True,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
timeout=timeout,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_importing_mcp_sets_disable_progress(self):
|
||||||
|
"""Importing the mcp package must set SEMANTICA_DISABLE_PROGRESS=1
|
||||||
|
before any tool handler runs."""
|
||||||
|
code = (
|
||||||
|
"import os; "
|
||||||
|
"import mcp; " # triggers mcp/__init__.py
|
||||||
|
"print(os.environ.get('SEMANTICA_DISABLE_PROGRESS', 'NOT SET'))"
|
||||||
|
)
|
||||||
|
result = self._run_in_subprocess(code)
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
self.assertIn("1", result.stdout)
|
||||||
|
|
||||||
|
def test_rdf_export_writes_no_progress_to_stdout(self):
|
||||||
|
"""An RDF export via handle_export_graph() must not write any Semantica
|
||||||
|
progress bytes to stdout. The only stdout bytes should be the explicit
|
||||||
|
print() call at the end of the snippet."""
|
||||||
|
code = """
|
||||||
|
import os, sys
|
||||||
|
# Ensure clean state
|
||||||
|
os.environ.pop("SEMANTICA_DISABLE_PROGRESS", None)
|
||||||
|
|
||||||
|
import mcp # sets SEMANTICA_DISABLE_PROGRESS=1
|
||||||
|
import mcp.session as session
|
||||||
|
from semantica.context.context_graph import ContextGraph
|
||||||
|
|
||||||
|
g = ContextGraph()
|
||||||
|
g.add_node("n1", node_type="entity")
|
||||||
|
g.add_node("n2", node_type="entity")
|
||||||
|
g.add_edge("n1", "n2", "related_to")
|
||||||
|
session._graph = g
|
||||||
|
|
||||||
|
# Intercept stdout writes to detect any progress output
|
||||||
|
written = []
|
||||||
|
_orig = sys.stdout.write
|
||||||
|
def _capture(s):
|
||||||
|
written.append(s)
|
||||||
|
return _orig(s)
|
||||||
|
sys.stdout.write = _capture
|
||||||
|
|
||||||
|
from mcp.tools.export import handle_export_graph
|
||||||
|
result = handle_export_graph({"format": "turtle"})
|
||||||
|
|
||||||
|
sys.stdout.write = _orig
|
||||||
|
|
||||||
|
# Only our explicit output below should be in written
|
||||||
|
# (the sentinel line is added after restoring stdout)
|
||||||
|
progress_writes = [s for s in written]
|
||||||
|
print("RESULT_OK:" + str("error" not in result))
|
||||||
|
print("STDOUT_WRITES:" + str(len(progress_writes)))
|
||||||
|
"""
|
||||||
|
proc = self._run_in_subprocess(code)
|
||||||
|
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||||
|
# Extract the printed lines
|
||||||
|
lines = proc.stdout.strip().splitlines()
|
||||||
|
result_ok_line = next((l for l in lines if l.startswith("RESULT_OK:")), None)
|
||||||
|
writes_line = next((l for l in lines if l.startswith("STDOUT_WRITES:")), None)
|
||||||
|
self.assertIsNotNone(result_ok_line, f"stdout: {proc.stdout!r}")
|
||||||
|
self.assertIsNotNone(writes_line, f"stdout: {proc.stdout!r}")
|
||||||
|
self.assertEqual(result_ok_line, "RESULT_OK:True",
|
||||||
|
f"export returned error; stdout={proc.stdout!r}, stderr={proc.stderr!r}")
|
||||||
|
n_writes = int(writes_line.split(":")[1])
|
||||||
|
self.assertEqual(n_writes, 0,
|
||||||
|
f"Expected 0 progress writes to stdout, got {n_writes}; "
|
||||||
|
f"stdout={proc.stdout!r}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
"""Regression tests for the MCP export_graph tool (issue: all branches broken).
|
||||||
|
|
||||||
|
The MCP server's export_graph tool failed on every format in 0.6.5/0.6.6:
|
||||||
|
- json: JSONExporter().export(graph) called without the required file_path
|
||||||
|
argument -> TypeError, surfaced as {"error": ...}
|
||||||
|
- RDF: RDFExporter().export_to_rdf(graph, ...) received the ContextGraph
|
||||||
|
object instead of the canonical kg dict -> AttributeError
|
||||||
|
- all: the RDF path printed a rich progress bar to stdout, corrupting the
|
||||||
|
stdio JSON-RPC framing and hanging the client (observed: 300s
|
||||||
|
timeout over MCP, <1s directly).
|
||||||
|
|
||||||
|
The fix: convert the graph with ContextGraph.to_kg_dict() before handing it to
|
||||||
|
the exporters, serialize json to a string, and force SEMANTICA_DISABLE_PROGRESS
|
||||||
|
for the server process (stdout is the protocol channel, not a console).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from semantica import mcp_server
|
||||||
|
from semantica.context import ContextGraph
|
||||||
|
|
||||||
|
|
||||||
|
def _graph_with_content() -> ContextGraph:
|
||||||
|
graph = ContextGraph(advanced_analytics=True)
|
||||||
|
graph.add_node("n1", node_type="entity", properties={"text": "hello"})
|
||||||
|
graph.add_node("n2", node_type="entity", properties={"text": "world"})
|
||||||
|
graph.add_edge("n1", "n2", "related_to")
|
||||||
|
return graph
|
||||||
|
|
||||||
|
|
||||||
|
class TestExportGraphTool(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self._old_graph = mcp_server._graph
|
||||||
|
mcp_server._graph = _graph_with_content()
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
mcp_server._graph = self._old_graph
|
||||||
|
|
||||||
|
def test_json_branch_returns_string_data_not_error(self):
|
||||||
|
result = mcp_server._tool_export_graph({"format": "json"})
|
||||||
|
self.assertNotIn("error", result)
|
||||||
|
self.assertEqual(result["format"], "json")
|
||||||
|
payload = json.loads(result["data"])
|
||||||
|
self.assertEqual(len(payload["entities"]), 2)
|
||||||
|
self.assertEqual(len(payload["relationships"]), 1)
|
||||||
|
|
||||||
|
def test_jsonld_branch_returns_string_data_not_error(self):
|
||||||
|
result = mcp_server._tool_export_graph({"format": "json-ld"})
|
||||||
|
self.assertNotIn("error", result)
|
||||||
|
self.assertEqual(result["format"], "json-ld")
|
||||||
|
self.assertIsInstance(result["data"], str)
|
||||||
|
self.assertGreater(len(result["data"]), 0)
|
||||||
|
|
||||||
|
def test_turtle_branch_returns_string_data_not_error(self):
|
||||||
|
result = mcp_server._tool_export_graph({"format": "turtle"})
|
||||||
|
self.assertNotIn("error", result)
|
||||||
|
self.assertIsInstance(result["data"], str)
|
||||||
|
self.assertIn("@prefix", result["data"])
|
||||||
|
|
||||||
|
def test_all_rdf_formats_succeed(self):
|
||||||
|
for fmt in ("turtle", "ttl", "nt", "xml", "json-ld"):
|
||||||
|
with self.subTest(fmt=fmt):
|
||||||
|
result = mcp_server._tool_export_graph({"format": fmt})
|
||||||
|
self.assertNotIn("error", result, fmt)
|
||||||
|
self.assertIsInstance(result["data"], str)
|
||||||
|
|
||||||
|
def test_progress_is_disabled_for_the_server_process(self):
|
||||||
|
self.assertEqual(os.environ.get("SEMANTICA_DISABLE_PROGRESS"), "1")
|
||||||
|
|
||||||
|
def test_unsupported_format_returns_error_not_mislabeled_json(self):
|
||||||
|
"""A format outside the declared enum (typo, unsupported value, or a
|
||||||
|
client that skips schema validation) must error, not silently return
|
||||||
|
JSON data mislabeled with the requested format string."""
|
||||||
|
result = mcp_server._tool_export_graph({"format": "yaml"})
|
||||||
|
self.assertIn("error", result)
|
||||||
|
self.assertIn("yaml", result["error"])
|
||||||
|
|
||||||
|
def test_export_graph_schema_enum_matches_handled_formats(self):
|
||||||
|
"""The tool's declared inputSchema enum must not drift from the set
|
||||||
|
of formats the handler actually accepts."""
|
||||||
|
tool = next(t for t in mcp_server.TOOLS if t["name"] == "export_graph")
|
||||||
|
schema_enum = set(tool["inputSchema"]["properties"]["format"]["enum"])
|
||||||
|
self.assertEqual(schema_enum, set(mcp_server._EXPORT_GRAPH_FORMATS))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,334 @@
|
|||||||
|
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
from semantica.semantic_extract.methods import extract_relations_llm, extract_entities_llm, extract_triplets_llm
|
||||||
|
from semantica.semantic_extract.ner_extractor import Entity
|
||||||
|
|
||||||
|
class TestMaxTokensPropagation(unittest.TestCase):
|
||||||
|
@patch("semantica.semantic_extract.methods.create_provider")
|
||||||
|
def test_max_tokens_propagation_relations(self, mock_create_provider):
|
||||||
|
"""Test that max_tokens is passed to generate_typed in extract_relations_llm."""
|
||||||
|
# Setup mock
|
||||||
|
mock_llm = MagicMock()
|
||||||
|
mock_create_provider.return_value = mock_llm
|
||||||
|
mock_llm.is_available.return_value = True
|
||||||
|
|
||||||
|
# Setup return value to avoid pydantic validation errors
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.relations = []
|
||||||
|
mock_llm.generate_typed.return_value = mock_response
|
||||||
|
|
||||||
|
# Create dummy entities
|
||||||
|
entities = [Entity(text="Foo", label="ORG", start_char=0, end_char=3)]
|
||||||
|
|
||||||
|
# Call the function with max_tokens
|
||||||
|
extract_relations_llm(
|
||||||
|
text="some text",
|
||||||
|
entities=entities,
|
||||||
|
provider="openai",
|
||||||
|
model="gpt-4",
|
||||||
|
max_tokens=128000
|
||||||
|
)
|
||||||
|
|
||||||
|
# Check if generate_typed was called with max_tokens
|
||||||
|
args, kwargs = mock_llm.generate_typed.call_args
|
||||||
|
|
||||||
|
print(f"Relations Call kwargs: {kwargs}")
|
||||||
|
|
||||||
|
self.assertIn("max_tokens", kwargs)
|
||||||
|
self.assertEqual(kwargs["max_tokens"], 128000)
|
||||||
|
|
||||||
|
@patch("semantica.semantic_extract.methods.create_provider")
|
||||||
|
def test_max_tokens_propagation_entities(self, mock_create_provider):
|
||||||
|
"""Test that max_tokens is passed to generate_typed in extract_entities_llm."""
|
||||||
|
# Setup mock
|
||||||
|
mock_llm = MagicMock()
|
||||||
|
mock_create_provider.return_value = mock_llm
|
||||||
|
mock_llm.is_available.return_value = True
|
||||||
|
|
||||||
|
# Setup return value to avoid pydantic validation errors
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.entities = []
|
||||||
|
mock_llm.generate_typed.return_value = mock_response
|
||||||
|
|
||||||
|
# Call the function with max_tokens
|
||||||
|
extract_entities_llm(
|
||||||
|
text="some text",
|
||||||
|
provider="openai",
|
||||||
|
model="gpt-4",
|
||||||
|
max_tokens=128000
|
||||||
|
)
|
||||||
|
|
||||||
|
# Check if generate_typed was called with max_tokens
|
||||||
|
args, kwargs = mock_llm.generate_typed.call_args
|
||||||
|
|
||||||
|
print(f"Entities Call kwargs: {kwargs}")
|
||||||
|
|
||||||
|
self.assertIn("max_tokens", kwargs)
|
||||||
|
self.assertEqual(kwargs["max_tokens"], 128000)
|
||||||
|
|
||||||
|
@patch("semantica.semantic_extract.methods.create_provider")
|
||||||
|
def test_max_tokens_propagation_triplets(self, mock_create_provider):
|
||||||
|
"""Test that max_tokens is passed to generate_typed in extract_triplets_llm."""
|
||||||
|
# Setup mock
|
||||||
|
mock_llm = MagicMock()
|
||||||
|
mock_create_provider.return_value = mock_llm
|
||||||
|
mock_llm.is_available.return_value = True
|
||||||
|
|
||||||
|
# Setup return value to avoid pydantic validation errors
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.triplets = []
|
||||||
|
mock_llm.generate_typed.return_value = mock_response
|
||||||
|
|
||||||
|
# Call the function with max_tokens
|
||||||
|
extract_triplets_llm(
|
||||||
|
text="some text",
|
||||||
|
provider="openai",
|
||||||
|
model="gpt-4",
|
||||||
|
max_tokens=128000
|
||||||
|
)
|
||||||
|
|
||||||
|
# Check if generate_typed was called with max_tokens
|
||||||
|
args, kwargs = mock_llm.generate_typed.call_args
|
||||||
|
|
||||||
|
print(f"Triplets Call kwargs: {kwargs}")
|
||||||
|
|
||||||
|
self.assertIn("max_tokens", kwargs)
|
||||||
|
self.assertEqual(kwargs["max_tokens"], 128000)
|
||||||
|
|
||||||
|
|
||||||
|
class TestCacheKeyIncludesGenerationParams(unittest.TestCase):
|
||||||
|
"""Regression tests for the cache-key bug: two calls with identical extraction
|
||||||
|
inputs but different generation settings must NOT share a cached result.
|
||||||
|
|
||||||
|
Before the fix, extract_relations_llm (and entities/triplets) built
|
||||||
|
cache_params without generation kwargs, so max_tokens=4096 and
|
||||||
|
max_tokens=128000 hashed to the same key. The second call would return the
|
||||||
|
first cached result without ever running generate_typed again.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _make_mock_llm(self, relations=None, entities=None, triplets=None):
|
||||||
|
mock_llm = MagicMock()
|
||||||
|
mock_llm.is_available.return_value = True
|
||||||
|
resp = MagicMock()
|
||||||
|
resp.relations = relations if relations is not None else []
|
||||||
|
resp.entities = entities if entities is not None else []
|
||||||
|
resp.triplets = triplets if triplets is not None else []
|
||||||
|
mock_llm.generate_typed.return_value = resp
|
||||||
|
return mock_llm
|
||||||
|
|
||||||
|
@patch("semantica.semantic_extract.methods.create_provider")
|
||||||
|
def test_relations_different_max_tokens_bypass_cache(self, mock_create_provider):
|
||||||
|
"""Two relation extraction calls with the same text/entities but different
|
||||||
|
max_tokens must each call generate_typed (2 calls total), not reuse the
|
||||||
|
first cached result."""
|
||||||
|
from semantica.semantic_extract.methods import _result_cache
|
||||||
|
_result_cache.clear("relations")
|
||||||
|
|
||||||
|
mock_llm = self._make_mock_llm()
|
||||||
|
mock_create_provider.return_value = mock_llm
|
||||||
|
|
||||||
|
entities = [Entity(text="Foo", label="ORG", start_char=0, end_char=3)]
|
||||||
|
|
||||||
|
extract_relations_llm(
|
||||||
|
text="some text", entities=entities,
|
||||||
|
provider="openai", model="gpt-4", max_tokens=4096
|
||||||
|
)
|
||||||
|
extract_relations_llm(
|
||||||
|
text="some text", entities=entities,
|
||||||
|
provider="openai", model="gpt-4", max_tokens=128000
|
||||||
|
)
|
||||||
|
|
||||||
|
# generate_typed must have been called twice — once per unique key
|
||||||
|
self.assertEqual(
|
||||||
|
mock_llm.generate_typed.call_count, 2,
|
||||||
|
"Different max_tokens values must produce different cache keys; "
|
||||||
|
"second call must not reuse the first cached result."
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch("semantica.semantic_extract.methods.create_provider")
|
||||||
|
def test_relations_same_max_tokens_uses_cache(self, mock_create_provider):
|
||||||
|
"""Two identical calls must reuse the cache (generate_typed called once)."""
|
||||||
|
from semantica.semantic_extract.methods import _result_cache
|
||||||
|
_result_cache.clear("relations")
|
||||||
|
|
||||||
|
mock_llm = self._make_mock_llm()
|
||||||
|
mock_create_provider.return_value = mock_llm
|
||||||
|
|
||||||
|
entities = [Entity(text="Foo", label="ORG", start_char=0, end_char=3)]
|
||||||
|
|
||||||
|
extract_relations_llm(
|
||||||
|
text="some text", entities=entities,
|
||||||
|
provider="openai", model="gpt-4", max_tokens=4096
|
||||||
|
)
|
||||||
|
extract_relations_llm(
|
||||||
|
text="some text", entities=entities,
|
||||||
|
provider="openai", model="gpt-4", max_tokens=4096
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
mock_llm.generate_typed.call_count, 1,
|
||||||
|
"Identical calls must reuse the cache."
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch("semantica.semantic_extract.methods.create_provider")
|
||||||
|
def test_relations_different_temperature_bypass_cache(self, mock_create_provider):
|
||||||
|
"""Different temperature values must also produce different cache keys."""
|
||||||
|
from semantica.semantic_extract.methods import _result_cache
|
||||||
|
_result_cache.clear("relations")
|
||||||
|
|
||||||
|
mock_llm = self._make_mock_llm()
|
||||||
|
mock_create_provider.return_value = mock_llm
|
||||||
|
|
||||||
|
entities = [Entity(text="Bar", label="PERSON", start_char=0, end_char=3)]
|
||||||
|
|
||||||
|
extract_relations_llm(
|
||||||
|
text="other text", entities=entities,
|
||||||
|
provider="openai", model="gpt-4", temperature=0.0
|
||||||
|
)
|
||||||
|
extract_relations_llm(
|
||||||
|
text="other text", entities=entities,
|
||||||
|
provider="openai", model="gpt-4", temperature=1.0
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(mock_llm.generate_typed.call_count, 2)
|
||||||
|
|
||||||
|
@patch("semantica.semantic_extract.methods.create_provider")
|
||||||
|
def test_entities_different_max_tokens_bypass_cache(self, mock_create_provider):
|
||||||
|
"""extract_entities_llm: different max_tokens must bypass cache."""
|
||||||
|
from semantica.semantic_extract.methods import _result_cache
|
||||||
|
_result_cache.clear("entities")
|
||||||
|
|
||||||
|
mock_llm = self._make_mock_llm()
|
||||||
|
mock_create_provider.return_value = mock_llm
|
||||||
|
|
||||||
|
extract_entities_llm(
|
||||||
|
text="some entity text", provider="openai", model="gpt-4",
|
||||||
|
max_tokens=4096
|
||||||
|
)
|
||||||
|
extract_entities_llm(
|
||||||
|
text="some entity text", provider="openai", model="gpt-4",
|
||||||
|
max_tokens=128000
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(mock_llm.generate_typed.call_count, 2)
|
||||||
|
|
||||||
|
@patch("semantica.semantic_extract.methods.create_provider")
|
||||||
|
def test_triplets_different_max_tokens_bypass_cache(self, mock_create_provider):
|
||||||
|
"""extract_triplets_llm: different max_tokens must bypass cache."""
|
||||||
|
from semantica.semantic_extract.methods import _result_cache
|
||||||
|
_result_cache.clear("triplets")
|
||||||
|
|
||||||
|
mock_llm = self._make_mock_llm()
|
||||||
|
mock_create_provider.return_value = mock_llm
|
||||||
|
|
||||||
|
extract_triplets_llm(
|
||||||
|
text="some triplet text", provider="openai", model="gpt-4",
|
||||||
|
max_tokens=4096
|
||||||
|
)
|
||||||
|
extract_triplets_llm(
|
||||||
|
text="some triplet text", provider="openai", model="gpt-4",
|
||||||
|
max_tokens=128000
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(mock_llm.generate_typed.call_count, 2)
|
||||||
|
|
||||||
|
|
||||||
|
class TestCacheKeyIncludesProviderSpecificGenerationParams(unittest.TestCase):
|
||||||
|
"""Regression tests for provider-specific generation params that aren't part
|
||||||
|
of the common OpenAI-shaped kwargs (max_tokens, temperature, etc.) but still
|
||||||
|
change provider output and must therefore also change the cache key.
|
||||||
|
|
||||||
|
See providers.py: AnthropicProvider.generate/generate_structured read
|
||||||
|
'system' and 'stop_sequences' via a manual pass-through loop (not
|
||||||
|
_add_if_set); GeminiProvider.generate reads 'candidate_count' and
|
||||||
|
'stop_sequences'; OllamaProvider._build_options reads 'repeat_penalty' and
|
||||||
|
'num_ctx'/'context_window'.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _make_mock_llm(self):
|
||||||
|
mock_llm = MagicMock()
|
||||||
|
mock_llm.is_available.return_value = True
|
||||||
|
resp = MagicMock()
|
||||||
|
resp.relations = []
|
||||||
|
mock_llm.generate_typed.return_value = resp
|
||||||
|
return mock_llm
|
||||||
|
|
||||||
|
@patch("semantica.semantic_extract.methods.create_provider")
|
||||||
|
def test_relations_different_system_prompt_bypass_cache(self, mock_create_provider):
|
||||||
|
"""Anthropic 'system' prompt changes output; must not share a cache entry."""
|
||||||
|
from semantica.semantic_extract.methods import _result_cache
|
||||||
|
_result_cache.clear("relations")
|
||||||
|
|
||||||
|
mock_llm = self._make_mock_llm()
|
||||||
|
mock_create_provider.return_value = mock_llm
|
||||||
|
|
||||||
|
entities = [Entity(text="Foo", label="ORG", start_char=0, end_char=3)]
|
||||||
|
|
||||||
|
extract_relations_llm(
|
||||||
|
text="some text", entities=entities,
|
||||||
|
provider="anthropic", model="claude-3-sonnet-20240229",
|
||||||
|
system="Extract only ORG relations."
|
||||||
|
)
|
||||||
|
extract_relations_llm(
|
||||||
|
text="some text", entities=entities,
|
||||||
|
provider="anthropic", model="claude-3-sonnet-20240229",
|
||||||
|
system="Extract only PERSON relations."
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
mock_llm.generate_typed.call_count, 2,
|
||||||
|
"Different 'system' prompts must produce different cache keys."
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch("semantica.semantic_extract.methods.create_provider")
|
||||||
|
def test_relations_different_stop_sequences_bypass_cache(self, mock_create_provider):
|
||||||
|
"""Anthropic/Gemini 'stop_sequences' must also be part of the cache key."""
|
||||||
|
from semantica.semantic_extract.methods import _result_cache
|
||||||
|
_result_cache.clear("relations")
|
||||||
|
|
||||||
|
mock_llm = self._make_mock_llm()
|
||||||
|
mock_create_provider.return_value = mock_llm
|
||||||
|
|
||||||
|
entities = [Entity(text="Foo", label="ORG", start_char=0, end_char=3)]
|
||||||
|
|
||||||
|
extract_relations_llm(
|
||||||
|
text="some text", entities=entities,
|
||||||
|
provider="anthropic", model="claude-3-sonnet-20240229",
|
||||||
|
stop_sequences=["\n\n"]
|
||||||
|
)
|
||||||
|
extract_relations_llm(
|
||||||
|
text="some text", entities=entities,
|
||||||
|
provider="anthropic", model="claude-3-sonnet-20240229",
|
||||||
|
stop_sequences=["STOP"]
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(mock_llm.generate_typed.call_count, 2)
|
||||||
|
|
||||||
|
@patch("semantica.semantic_extract.methods.create_provider")
|
||||||
|
def test_relations_different_repeat_penalty_bypass_cache(self, mock_create_provider):
|
||||||
|
"""Ollama 'repeat_penalty' must also be part of the cache key."""
|
||||||
|
from semantica.semantic_extract.methods import _result_cache
|
||||||
|
_result_cache.clear("relations")
|
||||||
|
|
||||||
|
mock_llm = self._make_mock_llm()
|
||||||
|
mock_create_provider.return_value = mock_llm
|
||||||
|
|
||||||
|
entities = [Entity(text="Foo", label="ORG", start_char=0, end_char=3)]
|
||||||
|
|
||||||
|
extract_relations_llm(
|
||||||
|
text="some text", entities=entities,
|
||||||
|
provider="ollama", model="llama2",
|
||||||
|
repeat_penalty=1.0
|
||||||
|
)
|
||||||
|
extract_relations_llm(
|
||||||
|
text="some text", entities=entities,
|
||||||
|
provider="ollama", model="llama2",
|
||||||
|
repeat_penalty=1.5
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(mock_llm.generate_typed.call_count, 2)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -5,6 +5,9 @@ import json
|
|||||||
import csv
|
import csv
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
from semantica.seed.seed_manager import SeedDataManager, SeedDataSource, SeedData
|
from semantica.seed.seed_manager import SeedDataManager, SeedDataSource, SeedData
|
||||||
from semantica.utils.exceptions import ProcessingError
|
from semantica.utils.exceptions import ProcessingError
|
||||||
|
|
||||||
@@ -263,6 +266,61 @@ def test_load_from_api_does_not_mutate_empty_headers_dict(mock_guard, seed_manag
|
|||||||
guard_headers = call_kwargs.get("headers", {})
|
guard_headers = call_kwargs.get("headers", {})
|
||||||
assert guard_headers.get("Authorization") == "Bearer key"
|
assert guard_headers.get("Authorization") == "Bearer key"
|
||||||
|
|
||||||
|
|
||||||
|
# requests.exceptions.RequestException subclasses OSError, so network failures raised
|
||||||
|
# by request_with_ssrf_guard used to be reported as "requests library not available"
|
||||||
|
# by the obsolete ImportError / OSError handler. They must surface the real cause.
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"error",
|
||||||
|
[
|
||||||
|
requests.exceptions.ConnectionError("connection refused"),
|
||||||
|
requests.exceptions.Timeout("timed out"),
|
||||||
|
requests.exceptions.HTTPError("500 Server Error"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
@patch("semantica.seed.seed_manager.request_with_ssrf_guard")
|
||||||
|
def test_load_from_api_request_failure_reports_real_cause(mock_guard, error, seed_manager):
|
||||||
|
mock_guard.side_effect = error
|
||||||
|
|
||||||
|
with pytest.raises(ProcessingError) as excinfo:
|
||||||
|
seed_manager.load_from_api(api_url="http://api.example.com", endpoint="users")
|
||||||
|
|
||||||
|
message = str(excinfo.value)
|
||||||
|
assert "Failed to load from API" in message
|
||||||
|
assert str(error) in message
|
||||||
|
assert "requests library not available" not in message
|
||||||
|
assert excinfo.value.__cause__ is error
|
||||||
|
|
||||||
|
|
||||||
|
@patch("semantica.seed.seed_manager.request_with_ssrf_guard")
|
||||||
|
def test_load_from_api_http_status_error_reports_real_cause(mock_guard, seed_manager):
|
||||||
|
http_error = requests.exceptions.HTTPError("404 Client Error: Not Found")
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.raise_for_status.side_effect = http_error
|
||||||
|
mock_guard.return_value = mock_response
|
||||||
|
|
||||||
|
with pytest.raises(ProcessingError) as excinfo:
|
||||||
|
seed_manager.load_from_api(api_url="http://api.example.com", endpoint="users")
|
||||||
|
|
||||||
|
message = str(excinfo.value)
|
||||||
|
assert "404 Client Error: Not Found" in message
|
||||||
|
assert "requests library not available" not in message
|
||||||
|
mock_response.json.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
@patch("semantica.seed.seed_manager.request_with_ssrf_guard")
|
||||||
|
def test_load_from_api_invalid_json_reports_real_cause(mock_guard, seed_manager):
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.json.side_effect = ValueError("Expecting value: line 1 column 1")
|
||||||
|
mock_guard.return_value = mock_response
|
||||||
|
|
||||||
|
with pytest.raises(ProcessingError) as excinfo:
|
||||||
|
seed_manager.load_from_api(api_url="http://api.example.com")
|
||||||
|
|
||||||
|
message = str(excinfo.value)
|
||||||
|
assert "Failed to load from API" in message
|
||||||
|
assert "Expecting value" in message
|
||||||
|
|
||||||
def test_load_source(seed_manager, temp_data_dir):
|
def test_load_source(seed_manager, temp_data_dir):
|
||||||
json_file = temp_data_dir / "source.json"
|
json_file = temp_data_dir / "source.json"
|
||||||
with open(json_file, "w") as f:
|
with open(json_file, "w") as f:
|
||||||
|
|||||||
@@ -159,3 +159,89 @@ def test_missing_optional_dependency_has_install_hint():
|
|||||||
):
|
):
|
||||||
with pytest.raises(ImportError, match="tripletstore-oxigraph"):
|
with pytest.raises(ImportError, match="tripletstore-oxigraph"):
|
||||||
_store()
|
_store()
|
||||||
|
|
||||||
|
|
||||||
|
def test_on_disk_add_triplets_calls_flush(tmp_path):
|
||||||
|
"""add_triplets on a disk-backed store must flush once after the batch.
|
||||||
|
|
||||||
|
The pyoxigraph background-thread flush "might lag a little bit"; an
|
||||||
|
explicit flush after the batch closes that race without fsyncing on
|
||||||
|
every individual write. This test verifies the contract directly
|
||||||
|
without relying on CPython destructor timing.
|
||||||
|
"""
|
||||||
|
store = OxigraphStore(path=tmp_path / "oxigraph")
|
||||||
|
with patch.object(store, "flush") as mock_flush:
|
||||||
|
store.add_triplets([
|
||||||
|
Triplet(EX + "alice", EX + "knows", EX + "bob"),
|
||||||
|
Triplet(EX + "bob", EX + "knows", EX + "carol"),
|
||||||
|
])
|
||||||
|
mock_flush.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_on_disk_add_triplet_does_not_flush(tmp_path):
|
||||||
|
"""add_triplet (single write) must NOT flush on every call.
|
||||||
|
|
||||||
|
Individual writes are committed to the store in memory; the caller is
|
||||||
|
responsible for calling flush() when a hard durability boundary is
|
||||||
|
needed. Flushing on every add_triplet() call would fsync on every
|
||||||
|
write, causing a severe throughput regression for workloads that write
|
||||||
|
triplets one at a time.
|
||||||
|
"""
|
||||||
|
store = OxigraphStore(path=tmp_path / "oxigraph")
|
||||||
|
with patch.object(store, "flush") as mock_flush:
|
||||||
|
store.add_triplet(Triplet(EX + "alice", EX + "knows", EX + "bob"))
|
||||||
|
mock_flush.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_in_memory_add_triplets_does_not_flush(tmp_path):
|
||||||
|
"""In-memory stores must not call flush() — there is nothing to flush."""
|
||||||
|
store = OxigraphStore() # no path → in-memory
|
||||||
|
with patch.object(store, "flush") as mock_flush:
|
||||||
|
store.add_triplet(Triplet(EX + "alice", EX + "knows", EX + "bob"))
|
||||||
|
store.add_triplets([Triplet(EX + "bob", EX + "knows", EX + "carol")])
|
||||||
|
mock_flush.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_on_disk_add_triplets_is_durable_on_reopen(tmp_path):
|
||||||
|
"""End-to-end durability: a batch written via add_triplets and closed
|
||||||
|
cleanly survives a reopen.
|
||||||
|
|
||||||
|
This is an integration test for the full add_triplets → flush → close →
|
||||||
|
reopen lifecycle. The durability contract here is provided by the
|
||||||
|
explicit ``store.flush()`` call before deletion; the internal flush
|
||||||
|
inside add_triplets reduces (but does not eliminate) the crash-window
|
||||||
|
race. The authoritative unit test for the internal flush behaviour is
|
||||||
|
``test_on_disk_add_triplets_calls_flush``.
|
||||||
|
"""
|
||||||
|
path = tmp_path / "oxigraph"
|
||||||
|
store = OxigraphStore(path=path)
|
||||||
|
store.add_triplets([
|
||||||
|
Triplet(EX + "alice", EX + "knows", EX + "bob"),
|
||||||
|
Triplet(EX + "bob", EX + "knows", EX + "carol"),
|
||||||
|
])
|
||||||
|
store.flush() # belt-and-suspenders: ensures close is clean
|
||||||
|
del store
|
||||||
|
gc.collect()
|
||||||
|
|
||||||
|
reopened = OxigraphStore(path=path)
|
||||||
|
assert len(reopened.get_triplets()) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_storage_path_is_accepted_as_alias_for_path(tmp_path):
|
||||||
|
"""Regression: ``storage_path=...`` used to be silently swallowed by
|
||||||
|
``**config`` (the __init__ parameter is named ``path``), so the store
|
||||||
|
silently degraded to in-memory with no warning. It must now be accepted
|
||||||
|
as an alias consistent with other Semantica stores (e.g. ProvenanceManager)."""
|
||||||
|
storage_path = tmp_path / "oxigraph"
|
||||||
|
|
||||||
|
store = OxigraphStore(storage_path=str(storage_path))
|
||||||
|
|
||||||
|
assert store.path == str(storage_path)
|
||||||
|
# and it must actually persist (proves the alias wired through to the
|
||||||
|
# on-disk path, not just set the attribute)
|
||||||
|
store.add_triplet(Triplet(EX + "alice", EX + "knows", EX + "bob"))
|
||||||
|
del store
|
||||||
|
gc.collect()
|
||||||
|
|
||||||
|
reopened = OxigraphStore(storage_path=str(storage_path))
|
||||||
|
assert len(reopened.get_triplets()) == 1
|
||||||
|
|||||||
Reference in New Issue
Block a user