diff --git a/.github/DISCUSSION_TEMPLATE/ideas.md b/.github/DISCUSSION_TEMPLATE/ideas.md index 6d28cd2a..c6e4e3fa 100644 --- a/.github/DISCUSSION_TEMPLATE/ideas.md +++ b/.github/DISCUSSION_TEMPLATE/ideas.md @@ -69,5 +69,5 @@ If you have ideas on how this could be implemented, please share. --- -**Note**: For feature requests that are ready to be implemented, consider creating a [Feature Request issue](https://github.com/Hawksight-AI/semantica/issues/new?template=feature_request.md) instead. +**Note**: For feature requests that are ready to be implemented, consider creating a [Feature Request issue](https://github.com/semantica-agi/semantica/issues/new?template=feature_request.md) instead. diff --git a/.github/DISCUSSION_TEMPLATE/qa.md b/.github/DISCUSSION_TEMPLATE/qa.md index fc019e53..76fc1959 100644 --- a/.github/DISCUSSION_TEMPLATE/qa.md +++ b/.github/DISCUSSION_TEMPLATE/qa.md @@ -46,8 +46,8 @@ If applicable, paste any error messages or describe unexpected behavior: ## Checklist -- [ ] I have searched existing [discussions](https://github.com/Hawksight-AI/semantica/discussions) and [issues](https://github.com/Hawksight-AI/semantica/issues) -- [ ] I have checked the [documentation](https://github.com/Hawksight-AI/semantica/tree/main/docs) and [FAQ](https://github.com/Hawksight-AI/semantica/blob/main/docs/faq.md) +- [ ] I have searched existing [discussions](https://github.com/semantica-agi/semantica/discussions) and [issues](https://github.com/semantica-agi/semantica/issues) +- [ ] I have checked the [documentation](https://github.com/semantica-agi/semantica/tree/main/docs) and [FAQ](https://github.com/semantica-agi/semantica/blob/main/docs/faq.md) - [ ] I have provided a minimal code example (if applicable) - [ ] I have included error messages (if applicable) - [ ] I have provided environment details diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 5699f569..35fd4d5c 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,3 +1,3 @@ # Funding options for Semantica -github: Hawksight-AI +github: semantica-agi diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 2b702ee7..3ff95742 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,8 +1,8 @@ blank_issues_enabled: true contact_links: - name: ๐Ÿ“š Documentation - url: https://github.com/Hawksight-AI/semantica/tree/main/docs + url: https://github.com/semantica-agi/semantica/tree/main/docs about: Browse the documentation - name: ๐Ÿ’ฌ Discussions - url: https://github.com/Hawksight-AI/semantica/discussions + url: https://github.com/semantica-agi/semantica/discussions about: Ask questions and discuss with the community diff --git a/.github/SUPPORT.md b/.github/SUPPORT.md index 148ceab1..4d484c62 100644 --- a/.github/SUPPORT.md +++ b/.github/SUPPORT.md @@ -3,31 +3,31 @@ ## Getting Help ### ๐Ÿ“š Documentation -Check the [docs folder](https://github.com/Hawksight-AI/semantica/tree/main/docs) and [README](https://github.com/Hawksight-AI/semantica/blob/main/README.md) for guides and examples. +Check the [docs folder](https://github.com/semantica-agi/semantica/tree/main/docs) and [README](https://github.com/semantica-agi/semantica/blob/main/README.md) for guides and examples. ### ๐Ÿ’ฌ Community Support -- **GitHub Discussions**: [Ask questions](https://github.com/Hawksight-AI/semantica/discussions) +- **GitHub Discussions**: [Ask questions](https://github.com/semantica-agi/semantica/discussions) - **Discord**: Join our [Discord server](https://discord.gg/sV34vps5hH) for real-time chat ### ๐Ÿ’ญ Discussions -Join the conversation on [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions): +Join the conversation on [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions): - **Q&A**: Ask questions and get help from the community - **Ideas**: Share feature requests and suggestions - **Show and Tell**: Showcase your projects and use cases - **General**: General discussions about Semantica ### ๐Ÿ› Bug Reports -Found a bug? [Create an issue](https://github.com/Hawksight-AI/semantica/issues/new/choose) +Found a bug? [Create an issue](https://github.com/semantica-agi/semantica/issues/new/choose) ### ๐Ÿ“– Resources -- [Quick Start Guide](https://github.com/Hawksight-AI/semantica/blob/main/docs/quickstart.md) -- [FAQ](https://github.com/Hawksight-AI/semantica/blob/main/docs/faq.md) -- [Cookbook Examples](https://github.com/Hawksight-AI/semantica/tree/main/cookbook) +- [Quick Start Guide](https://github.com/semantica-agi/semantica/blob/main/docs/quickstart.md) +- [FAQ](https://github.com/semantica-agi/semantica/blob/main/docs/faq.md) +- [Cookbook Examples](https://github.com/semantica-agi/semantica/tree/main/cookbook) ## Commercial Support For enterprise support, custom development, or consulting services: -- Contact us through [GitHub Issues](https://github.com/Hawksight-AI/semantica/issues) +- Contact us through [GitHub Issues](https://github.com/semantica-agi/semantica/issues) - Include "Commercial Support" in the title ## Sponsorship @@ -35,7 +35,7 @@ For enterprise support, custom development, or consulting services: ### Sponsor this project Support Semantica development: -- [GitHub Sponsors](https://github.com/sponsors/Hawksight-AI) +- [GitHub Sponsors](https://github.com/sponsors/semantica-agi) Your sponsorship helps us: - Maintain and improve the framework diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ac9df42..3910a3f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **First-class LangChain integration** (closes #963; recreates #969) + - New `pip install semantica[langchain]` extra (`langchain-core>=0.3.0`), included in the `all` bundle + - `integrations/langchain/SemanticaRetriever` โ€” LangChain `BaseRetriever` that seeds from `HybridSearch` then walks graph edges (`hops=2` default) for GraphRAG-style retrieval; falls back to `ContextGraph.query` when hybrid search is unavailable + - `integrations/langchain/SemanticaVectorStore` โ€” LangChain `VectorStore` adapter over `HybridSearch` (`add_texts`, `similarity_search`, `similarity_search_with_score`, `from_texts`) + - `integrations/langchain/SemanticaKGTool` / `SemanticaDecisionTool` โ€” `BaseTool` subclasses with Pydantic `args_schema` (`semantica_query_graph`, `semantica_query_decisions`); `build()` returns the tool, or `None` when langchain-core is absent + - Retriever and VectorStore read HybridSearch nested `metadata` (`content`, `node_id`, `node_type`) rather than top-level fields that HybridSearch does not set + - All adapters remain importable without langchain-core (`LANGCHAIN_AVAILABLE` flag) + - Docs: `docs/integrations/langchain.md`, README native-integration matrix, and `docs.json` nav entry + ## [0.6.6] - 2026-08-20 ### Added @@ -108,6 +119,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **The temporal-evolution `stability` metric was a hardcoded placeholder, not a duration** + - `TemporalGraphQuery.analyze_evolution()` documents `stability` as a "relationship duration/stability measure", but the implementation appended a constant `1` for every relationship with both `valid_from` and `valid_until` set (`durations.append(1) # Placeholder`). The reported stability was therefore always `1.0` when any bounded relationship existed and `0` otherwise โ€” it never reflected how long relationships actually stayed valid, so it could not distinguish a graph of decade-long relationships from one of one-second relationships + - `stability` now computes the mean valid-time duration in seconds (`(valid_until - valid_from).total_seconds()`) across relationships that have both bounds set. Relationships with a missing or open `valid_from`/`valid_until` are skipped (their duration is unbounded), and non-positive intervals are clamped to `0`; an empty set still reports `0` + - New tests in `tests/kg/test_kg.py` assert the mean-duration result, the skipping of unbounded/half-open intervals, and the empty-graph zero case + - **Every timestamp an export or a provenance record wrote was timezone-naive** (closes #1114) by @fabio-rovai - `semantica/export/` stamped with `datetime.now().isoformat()`, which reads the machine's **local** clock; `semantica/provenance/` stamped with `datetime.utcnow().isoformat()`, which reads **UTC**. Both produce a naive value and both serialize identically, so nothing downstream can tell which zone a given timestamp belongs to โ€” the same string means two different instants depending on which module wrote it - In RDF the consequence is silent rather than loud. Under XSD 1.1 a value with no timezone compared against one with a timezone is indeterminate whenever the two fall inside the ยฑ14 hour window; SPARQL turns an indeterminate comparison into an error, and `FILTER` discards errors as non-matches. A timezone-qualified query over an Oxigraph store returns an answer with every Semantica-written record quietly absent from it, which is a poor property for `prov:generatedAtTime`, `prov:startedAtTime`, `prov:endedAtTime` and `prov:atTime` to have @@ -116,6 +132,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - New `tests/export/test_timestamp_timezones.py` and `tests/provenance/test_timestamp_timezones.py`: offset presence on every export and provenance path, PROV-O literals valid as `xsd:dateTimeStamp`, comparison against a timezone-aware instant without `TypeError`, the Oxigraph filter that dropped the naive value (with a bound inside the indeterminate window, so the test cannot pass by accident), and the document `@id` remaining a valid IRI with `+00:00` in it. 11 of the 13 fail on the parent commit - **Fixed during review** (Qodo): once new entries carry `+00:00` and stored ones do not, `ProvenanceManager.query_recorded_between` and `audit_log` compared ISO timestamps as raw strings, so they ordered by spelling rather than by instant โ€” an inclusive naive bound naming a stored offset-bearing timestamp sorted *below* it and dropped the record, and a bound written in another offset landed wherever its digits fell (`19:45+05:30` is 14:15Z, but sorted after 14:19Z). Both now compare instants through a new `to_utc_datetime()` helper that reads a missing offset as UTC, which is what the values written before this change actually were; a bound that cannot be read as a timestamp keeps the historical string comparison rather than raising on a call that used to work - The remaining 147 naive call sites are in `context/`, `vector_store/`, `seed/` and elsewhere, where timestamps are compared against values parsed from previously stored naive strings. Converting those without a read-side migration would raise `TypeError: can't compare offset-naive and offset-aware datetimes` on existing data, so they are deliberately left for a separate change +- **`SHACLGenerator` mangles `#`-terminated namespaces into `#/`, so generated shapes target nothing** (#1082) by @changshenhan + - `__init__` normalized `base_uri` with `rstrip("/") + "/"`, which turns `http://example.org/manufacturing#` into `...manufacturing#/` โ€” the most common RDF namespace convention. Every generated URI (`sh:targetClass`, `sh:path`, shape URIs) then landed in a different namespace than the instance data, and SHACL validation silently passed because the shapes targeted nothing + - `__init__` now preserves a namespace already ending in `/` or `#`, matching the `#`-aware normalization `generate()` already applies; `shapes_uri` inherits the fix + - New `test_hash_namespace_base_uri_is_not_mangled` in `tests/ontology/test_ontology_advanced.py` fails on the pre-fix normalization and passes with it; full ontology suite (76 tests) green - **`split`/chunking paths bypassed the centralized spaCy model cache, reloading the model on every call** (#1042, closes #998) by @Accute9, reviewed by @Sameer6305 - `semantica/split/methods.py`'s `split_by_sentences()` and `semantica/split/semantic_chunker.py`'s `SemanticChunker.__init__` each called `spacy.load()` directly instead of reusing the process-level cache added in #889/`semantic_extract/methods.py`'s `load_spacy_model()` โ€” every call/construction re-paid the ~120ms model-load cost independently of `NERExtractor`, which already used the cache @@ -395,8 +415,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Entities and relationships round-trip as memory-local provenance only โ€” Markdown import intentionally does not write into `ContextGraph`, matching the MVP scope agreed on in #765 - Documented the file contract and workflow in `docs/reference/context.md`; 43 new tests in `tests/context/test_agent_memory_markdown.py` cover round-trip losslessness, idempotency, validation errors, rollback on failure, and vector-store sync ordering +- **Markdown directory round trips for `ContextGraph`** (#852) by @SaurabhScripts + - `ContextGraph.save_to_file(..., format="markdown")` and `load_from_file(..., format="markdown")` persist a deterministic `graph.md` relationship manifest plus one human-editable Markdown file per node, preserving graph, node, edge, family, temporal, and cross-graph link identities + - Imports validate the complete directory before replacing graph state, rebuild indexes and analytics state atomically, create JSON-compatible stub nodes for dangling edge endpoints, and emit the same granular node/edge audit events as JSON loading + - Existing exports are replaced atomically only after their complete canonical layout is validated; untracked files, renamed node files, symlinks, Windows directory junctions, and other reparse points cause a fail-closed error instead of authorizing directory deletion + - Added 30 focused tests covering deterministic round trips, manual edits, validation rollback, managed-directory identity, publish rollback, audit-manager compatibility, stale-cache clearing, mocked and real Windows junctions, and missing-path behavior + ### Fixed +- **Markdown import followed filesystem links even though Markdown export already refused to overwrite them** (#851, follow-up to #765, #786) by @SaurabhScripts + - `AgentMemory._read_markdown_path()` now rejects symlink files, broken symlinks, symlinked directories, Windows directory junctions, and other Windows reparse points supplied directly; linked entries discovered inside an otherwise valid directory are safely skipped, preserving the current directory-import contract + - `_read_markdown_file_content()` re-checks the file and parent directory immediately before and after opening, uses `O_NOFOLLOW` where available, and verifies the resulting descriptor is a regular file via `fstat`/`S_ISREG`, so link swaps are rejected rather than silently followed + - Junction detection uses `os.path.isjunction()` where available and falls back to the Windows reparse-point file attribute on older Python versions; export applies the same link check before replacing a Markdown file + - Documented the import restriction in `docs/reference/context.md`; added 11 tests to `tests/context/test_agent_memory_markdown.py` covering file/directory/broken-symlink rejection, simulated open races, mocked and real Windows junctions, and the reparse-point fallback + - Any additional review follow-up commits land in this same PR/entry rather than as a separate changelog item + - **`PipelineWithProvenance` raised `ModuleNotFoundError` on import and `AttributeError` on `.run()`** (#858, closes #858) by @Karunasagar12 - `from .pipeline import Pipeline` failed because `semantica/pipeline/pipeline.py` does not exist; corrected to `from .pipeline_builder import Pipeline` - `.run()` called `self._pipeline.run()` on the `Pipeline` dataclass, which has no such method; replaced with `self._engine.execute_pipeline(self._pipeline, ...)` delegating to `ExecutionEngine` @@ -1513,4 +1546,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 --- -For detailed release notes, see [GitHub Releases](https://github.com/Hawksight-AI/semantica/releases). +For detailed release notes, see [GitHub Releases](https://github.com/semantica-agi/semantica/releases). diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 0197824a..7b3304eb 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -58,7 +58,7 @@ representative at an online or offline event. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement through -[GitHub Issues](https://github.com/Hawksight-AI/semantica/issues) with "[CoC]" prefix. +[GitHub Issues](https://github.com/semantica-agi/semantica/issues) with "[CoC]" prefix. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 336ece42..b5ec8b37 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -44,7 +44,7 @@ We recognize all types of contributions: All contributors are recognized in: - This contributors list -- [GitHub contributors page](https://github.com/Hawksight-AI/semantica/graphs/contributors) +- [GitHub contributors page](https://github.com/semantica-agi/semantica/graphs/contributors) - Release notes for significant contributions - Community appreciation @@ -54,7 +54,7 @@ All contributors are recognized in: ### Automatic Recognition -If you've made a commit, you'll automatically appear in [GitHub's contributors graph](https://github.com/Hawksight-AI/semantica/graphs/contributors). +If you've made a commit, you'll automatically appear in [GitHub's contributors graph](https://github.com/semantica-agi/semantica/graphs/contributors). ### Using All-Contributors Bot @@ -111,4 +111,4 @@ Every contribution, no matter how small, helps make Semantica better. Thank you **Want to contribute?** -โญ Give us a Star โ€ข ๐Ÿด [Fork us](https://github.com/Hawksight-AI/semantica/fork) โ€ข Check out our [Contributing Guide](CONTRIBUTING.md) to get started! +โญ Give us a Star โ€ข ๐Ÿด [Fork us](https://github.com/semantica-agi/semantica/fork) โ€ข Check out our [Contributing Guide](CONTRIBUTING.md) to get started! diff --git a/Dockerfile b/Dockerfile index 0cb1f418..a462509e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN npm ci COPY explorer/ ./ RUN mkdir -p /app/semantica && npm run build -FROM python:3.14-slim AS runtime +FROM python:3.13-slim AS runtime ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ diff --git a/LICENSE b/LICENSE index d0dbcb9a..c66f5086 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2026 Hawksight AI +Copyright (c) 2026 Semantica Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index a2646bde..85833b2c 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ Semantica sits underneath your LLM, vector store, and agent framework as a deter - **Graph Analytics:** Centrality, community detection, link prediction, and shortest-path queries over the graph you just built - **Polyglot Graph Storage:** Native RDF (embedded Oxigraph, Blazegraph, Apache Jena, Eclipse RDF4J via SPARQL) and Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune via Cypher), plus vector stores, all swappable without touching your code - **Visualization:** Explore any graph, ontology, or timeline in an interactive browser workbench -- **Drop-in Integrations:** Native Agno and CrewAI support, a full-featured MCP server, a comprehensive CLI, a REST API, and plugins across major editors +- **Drop-in Integrations:** Native Agno, CrewAI, and LangChain support, a full-featured MCP server, a comprehensive CLI, a REST API, and plugins across major editors --- @@ -147,6 +147,8 @@ semantica doctor # Config file pass ~/.semantica/config.yaml ``` +**Running in a script or CI?** Progress bars are written only when stdout is an interactive terminal (or a Jupyter notebook), so piping and redirecting stay clean by default. Override with `SEMANTICA_DISABLE_PROGRESS=1` to silence progress everywhere, or `SEMANTICA_FORCE_PROGRESS=1` to keep it when stdout is redirected. `SEMANTICA_DISABLE_PROGRESS` takes precedence. +
If Semantica solves a real problem for you, a star helps others find it. @@ -1186,7 +1188,7 @@ Start with `semantica`, verify with `doctor`, build a graph, and explore the com ## Integrations -Native plugin bundles for Claude Code, Cursor, Codex, Windsurf, Cline, Continue, VS Code, and OpenClaw; a full-featured MCP server for any MCP-compatible client; a comprehensive REST API; and first-class Agno and CrewAI support for agentic frameworks. Every major LLM provider is already supported via `semantica.llms` and LiteLLM: OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Azure, Bedrock, Ollama, DeepSeek, HuggingFace, and more. +Native plugin bundles for Claude Code, Cursor, Codex, Windsurf, Cline, Continue, VS Code, and OpenClaw; a full-featured MCP server for any MCP-compatible client; a comprehensive REST API; and first-class Agno, CrewAI, and LangChain support for agentic frameworks. Every major LLM provider is already supported via `semantica.llms` and LiteLLM: OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Azure, Bedrock, Ollama, DeepSeek, HuggingFace, and more. MCP setup takes 30 seconds โ€” see [MCP Server](#mcp-server) below. @@ -1305,17 +1307,17 @@ MCP setup takes 30 seconds โ€” see [MCP Server](#mcp-server) below. CrewAI
First-class ยท pip install semantica[crewai] + +LangChain
+LangChain
+First-class ยท pip install semantica[langchain] + Already Supported via REST API & MCP -LangChain
-LangChain
-REST API ยท MCP - - LangGraph
LangGraph
REST API ยท MCP @@ -1346,11 +1348,6 @@ MCP setup takes 30 seconds โ€” see [MCP Server](#mcp-server) below. -LangChain
-LangChain
-Dedicated toolkit - - LlamaIndex
LlamaIndex
Dedicated toolkit @@ -1509,6 +1506,7 @@ pip install semantica[all] # everything ```bash pip install semantica[agno] # Agno multi-agent integration pip install semantica[crewai] # CrewAI integration +pip install semantica[langchain] # LangChain / LangGraph integration pip install semantica[llm-litellm] # OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Bedrock, Ollama, DeepSeek, and more pip install semantica[graph-neo4j] # Neo4j graph store (LPG) pip install semantica[graph-falkordb] # FalkorDB graph store (LPG) @@ -1561,11 +1559,11 @@ On-premises deployment ยท Private cloud ยท Custom domain implementations ยท SLA- ## Star History - + - - - Star History Chart + + + Star History Chart @@ -1594,6 +1592,23 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for full guidelines. --- +## Cite Us + +If you use Semantica in your research or production systems, please cite it as: + +```bibtex +@software{semantica2026, + title = {Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems}, + author = {Semantica}, + year = {2026}, + url = {https://github.com/semantica-agi/semantica} +} +``` + +All citation formats (APA, MLA, Chicago, IEEE) live on the [Citation](https://docs.getsemantica.ai/citation) page โ€” every format attributes authorship to **Semantica**, not individual contributors. + +--- +
MIT License ยท Built by [Semantica](https://github.com/semantica-agi) diff --git a/cookbook/advanced/01_Advanced_Extraction.ipynb b/cookbook/advanced/01_Advanced_Extraction.ipynb index e4989da6..49113115 100644 --- a/cookbook/advanced/01_Advanced_Extraction.ipynb +++ b/cookbook/advanced/01_Advanced_Extraction.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/01_Advanced_Extraction.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/01_Advanced_Extraction.ipynb)\n", "\n", "# Advanced Extraction\n", "\n", diff --git a/cookbook/advanced/03_Complete_Visualization_Suite.ipynb b/cookbook/advanced/03_Complete_Visualization_Suite.ipynb index d081b2df..655e723a 100644 --- a/cookbook/advanced/03_Complete_Visualization_Suite.ipynb +++ b/cookbook/advanced/03_Complete_Visualization_Suite.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/03_Complete_Visualization_Suite.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/03_Complete_Visualization_Suite.ipynb)\n", "\n", "# Complete Visualization Suite\n", "\n", diff --git a/cookbook/advanced/05_Multi_Format_Export.ipynb b/cookbook/advanced/05_Multi_Format_Export.ipynb index 197ce788..306409c7 100644 --- a/cookbook/advanced/05_Multi_Format_Export.ipynb +++ b/cookbook/advanced/05_Multi_Format_Export.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/05_Multi_Format_Export.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/05_Multi_Format_Export.ipynb)\n", "\n", "# Advanced Multi-Format Export\n", "\n", diff --git a/cookbook/advanced/08_Reasoning_and_Inference.ipynb b/cookbook/advanced/08_Reasoning_and_Inference.ipynb index 2f86fe1c..5854f4bf 100644 --- a/cookbook/advanced/08_Reasoning_and_Inference.ipynb +++ b/cookbook/advanced/08_Reasoning_and_Inference.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/08_Reasoning_and_Inference.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/08_Reasoning_and_Inference.ipynb)\n", "\n", "# Reasoning and Inference\n", "\n", diff --git a/cookbook/advanced/09_Semantic_Layer_Construction.ipynb b/cookbook/advanced/09_Semantic_Layer_Construction.ipynb index d8c090c9..de1ecd78 100644 --- a/cookbook/advanced/09_Semantic_Layer_Construction.ipynb +++ b/cookbook/advanced/09_Semantic_Layer_Construction.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/09_Semantic_Layer_Construction.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/09_Semantic_Layer_Construction.ipynb)\n", "\n", "# Semantic Layer Construction\n", "\n", diff --git a/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb b/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb index 21597361..03843a60 100644 --- a/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb +++ b/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb)\n", "\n", "# Deep Dive: Temporal Knowledge Graphs\n", "\n", diff --git a/cookbook/advanced/12_Unstructured_to_Ontology.ipynb b/cookbook/advanced/12_Unstructured_to_Ontology.ipynb index f683c7e0..6eddba5b 100644 --- a/cookbook/advanced/12_Unstructured_to_Ontology.ipynb +++ b/cookbook/advanced/12_Unstructured_to_Ontology.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/12_Unstructured_to_Ontology.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/12_Unstructured_to_Ontology.ipynb)\n", "\n", "# Unstructured Text to Ontology\n", "\n", diff --git a/cookbook/advanced/13_Manual_Ontology_Snowflake_Mapping.ipynb b/cookbook/advanced/13_Manual_Ontology_Snowflake_Mapping.ipynb index 392cdc28..0647f3b9 100644 --- a/cookbook/advanced/13_Manual_Ontology_Snowflake_Mapping.ipynb +++ b/cookbook/advanced/13_Manual_Ontology_Snowflake_Mapping.ipynb @@ -18,7 +18,7 @@ "id": "cell-0", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/13_Manual_Ontology_Snowflake_Mapping.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/13_Manual_Ontology_Snowflake_Mapping.ipynb)\n", "\n", "# Manual Ontology + Snowflake Mapping\n", "\n", diff --git a/cookbook/advanced/14_Datalog_Style_Reasoning.ipynb b/cookbook/advanced/14_Datalog_Style_Reasoning.ipynb index 5382c03e..2898f9d6 100644 --- a/cookbook/advanced/14_Datalog_Style_Reasoning.ipynb +++ b/cookbook/advanced/14_Datalog_Style_Reasoning.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/14_Datalog_Style_Reasoning.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/14_Datalog_Style_Reasoning.ipynb)\n", "\n", "# Datalog-Style Reasoning\n", "\n", diff --git a/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb b/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb index 1823a434..175736fe 100644 --- a/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb +++ b/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb)\n", "\n", "# Advanced Vector Store - Made Easy\n", "\n", @@ -352,7 +352,7 @@ "- Build a multi-user application\n", "- Explore the [introduction notebook](../introduction/13_Vector_Store.ipynb) for more basics\n", "\n", - "**Need Help?** Check our [documentation](https://semantica.readthedocs.io) or ask on [GitHub](https://github.com/Hawksight-AI/semantica)." + "**Need Help?** Check our [documentation](https://semantica.readthedocs.io) or ask on [GitHub](https://github.com/semantica-agi/semantica)." ] } ], diff --git a/cookbook/introduction/01_Welcome_to_Semantica.ipynb b/cookbook/introduction/01_Welcome_to_Semantica.ipynb index 05417881..21677088 100644 --- a/cookbook/introduction/01_Welcome_to_Semantica.ipynb +++ b/cookbook/introduction/01_Welcome_to_Semantica.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)\n", "\n", "Semantica is a **semantic intelligence and knowledge engineering framework**. It helps you:\n", "\n", diff --git a/cookbook/introduction/02_Data_Ingestion.ipynb b/cookbook/introduction/02_Data_Ingestion.ipynb index f343a5a2..a8d9e5d6 100644 --- a/cookbook/introduction/02_Data_Ingestion.ipynb +++ b/cookbook/introduction/02_Data_Ingestion.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb)\n", "\n", "# Data Ingestion - Comprehensive Guide\n", "\n", diff --git a/cookbook/introduction/03_Document_Parsing.ipynb b/cookbook/introduction/03_Document_Parsing.ipynb index 639c076b..171f111a 100644 --- a/cookbook/introduction/03_Document_Parsing.ipynb +++ b/cookbook/introduction/03_Document_Parsing.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/04_Document_Parsing.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/03_Document_Parsing.ipynb)\n", "\n", "# Document Parsing\n", "\n", diff --git a/cookbook/introduction/04_Data_Normalization.ipynb b/cookbook/introduction/04_Data_Normalization.ipynb index a4a668ee..6cdd07db 100644 --- a/cookbook/introduction/04_Data_Normalization.ipynb +++ b/cookbook/introduction/04_Data_Normalization.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/05_Data_Normalization.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/04_Data_Normalization.ipynb)\n", "\n", "# Data Normalization\n", "\n", diff --git a/cookbook/introduction/05_Entity_Extraction.ipynb b/cookbook/introduction/05_Entity_Extraction.ipynb index 4b78e19c..78cabe22 100644 --- a/cookbook/introduction/05_Entity_Extraction.ipynb +++ b/cookbook/introduction/05_Entity_Extraction.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/05_Entity_Extraction.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/05_Entity_Extraction.ipynb)\n", "\n", "# Entity Extraction - Comprehensive Guide\n", "\n", @@ -622,7 +622,7 @@ "\n", "---\n", "\n", - "**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)." + "**Questions or Issues?** Check out our [GitHub repository](https://github.com/semantica-agi/semantica) or [documentation](https://semantica.readthedocs.io)." ] } ], diff --git a/cookbook/introduction/06_Relation_Extraction.ipynb b/cookbook/introduction/06_Relation_Extraction.ipynb index e11566c6..8015f86f 100644 --- a/cookbook/introduction/06_Relation_Extraction.ipynb +++ b/cookbook/introduction/06_Relation_Extraction.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/06_Relation_Extraction.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/06_Relation_Extraction.ipynb)\n", "\n", "# Relation Extraction - Comprehensive Guide\n", "\n", @@ -599,7 +599,7 @@ "\n", "---\n", "\n", - "**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)." + "**Questions or Issues?** Check out our [GitHub repository](https://github.com/semantica-agi/semantica) or [documentation](https://semantica.readthedocs.io)." ] } ], diff --git a/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb b/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb index bd3c7ba2..5c27420a 100644 --- a/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb +++ b/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Building_Knowledge_Graphs.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb)\n", "\n", "# Building Knowledge Graphs\n", "\n", diff --git a/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb b/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb index fb6e1c2f..f159efb6 100644 --- a/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb +++ b/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/09_Your_First_Knowledge_Graph.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)\n", "\n", "# ๐Ÿš€ Your First Knowledge Graph\n", "\n", diff --git a/cookbook/introduction/10_Graph_Analytics.ipynb b/cookbook/introduction/10_Graph_Analytics.ipynb index f15ee443..e0307e04 100644 --- a/cookbook/introduction/10_Graph_Analytics.ipynb +++ b/cookbook/introduction/10_Graph_Analytics.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/11_Graph_Analytics.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/10_Graph_Analytics.ipynb)\n", "\n", "# Graph Analytics\n", "\n", diff --git a/cookbook/introduction/11_Chunking_and_Splitting.ipynb b/cookbook/introduction/11_Chunking_and_Splitting.ipynb index 9bb5cb3f..27101493 100644 --- a/cookbook/introduction/11_Chunking_and_Splitting.ipynb +++ b/cookbook/introduction/11_Chunking_and_Splitting.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/11_Chunking_and_Splitting.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/11_Chunking_and_Splitting.ipynb)\n", "\n", "# Chunking and Splitting - Comprehensive Guide\n", "\n", @@ -817,7 +817,7 @@ "\n", "---\n", "\n", - "**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)." + "**Questions or Issues?** Check out our [GitHub repository](https://github.com/semantica-agi/semantica) or [documentation](https://semantica.readthedocs.io)." ] } ], diff --git a/cookbook/introduction/12_Embedding_Generation.ipynb b/cookbook/introduction/12_Embedding_Generation.ipynb index b1ad1081..4b334fcc 100644 --- a/cookbook/introduction/12_Embedding_Generation.ipynb +++ b/cookbook/introduction/12_Embedding_Generation.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/13_Embedding_Generation.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/12_Embedding_Generation.ipynb)\n", "\n", "# Embedding Generation\n", "\n", diff --git a/cookbook/introduction/13_Vector_Store.ipynb b/cookbook/introduction/13_Vector_Store.ipynb index f2424104..32baeab3 100644 --- a/cookbook/introduction/13_Vector_Store.ipynb +++ b/cookbook/introduction/13_Vector_Store.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/13_Vector_Store.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/13_Vector_Store.ipynb)\n", "\n", "# Vector Store - Comprehensive Guide\n", "\n", @@ -492,7 +492,7 @@ "\n", "---\n", "\n", - "**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)." + "**Questions or Issues?** Check out our [GitHub repository](https://github.com/semantica-agi/semantica) or [documentation](https://semantica.readthedocs.io)." ] } ], diff --git a/cookbook/introduction/14_Ontology.ipynb b/cookbook/introduction/14_Ontology.ipynb index 3c112404..64bbee06 100644 --- a/cookbook/introduction/14_Ontology.ipynb +++ b/cookbook/introduction/14_Ontology.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/14_Ontology.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/14_Ontology.ipynb)\n", "\n", "# Ontology Generation \n", "\n", diff --git a/cookbook/introduction/15_Export.ipynb b/cookbook/introduction/15_Export.ipynb index 3d6a8c10..224c63ea 100644 --- a/cookbook/introduction/15_Export.ipynb +++ b/cookbook/introduction/15_Export.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/15_Export.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/15_Export.ipynb)\n", "\n", "# Export Module - Comprehensive Guide\n", "\n", diff --git a/cookbook/introduction/16_Visualization.ipynb b/cookbook/introduction/16_Visualization.ipynb index 05c55b61..31483278 100644 --- a/cookbook/introduction/16_Visualization.ipynb +++ b/cookbook/introduction/16_Visualization.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/17_Visualization.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/16_Visualization.ipynb)\n", "\n", "# Visualization\n", "\n", diff --git a/cookbook/introduction/18_Deduplication.ipynb b/cookbook/introduction/18_Deduplication.ipynb index 087817a7..53e03683 100644 --- a/cookbook/introduction/18_Deduplication.ipynb +++ b/cookbook/introduction/18_Deduplication.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/18_Deduplication.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/18_Deduplication.ipynb)\n", "\n", "# Deduplication in Semantica\n", "\n", diff --git a/cookbook/introduction/19_Context_Module.ipynb b/cookbook/introduction/19_Context_Module.ipynb index 2bbe81ce..d2ec4cf4 100644 --- a/cookbook/introduction/19_Context_Module.ipynb +++ b/cookbook/introduction/19_Context_Module.ipynb @@ -5,7 +5,7 @@ "id": "c21e9c8d", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/19_Context_Module.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/19_Context_Module.ipynb)\n", "\n", "# Context Module โ€” Practical Guide\n", "\n", diff --git a/cookbook/introduction/22_Provenance_Tracking.ipynb b/cookbook/introduction/22_Provenance_Tracking.ipynb new file mode 100644 index 00000000..de54ddf1 --- /dev/null +++ b/cookbook/introduction/22_Provenance_Tracking.ipynb @@ -0,0 +1,253 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Provenance Tracking (W3C PROV-O)\n", + "\n", + "## Overview\n", + "\n", + "In high-stakes domains โ€” healthcare, legal, finance, research โ€” a Knowledge Graph is only as trustworthy as its ability to answer **\"where did this fact come from?\"**. Semantica's `provenance` module provides audit-grade, W3C PROV-O-aligned tracking for every entity, relationship and chunk that flows through your pipeline.\n", + "\n", + "In this cookbook you will learn how to:\n", + "\n", + "- Track entities and relationships with **source details** (DOI, page, verbatim quote, confidence)\n", + "- Walk the full **lineage** of a fact (document โ†’ chunk โ†’ entity โ†’ KG)\n", + "- Audit **revision history** and **all sources** behind an entity\n", + "- **Invalidate** a fact without deleting it (prov:Invalidation) โ€” corrections stay provable\n", + "- Verify **tamper-evidence** with chained SHA-256 checksums\n", + "\n", + "**The Scenario:** a research team ingests findings from two scientific papers (with DOIs) into a Knowledge Graph. A regulator later asks: *\"Which paper, which figure, and which exact sentence supports the claim that fish biomass increased by 463%? And was that fact ever corrected?\"*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -q semantica" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from semantica.provenance import (\n", + " ProvenanceManager,\n", + " compute_checksum,\n", + " verify_checksum,\n", + ")\n", + "\n", + "# In-memory storage for this demo; pass storage_path=\"provenance.db\"\n", + "# (or a config with provenance.storage_path) for a persistent SQLite backend.\n", + "prov = ProvenanceManager()\n", + "print(\"ProvenanceManager ready (in-memory storage)\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Track Entities with Audit-Grade Source Details\n", + "\n", + "Every fact we ingest carries its evidence with it: the **source identifier** (a DOI here), the **location** inside the source (a figure), the **verbatim quote**, and the extractor's **confidence**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Finding from paper #1\n", + "entry_biomass = prov.track_entity(\n", + " entity_id=\"claim_biomass_increase\",\n", + " source=\"DOI:10.1371/journal.pone.0023601\",\n", + " confidence=0.92,\n", + " source_location=\"Figure 2\",\n", + " source_quote=\"Total fish biomass increased by 463% ...\",\n", + ")\n", + "\n", + "# Supporting entity from paper #2\n", + "entry_reserve = prov.track_entity(\n", + " entity_id=\"marine_reserve_1\",\n", + " source=\"DOI:10.1126/science.1088121\",\n", + " confidence=0.88,\n", + " source_location=\"Table 1\",\n", + " source_quote=\"... no-take marine reserve at Cabo Pulmo ...\",\n", + ")\n", + "\n", + "print(\"Tracked:\", entry_biomass.entity_id, \"|\", entry_reserve.entity_id)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Track the Relationship Between Facts\n", + "\n", + "Facts rarely stand alone. The claim about biomass increase is *about* the marine reserve โ€” that relationship is a first-class provenance-tracked object too.\n", + "\n", + "`track_relationship()` has no dedicated subject/object fields, so by convention we record which two entities it connects inside `metadata`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rel = prov.track_relationship(\n", + " relationship_id=\"rel_biomass_about_reserve\",\n", + " source=\"DOI:10.1371/journal.pone.0023601\",\n", + " metadata={\n", + " \"type\": \"measured_at\",\n", + " # No dedicated endpoint fields on track_relationship() yet -- record\n", + " # which entities this relationship connects here by convention.\n", + " \"subject_entity_id\": \"claim_biomass_increase\",\n", + " \"object_entity_id\": \"marine_reserve_1\",\n", + " },\n", + ")\n", + "\n", + "print(\"Relationship tracked:\", rel.entity_id, \"|\", rel.metadata[\"subject_entity_id\"], \"->\", rel.metadata[\"object_entity_id\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Walk the Lineage\n", + "\n", + "`get_lineage` reconstructs everything known about a fact; `trace_lineage` returns the ordered chain of `ProvenanceEntry` records โ€” every version, every activity, every agent that touched it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "lineage = prov.get_lineage(\"claim_biomass_increase\")\n", + "print(json.dumps(lineage, indent=2, default=str)[:800])\n", + "\n", + "print(\"\\n--- ordered chain ---\")\n", + "for e in prov.trace_lineage(\"claim_biomass_increase\"):\n", + " print(f\"{e.entity_id} | seq#{e.sequence_id} | {e.activity_id}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Audit Sources and Revision History\n", + "\n", + "When the regulator asks *\"has this fact ever been corrected?\"*, `revision_history` answers with the full version chain, and `get_all_sources` lists every source document that ever supported the entity." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "revisions = prov.revision_history(\"claim_biomass_increase\")\n", + "print(f\"{len(revisions)} revision(s) on record\")\n", + "\n", + "for s in prov.get_all_sources(\"claim_biomass_increase\"):\n", + " print(\"source:\", s)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Invalidate โ€” Correct Without Deleting\n", + "\n", + "Suppose paper #1 is retracted in part. An audit trail must **not** silently delete the fact: `invalidate` archives the pre-invalidation state and appends a fresh `prov:Invalidation` entry naming **who** retracted it and **why**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "invalidated = prov.invalidate(\n", + " entity_id=\"claim_biomass_increase\",\n", + " agent_id=\"reviewer_dr_chen\",\n", + " reason=\"Partial retraction: Figure 2 statistics corrected by publisher (see erratum).\",\n", + ")\n", + "print(\"Invalidated:\", invalidated.entity_id, \"| invalidated flag:\", getattr(invalidated, \"invalidated\", True))\n", + "\n", + "stats = prov.get_statistics()\n", + "print(\"\\nStorage statistics:\", json.dumps(stats, indent=2, default=str))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Verify Tamper-Evidence\n", + "\n", + "Each entry carries a deterministic SHA-256 checksum chained to the previous entry. Recompute and compare to detect any after-the-fact corruption of the provenance record." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# entry_biomass was returned by track_entity in Step 1\n", + "ok = verify_checksum(entry_biomass)\n", + "print(\"Checksum verified:\", ok)\n", + "\n", + "print(\"Computed:\", compute_checksum(entry_biomass)[:16], \"...\")\n", + "print(\"Stored: \", entry_biomass.checksum[:16] if getattr(entry_biomass, 'checksum', None) else \"(see entry fields)\")\n", + "chain = prov.verify_chain()\n", + "print(\"Chain verification:\", json.dumps(chain, default=str)[:200])\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "| Need | Call |\n", + "|---|---|\n", + "| Record a fact's evidence | `prov.track_entity(entity_id, source, confidence=..., source_location=..., source_quote=...)` |\n", + "| Record a relationship | `prov.track_relationship(relationship_id, source, metadata=...)` |\n", + "| Full lineage of a fact | `prov.get_lineage(entity_id)` / `prov.trace_lineage(entity_id)` |\n", + "| \"Was it ever corrected?\" | `prov.revision_history(entity_id)` |\n", + "| \"Which sources support it?\" | `prov.get_all_sources(entity_id)` |\n", + "| Retract without deleting | `prov.invalidate(entity_id, agent_id, reason=...)` |\n", + "| Tamper check | `verify_checksum(entry)` |\n", + "\n", + "### Where to go next\n", + "\n", + "- **Conflict Detection and Resolution** (notebook 17) โ€” what happens when two sources disagree.\n", + "- **Your First Knowledge Graph** (notebook 08) โ€” plug `provenance=True` into extractors so tracking happens automatically during ingestion.\n", + "- The module docstring (`help(semantica.provenance)`) documents opt-in integration with `kg`, `split` and `conflicts` trackers." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cookbook/introduction/23_Reasoning.ipynb b/cookbook/introduction/23_Reasoning.ipynb new file mode 100644 index 00000000..b685e46c --- /dev/null +++ b/cookbook/introduction/23_Reasoning.ipynb @@ -0,0 +1,383 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "b76a5997", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/23_Reasoning.ipynb)\n", + "\n", + "# Reasoning Module โ€” Practical Guide\n", + "\n", + "Semantica's `reasoning` module derives new knowledge from existing facts and knowledge graphs. It ships several strategies behind one facade:\n", + "\n", + "- **`Reasoner`** โ€” unified facade with forward chaining, backward chaining, and one-shot `infer_facts`\n", + "- **`DatalogReasoner`** โ€” semi-naive Datalog fixpoint evaluation with variable queries\n", + "- **`ExplanationGenerator`** โ€” human-readable explanations and reasoning paths for inferred conclusions\n", + "- Plus lower-level engines: `ReteEngine`, `SPARQLReasoner`, `GraphReasoner`, temporal reasoning\n", + "\n", + "This notebook walks through the facade, the Datalog engine, and explanations. All APIs are verified against `semantica/reasoning/`." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "52073af7", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:45:55.427457Z", + "iopub.status.busy": "2026-08-26T18:45:55.427247Z", + "iopub.status.idle": "2026-08-26T18:45:57.266607Z", + "shell.execute_reply": "2026-08-26T18:45:57.264783Z" + } + }, + "outputs": [], + "source": [ + "!pip install -q semantica" + ] + }, + { + "cell_type": "markdown", + "id": "06deb916", + "metadata": {}, + "source": [ + "## 1) Forward chaining with the `Reasoner` facade\n", + "\n", + "Facts are simple `Predicate(args)` strings. Rules use `IF THEN ` with `?x`-style variables. `forward_chain()` derives everything possible and returns a list of `InferenceResult` objects." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "519ca92d", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:45:57.270791Z", + "iopub.status.busy": "2026-08-26T18:45:57.270352Z", + "iopub.status.idle": "2026-08-26T18:45:59.991941Z", + "shell.execute_reply": "2026-08-26T18:45:59.990678Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "

๐Ÿง  Semantica - ๐Ÿ“Š Current Progress

StatusActionModuleSubmoduleProgressETARateTimeExtracted
โœ…Semantica is reasoning๐Ÿค” reasoningReasoner100.0%--0.00s-
โœ…Semantica is reasoning๐Ÿค” reasoningDatalogReasoner100.0%--0.00s-
โœ…Semantica is reasoning๐Ÿค” reasoningExplanationGenerator100.0%--0.00s-
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "๐Ÿ”„ Semantica is reasoning: Performing forward chaining ๐Ÿค” reasoning Reasoner |โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘| 0.0% ETA: - Rate: - Time: 0.00s Extracted: -" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inferred 2 new facts\n", + " Human(Jane) (rule: Rule 1, confidence: 1.0)\n", + " Human(John) (rule: Rule 1, confidence: 1.0)\n" + ] + } + ], + "source": [ + "from semantica.reasoning import Reasoner\n", + "\n", + "reasoner = Reasoner()\n", + "\n", + "reasoner.add_fact(\"Person(John)\")\n", + "reasoner.add_fact(\"Person(Jane)\")\n", + "reasoner.add_rule(\"IF Person(?x) THEN Human(?x)\")\n", + "\n", + "results = reasoner.forward_chain()\n", + "print(f\"Inferred {len(results)} new facts\")\n", + "for res in results:\n", + " print(f\" {res.conclusion} (rule: {res.rule_used.name}, confidence: {res.confidence})\")" + ] + }, + { + "cell_type": "markdown", + "id": "c1131c45", + "metadata": {}, + "source": [ + "## 2) One-shot inference with `infer_facts`\n", + "\n", + "`infer_facts(facts, rules)` **adds** the given facts and rules to this `Reasoner` instance, runs forward chaining to fixpoint, and returns the derived facts as strings. It does not reset the instance's existing state โ€” create a fresh `Reasoner()` first if you need isolation between runs." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "26249990", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:45:59.995447Z", + "iopub.status.busy": "2026-08-26T18:45:59.995069Z", + "iopub.status.idle": "2026-08-26T18:46:00.004107Z", + "shell.execute_reply": "2026-08-26T18:46:00.002873Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['Employee(Jane, Acme)', 'Employee(John, Acme)']" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from semantica.reasoning import Reasoner\n", + "\n", + "derived = Reasoner().infer_facts(\n", + " facts=[\"WorksFor(John, Acme)\", \"WorksFor(Jane, Acme)\"],\n", + " rules=[\"IF WorksFor(?x, ?y) THEN Employee(?x, ?y)\"],\n", + ")\n", + "derived" + ] + }, + { + "cell_type": "markdown", + "id": "d5504a38", + "metadata": {}, + "source": [ + "## 3) Backward chaining: proving a goal\n", + "\n", + "`backward_chain(goal)` works backwards from a conclusion through the rules. It returns the `InferenceResult` that proves the goal, or `None`." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "c4ef85dd", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:46:00.007740Z", + "iopub.status.busy": "2026-08-26T18:46:00.007346Z", + "iopub.status.idle": "2026-08-26T18:46:00.015561Z", + "shell.execute_reply": "2026-08-26T18:46:00.014145Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Human(John)\n", + "premises: ['Person(John)']\n" + ] + } + ], + "source": [ + "from semantica.reasoning import Reasoner\n", + "\n", + "reasoner = Reasoner()\n", + "reasoner.add_fact(\"Person(John)\")\n", + "reasoner.add_rule(\"IF Person(?x) THEN Human(?x)\")\n", + "\n", + "proof = reasoner.backward_chain(\"Human(John)\")\n", + "print(proof.conclusion if proof else \"not provable\")\n", + "print(\"premises:\", proof.premises if proof else None)" + ] + }, + { + "cell_type": "markdown", + "id": "b245581d", + "metadata": {}, + "source": [ + "## 4) Re-run safety\n", + "\n", + "`add_rule` deduplicates rules with identical conditions and conclusion, so re-executing a setup cell (the common Jupyter re-run) does not duplicate rules โ€” see issue #732." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "fb2aeb39", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:46:00.019091Z", + "iopub.status.busy": "2026-08-26T18:46:00.018881Z", + "iopub.status.idle": "2026-08-26T18:46:00.024042Z", + "shell.execute_reply": "2026-08-26T18:46:00.022836Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Skipping duplicate rule (same conditions/conclusion as 'rule_1'): IF Person(?x) THEN Human(?x)\n" + ] + }, + { + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from semantica.reasoning import Reasoner\n", + "\n", + "reasoner = Reasoner()\n", + "reasoner.add_fact(\"Person(John)\")\n", + "\n", + "# Simulate a Jupyter cell re-run: add the same rule twice\n", + "r1 = reasoner.add_rule(\"IF Person(?x) THEN Human(?x)\")\n", + "r2 = reasoner.add_rule(\"IF Person(?x) THEN Human(?x)\")\n", + "\n", + "len(reasoner.rules)" + ] + }, + { + "cell_type": "markdown", + "id": "ba2e5c4a", + "metadata": {}, + "source": [ + "## 5) Datalog reasoning\n", + "\n", + "`DatalogReasoner` uses classic Datalog syntax (`head :- body.`) and semi-naive fixpoint evaluation. Queries return variable bindings as a list of dicts โ€” use uppercase variables to ask *which* facts hold." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "9ec5c0c4", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:46:00.026769Z", + "iopub.status.busy": "2026-08-26T18:46:00.026588Z", + "iopub.status.idle": "2026-08-26T18:46:00.034963Z", + "shell.execute_reply": "2026-08-26T18:46:00.032672Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'X': 'tom', 'Z': 'ann'}]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from semantica.reasoning import DatalogReasoner\n", + "\n", + "datalog = DatalogReasoner()\n", + "datalog.add_fact(\"parent(tom, mary)\")\n", + "datalog.add_fact(\"parent(mary, ann)\")\n", + "datalog.add_rule(\"grandparent(X, Z) :- parent(X, Y), parent(Y, Z)\")\n", + "\n", + "datalog.derive_all()\n", + "datalog.query(\"grandparent(X, Z)\")" + ] + }, + { + "cell_type": "markdown", + "id": "d4f0689b", + "metadata": {}, + "source": [ + "## 6) Explanations for inferred conclusions\n", + "\n", + "`ExplanationGenerator` turns `InferenceResult` objects into structured `Explanation` and `ReasoningPath` records, so agents can show *why* they believe a derived fact." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "19dcd3a7", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:46:00.038649Z", + "iopub.status.busy": "2026-08-26T18:46:00.038396Z", + "iopub.status.idle": "2026-08-26T18:46:00.059188Z", + "shell.execute_reply": "2026-08-26T18:46:00.057805Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "('Explanation', 'ReasoningPath')" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from semantica.reasoning import Reasoner, ExplanationGenerator\n", + "\n", + "reasoner = Reasoner()\n", + "reasoner.add_fact(\"Person(John)\")\n", + "reasoner.add_rule(\"IF Person(?x) THEN Human(?x)\")\n", + "results = reasoner.forward_chain()\n", + "\n", + "gen = ExplanationGenerator()\n", + "explanation = gen.generate_explanation(results[0])\n", + "path = gen.show_reasoning_path(results[0])\n", + "\n", + "type(explanation).__name__, type(path).__name__" + ] + }, + { + "cell_type": "markdown", + "id": "fb882ee4", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "| Task | API |\n", + "|---|---|\n", + "| Derive all new facts | `Reasoner.forward_chain()` |\n", + "| One-shot inference | `Reasoner.infer_facts(facts, rules)` |\n", + "| Prove a goal | `Reasoner.backward_chain(goal)` |\n", + "| Datalog fixpoint | `DatalogReasoner.derive_all()` + `query(\"p(X, Y)\")` |\n", + "| Explain a conclusion | `ExplanationGenerator.generate_explanation(result)` |\n", + "\n", + "See also `semantica/reasoning/reasoning_usage.md` and the module docstrings for `ReteEngine`, `SPARQLReasoner`, and temporal reasoning." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cookbook/introduction/24_Change_Management.ipynb b/cookbook/introduction/24_Change_Management.ipynb new file mode 100644 index 00000000..3f8d3db8 --- /dev/null +++ b/cookbook/introduction/24_Change_Management.ipynb @@ -0,0 +1,299 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "8d7096ea", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/24_Change_Management.ipynb)\n", + "\n", + "# Change Management โ€” Practical Guide\n", + "\n", + "Semantica's `change_management` module provides versioning, audit trails, and data-integrity checks for knowledge graphs and ontologies:\n", + "\n", + "- **`ChangeLogEntry`** โ€” standardized change metadata (validated timestamp/author)\n", + "- **`InMemoryVersionStorage` / `SQLiteVersionStorage`** โ€” version snapshot storage with named tags\n", + "- **`compute_checksum` / `verify_checksum`** โ€” SHA-256 integrity verification\n", + "\n", + "This notebook runs a complete save โ†’ tag โ†’ verify โ†’ tamper-detect cycle. All outputs are real executed results verified against the repository's `semantica/change_management/` source at the time of writing (the `pip install` cell may fetch a newer release with slightly different behavior)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "7bdffec1", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:46:37.171333Z", + "iopub.status.busy": "2026-08-26T18:46:37.171183Z", + "iopub.status.idle": "2026-08-26T18:46:39.060860Z", + "shell.execute_reply": "2026-08-26T18:46:39.059594Z" + } + }, + "outputs": [], + "source": [ + "!pip install -q semantica" + ] + }, + { + "cell_type": "markdown", + "id": "169efee1", + "metadata": {}, + "source": [ + "## 1) A `ChangeLogEntry` records *who* changed *what*, *when*\n", + "\n", + "`author` must be a valid email โ€” the dataclass validates on construction (`ValidationError` otherwise), which keeps audit trails clean." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "5b17acdb", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:46:39.064077Z", + "iopub.status.busy": "2026-08-26T18:46:39.063818Z", + "iopub.status.idle": "2026-08-26T18:46:39.321881Z", + "shell.execute_reply": "2026-08-26T18:46:39.321036Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "ChangeLogEntry(timestamp='2026-08-15T09:00:00Z', author='demo@example.com', description='initial version', change_id=None, related_changes=[])" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from semantica.change_management import ChangeLogEntry\n", + "\n", + "entry = ChangeLogEntry(\n", + " timestamp=\"2026-08-15T09:00:00Z\",\n", + " author=\"demo@example.com\",\n", + " description=\"initial version\",\n", + ")\n", + "entry" + ] + }, + { + "cell_type": "markdown", + "id": "53d8df5c", + "metadata": {}, + "source": [ + "## 2) Save a versioned snapshot\n", + "\n", + "A snapshot is a dict with a required `label` plus your payload. Here we attach the KG data, the change log, and a SHA-256 `checksum` computed over everything except the checksum field itself." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "fec16f24", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:46:39.325528Z", + "iopub.status.busy": "2026-08-26T18:46:39.325140Z", + "iopub.status.idle": "2026-08-26T18:46:39.331480Z", + "shell.execute_reply": "2026-08-26T18:46:39.330586Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from semantica.change_management import InMemoryVersionStorage, compute_checksum\n", + "\n", + "storage = InMemoryVersionStorage()\n", + "\n", + "snapshot = {\n", + " \"label\": \"v1.0.0\",\n", + " \"data\": {\"entities\": {\"acme\": {\"type\": \"Company\"}}},\n", + " \"change_log\": {\n", + " \"timestamp\": entry.timestamp,\n", + " \"author\": entry.author,\n", + " \"description\": entry.description,\n", + " },\n", + "}\n", + "snapshot[\"checksum\"] = compute_checksum({k: v for k, v in snapshot.items() if k != \"checksum\"})\n", + "\n", + "storage.save(snapshot)\n", + "storage.exists(\"v1.0.0\")" + ] + }, + { + "cell_type": "markdown", + "id": "0f1c603b", + "metadata": {}, + "source": [ + "## 3) Named tags pin a version for releases\n", + "\n", + "`save_tag` / `get_tag` map stable names (e.g. `release`) to version labels, decoupling consumers from label churn." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "62f7643e", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:46:39.335182Z", + "iopub.status.busy": "2026-08-26T18:46:39.334886Z", + "iopub.status.idle": "2026-08-26T18:46:39.339586Z", + "shell.execute_reply": "2026-08-26T18:46:39.338568Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "('v1.0.0', ['v1.0.0'])" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "storage.save_tag(\"release\", \"v1.0.0\")\n", + "\n", + "storage.get_tag(\"release\"), [s[\"label\"] for s in storage.list_all()]" + ] + }, + { + "cell_type": "markdown", + "id": "96df12da", + "metadata": {}, + "source": [ + "## 4) Verify integrity โ€” and catch tampering\n", + "\n", + "`verify_checksum(snapshot)` recomputes the SHA-256 over the snapshot (minus its `checksum` field) and compares. A single mutated character in the data flips the result to `False`." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "26d0de85", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:46:39.342653Z", + "iopub.status.busy": "2026-08-26T18:46:39.342466Z", + "iopub.status.idle": "2026-08-26T18:46:39.346714Z", + "shell.execute_reply": "2026-08-26T18:46:39.345623Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "intact: True\n", + "tampered: False\n" + ] + } + ], + "source": [ + "from semantica.change_management import verify_checksum\n", + "\n", + "stored = storage.get(\"v1.0.0\")\n", + "print(\"intact:\", verify_checksum(stored))\n", + "\n", + "tampered = storage.get(\"v1.0.0\")\n", + "tampered[\"data\"][\"entities\"][\"acme\"][\"note\"] = \"mutated after the fact\"\n", + "print(\"tampered:\", verify_checksum(tampered))" + ] + }, + { + "cell_type": "markdown", + "id": "bd14c3e4", + "metadata": {}, + "source": [ + "## 5) Retiring a version\n", + "\n", + "`delete(label)` removes a snapshot; tags pointing at it are your responsibility to update." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "de9fe3e5", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:46:39.349814Z", + "iopub.status.busy": "2026-08-26T18:46:39.349513Z", + "iopub.status.idle": "2026-08-26T18:46:39.354710Z", + "shell.execute_reply": "2026-08-26T18:46:39.353669Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "storage.delete(\"v1.0.0\")\n", + "storage.exists(\"v1.0.0\")" + ] + }, + { + "cell_type": "markdown", + "id": "ab667b32", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "| Task | API |\n", + "|---|---|\n", + "| Record audit metadata | `ChangeLogEntry(timestamp, author=email, description)` |\n", + "| Persist a version | `InMemoryVersionStorage().save({\"label\": ..., ...})` |\n", + "| Pin a release name | `save_tag(\"release\", \"v1.0.0\")` / `get_tag(\"release\")` |\n", + "| Integrity check | `compute_checksum(snap)` / `verify_checksum(snap)` |\n", + "| Persistent backend | `SQLiteVersionStorage(path)` โ€” same interface |\n", + "\n", + "See also `semantica/change_management/change_management_usage.md` for the manager classes (`TemporalVersionManager`, `OntologyVersionManager`)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cookbook/introduction/25_Seed_Data.ipynb b/cookbook/introduction/25_Seed_Data.ipynb new file mode 100644 index 00000000..bf962e5d --- /dev/null +++ b/cookbook/introduction/25_Seed_Data.ipynb @@ -0,0 +1,314 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6eb4dfba", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/25_Seed_Data.ipynb)\n", + "\n", + "# Seed Data โ€” Practical Guide\n", + "\n", + "The `seed` module bootstraps a knowledge graph from **trusted, pre-known data** (CSV/JSON/database/API sources) before any extraction runs. This gives extraction a foundation to link against instead of starting from an empty graph.\n", + "\n", + "Key pieces:\n", + "\n", + "- **`SeedDataManager`** โ€” registers data sources and builds foundation graphs\n", + "- **`create_foundation_graph()`** โ€” turns registered sources into `entities` + `relationships` + `metadata`\n", + "- **`validate_quality()`** โ€” checks a foundation graph before you commit it\n", + "\n", + "All examples below were executed against `semantica/seed/seed_manager.py`." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "32f80cc6", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:51:18.716466Z", + "iopub.status.busy": "2026-08-26T18:51:18.716264Z", + "iopub.status.idle": "2026-08-26T18:51:20.533828Z", + "shell.execute_reply": "2026-08-26T18:51:20.531402Z" + } + }, + "outputs": [], + "source": [ + "!pip install -q semantica" + ] + }, + { + "cell_type": "markdown", + "id": "75136e5f", + "metadata": {}, + "source": [ + "## 1) Prepare a seed CSV and register the source\n", + "\n", + "`register_source(name, format, location, entity_type=...)` records where trusted data lives. `verified=True` (the default) marks the source as pre-validated." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "a8089e1f", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:51:20.538772Z", + "iopub.status.busy": "2026-08-26T18:51:20.538323Z", + "iopub.status.idle": "2026-08-26T18:51:20.675403Z", + "shell.execute_reply": "2026-08-26T18:51:20.674060Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import csv\n", + "import tempfile\n", + "from pathlib import Path\n", + "from semantica.seed import SeedDataManager\n", + "\n", + "# Write the sample CSV into a session-scoped temp directory so we never\n", + "# clobber a companies.csv that might exist in the user's working directory.\n", + "seed_csv = Path(tempfile.mkdtemp(prefix=\"semantica-seed-\")) / \"companies.csv\"\n", + "with open(seed_csv, \"w\", newline=\"\") as f:\n", + " writer = csv.DictWriter(f, fieldnames=[\"id\", \"name\", \"type\", \"industry\"])\n", + " writer.writeheader()\n", + " writer.writerow({\"id\": \"c1\", \"name\": \"Acme\", \"type\": \"Company\", \"industry\": \"robotics\"})\n", + " writer.writerow({\"id\": \"c2\", \"name\": \"Globex\", \"type\": \"Company\", \"industry\": \"energy\"})\n", + "\n", + "manager = SeedDataManager()\n", + "manager.register_source(\"companies\", format=\"csv\", location=str(seed_csv), entity_type=\"Company\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "e87221ba", + "metadata": {}, + "source": [ + "## 2) Load records from a registered source\n", + "\n", + "`load_source(name)` reads the source and enriches each record with `entity_type` and `source` provenance keys." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "f932e550", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:51:20.679424Z", + "iopub.status.busy": "2026-08-26T18:51:20.679156Z", + "iopub.status.idle": "2026-08-26T18:51:20.690659Z", + "shell.execute_reply": "2026-08-26T18:51:20.688812Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "

๐Ÿง  Semantica - ๐Ÿ“Š Current Progress

StatusActionModuleSubmoduleProgressETARateTimeExtracted
โœ…Semantica is seeding๐ŸŒฑ seedSeedDataManager100.0%--0.00s-
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "๐Ÿ”„ Semantica is seeding: Loading seed data from CSV: /var/folders/7s/bvvstgs10y963tz6_4bbnklr0000gn/T/semantica-seed-eu9__ep1/companies.csv ๐ŸŒฑ seed SeedDataManager |โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘| 0.0% ETA: - Rate: - Time: 0.00s Extracted: -" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loaded 2 records\n" + ] + }, + { + "data": { + "text/plain": [ + "{'id': 'c1',\n", + " 'name': 'Acme',\n", + " 'type': 'Company',\n", + " 'industry': 'robotics',\n", + " 'entity_type': 'Company',\n", + " 'source': 'companies'}" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "records = manager.load_source(\"companies\")\n", + "print(f\"loaded {len(records)} records\")\n", + "records[0]" + ] + }, + { + "cell_type": "markdown", + "id": "f2ebce64", + "metadata": {}, + "source": [ + "## 3) Build the foundation graph\n", + "\n", + "`create_foundation_graph()` converts every registered source into graph-ready entities and relationships. Entities carry `confidence: 1.0` โ€” seed data is trusted by definition." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "09388c31", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:51:20.695259Z", + "iopub.status.busy": "2026-08-26T18:51:20.694928Z", + "iopub.status.idle": "2026-08-26T18:51:20.708595Z", + "shell.execute_reply": "2026-08-26T18:51:20.707072Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['entities', 'metadata', 'relationships']" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "foundation = manager.create_foundation_graph()\n", + "sorted(foundation.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "4610a59f", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:51:20.713136Z", + "iopub.status.busy": "2026-08-26T18:51:20.712795Z", + "iopub.status.idle": "2026-08-26T18:51:20.718637Z", + "shell.execute_reply": "2026-08-26T18:51:20.716835Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'c1',\n", + " 'text': 'Acme',\n", + " 'type': 'Company',\n", + " 'confidence': 1.0,\n", + " 'metadata': {'industry': 'robotics', 'source': 'companies'}}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "foundation[\"entities\"][0]" + ] + }, + { + "cell_type": "markdown", + "id": "f3a52dc7", + "metadata": {}, + "source": [ + "## 4) Validate quality before committing\n", + "\n", + "`validate_quality(foundation_graph)` returns `valid`, `errors`, `warnings`, and `metrics` so you can gate bad seed data before it pollutes the graph." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "4eb7e664", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-26T18:51:20.722674Z", + "iopub.status.busy": "2026-08-26T18:51:20.722118Z", + "iopub.status.idle": "2026-08-26T18:51:20.732003Z", + "shell.execute_reply": "2026-08-26T18:51:20.730170Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "quality = manager.validate_quality(foundation)\n", + "quality[\"valid\"]" + ] + }, + { + "cell_type": "markdown", + "id": "b534be89", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "| Task | API |\n", + "|---|---|\n", + "| Register a trusted source | `register_source(name, format, location, entity_type=...)` |\n", + "| Load records | `load_source(name)` โ€” adds `entity_type` / `source` keys |\n", + "| Direct file load | `load_from_csv(path)` / `load_from_json(path)` |\n", + "| Build the graph | `create_foundation_graph()` โ†’ `entities` / `relationships` / `metadata` |\n", + "| Gate bad data | `validate_quality(graph)` โ†’ `valid` / `errors` / `warnings` / `metrics` |\n", + "\n", + "See also `semantica/seed/seed_usage.md` for `load_from_database`, `load_from_api`, and `integrate_with_extracted`." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/citation.md b/docs/citation.md index 45ac350b..b798677a 100644 --- a/docs/citation.md +++ b/docs/citation.md @@ -13,26 +13,25 @@ icon: "quote-left" ```bibtex @software{semantica2026, - title = {Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems}, - author = {Semantica}, - year = {2026}, - url = {https://github.com/semantica-agi/semantica}, - version = {0.6.6}, - doi = {10.5281/zenodo.XXXXXXX} + title = {Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems}, + author = {Semantica}, + year = {2026}, + url = {https://github.com/semantica-agi/semantica}, + doi = {10.5281/zenodo.XXXXXXX} } ``` - Semantica. (2026). *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems* (Version 0.6.6) \[Computer software\]. https://github.com/semantica-agi/semantica + Semantica. (2026). *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems* \[Computer software\]. https://github.com/semantica-agi/semantica - Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.6, GitHub, 2026, https://github.com/semantica-agi/semantica. + Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. GitHub, 2026, https://github.com/semantica-agi/semantica. - Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.6. GitHub, 2026. https://github.com/semantica-agi/semantica. + Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. GitHub, 2026. https://github.com/semantica-agi/semantica. - Semantica, "Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems," Version 0.6.6, GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica + Semantica, "Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems," GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica diff --git a/docs/cookbook.md b/docs/cookbook.md index 443aae7c..b556eb4f 100644 --- a/docs/cookbook.md +++ b/docs/cookbook.md @@ -35,6 +35,7 @@ Essential guides to master the Semantica framework. - **[Vector Store](https://github.com/semantica-agi/semantica/blob/main/cookbook/introduction/13_Vector_Store.ipynb)** โ€” Setting up vector stores for similarity search and retrieval. *Intermediate* - **[Graph Store](https://github.com/semantica-agi/semantica/blob/main/cookbook/introduction/09_Graph_Store.ipynb)** โ€” Persisting knowledge graphs in Neo4j or FalkorDB. Topics: Neo4j, Cypher, Persistence ยท *Intermediate* - **[Ontology](https://github.com/semantica-agi/semantica/blob/main/cookbook/introduction/14_Ontology.ipynb)** โ€” Defining domain schemas and ontologies to structure your data. Topics: OWL, RDF, Schema Design ยท *Intermediate* +- **[Seed Data](https://github.com/semantica-agi/semantica/blob/main/cookbook/introduction/25_Seed_Data.ipynb)** โ€” Bootstrapping a knowledge graph from trusted CSV, JSON, database, and API sources before extraction runs. Topics: SeedDataManager, Foundation Graphs ยท *Intermediate* ## Advanced Concepts @@ -50,6 +51,9 @@ Deep dive into advanced features, customization, and complex workflows. - **[Multi-Source Integration](https://github.com/semantica-agi/semantica/blob/main/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb)** โ€” Merging data from disparate sources into a unified graph. Topics: Entity Resolution, Merging, Fusion ยท *Advanced* - **[Reasoning and Inference](https://github.com/semantica-agi/semantica/blob/main/cookbook/advanced/08_Reasoning_and_Inference.ipynb)** โ€” Using logical reasoning to infer new knowledge from existing facts. Topics: Logic Rules, Inference Engines ยท *Advanced* - **[Temporal Knowledge Graphs](https://github.com/semantica-agi/semantica/blob/main/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb)** โ€” Modeling and querying data that changes over time. Topics: Time Series, Temporal Logic, Allen Algebra ยท *Advanced* +- **[Provenance Tracking](https://github.com/semantica-agi/semantica/blob/main/cookbook/introduction/22_Provenance_Tracking.ipynb)** โ€” Audit-grade, W3C PROV-O-aligned tracking of where every entity, relationship, and chunk came from. Topics: PROV-O, Lineage, Checksums, Invalidation ยท *Advanced* +- **[Reasoning Module](https://github.com/semantica-agi/semantica/blob/main/cookbook/introduction/23_Reasoning.ipynb)** โ€” Deriving new knowledge from existing facts with forward chaining, backward chaining, and Datalog strategies. Topics: Reasoner, Datalog, Explanations ยท *Advanced* +- **[Change Management](https://github.com/semantica-agi/semantica/blob/main/cookbook/introduction/24_Change_Management.ipynb)** โ€” Versioning, audit trails, and data-integrity checks for knowledge graphs and ontologies. Topics: ChangeLogEntry, Version Storage, Data Integrity ยท *Advanced* ## How to Run @@ -80,6 +84,6 @@ Deep dive into advanced features, customization, and complex workflows. You can also run the cookbook using Docker: ```bash - docker run -p 8888:8888 hawksight/semantica-cookbook + docker run -p 8888:8888 semantica/semantica-cookbook ``` diff --git a/docs/docs.json b/docs/docs.json index d2ad5da2..d5713f4d 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -103,6 +103,7 @@ "pages": [ "integrations/agno", "integrations/crewai", + "integrations/langchain", "integrations/docling", "integrations/snowflake", "integrations/databricks" diff --git a/docs/governance.md b/docs/governance.md index 332cf5a2..e1df0508 100644 --- a/docs/governance.md +++ b/docs/governance.md @@ -4,12 +4,12 @@ description: "Project governance model: roles, decision process, release cadence icon: "scale-balanced" --- -> Semantica is maintained by Hawksight AI with community contributions under an open governance model. +> Semantica is maintained by the Semantica team with community contributions under an open governance model. ## Roles -- **Maintainers** โ€” Hawksight AI team: review and merge PRs, manage releases and code quality, set project direction and community standards. +- **Maintainers** โ€” Semantica team: review and merge PRs, manage releases and code quality, set project direction and community standards. - **Contributors** โ€” Submit code, documentation, and bug reports. Help with issues and reviews. Recognized in [CONTRIBUTORS.md](https://github.com/semantica-agi/semantica/blob/main/CONTRIBUTORS.md). - **Community Members** โ€” Use Semantica, provide feedback, share use cases, and participate in GitHub Discussions and Discord. diff --git a/docs/guides/context-graphs.md b/docs/guides/context-graphs.md index 8521db7e..26ec5174 100644 --- a/docs/guides/context-graphs.md +++ b/docs/guides/context-graphs.md @@ -436,6 +436,35 @@ d = graph.to_dict() # d["statistics"] โ†’ {"node_count": int, "edge_count": int} ``` +For a human-editable, version-control-friendly representation, save a Markdown +directory instead: + +```python +graph.save_to_file("context_graph/", format="markdown") + +restored = ContextGraph(advanced_analytics=True) +restored.load_from_file("context_graph/", format="markdown") +``` + +The directory contains a versioned `graph.md` manifest for graph identity, +relationships, and cross-graph link descriptors, plus one file per node under +`nodes/`. A node's content is its Markdown body; its ID, type, properties, +metadata, and temporal validity are YAML frontmatter. Node, edge, family, graph, +and cross-graph link IDs are preserved across round trips. + +Markdown loading uses replacement semantics, like `from_dict()`: it parses and +validates the complete directory before replacing the current graph. Invalid YAML, +duplicate IDs, unsupported versions, and unsafe filesystem links fail without +partially mutating the graph. As with JSON loading, an edge endpoint without a node +file creates an `entity` stub node. Symlinks, Windows directory junctions, and other +Windows reparse points are rejected. + +Re-exporting to an existing managed directory atomically replaces it, removing stale +node files. Before replacement, Semantica validates the complete canonical export +layout, not just the manifest header. Untracked files, assets, extra directories, or +renamed node files therefore cause the export to fail closed instead of being deleted. +Keep attachments and hand-written indexes outside the managed export directory. + If the graph had cross-graph links created with `link_graph()`, call `resolve_links()` after loading to restore live navigation โ€” object references cannot be serialized, so they must be reconnected manually: ```python diff --git a/docs/guides/pipeline.md b/docs/guides/pipeline.md index ffbfe163..e8e86107 100644 --- a/docs/guides/pipeline.md +++ b/docs/guides/pipeline.md @@ -222,10 +222,10 @@ builder.register_step_handler("ner_extract", run_ner) builder.register_step_handler("triplet_extract", run_triplets) builder.register_step_handler("kg_merge", merge_into_graph) -builder.add_step("ingest", "file_ingest", handler=ingest_stix_bundles, path="./stix_bundles/") -builder.add_step("ner", "ner_extract", handler=run_ner, confidence_threshold=0.75) -builder.add_step("triplets", "triplet_extract", handler=run_triplets, include_temporal=True) -builder.add_step("store", "kg_merge", handler=merge_into_graph, output_path="./cti_output/") +builder.add_step("ingest", "file_ingest", path="./stix_bundles/") +builder.add_step("ner", "ner_extract", confidence_threshold=0.75) +builder.add_step("triplets", "triplet_extract", include_temporal=True) +builder.add_step("store", "kg_merge", output_path="./cti_output/") # ingest feeds both ner and triplets in parallel builder.connect_steps("ingest", "ner") diff --git a/docs/guides/reasoning.md b/docs/guides/reasoning.md index 6e43e23f..dc9aa744 100644 --- a/docs/guides/reasoning.md +++ b/docs/guides/reasoning.md @@ -150,6 +150,12 @@ HighRiskSupplier(DELTA-3) conf=100% rule=Rule 3 DELTA-3 is flagged even though no document described it that way โ€” the system traced: DELTA-3 supplied GAMMA-7, and GAMMA-7 exploits critical CVEs. For rules that need priority ordering or graded confidence, use the `Rule` dataclass: +If a rule has side-effecting actions, one concrete activation runs those +actions at most once on a Reasoner instance. Re-running `forward_chain()` is +therefore safe: already-attempted actions are not repeated. Use +`reasoner.reset_action_history()` when you intentionally want to replay them; +`reasoner.clear()` and `reasoner.reset()` also clear the history. + ```python # Higher priority rules fire first; confidence propagates into InferenceResult.confidence reasoner.add_rule(Rule( @@ -269,7 +275,7 @@ print("Loaded {} facts from graph".format(count)) ## Step 5 โ€” SPARQL queries over enriched working memory -After forward chaining has derived new facts, `SPARQLReasoner` lets you query the enriched working memory using SPARQL triple-pattern matching with optional inference expansion: +After forward chaining has derived new facts, `SPARQLReasoner` prepares SPARQL queries over the enriched working memory with optional inference expansion: ```python from semantica.reasoning import SPARQLReasoner @@ -288,22 +294,13 @@ query = """ } """ -# execute_query() runs: expansion โ†’ inference โ†’ deduplication -result = sparql.execute_query(query) - -for binding in result.bindings: - print("Actor: {:15s} CVE: {}".format( - binding.get("actor", "?"), - binding.get("cve", "?"), - )) - -# metadata shows how many results came from inference vs ground facts -print("Original: {} Inferred: {}".format( - result.metadata.get("original_count", 0), - result.metadata.get("inferred_count", 0), -)) +# expand_query() applies inference rules to the query text: +expanded = sparql.expand_query(query) +print(expanded) ``` +`execute_query()` is not implemented yet: no triplet-store execution path exists, so it raises `NotImplementedError` rather than returning an empty result set that callers would misread as "no matches". Until execution lands, run the expanded query against your RDF store directly (for example with `rdflib`). + Inspect the expanded query before running it: ```python @@ -369,6 +366,13 @@ engine.reset() The rule network is compiled once by `build_network()`. Each subsequent `add_fact()` call propagates incrementally through only the nodes whose conditions it satisfies โ€” not the full rule set โ€” which keeps evaluation cost proportional to the number of new activations rather than the total rule count. +With a Reasoner bound, Rete action side effects are attempted once per rule, +bindings, and matched fact identity. Passing the same match to +`execute_matches()` again still returns the same conclusion, but does not repeat +its actions. Call `engine.reset_action_history()` to replay actions without +clearing working memory. `engine.reset()` and `engine.build_network()` also +clear the action history. + ## Step 7 โ€” Temporal interval reasoning `TemporalReasoningEngine` computes Allen interval relations between time windows, letting you identify whether two events overlap, one contains the other, they meet at a boundary, and so on across your graph: diff --git a/docs/guides/shacl-validation.md b/docs/guides/shacl-validation.md index a2b81ac2..76c07b89 100644 --- a/docs/guides/shacl-validation.md +++ b/docs/guides/shacl-validation.md @@ -8,7 +8,7 @@ icon: "shield-check" SHACL (Shapes Constraint Language) is a standard for validating graph-based data. While an ontology defines the conceptual *schema* (the "what" exists in your domain), SHACL defines the structural *rules and constraints* (the "how" it should be structured). -In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and `_run_pyshacl` evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated. +In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and the public `run_shacl_validation` function evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated. The historical `_run_pyshacl` name remains available as a compatibility alias. ## Why Use SHACL Validation? @@ -55,7 +55,7 @@ Let's look at a simple, universally understood example: ensuring every `Employee ```python from semantica.context import ContextGraph from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape -from semantica.ontology.ontology_validator import _run_pyshacl +from semantica.ontology import run_shacl_validation # 1. Prepare your data graph graph = ContextGraph() @@ -95,7 +95,7 @@ data_ttl = """ """ # 5. Run Validation -report = _run_pyshacl(data_ttl, shacl_ttl) +report = run_shacl_validation(data_ttl, shacl_ttl) # 6. Analyze the Report print(f"Graph conforms: {report.conforms}") @@ -265,10 +265,10 @@ cve_id_shape = NodeShape( ## Step 4 โ€” Run validation and read the report -Serialize the graph to RDF, then run `_run_pyshacl` against the shapes. +Serialize the graph to RDF, then run `run_shacl_validation` against the shapes. ```python -from semantica.ontology.ontology_validator import _run_pyshacl +from semantica.ontology import run_shacl_validation # Prepare your RDF data string (since export_rdf primarily exports structural metadata, # you typically serialize your custom data graph to Turtle using rdflib or similar). @@ -281,7 +281,7 @@ data_ttl = """ """ # Run SHACL validation -report = _run_pyshacl( +report = run_shacl_validation( data_ttl, shacl_ttl, data_graph_format="turtle", @@ -366,8 +366,8 @@ print(f"Malware nodes missing 'family': {len(missing_family)}") # e.g. graph.update_node(node_id, {"family": "UNKNOWN โ€” requires triage"}) # After remediation, re-run validation to confirm the fix -# (re-export the patched graph to Turtle first, then call _run_pyshacl again) -report2 = _run_pyshacl(patched_data_ttl, shacl_ttl) +# (re-export the patched graph to Turtle first, then call run_shacl_validation again) +report2 = run_shacl_validation(patched_data_ttl, shacl_ttl) print(f"Violations after remediation: {report2.violation_count}") # Violations after remediation: 0 ``` @@ -377,10 +377,49 @@ print(f"Violations after remediation: {report2.violation_count}") ## Common Pitfalls - **Assuming the ontology automatically enforces data quality**: `SHACLGenerator` generates shapes based on what it observes in the data. If your data is missing a field, the generator won't know it was mandatory unless you explicitly inject the constraint (as shown in Step 3). -- **Passing `ContextGraph` directly to SHACL validators**: The `_run_pyshacl` function expects an RDF string (like Turtle format), not a raw Python dictionary or `ContextGraph` object. +- **Passing `ContextGraph` directly to SHACL validators**: The `run_shacl_validation` function expects an RDF string (like Turtle format), not a raw Python dictionary or `ContextGraph` object. - **Forgetting RDF serialization**: You must serialize your graph (often via a temporary file using `export_rdf`) before validating it. - **Treating validation as a one-time step**: Validation should be integrated as an automated step in your CI/CD pipeline or data ingestion flow, acting as a recurring gatekeeper rather than a one-off script. - **Ignoring validation reports**: A graph that does not conform must be remediated. Failing to review the `violation_count` and address the issues negates the purpose of SHACL validation. +- **Validating `sh:class`/`sh:node` range checks on a property that declares `rdfs:range` with RDFS entailment on**: RDFS is an entailment rule, not a constraint. When pyshacl runs with `inference="rdfs"`, it infers the range class onto every object of the property, so class-based constraints on that property can never fail โ€” the report says `conforms: True` on data that does not conform: + + ```python + from pyshacl import validate + from rdflib import Graph + + data = Graph() + data.parse( + data=""" + @prefix ex: . + @prefix rdfs: . + ex:contains rdfs:domain ex:Container ; rdfs:range ex:Item . + ex:box a ex:Container ; ex:contains ex:notAnItem . + ex:notAnItem a ex:Fish . + """, + format="turtle", + ) + + shapes = Graph() + shapes.parse( + data=""" + @prefix ex: . + @prefix sh: . + ex:ContainerShape a sh:NodeShape ; + sh:targetClass ex:Container ; + sh:property [ sh:path ex:contains ; sh:class ex:Item ] . + """, + format="turtle", + ) + + for inference in ("none", "rdfs"): + conforms, _, _ = validate(data, shacl_graph=shapes, inference=inference) + print(inference, conforms) + # none False <- correct: notAnItem is a Fish, not an Item + # rdfs True <- the entailment manufactured the type + ``` + + Mitigations: prefer not to declare `rdfs:range` on properties you intend to constrain with `sh:class`; when class membership is the thing under test, run validation without RDFS entailment (`inference="none"`); or express the check as a constraint the entailment cannot satisfy (for example a literal property constraint). Note the trade-off: with entailment off, `sh:targetClass` no longer reaches subclasses, so subclass hierarchies need explicit typing or inference-aware target selection. Semantica's own `run_shacl_validation` wrapper already calls pyshacl with `inference="none"`, so this pitfall only bites when calling `pyshacl.validate` directly with entailment enabled. +- **Trusting `conforms: True` without checking the inference mode**: an inference-enabled run can hide the exact violations the shapes were written to catch (see above). Record which inference mode validation ran under alongside the result, and re-run shape sets that contain `sh:class`/`sh:node` with entailment off before treating a pass as authoritative. --- @@ -396,7 +435,7 @@ A DoD CTI team enforces STIX-compatible constraints on a threat graph before sha from semantica.context import AgentContext, ContextGraph from semantica.vector_store import VectorStore from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape -from semantica.ontology.ontology_validator import _run_pyshacl +from semantica.ontology import run_shacl_validation graph = ContextGraph() ctx = AgentContext( @@ -448,7 +487,7 @@ data_ttl = """ a ex:Malware . """ -report = _run_pyshacl(data_ttl, shacl_ttl) +report = run_shacl_validation(data_ttl, shacl_ttl) print(f"CTI graph conforms : {report.conforms}") print(f"Violations : {report.violation_count}") print(f"Warnings : {report.warning_count}") @@ -469,7 +508,7 @@ A SOC team validates zero-trust policy nodes before publishing them to the polic ```python from semantica.context import ContextGraph from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape -from semantica.ontology.ontology_validator import _run_pyshacl +from semantica.ontology import run_shacl_validation graph = ContextGraph() graph.add_node("policy-001", "Policy", "MFA Required for Tier-1 Resources", @@ -516,7 +555,7 @@ data_ttl = """ a ex:Policy . """ -report = _run_pyshacl(data_ttl, shacl_ttl) +report = run_shacl_validation(data_ttl, shacl_ttl) print(f"Policy graph conforms: {report.conforms}") # Policy graph conforms: False @@ -534,7 +573,7 @@ A clinical informatics team validates trial ontology nodes before loading them i ```python from semantica.ontology import LLMOntologyGenerator, SHACLGenerator, PropertyShape -from semantica.ontology.ontology_validator import _run_pyshacl +from semantica.ontology import run_shacl_validation from semantica.export import export_rdf import tempfile, os @@ -586,7 +625,7 @@ with open(tmp.name) as f: data_ttl = f.read() os.unlink(tmp.name) -report = _run_pyshacl(data_ttl, shacl_ttl) +report = run_shacl_validation(data_ttl, shacl_ttl) print(f"Trial data conforms: {report.conforms}") print(f"Warnings : {report.warning_count}") ``` @@ -600,7 +639,7 @@ A credit risk team validates every `LoanApplication` node against Basel III CRE2 ```python from semantica.context import ContextGraph from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape -from semantica.ontology.ontology_validator import _run_pyshacl +from semantica.ontology import run_shacl_validation graph = ContextGraph() graph.add_node("loan-001", "LoanApplication", "Prime mortgage APP-2025-88421", @@ -645,7 +684,7 @@ data_ttl = """ ex:ltv "0.65" . """ -report = _run_pyshacl(data_ttl, shacl_ttl) +report = run_shacl_validation(data_ttl, shacl_ttl) print(f"Loan portfolio conforms: {report.conforms}") # Loan portfolio conforms: False @@ -675,14 +714,14 @@ Call this function as a pre-publish gate; exit code 1 blocks the pipeline. ```python import sys from semantica.ontology import OntologyGenerator, SHACLGenerator -from semantica.ontology.ontology_validator import _run_pyshacl +from semantica.ontology import run_shacl_validation def validate_before_publish(data_graph_str: str, ontology: dict) -> None: shacl_gen = SHACLGenerator(base_uri="https://example.org/shapes/") shacl_graph = shacl_gen.generate(ontology) shacl_ttl = shacl_gen.serialize(shacl_graph, format="turtle") - report = _run_pyshacl(data_graph_str, shacl_ttl) + report = run_shacl_validation(data_graph_str, shacl_ttl) if not report.conforms: print(f"Graph validation FAILED โ€” {report.violation_count} violation(s)") @@ -700,7 +739,6 @@ def validate_before_publish(data_graph_str: str, ontology: dict) -> None: - [Ontology Management](ontology) โ€” generate the OWL ontology that SHACL shapes are derived from - [Reasoning & Rules](reasoning) โ€” complement SHACL structural constraints with logical inference rules -- [Export & Serialization](export) โ€” serialize graph data to Turtle/RDF/XML for `_run_pyshacl` input +- [Export & Serialization](export) โ€” serialize graph data to Turtle/RDF/XML for `run_shacl_validation` input - [Conflict Resolution](conflict-resolution) โ€” detect and resolve data conflicts before SHACL validation - [Change Management](change-management) โ€” version-gate SHACL shapes alongside ontology versions - diff --git a/docs/integrations/langchain.md b/docs/integrations/langchain.md new file mode 100644 index 00000000..e10fdc0b --- /dev/null +++ b/docs/integrations/langchain.md @@ -0,0 +1,81 @@ +--- +title: "LangChain Integration" +description: "Drop Semantica into LangChain / LangGraph pipelines via a GraphRAG retriever, VectorStore adapter, and agent tools." +icon: "link" +--- + +> Three drop-in adapters that bring Semantica's context graph and hybrid search into LangChain chains and LangGraph agents. + +## Installation + +```bash +pip install "semantica[langchain]" +``` + +Requires `langchain-core >= 0.3`. If langchain-core is not installed, the integration still imports โ€” every class carries the full Semantica API and degrades gracefully (`build()` returns `None`; branch on `LANGCHAIN_AVAILABLE`). + +## Components at a Glance + +- **SemanticaRetriever** โ€” `BaseRetriever`: hybrid-search seeds retrieval, then graph edges are walked `hops` steps (default 2) for GraphRAG-style results. +- **SemanticaVectorStore** โ€” `VectorStore`: `add_texts` / `similarity_search` / `similarity_search_with_score` / `from_texts` over `HybridSearch`. +- **SemanticaKGTool** / **SemanticaDecisionTool** โ€” `BaseTool` subclasses: `semantica_query_graph` and `semantica_query_decisions` for LangGraph / tool-calling agents. + +## Component Details + + + + Hybrid search seeds retrieval; then graph edges are walked `hops` steps so results go beyond flat vector similarity. If hybrid search is omitted or fails, the retriever falls back to a `ContextGraph.query` keyword scan. + + ```python + from integrations.langchain import SemanticaRetriever + from semantica.context import ContextGraph + from semantica.vector_store import HybridSearch + + graph = ContextGraph() + hybrid = HybridSearch() + + retriever = SemanticaRetriever(graph=graph, hybrid=hybrid, hops=2, top_k=10) + + from langchain.chains import RetrievalQA + + qa = RetrievalQA.from_chain_type(llm=llm, retriever=retriever) + ``` + + + Drop-in `VectorStore` for RetrievalQA / LCEL chains. `from_texts` requires a pre-configured `hybrid` instance. + + ```python + from integrations.langchain import SemanticaVectorStore + + store = SemanticaVectorStore(hybrid=hybrid) + store.add_texts( + ["document one", "document two"], + metadatas=[{"source": "a"}, {"source": "b"}], + ) + docs = store.similarity_search("document", k=2) + docs, scores = store.similarity_search_with_score("document", k=2) + ``` + + `add_texts` delegates to a Semantica vector store with `add_documents` (pass `vector_store=` to `HybridSearch` or to `SemanticaVectorStore`). + + + Instances are LangChain `BaseTool`s and can be passed to an agent directly. + `.build()` returns the tool, or `None` when langchain-core is absent. + + ```python + from integrations.langchain import SemanticaKGTool, SemanticaDecisionTool + from langgraph.prebuilt import create_react_agent + + tools = [ + SemanticaKGTool(graph), + SemanticaDecisionTool(graph), + ] + agent = create_react_agent(model, tools) + ``` + + | Tool | Description | + | :------ | :------------- | + | `semantica_query_graph` | Keyword / NL query over the shared context graph | + | `semantica_query_decisions` | Search the recorded decision log | + + diff --git a/docs/project-license.md b/docs/project-license.md index b1fb31ef..220f0f44 100644 --- a/docs/project-license.md +++ b/docs/project-license.md @@ -12,7 +12,7 @@ icon: "file-contract" ``` MIT License -Copyright (c) 2026 Hawksight AI +Copyright (c) 2026 Semantica Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docs/reference/context.md b/docs/reference/context.md index 18950e9e..24f09935 100644 --- a/docs/reference/context.md +++ b/docs/reference/context.md @@ -435,8 +435,8 @@ print("Nodes: {}, Edges: {}".format(stats["node_count"], stats["edge_count"])) | `query(query, skip, limit)` | `List[Dict]` | Full-text search over node content | | `stats()` | `Dict` | Node/edge counts, type breakdowns, graph density | | `density()` | `float` | Graph density score | -| `save_to_file(path)` | `None` | Persist graph to JSON | -| `load_from_file(path)` | `None` | Load graph from JSON | +| `save_to_file(path, format="json")` | `None` | Persist graph as JSON or a Markdown directory | +| `load_from_file(path, format="json")` | `None` | Replace graph state from JSON or a Markdown directory | | `build_from_conversations(conversations, link_entities)` | `Dict` | Build graph from conversation data | | `link_graph(other_graph, source_node_id, target_node_id, link_type)` | `str` | Create cross-graph navigation link; returns `link_id` | | `navigate_to(link_id)` | `Tuple` | Follow a cross-graph link to `(target_graph, target_node_id)` | @@ -625,8 +625,10 @@ malformed or duplicate fields before changing memory, and re-importing unchanged files is idempotent. Memory-local `entities` and `relationships` are preserved as provenance but are not applied to `ContextGraph` by Markdown import. Use a dedicated export directory: matching files are overwritten, but unrelated or stale Markdown -files are not deleted automatically. Export refuses to overwrite symbolic links and -uses atomic file replacement. Timestamp offsets are preserved in Markdown and +files are not deleted automatically. Export refuses to overwrite filesystem links and +uses atomic file replacement; import also refuses symlinks, Windows directory +junctions, and other Windows reparse points. +Timestamp offsets are preserved in Markdown and normalized to UTC only for comparisons, so aware and local-naive records can be queried together safely. Vector-store writes are deferred until the in-memory import commits; adapter synchronization remains best-effort and logs failures. diff --git a/docs/reference/reasoning.md b/docs/reference/reasoning.md index 41a05190..ee22e522 100644 --- a/docs/reference/reasoning.md +++ b/docs/reference/reasoning.md @@ -127,9 +127,19 @@ conclusions = reasoner.infer_facts( | `forward_chain()` | `List[InferenceResult]` | Derive all possible conclusions iteratively until fixpoint | | `backward_chain(goal, max_depth)` | `InferenceResult \| None` | Prove a specific goal string, returns `None` if unprovable | | `infer_facts(facts, rules)` | `List[str]` | Load facts and rules then run `forward_chain()`, returns conclusion strings | -| `clear()` | `None` | Clear all facts and rules | +| `reset_action_history()` | `None` | Allow actions for previously fired activations to run again | +| `clear()` | `None` | Clear all facts, rules, and action activation history | | `reset()` | `None` | Alias for `clear()` | +Rules with actions use at-most-once attempt semantics per concrete activation +(rule ID, bindings, and matched facts). Calling `forward_chain()` again on the +same instance does not repeat side effects for an activation that was already +attempted, even when an action raised an exception. Call +`reset_action_history()` to deliberately retry without clearing facts or rules; +`clear()` and `reset()` also clear this history. Replacing a rule's actions in +place does not invalidate an existing activation; reset the history explicitly +when the replacement should be replayed. + ### Rule and Fact dataclass fields ```python @@ -230,9 +240,16 @@ engine.reset() | `add_fact(fact)` | `None` | Add a `Fact` to working memory and propagate through the network | | `match_patterns(facts)` | `List[Match]` | Match all patterns; optionally add facts before matching | | `execute_matches(matches)` | `List[Any]` | Execute matched rules and return their conclusion values | -| `reset()` | `None` | Clear facts and all node activation state | +| `reset_action_history()` | `None` | Allow actions for previously executed activations to run again | +| `reset()` | `None` | Clear facts, node activation state, and action activation history | | `get_network_stats()` | `dict` | Return counts of alpha, beta, terminal nodes and facts | +When a Reasoner is bound, `execute_matches()` deduplicates action side effects +by rule ID, bindings, and matched fact identity. Re-executing a match still +returns its conclusion for compatibility, but its actions are skipped after the +first attempt. `reset_action_history()`, `reset()`, and `build_network()` allow +those actions to run again. + ## SPARQLReasoner diff --git a/docs/reference/triplet_store.md b/docs/reference/triplet_store.md index ad7a0645..ee5c24e6 100644 --- a/docs/reference/triplet_store.md +++ b/docs/reference/triplet_store.md @@ -182,7 +182,7 @@ for row in result.bindings: store = TripletStore( backend="rdf4j", endpoint="http://localhost:8080/rdf4j-server", - repository_id="semantica", # passed through **config + repository_id="semantica", # selects the remote repository ) ``` diff --git a/docs/reference/utils.md b/docs/reference/utils.md index 83f9cc97..f7e97789 100644 --- a/docs/reference/utils.md +++ b/docs/reference/utils.md @@ -77,7 +77,17 @@ Most users won't call utils directly: it's the **shared foundation** for all mod export SEMANTICA_LOG_LEVEL=DEBUG export SEMANTICA_LOG_FORMAT=json # "json" | "text" export SEMANTICA_DISABLE_PROGRESS=true + export SEMANTICA_FORCE_PROGRESS=true ``` + + + **Progress bars follow your terminal.** Console progress is written only when + stdout is an interactive terminal (or a Jupyter notebook), so piping or + redirecting output no longer fills logs with progress bars and escape + sequences. Set `SEMANTICA_DISABLE_PROGRESS` to silence progress even in a + terminal, or `SEMANTICA_FORCE_PROGRESS` to keep it when stdout is redirected. + `SEMANTICA_DISABLE_PROGRESS` wins if both are set. + diff --git a/docs/storage-backends.md b/docs/storage-backends.md new file mode 100644 index 00000000..dc211ef0 --- /dev/null +++ b/docs/storage-backends.md @@ -0,0 +1,154 @@ +# Graph storage backends and feature matrix + +Semantica separates graph modeling from physical storage. LPG backends are accessed through `graph_store` adapters; RDF backends are accessed through `triplet_store` adapters. + +This page is intentionally conservative: it distinguishes between an adapter existing, a feature being generally available with that model, and a backend needing user-supplied wiring. + +## Status labels + +- `built-in`: adapter implementation exists in Semantica core. +- `tested`: covered by automated integration fixtures or tests. +- `example-only`: usable example exists, but support is not asserted by integration tests. +- `interface/BYO`: interface or integration point exists; bring your own backend wiring. + +## Adapter inventory + +| Backend | Model | Adapter | Status | Reference | +| --- | --- | --- | --- | --- | +| Neo4j | LPG | `semantica.graph_store.Neo4jStore` | built-in | `cookbook/introduction/09_Graph_Store.ipynb` | +| FalkorDB | LPG | `semantica.graph_store.FalkorDBStore` | built-in | `docs/reference/graph_store.md` | +| Amazon Neptune | LPG | `semantica.graph_store.AmazonNeptuneStore` | built-in | `cookbook/introduction/21_Amazon_Neptune_Store.ipynb` | +| Apache AGE | LPG | `semantica.graph_store.ApacheAgeStore` | built-in | `docs/graph_stores/apache_age.md` | +| RDF4J | RDF | `semantica.triplet_store.RDF4JStore` | built-in | `cookbook/introduction/20_Triplet_Store.ipynb` | +| Apache Jena | RDF | `semantica.triplet_store.JenaStore` | built-in | `cookbook/introduction/20_Triplet_Store.ipynb` | +| Blazegraph | RDF | `semantica.triplet_store.BlazegraphStore` | built-in | `cookbook/introduction/20_Triplet_Store.ipynb` | +| Anzo | RDF | `semantica.triplet_store.AnzoStore` | built-in | `cookbook/introduction/20_Triplet_Store.ipynb` | +| Oxigraph | RDF | `semantica.triplet_store.OxigraphStore` | built-in | `docs/reference/triplet_store.md` | + +## Feature matrix + +`Yes` means the capability is expected to work with the adapter and graph model. `Partial` means the capability works with model-specific constraints. `BYO` means the user must supply or validate wiring for the backend. + +| Backend | Model | Ingestion | Context graph construction | Reasoning/analytics | Provenance | Known limitations | +| --- | --- | --- | --- | --- | --- | --- | +| Neo4j | LPG | Yes | Yes | Yes | Partial | Provenance and context metadata are stored as node and edge properties; relationship properties and stable node identifiers are required. | +| FalkorDB | LPG | Yes | Yes | Partial | Partial | Redis-based; provenance depends on node/edge properties, and multi-graph isolation depends on the selected graph name. | +| Amazon Neptune | LPG | Yes | Yes | Partial | Partial | Use the property-graph endpoint; AWS auth, VPC, and endpoint configuration can affect local tests. Provenance depends on node/edge properties. | +| Apache AGE | LPG | Yes | Yes | Partial | Partial | Runs through PostgreSQL/AGE; Cypher compatibility and property handling can differ from standalone LPG engines. | +| RDF4J | RDF | Yes | Partial | Partial | Partial | Context separation relies on named graphs; triple-level provenance may require reification or graph-level metadata. | +| Apache Jena | RDF | Yes | Partial | Partial | Partial | Named graphs are needed for context separation; backend configuration and transaction behavior matter. | +| Blazegraph | RDF | Yes | Partial | Partial | Partial | Use quads/named graphs for context; IRI stability and graph naming matter for provenance. | +| Anzo | RDF | Yes | Partial | Partial | Partial | Anzo deployments are environment-specific; validate `dataset_uri`/graphmart naming, named-graph support, and provenance mapping. | +| Oxigraph | RDF | Yes | Partial | Partial | Partial | Embedded, single-process store (in-memory or on-disk); named graphs are supported, but there is no separate server process to scale independently. | + +## RDF and LPG differences + +- LPG backends store context and provenance as graph elements and properties. If a backend does not support relationship properties, some provenance patterns may be degraded. +- RDF backends rely on IRIs, named graphs, and optional reification. Context graphs and provenance are easiest to preserve when the store supports named graphs/quads. +- Ingestion works across both models, but the physical representation differs: LPG stores nodes/edges directly, while RDF stores subject-predicate-object statements. +- Reasoning and analytics should be validated against the adapter's query capabilities, especially for path traversal, property filters, and named-graph queries. + +## Minimal connection examples + +Prefer the referenced notebook cells for a working setup. The examples below show the intended adapter entrypoints, not a universal connection DSL. + +### Neo4j + +```python +import os +from semantica.graph_store import Neo4jStore + +store = Neo4jStore( + uri='bolt://localhost:7687', + user='neo4j', + password=os.environ['NEO4J_PASSWORD'] +) +``` + +### FalkorDB + +```python +from semantica.graph_store import FalkorDBStore + +store = FalkorDBStore( + host='localhost', + port=6379, + graph_name='semantica' +) +``` + +### Amazon Neptune + +```python +from semantica.graph_store import AmazonNeptuneStore + +store = AmazonNeptuneStore( + endpoint='your-neptune-cluster-endpoint', + port=8182, + region='us-east-1' +) +``` + +### Apache AGE + +```python +from semantica.graph_store import ApacheAgeStore + +store = ApacheAgeStore( + connection_string='host=localhost dbname=agedb user=postgres password=postgres', + graph_name='semantica' +) +``` + +### RDF4J + +```python +from semantica.triplet_store import RDF4JStore + +store = RDF4JStore( + endpoint='http://localhost:8080/rdf4j-server', + repository_id='semantica' +) +``` + +### Apache Jena + +```python +from semantica.triplet_store import JenaStore + +store = JenaStore( + endpoint='http://localhost:3030/ds' +) +``` + +### Blazegraph + +```python +from semantica.triplet_store import BlazegraphStore + +store = BlazegraphStore( + endpoint='http://localhost:9999/blazegraph/sparql' +) +``` + +### Anzo + +```python +from semantica.triplet_store import AnzoStore + +store = AnzoStore( + endpoint='http://anzo-host:8080', + dataset_uri='http://cambridgesemantics.com/Graphmart/your-graphmart-id' +) +``` + +### Oxigraph + +```python +from semantica.triplet_store import OxigraphStore + +# Omit `path` for an in-memory store; pass a directory for on-disk persistence. +store = OxigraphStore(path='./semantica-oxigraph-data') +``` + +Replace hostnames, ports, repositories, graphs, and credentials with values from your environment. For regulated or self-hosted deployments, keep credentials in environment variables or secret storage rather than source code. diff --git a/explorer/src/workspaces/GraphWorkspace/MarkdownContentViewer.tsx b/explorer/src/workspaces/GraphWorkspace/MarkdownContentViewer.tsx index f00d3dc5..f8121cf5 100644 --- a/explorer/src/workspaces/GraphWorkspace/MarkdownContentViewer.tsx +++ b/explorer/src/workspaces/GraphWorkspace/MarkdownContentViewer.tsx @@ -1,8 +1,9 @@ -import { useState, useRef, useEffect, type CSSProperties } from "react"; -import ReactMarkdown from "react-markdown"; +import { useState, useRef, useEffect, useMemo, type CSSProperties } from "react"; +import ReactMarkdown, { type Components } from "react-markdown"; import remarkGfm from "remark-gfm"; import { Check, Copy, Code2, Eye, ExternalLink, Image as ImageIcon } from "lucide-react"; import { GRAPH_THEME } from "./graphTheme"; +import { isSafeUrl } from "./markdownUrlSafety"; export interface MarkdownContentViewerProps { content?: string | null; @@ -10,25 +11,6 @@ export interface MarkdownContentViewerProps { defaultMode?: "preview" | "source"; } -export function isSafeUrl(url?: string): boolean { - if (!url) return false; - const trimmed = url.trim(); - // Reject whitespace-only strings โ€” new URL("", base) would resolve to the base - // protocol and produce a false positive. This guards direct callers of the exported - // function; markdown parsers normalise whitespace-only destinations to "" which - // already fails the !url check above. - if (!trimmed) return false; - if (trimmed.startsWith("//")) return false; - if (trimmed.startsWith("#")) return true; - if (trimmed.startsWith("/")) return true; - try { - const parsed = new URL(trimmed, "http://localhost"); - return ["http:", "https:", "mailto:"].includes(parsed.protocol); - } catch { - return false; - } -} - export function MarkdownContentViewer({ content, className, @@ -65,6 +47,20 @@ export function MarkdownContentViewer({ const rawContent = typeof content === "string" ? content : ""; const hasContent = rawContent.trim().length > 0; + // react-markdown runs the whole remark pipeline synchronously inside its own + // render, so without this memo every unrelated re-render of this component -- + // clicking Copy, toggling Preview/Source -- re-parses the entire document. + // Measured at ~364ms per re-render for a 1000-row GFM table (issue #1118). + // Keyed on rawContent so a genuine node change still re-parses exactly once. + const renderedMarkdown = useMemo( + () => ( + + {rawContent} + + ), + [rawContent], + ); + const handleCopy = async () => { if (!hasContent) return; try { @@ -130,98 +126,103 @@ export function MarkdownContentViewer({ {rawContent} ) : ( -
- { - if (!isSafeUrl(href)) { - return {children}; - } - // isSafeUrl returning true guarantees href is a non-empty string. - const safeHref = href ?? ""; - // Fragment links (#section, footnote backlinks like - // #user-content-fnref-1) are in-document anchors. Opening them - // in a new tab would break GFM footnote back-navigation. - const isFragment = safeHref.startsWith("#"); - if (isFragment) { - return ( - - {children} - - ); - } - return ( - - {children} - - - ); - }, - img: ({ src, alt }) => ( - - - Image: {alt || src || "unlabeled"} - - ), - h1: ({ children }) =>

{children}

, - h2: ({ children }) =>

{children}

, - h3: ({ children }) =>

{children}

, - h4: ({ children }) =>

{children}

, - p: ({ children }) =>

{children}

, - ul: ({ children }) =>
    {children}
, - ol: ({ children }) =>
    {children}
, - li: ({ children }) =>
  • {children}
    • , - blockquote: ({ children }) =>
    {children}
    , - hr: () =>
    , - table: ({ children }) => ( -
    - {children}
    -
    - ), - thead: ({ children }) => {children}, - tbody: ({ children }) => {children}, - tr: ({ children }) => {children}, - th: ({ children }) => {children}, - td: ({ children }) => {children}, - pre: ({ children }) => 
    {children}
    , - // C-1: discard `node` here too โ€” code elements are custom components - // and would otherwise receive node="[object Object]" in the DOM. - code: ({ className: codeClass, children }) => { - const isInline = !codeClass && typeof children === "string" && !children.includes("\n"); - return ( - - {children} - - ); - }, - }} - > - {rawContent} -     -
    +
    {renderedMarkdown}
    )}
    ); } +/* โ”€โ”€โ”€ Markdown rendering config โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ + +// Both props are hoisted to module scope so they keep a stable identity across +// renders. As inline literals they allocated a fresh plugin array and ~20 fresh +// arrow components on every render, which made React treat every mapped tag as a +// new element type and remount the entire rendered subtree instead of updating +// it (issue #1118). The arrow bodies only read the style constants below at call +// time, so declaring the map before them is safe. +const REMARK_PLUGINS = [remarkGfm]; + +const MARKDOWN_COMPONENTS: Components = { + // C-1: react-markdown passes a HAST `node` prop (the raw AST + // Element) to every custom component override via passNode:true. + // In React 19 any unknown prop spreads onto a native element are + // serialised as HTML attributes, producing node="[object Object]" + // on every rendered link. Fix: destructure `node` by name so it + // is explicitly discarded, then spread `...rest` to preserve all + // other legitimate HAST/remark-gfm attributes โ€” e.g. the `id`, + // `aria-describedby`, `aria-label`, `data-footnote-ref`, + // `data-footnote-backref`, and `class` attrs that GFM footnotes + // require for correct in-page navigation and accessibility. + // + // C-2: fragment links (#anchor, GFM footnote backlinks) must + // navigate within the current document. External links continue + // to use target="_blank" with noopener noreferrer. + // + // eslint-disable-next-line @typescript-eslint/no-unused-vars + a: ({ href, children, title, node: _node, ...rest }) => { + if (!isSafeUrl(href)) { + return {children}; + } + // isSafeUrl returning true guarantees href is a non-empty string. + const safeHref = href ?? ""; + // Fragment links (#section, footnote backlinks like + // #user-content-fnref-1) are in-document anchors. Opening them + // in a new tab would break GFM footnote back-navigation. + const isFragment = safeHref.startsWith("#"); + if (isFragment) { + return ( + + {children} + + ); + } + return ( + + {children} + + + ); + }, + img: ({ src, alt }) => ( + + + Image: {alt || src || "unlabeled"} + + ), + h1: ({ children }) =>

    {children}

    , + h2: ({ children }) =>

    {children}

    , + h3: ({ children }) =>

    {children}

    , + h4: ({ children }) =>

    {children}

    , + p: ({ children }) =>

    {children}

    , + ul: ({ children }) => , + ol: ({ children }) =>
      {children}
    , + li: ({ children }) =>
  • {children}
    • , + blockquote: ({ children }) =>
    {children}
    , + hr: () =>
    , + table: ({ children }) => ( +
    + {children}
    +
    + ), + thead: ({ children }) => {children}, + tbody: ({ children }) => {children}, + tr: ({ children }) => {children}, + th: ({ children }) => {children}, + td: ({ children }) => {children}, + pre: ({ children }) => 
    {children}
    , + // C-1: discard `node` here too โ€” code elements are custom components + // and would otherwise receive node="[object Object]" in the DOM. + code: ({ className: codeClass, children }) => { + const isInline = !codeClass && typeof children === "string" && !children.includes("\n"); + return ( + + {children} + + ); + }, +}; + /* โ”€โ”€โ”€ Styles โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ */ const viewerContainerStyle: CSSProperties = { diff --git a/explorer/src/workspaces/GraphWorkspace/markdownUrlSafety.ts b/explorer/src/workspaces/GraphWorkspace/markdownUrlSafety.ts new file mode 100644 index 00000000..3944a86b --- /dev/null +++ b/explorer/src/workspaces/GraphWorkspace/markdownUrlSafety.ts @@ -0,0 +1,29 @@ +/** + * URL-safety predicate for the Markdown content viewer. + * + * Extracted into a pure module so the check can be unit-tested without + * importing the MarkdownContentViewer React component, and so the component + * module exports only components (react-refresh/only-export-components, + * issue #1119). The behaviour is unchanged from the original in-component + * implementation: only http, https, mailto, in-document fragments, and + * root-relative paths are permitted. + */ + +export function isSafeUrl(url?: string): boolean { + if (!url) return false; + const trimmed = url.trim(); + // Reject whitespace-only strings โ€” new URL("", base) would resolve to the base + // protocol and produce a false positive. This guards direct callers of the exported + // function; markdown parsers normalise whitespace-only destinations to "" which + // already fails the !url check above. + if (!trimmed) return false; + if (trimmed.startsWith("//")) return false; + if (trimmed.startsWith("#")) return true; + if (trimmed.startsWith("/")) return true; + try { + const parsed = new URL(trimmed, "http://localhost"); + return ["http:", "https:", "mailto:"].includes(parsed.protocol); + } catch { + return false; + } +} diff --git a/explorer/tests/markdownContentViewer.test.ts b/explorer/tests/markdownContentViewer.test.ts index aa89f3cb..0435578a 100644 --- a/explorer/tests/markdownContentViewer.test.ts +++ b/explorer/tests/markdownContentViewer.test.ts @@ -5,7 +5,8 @@ import { renderToString } from "react-dom/server"; (globalThis as any).React = React; -import { isSafeUrl, MarkdownContentViewer } from "../src/workspaces/GraphWorkspace/MarkdownContentViewer.tsx"; +import { MarkdownContentViewer } from "../src/workspaces/GraphWorkspace/MarkdownContentViewer.tsx"; +import { isSafeUrl } from "../src/workspaces/GraphWorkspace/markdownUrlSafety.ts"; test("isSafeUrl permits safe http, https, and mailto URLs and relative paths", () => { assert.equal(isSafeUrl("https://example.com"), true); diff --git a/integrations/__init__.py b/integrations/__init__.py index 06219ff5..080383fc 100644 --- a/integrations/__init__.py +++ b/integrations/__init__.py @@ -1,7 +1,7 @@ """ Semantica Framework Integrations -Optional integration packages for agentic frameworks (Google ADK, Claude Agent SDK, Agno, etc.). +Optional integration packages for agentic frameworks (Google ADK, Claude Agent SDK, Agno, CrewAI, LangChain, etc.). Each integration is self-contained, independently installable via extras_require, and maintains zero impact on core Semantica - keeping the semantic layer lean while maximizing ecosystem reach. """ diff --git a/integrations/agno/knowledge_graph.py b/integrations/agno/knowledge_graph.py index 21b6b280..78004bc3 100644 --- a/integrations/agno/knowledge_graph.py +++ b/integrations/agno/knowledge_graph.py @@ -277,25 +277,22 @@ class AgnoKnowledgeGraph(_KnowledgeBase): # type: ignore[misc] def load_urls(self, urls: List[str]) -> None: """Fetch each URL and ingest the response body. - Only ``http`` and ``https`` schemes are permitted to prevent SSRF. + Uses the shared SSRF guard so that ``http`` and ``https`` are the only + permitted schemes, private/loopback/link-local/cloud-metadata addresses + are blocked by default, DNS resolution is validated, and every redirect + hop is re-checked before being followed. """ - import urllib.request - from urllib.parse import urlparse + from semantica.ingest.ssrf import request_with_ssrf_guard + from semantica.utils.exceptions import ValidationError for url in urls: - parsed = urlparse(url) - if parsed.scheme not in ("http", "https"): - logger.warning( - "Skipping URL with disallowed scheme '%s': %s", - parsed.scheme, - url, - ) - continue try: - with urllib.request.urlopen(url, timeout=10) as resp: # noqa: S310 - text = resp.read().decode("utf-8", errors="replace") + response = request_with_ssrf_guard("GET", url, timeout=10) + text = response.text self._ingest_text(text, source=url) logger.info("Loaded URL: %s", url) + except ValidationError as exc: + logger.warning("Skipping URL (SSRF check failed) %s: %s", url, exc) except Exception as exc: logger.warning("Failed to fetch %s: %s", url, exc) diff --git a/integrations/langchain/README.md b/integrations/langchain/README.md new file mode 100644 index 00000000..d34b002b --- /dev/null +++ b/integrations/langchain/README.md @@ -0,0 +1,67 @@ +# Semantica ร— LangChain + +Drop Semantica into existing LangChain / LangGraph pipelines: GraphRAG-style +retrieval, a `VectorStore` adapter, and agent tools. + +## Install + +```bash +pip install semantica[langchain] +# or just the core adapter dependency: +pip install langchain-core +``` + +## Retriever (GraphRAG) + +```python +from integrations.langchain import SemanticaRetriever +from semantica.context import ContextGraph +from semantica.vector_store import HybridSearch + +graph = ContextGraph() +hybrid = HybridSearch() + +retriever = SemanticaRetriever(graph=graph, hybrid=hybrid, hops=2, top_k=10) + +# Use with any LangChain chain that accepts a retriever: +from langchain.chains import RetrievalQA + +qa = RetrievalQA.from_chain_type(llm=llm, retriever=retriever) +``` + +Hybrid search seeds retrieval; then graph edges are walked `hops` steps so +results go beyond flat vector similarity. + +## VectorStore + +```python +from integrations.langchain import SemanticaVectorStore + +store = SemanticaVectorStore(hybrid=hybrid) +store.add_texts(["document one", "document two"], metadatas=[{"source": "a"}, {"source": "b"}]) +docs = store.similarity_search("document", k=2) +docs, scores = store.similarity_search_with_score("document", k=2) +``` + +## Agent tools (LangGraph / tool-calling agents) + +```python +from integrations.langchain import SemanticaKGTool, SemanticaDecisionTool +from langgraph.prebuilt import create_react_agent + +tools = [ + SemanticaKGTool(graph), + SemanticaDecisionTool(graph), +] +agent = create_react_agent(model, tools) +``` + +- `semantica_query_graph` โ€” query the shared context graph (keyword / NL) +- `semantica_query_decisions` โ€” search the recorded decision log + +## Compatibility + +- Requires `langchain-core >= 0.3`. +- All classes degrade gracefully when `langchain-core` is absent: they remain + importable (carrying the full Semantica API), and `build()` returns `None`, + so agents can branch on `LANGCHAIN_AVAILABLE`. diff --git a/integrations/langchain/__init__.py b/integrations/langchain/__init__.py new file mode 100644 index 00000000..875b5f66 --- /dev/null +++ b/integrations/langchain/__init__.py @@ -0,0 +1,48 @@ +""" +Semantica ร— LangChain Integration +================================= + +First-class integration between the Semantica semantic intelligence stack and +the `LangChain `_ / LangGraph +ecosystem. + +Public surface +-------------- +SemanticaRetriever โ€” ``BaseRetriever`` with multi-hop GraphRAG (walks graph + edges from hybrid-search hits) +SemanticaVectorStore โ€” ``VectorStore`` adapter over Semantica's hybrid search + (drop-in for RetrievalQA / LCEL chains) +SemanticaKGTool โ€” ``BaseTool`` for querying the context graph +SemanticaDecisionTool โ€” ``BaseTool`` exposing the recorded decision log + +Quick start +----------- + pip install semantica[langchain] + + >>> from integrations.langchain import ( + ... SemanticaRetriever, + ... SemanticaVectorStore, + ... SemanticaKGTool, + ... SemanticaDecisionTool, + ... ) + +Compatibility +------------- +Requires ``langchain-core >= 0.3``. All classes degrade gracefully when +``langchain-core`` is not installed โ€” they are still importable and carry the +full Semantica API, but cannot be bound to LangChain chains/agents. +""" + +from .retriever import LANGCHAIN_AVAILABLE, SemanticaRetriever +from .tools import SemanticaDecisionTool, SemanticaKGTool +from .vectorstore import SemanticaVectorStore + +__all__ = [ + "SemanticaRetriever", + "SemanticaVectorStore", + "SemanticaKGTool", + "SemanticaDecisionTool", + "LANGCHAIN_AVAILABLE", +] + +__version__ = "0.1.0" diff --git a/integrations/langchain/retriever.py b/integrations/langchain/retriever.py new file mode 100644 index 00000000..b2489725 --- /dev/null +++ b/integrations/langchain/retriever.py @@ -0,0 +1,216 @@ +""" +SemanticaRetriever โ€” LangChain ``BaseRetriever`` with multi-hop GraphRAG. + +Hybrid search seeds the retrieval, then graph edges are walked for ``hops`` +steps so results go beyond flat vector similarity. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple + +from semantica.utils.logging import get_logger + +logger = get_logger(__name__) + +# --------------------------------------------------------------------------- +# Optional: LangChain core +# --------------------------------------------------------------------------- +LANGCHAIN_AVAILABLE = False +LANGCHAIN_IMPORT_ERROR: Optional[str] = None + +_BaseRetriever: Any = object +_Document: Any = None + + +def _get_document(**kwargs: Any) -> Any: + """Instantiate a langchain Document lazily (keeps the import optional).""" + if _Document is None: # pragma: no cover - exercised only with langchain + raise RuntimeError(LANGCHAIN_IMPORT_ERROR or "langchain-core not installed") + return _Document(**kwargs) + + +try: + from langchain_core.documents import Document as _Document # type: ignore + from langchain_core.retrievers import ( + BaseRetriever as _BaseRetriever, # type: ignore + ) + + LANGCHAIN_AVAILABLE = True +except ImportError: # pragma: no cover - exercised only without langchain + LANGCHAIN_IMPORT_ERROR = ( + "langchain-core is not installed. Install with: pip install langchain-core" + ) + logger.debug(LANGCHAIN_IMPORT_ERROR) + + +def _hit_layers(hit: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]: + """Nested HybridSearch metadata and ContextGraph.query node, if present.""" + metadata = hit.get("metadata") if isinstance(hit.get("metadata"), dict) else {} + node = hit.get("node") if isinstance(hit.get("node"), dict) else {} + return metadata, node + + +def _hit_id(hit: Dict[str, Any]) -> Optional[str]: + """Graph node id, preferring metadata over a HybridSearch vector id.""" + metadata, node = _hit_layers(hit) + return ( + hit.get("node_id") + or metadata.get("node_id") + or node.get("id") + or node.get("node_id") + or hit.get("id") + ) + + +def _hit_content(hit: Dict[str, Any], fallback: str = "") -> str: + metadata, node = _hit_layers(hit) + props = node.get("properties") if isinstance(node.get("properties"), dict) else {} + return ( + hit.get("content") + or hit.get("text") + or metadata.get("content") + or metadata.get("text") + or props.get("content") + or fallback + ) + + +def _hit_type(hit: Dict[str, Any]) -> str: + metadata, node = _hit_layers(hit) + return ( + hit.get("node_type") + or hit.get("type") + or metadata.get("node_type") + or metadata.get("type") + or node.get("type") + or node.get("node_type") + or "node" + ) + + +def _hit_score(hit: Dict[str, Any], default: float = 1.0) -> float: + return float(hit.get("score") if hit.get("score") is not None else hit.get("distance") or default) + + +class SemanticaRetriever(_BaseRetriever): # type: ignore[misc] + """GraphRAG-style retriever over a Semantica ``ContextGraph``. + + Args: + graph: A semantica.context.ContextGraph instance. + hybrid: A semantica.vector_store.HybridSearch instance used to seed + retrieval. If omitted, a best-effort keyword search on the graph + is used. + hops: Number of graph-edge expansion hops (default 2). + top_k: Number of seed hits (default 10). + """ + + graph: Any + hybrid: Any = None + hops: int = 2 + top_k: int = 10 + + def __init__( + self, + graph: Any, + hybrid: Any = None, + hops: int = 2, + top_k: int = 10, + **kwargs: Any, + ) -> None: + """Explicit init so the retriever works with and without langchain.""" + if LANGCHAIN_AVAILABLE: + # BaseRetriever is a Pydantic model: pass the declared fields + # through so validation succeeds. + super().__init__( + graph=graph, + hybrid=hybrid, + hops=hops, + top_k=top_k, + **kwargs, + ) + else: + # Without langchain-core, BaseRetriever is a plain object + super().__init__() # type: ignore[call-arg] + self.graph = graph + self.hybrid = hybrid + self.hops = hops + self.top_k = top_k + + def _get_relevant_documents(self, query: str, **kwargs: Any) -> List[Any]: + """LangChain BaseRetriever entry point.""" + seed = self._seed_results(query) + if not seed: + return [] + + # Expand each seed node through the graph + expanded: Dict[str, Dict[str, Any]] = {} + for hit in seed: + node_id = _hit_id(hit) + if not node_id: + continue + metadata, _ = _hit_layers(hit) + expanded[node_id] = { + "content": _hit_content(hit, fallback=str(node_id)), + "node_type": _hit_type(hit), + "score": _hit_score(hit), + "metadata": metadata, + } + try: + neighbors = self.graph.get_neighbors(node_id, hops=self.hops) + for neighbor in neighbors: + nid = neighbor.get("node_id") or neighbor.get("id") + if nid and nid not in expanded: + expanded[nid] = { + "content": neighbor.get("content") + or neighbor.get("text") + or neighbor.get("name") + or str(nid), + "node_type": neighbor.get("node_type") + or neighbor.get("type") + or "node", + "score": float(neighbor.get("weight") or 0.5), + "metadata": {}, + } + except Exception as exc: # graph expansion is best-effort + logger.debug("graph expansion failed for %s: %s", node_id, exc) + + # Order: seed hits first (they have real scores), then neighbors. + # Keep a deterministic id->payload list (sets are unordered โ€” see Qodo). + ordered_pairs: List[tuple] = [] + seen_ids = set() + for hit in seed: + nid = _hit_id(hit) + if nid and nid in expanded and nid not in seen_ids: + ordered_pairs.append((nid, expanded[nid])) + seen_ids.add(nid) + for nid, item in expanded.items(): + if nid not in seen_ids: + ordered_pairs.append((nid, item)) + seen_ids.add(nid) + + return [ + _get_document( + page_content=item["content"], + metadata={ + **item["metadata"], + "node_id": nid, + "node_type": item["node_type"], + "score": item["score"], + }, + ) + for nid, item in ordered_pairs + ] + + def _seed_results(self, query: str) -> List[Dict[str, Any]]: + """Get seed results from hybrid search or a graph keyword scan.""" + if self.hybrid is not None: + try: + return self.hybrid.search(query, k=self.top_k) + except Exception as exc: + logger.debug("hybrid search failed, falling back: %s", exc) + # Best-effort keyword scan over graph nodes (ContextGraph.query) + try: + return self.graph.query(query, limit=self.top_k) + except Exception: + return [] diff --git a/integrations/langchain/tools.py b/integrations/langchain/tools.py new file mode 100644 index 00000000..9fa23f62 --- /dev/null +++ b/integrations/langchain/tools.py @@ -0,0 +1,133 @@ +""" +SemanticaKGTool / SemanticaDecisionTool โ€” LangChain ``BaseTool`` adapters +for LangChain / LangGraph agents. +""" + +from __future__ import annotations + +import json +from typing import Any, Optional, Type + +from pydantic import BaseModel, ConfigDict, Field + +from semantica.utils.logging import get_logger + +logger = get_logger(__name__) + +# --------------------------------------------------------------------------- +# Optional: LangChain core +# --------------------------------------------------------------------------- +LANGCHAIN_AVAILABLE = False +LANGCHAIN_IMPORT_ERROR: Optional[str] = None + +_BaseTool: Any = object + + +try: + from langchain_core.tools import BaseTool as _BaseTool # type: ignore + + LANGCHAIN_AVAILABLE = True +except ImportError: # pragma: no cover + LANGCHAIN_IMPORT_ERROR = ( + "langchain-core is not installed. Install with: pip install langchain-core" + ) + logger.debug(LANGCHAIN_IMPORT_ERROR) + + +def _json(payload: Any) -> str: + return json.dumps(payload, default=str, ensure_ascii=False) + + +class QueryGraphInput(BaseModel): + query: str = Field(..., description="Natural-language or keyword graph query") + limit: int = Field(10, description="Maximum matching nodes to return") + + +class QueryDecisionsInput(BaseModel): + category: str = Field( + "", + description="Keyword to search recorded decisions; empty returns insights", + ) + limit: int = Field(10, description="Maximum results when searching by keyword") + + +class SemanticaKGTool(_BaseTool): # type: ignore[misc] + """LangChain tool for querying a Semantica ``ContextGraph``. + + Args: + graph: A semantica.context.ContextGraph instance. + + Example: + >>> tool = SemanticaKGTool(graph) + >>> agent = create_react_agent(model, tools=[tool]) + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + name: str = "semantica_query_graph" + description: str = ( + "Query Semantica's shared context graph with a natural-language " + "keyword query. Returns matching entities and relationships." + ) + args_schema: Type[BaseModel] = QueryGraphInput + graph: Any = None + + def __init__(self, graph: Any = None, **kwargs: Any) -> None: + if LANGCHAIN_AVAILABLE: + super().__init__(graph=graph, **kwargs) + else: + super().__init__() + self.graph = graph + + def build(self) -> Any: + """Return this tool, or None if langchain-core is missing.""" + return self if LANGCHAIN_AVAILABLE else None + + def _run(self, query: str, limit: int = 10, **kwargs: Any) -> str: + try: + return _json(self.graph.query(query, limit=limit)) + except Exception as exc: + return _json({"error": str(exc)}) + + async def _arun(self, query: str, limit: int = 10, **kwargs: Any) -> str: + return self._run(query, limit=limit) + + +class SemanticaDecisionTool(_BaseTool): # type: ignore[misc] + """LangChain tool for searching Semantica's recorded decision log. + + Args: + graph: A semantica.context.ContextGraph instance. + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + name: str = "semantica_query_decisions" + description: str = ( + "Search Semantica's recorded decision log with a keyword query. " + "Returns decisions, rationale, and context." + ) + args_schema: Type[BaseModel] = QueryDecisionsInput + graph: Any = None + + def __init__(self, graph: Any = None, **kwargs: Any) -> None: + if LANGCHAIN_AVAILABLE: + super().__init__(graph=graph, **kwargs) + else: + super().__init__() + self.graph = graph + + def build(self) -> Any: + """Return this tool, or None if langchain-core is missing.""" + return self if LANGCHAIN_AVAILABLE else None + + def _run(self, category: str = "", limit: int = 10, **kwargs: Any) -> str: + try: + if category: + return _json(self.graph.query(category, limit=limit)) + return _json(self.graph.get_decision_insights()) + except Exception as exc: + return _json({"error": str(exc)}) + + async def _arun(self, category: str = "", limit: int = 10, **kwargs: Any) -> str: + return self._run(category=category, limit=limit) diff --git a/integrations/langchain/vectorstore.py b/integrations/langchain/vectorstore.py new file mode 100644 index 00000000..49473cce --- /dev/null +++ b/integrations/langchain/vectorstore.py @@ -0,0 +1,143 @@ +""" +SemanticaVectorStore โ€” LangChain ``VectorStore`` adapter over Semantica's +hybrid search (``semantica.vector_store.HybridSearch``). +""" + +from __future__ import annotations + +from typing import Any, Dict, Iterable, List, Optional + +from semantica.utils.logging import get_logger + +from .retriever import _hit_content, _hit_id, _hit_score, _hit_type, _hit_layers + +logger = get_logger(__name__) + +# --------------------------------------------------------------------------- +# Optional: LangChain core +# --------------------------------------------------------------------------- +LANGCHAIN_AVAILABLE = False +LANGCHAIN_IMPORT_ERROR: Optional[str] = None + +_VectorStoreBase: Any = object +_Document: Any = None + + +def _make_document(**kwargs: Any) -> Any: + if _Document is None: # pragma: no cover + raise RuntimeError(LANGCHAIN_IMPORT_ERROR or "langchain-core not installed") + return _Document(**kwargs) + + +try: + from langchain_core.documents import Document as _Document # type: ignore + from langchain_core.vectorstores import ( + VectorStore as _VectorStoreBase, # type: ignore + ) + + LANGCHAIN_AVAILABLE = True +except ImportError: # pragma: no cover + LANGCHAIN_IMPORT_ERROR = ( + "langchain-core is not installed. Install with: pip install langchain-core" + ) + logger.debug(LANGCHAIN_IMPORT_ERROR) + + +def _document_from_hit(hit: Dict[str, Any], include_score: bool = True) -> Any: + metadata, _ = _hit_layers(hit) + node_id = _hit_id(hit) + doc_meta = { + **metadata, + "node_id": node_id, + "node_type": _hit_type(hit), + } + if include_score: + doc_meta["score"] = _hit_score(hit, default=0.0) + return _make_document( + page_content=_hit_content(hit), + metadata=doc_meta, + ) + + +class SemanticaVectorStore(_VectorStoreBase): # type: ignore[misc] + """Wrap Semantica hybrid search as a LangChain ``VectorStore``. + + Args: + hybrid: A semantica.vector_store.HybridSearch instance. + vector_store: Optional Semantica vector store passed through to + ``HybridSearch.add_texts``. + """ + + hybrid: Any + vector_store: Any = None + + def __init__(self, hybrid: Any, vector_store: Any = None, **kwargs: Any) -> None: + if LANGCHAIN_AVAILABLE: + super().__init__(**kwargs) + else: + super().__init__() + self.hybrid = hybrid + self.vector_store = vector_store + + # -- required VectorStore API ------------------------------------------ + def add_texts( + self, + texts: Iterable[str], + metadatas: Optional[List[Dict[str, Any]]] = None, + **kwargs: Any, + ) -> List[str]: + """Embed and store texts; return the generated IDs. + + Delegates to the Semantica ``VectorStore.add_documents`` backing the + HybridSearch instance (or to ``hybrid.vector_store`` if provided). + """ + if self.vector_store is not None: + return self.vector_store.add_documents( + list(texts), metadata=metadatas, **kwargs + ) + vs = getattr(self.hybrid, "vector_store", None) + if vs is not None and hasattr(vs, "add_documents"): + return vs.add_documents(list(texts), metadata=metadatas, **kwargs) + raise ValueError( + "SemanticaVectorStore requires a Semantica vector store with " + "add_documents (pass vector_store=... to the HybridSearch or to " + "SemanticaVectorStore)" + ) + + def similarity_search(self, query: str, k: int = 4, **kwargs: Any) -> List[Any]: + """Return documents most similar to the query.""" + return [_document_from_hit(hit) for hit in self.hybrid.search(query, k=k)] + + def similarity_search_with_score( + self, query: str, k: int = 4, **kwargs: Any + ) -> List[Any]: + """Return (document, score) pairs.""" + return [ + ( + _document_from_hit(hit, include_score=False), + _hit_score(hit, default=0.0), + ) + for hit in self.hybrid.search(query, k=k) + ] + + @classmethod + def from_texts( + cls, + texts: List[str], + embedding: Any = None, + metadatas: Optional[List[Dict[str, Any]]] = None, + **kwargs: Any, + ) -> "SemanticaVectorStore": + """Build a store from a list of texts (LangChain convention). + + Requires a pre-configured ``hybrid`` instance passed via kwargs. + """ + hybrid = kwargs.pop("hybrid", None) + if hybrid is None: + raise ValueError( + "SemanticaVectorStore.from_texts requires a 'hybrid' " + "HybridSearch instance as a keyword argument" + ) + store = cls(hybrid=hybrid, **kwargs) + store.add_texts(texts, metadatas=metadatas) + return store diff --git a/integrations/openclaw/mcp_tool.py b/integrations/openclaw/mcp_tool.py index dff6b6db..626f8b95 100644 --- a/integrations/openclaw/mcp_tool.py +++ b/integrations/openclaw/mcp_tool.py @@ -116,7 +116,41 @@ class OpenClawKGTool: ) def __init__(self, base_url: str = "http://localhost:8000", timeout: int = 30) -> None: - self.base_url = base_url.rstrip("/") + # Validate base_url at construction time so callers get an immediate, + # actionable error rather than a cryptic failure on the first request. + # allow_private_ips=True because the documented default (localhost:8000) + # is intentionally a local Semantica server; the scheme check and + # URL-structure check still apply unconditionally. + try: + from semantica.ingest.ssrf import validate_url_for_request + validate_url_for_request(base_url, allow_private_ips=True) + except ImportError: + # semantica.ingest not installed in minimal openclaw-only environments; + # mirror the structural checks that validate_url_for_request performs + # unconditionally (before allow_private_ips is consulted), so the + # guarantee in the comment above โ€” "scheme check and URL-structure check + # still apply unconditionally" โ€” holds in this path too. + from urllib.parse import urlparse as _urlparse + if not isinstance(base_url, str) or not base_url.strip(): + raise ValueError("OpenClawKGTool base_url must be a non-empty string.") + _parsed = _urlparse(base_url.strip()) + _scheme = (_parsed.scheme or "").lower() + if _scheme not in ("http", "https"): + raise ValueError( + f"OpenClawKGTool base_url scheme '{_parsed.scheme}' is not permitted. " + "Only http and https are allowed." + ) + if not _parsed.netloc: + raise ValueError( + f"Invalid OpenClawKGTool base_url '{base_url}': " + "URL must include a netloc (domain or host)." + ) + if not _parsed.hostname: + raise ValueError( + f"Invalid OpenClawKGTool base_url '{base_url}': " + "URL must include a hostname." + ) + self.base_url = base_url.strip().rstrip("/") self.timeout = timeout self._session: Any = None diff --git a/mcp/__init__.py b/mcp/__init__.py index 6154f804..0d18af90 100644 --- a/mcp/__init__.py +++ b/mcp/__init__.py @@ -21,6 +21,17 @@ Configure in Claude Desktop, Windsurf, Cline, Continue, VS Code: } """ +import os + +# MCP stdio framing IS stdout: any progress bar or console renderer that writes +# to stdout would interleave with the JSON-RPC stream and corrupt framing for +# every client. This package is always used as an MCP stdio server, so force +# progress tracking off for the entire process. Set before importing server / +# tools so the Semantica progress-tracker singleton is never created with +# output enabled (the singleton reads this variable at construction time and +# the enabled.setter re-checks it, so later re-enable attempts are also blocked). +os.environ["SEMANTICA_DISABLE_PROGRESS"] = "1" + # `semantica.__version__` is the authoritative package version โ€” see # semantica/mcp_server/__init__.py for why it is used directly rather than # importlib.metadata.version("semantica"). diff --git a/mcp/tools/export.py b/mcp/tools/export.py index f435bf18..df39162b 100644 --- a/mcp/tools/export.py +++ b/mcp/tools/export.py @@ -80,7 +80,12 @@ def handle_export_graph(args: dict) -> dict: if rdf_fmt: try: from semantica.export import RDFExporter - rdf_str = RDFExporter().export_to_rdf(graph, format=rdf_fmt) + # RDFExporter.export_to_rdf() expects the canonical kg dict + # {"entities": [...], "relationships": [...]}, not a ContextGraph + # object. Convert before handing off; passing the raw graph + # caused AttributeError: 'ContextGraph' object has no attribute + # 'get' on every RDF format. + rdf_str = RDFExporter().export_to_rdf(graph.to_kg_dict(), format=rdf_fmt) return {"format": rdf_fmt, "data": rdf_str} except Exception as exc: return {"error": f"RDF export failed: {exc}"} diff --git a/plugins/.claude-plugin/README.md b/plugins/.claude-plugin/README.md index 6fa56eef..c685a65b 100644 --- a/plugins/.claude-plugin/README.md +++ b/plugins/.claude-plugin/README.md @@ -53,7 +53,7 @@ plugins/ ## Prerequisites ```bash -git clone https://github.com/Hawksight-AI/semantica.git +git clone https://github.com/semantica-agi/semantica.git cd semantica pip install semantica # Python 3.10+ ``` diff --git a/plugins/.claude-plugin/marketplace.json b/plugins/.claude-plugin/marketplace.json index cbe8b924..0afe6ddb 100644 --- a/plugins/.claude-plugin/marketplace.json +++ b/plugins/.claude-plugin/marketplace.json @@ -1,8 +1,8 @@ { "name": "semantica-local", "owner": { - "name": "Hawksight AI", - "url": "https://github.com/Hawksight-AI/semantica" + "name": "Semantica", + "url": "https://github.com/semantica-agi/semantica" }, "plugins": [ { diff --git a/plugins/.claude-plugin/plugin.json b/plugins/.claude-plugin/plugin.json index fd5d35e7..8328e3a9 100644 --- a/plugins/.claude-plugin/plugin.json +++ b/plugins/.claude-plugin/plugin.json @@ -5,8 +5,8 @@ "author": { "name": "Semantica Contributors" }, - "homepage": "https://github.com/Hawksight-AI/semantica", - "repository": "https://github.com/Hawksight-AI/semantica", + "homepage": "https://github.com/semantica-agi/semantica", + "repository": "https://github.com/semantica-agi/semantica", "license": "MIT", "keywords": [ "semantica", diff --git a/plugins/.cline-plugin/plugin.json b/plugins/.cline-plugin/plugin.json index 81c004c4..d4f09886 100644 --- a/plugins/.cline-plugin/plugin.json +++ b/plugins/.cline-plugin/plugin.json @@ -6,8 +6,8 @@ "author": { "name": "Semantica Contributors" }, - "homepage": "https://github.com/Hawksight-AI/semantica", - "repository": "https://github.com/Hawksight-AI/semantica", + "homepage": "https://github.com/semantica-agi/semantica", + "repository": "https://github.com/semantica-agi/semantica", "license": "MIT", "keywords": [ "semantica", diff --git a/plugins/.codex-plugin/plugin.json b/plugins/.codex-plugin/plugin.json index c12d66c2..eac91fc6 100644 --- a/plugins/.codex-plugin/plugin.json +++ b/plugins/.codex-plugin/plugin.json @@ -5,8 +5,8 @@ "author": { "name": "Semantica Contributors" }, - "homepage": "https://github.com/Hawksight-AI/semantica", - "repository": "https://github.com/Hawksight-AI/semantica", + "homepage": "https://github.com/semantica-agi/semantica", + "repository": "https://github.com/semantica-agi/semantica", "license": "MIT", "keywords": [ "semantica", diff --git a/plugins/.continue-plugin/plugin.json b/plugins/.continue-plugin/plugin.json index da93dc38..57d45e82 100644 --- a/plugins/.continue-plugin/plugin.json +++ b/plugins/.continue-plugin/plugin.json @@ -6,8 +6,8 @@ "author": { "name": "Semantica Contributors" }, - "homepage": "https://github.com/Hawksight-AI/semantica", - "repository": "https://github.com/Hawksight-AI/semantica", + "homepage": "https://github.com/semantica-agi/semantica", + "repository": "https://github.com/semantica-agi/semantica", "license": "MIT", "keywords": [ "semantica", diff --git a/plugins/.cursor-plugin/plugin.json b/plugins/.cursor-plugin/plugin.json index 3b73366a..0a221866 100644 --- a/plugins/.cursor-plugin/plugin.json +++ b/plugins/.cursor-plugin/plugin.json @@ -6,8 +6,8 @@ "author": { "name": "Semantica Contributors" }, - "homepage": "https://github.com/Hawksight-AI/semantica", - "repository": "https://github.com/Hawksight-AI/semantica", + "homepage": "https://github.com/semantica-agi/semantica", + "repository": "https://github.com/semantica-agi/semantica", "license": "MIT", "keywords": [ "semantica", diff --git a/plugins/.openclaw-plugin/plugin.json b/plugins/.openclaw-plugin/plugin.json index 0489b609..582145f5 100644 --- a/plugins/.openclaw-plugin/plugin.json +++ b/plugins/.openclaw-plugin/plugin.json @@ -6,8 +6,8 @@ "author": { "name": "Semantica Contributors" }, - "homepage": "https://github.com/Hawksight-AI/semantica", - "repository": "https://github.com/Hawksight-AI/semantica", + "homepage": "https://github.com/semantica-agi/semantica", + "repository": "https://github.com/semantica-agi/semantica", "license": "MIT", "keywords": [ "semantica", diff --git a/plugins/.vscode-plugin/plugin.json b/plugins/.vscode-plugin/plugin.json index a39c097e..771df46d 100644 --- a/plugins/.vscode-plugin/plugin.json +++ b/plugins/.vscode-plugin/plugin.json @@ -6,8 +6,8 @@ "author": { "name": "Semantica Contributors" }, - "homepage": "https://github.com/Hawksight-AI/semantica", - "repository": "https://github.com/Hawksight-AI/semantica", + "homepage": "https://github.com/semantica-agi/semantica", + "repository": "https://github.com/semantica-agi/semantica", "license": "MIT", "keywords": [ "semantica", diff --git a/plugins/.windsurf-plugin/plugin.json b/plugins/.windsurf-plugin/plugin.json index cbf45713..abb4831d 100644 --- a/plugins/.windsurf-plugin/plugin.json +++ b/plugins/.windsurf-plugin/plugin.json @@ -6,8 +6,8 @@ "author": { "name": "Semantica Contributors" }, - "homepage": "https://github.com/Hawksight-AI/semantica", - "repository": "https://github.com/Hawksight-AI/semantica", + "homepage": "https://github.com/semantica-agi/semantica", + "repository": "https://github.com/semantica-agi/semantica", "license": "MIT", "keywords": [ "semantica", diff --git a/poc_runner.py b/poc_runner.py index 49b9a952..49a06a40 100644 --- a/poc_runner.py +++ b/poc_runner.py @@ -187,15 +187,6 @@ def poc_vuln3(): }) return nodes - # Simulate the CSV parser โ€” mirrors export_import.py lines 131-133 - def parse_import_csv_row(row: dict) -> dict: - """Mirrors export_import.py CSV node ID extraction (no sanitization).""" - node_id = row.get("id") or row.get("node_id") or row.get(":ID") or row.get("_id") - return { - "id": str(node_id), # โ† UNSANITIZED - "type": row.get("type", "entity"), - } - # Attack payloads payloads = [ # Header injection payload (chained with VULN-1) diff --git a/pyproject.toml b/pyproject.toml index 8ffb9395..278b4c52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,8 @@ dependencies = [ "loguru>=0.7.3", "structlog>=22.1.0", "gensim>=4.4.0", - "httpx<0.29.0" + "httpx<0.29.0", + "pyarrow>=14.0.0" ] [project.urls] @@ -205,6 +206,7 @@ agno = ["agno>=1.0.0"] # needed (it pulls vulnerable transitive deps like chromadb) and would only # duplicate the prebuilt tooling users can install separately. crewai = ["crewai>=0.80.0"] +langchain = ["langchain-core>=0.3.0"] # ---- File Watching ---- watch = ["watchdog>=6.0.0"] @@ -252,7 +254,7 @@ explorer-lite = [ # dependency-audit/security gates. Install it explicitly via ``semantica[crewai]``. all = [ "semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,explorer]", - "semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,agno]" + "semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,agno,langchain]" ] # ---------------- ENTRYPOINTS ---------------- diff --git a/requirements-ci.txt b/requirements-ci.txt index 1126111c..559c42c1 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -6,9 +6,9 @@ accelerate==1.14.0 \ # via # docling-ibm-models # docling-slim -agno==2.8.7 \ - --hash=sha256:6a2763eb469163f7b79ab1da6ca2f22d8619f6b9d614574f975d9c12bb4323ea \ - --hash=sha256:d49396a2062ee6994ca82695b9bd1e1b95667fec432c544afa38133e564bf090 +agno==2.9.0 \ + --hash=sha256:7777674b3931b341fad4fcf02a61b185a08588c509101348facf87feb2144c0c \ + --hash=sha256:7d9c134703e3c2798023cd57dcb9caa8e1174f6914813f9b130becfc3521a46f # via semantica (pyproject.toml) agnoctl==0.1.3 \ --hash=sha256:6fce1d2482b1f2e0a3d14b0a7c12fbd49d8df4f0bf0a4fd9fd91753cbff5efdc \ @@ -403,9 +403,9 @@ boto3==1.43.69 \ --hash=sha256:4eb494d05b2bd08a7eee61b8ac4c34745c99e9bbce435c91f8d15d372dd8c2db \ --hash=sha256:76297a0b415849c63575ae08a4f1661b2dc8ee0100f104b86f98aa69b47fa2c7 # via semantica (pyproject.toml) -botocore==1.43.69 \ - --hash=sha256:5caa46b740d9a886137146ffbb69edb691f702bfe74c64e85621947ae00181fd \ - --hash=sha256:b1f0e01c53d6b84ee9c184ebf3636c3b3aef85e0ae8498c74afb8734ff224f87 +botocore==1.43.73 \ + --hash=sha256:068433028e011ccbeab1dd7c46b1090c24e378397693c66e67ca571176498daa \ + --hash=sha256:0fa1e63c24b3531be3e1bc1687a88b3be9e63a430153f24edd93efc162bb1c51 # via # boto3 # s3transfer @@ -581,179 +581,179 @@ chardet==7.5.1 \ --hash=sha256:f22396ad419f1e78594057e200aa7253be56840f5b04d07b86ccecd99e09c068 \ --hash=sha256:fad6fbc154113e3b17bb757c34b21477e4b6d69fdd4ce51ff2b3f29a42f08b5b # via semantica (pyproject.toml) -charset-normalizer==3.5.0 \ - --hash=sha256:054420b5db984971d886e5e4e2c37c760ae6682aedbd066687ff0949d9ed5f08 \ - --hash=sha256:06f4fb62a9139bef056b8b2da6773c94c2f259f90e4b8e53b166f3d0372d7cf6 \ - --hash=sha256:076cf9d3f3c7e410295c09d96355cf3b1bcae74990034d80e4371e20fe1ba4c6 \ - --hash=sha256:07f6f42b5a6325df35b458004fb5f9f29bf502d89287a33c7cdef3590e31de0f \ - --hash=sha256:0b2e44e6d42d1a4ff78ccc219a93c5449105d10b16198d1aea581080df8073f9 \ - --hash=sha256:0b373bab0b867b68b8eb249da9478cab9181a42993437cd2f5dba5fb0b4fbd1b \ - --hash=sha256:0c8953d9d1617794cfc40d81179571c9ba3805dd029623a15c93f1fb70e60a74 \ - --hash=sha256:0cce46dd29d73e135e8087b96eb62a4aca6d69391b7f97808c6588ebed3178f3 \ - --hash=sha256:0dfe83c1b4d00abbf433998117a14f56a5c2bc68226c0d331709eed0d1ce539b \ - --hash=sha256:0f211c21aa316cb6e2662e54a1194633a79d98a50a876addacfce7ba5b34b09f \ - --hash=sha256:0f76dc0a47f94cb9b69d86f01e477f4b0371ca70208b9ccea7e063c41eed9046 \ - --hash=sha256:125ee619611019471b177c70bc3e9d4cda9fad7e01d93523501d3b188df0193a \ - --hash=sha256:1328cc57dd4372be1265f68232cee890e087416e3e6e93e6ffb32c2bad4d36a4 \ - --hash=sha256:143792a43e06dc3b27fc891948406e251502dc19ff9216cd80182b79131be5c5 \ - --hash=sha256:14f6904a3cf870abf044df3a8c4924ac6c8ef77e9896586fd37e73ae96cff2af \ - --hash=sha256:168a0cb536b5123a77bc42ecf5e0bf6f923d0d9ae43c42a14eb0677c19ac6c19 \ - --hash=sha256:17a0fd0e23961c2c017372e37aabc7ca8fceb9e10ad898977dfb40ad3927baae \ - --hash=sha256:17db18db9a1374d5b9d9a3252f980b4243b0b4efd1df03fac78bb587f6ce98cd \ - --hash=sha256:196e270c4e80827b5072eed7d6aa661d133afada94fe366669f9609e718d305e \ - --hash=sha256:19e52bda45086df8a4be4bb5910af6f5d9d3b538c78712c8ae09ef10b85bf458 \ - --hash=sha256:1a573e1e428f93908e79e04b349717f400e720f2f82285f0aaaf3ee0ff7f4c79 \ - --hash=sha256:1c010dd86d3f4c4433c9634d33ce8147393b270dfa54f217f965540b8ae8e075 \ - --hash=sha256:1cdfed4d7a59333c8220c67dd3be4e7a6c887b67453a64394022dcc919570add \ - --hash=sha256:1d366548d2ee28a8cfdcc4296363978cc644a728333be9824d2de4652e83df0a \ - --hash=sha256:1f56ce84b317ef2a59d7d3461891c7597c79247d2192bb8114c68a1a1debfcc0 \ - --hash=sha256:1f99a8c3a1da5d955edbad18208b3d627bdd54c48a6e739fa877bdca98c686d6 \ - --hash=sha256:2080aa129a28267984cdc902898993d788c995c384e285d0d19199f56760d52e \ - --hash=sha256:22a1889f1c9b752c63c36758a0c2145458e3cadb20fced7a0790002e9dd12b26 \ - --hash=sha256:2401f7671242e921e604f609d429f6b282ea4ca787a6ffd22ed7372011ddb9d1 \ - --hash=sha256:2403b489c103e9a18c835863fc6dd54361355c8291d4cafdb37492b683440b9b \ - --hash=sha256:2df26d4134948616be0ece05d0b24d621d3990f37147b5883c52052b613ef1f5 \ - --hash=sha256:301bfc4877c4f4f62b344235ecc58d06c901683801636eef819f88769c315ba2 \ - --hash=sha256:30ae26a1adcd943690dcbbc47f28be762bae9e08ad7442b78c86b1c0dd5a626c \ - --hash=sha256:3288a560dc3114d5d2ebe309b1ef43f8af355eafe25856832415c2a8196c9db3 \ - --hash=sha256:32e6d56dd825205f81e5c45bcebb4df6a11fb2bbf4969a01ef156d6ced90c224 \ - --hash=sha256:3418edd0ecb72a0a3861cf72f31be0ad9b7fe338ce2b58fb5cc80b9aeb792700 \ - --hash=sha256:3587d94b5c9f05c2dc4c3f3d47aba6375ff141a21adae3051d8d4d53e8a937c0 \ - --hash=sha256:3684ebbdffd51329ac44245d1d227d90b965797aa1a8abd026568a1f6ae88811 \ - --hash=sha256:368eb2fc9482158b3a3386e8f01fa61f479c968e9a19ceab8f0188b86b312991 \ - --hash=sha256:38a395079f229a631dece74e24c69c1f612536dd51f345a7d6a98abe2d3e047a \ - --hash=sha256:3b08ebf9488c7ff5eff038e48e6ea938178dfd9dcc8598b5ca941e4ae27b20be \ - --hash=sha256:3bfbe543d957213fc9a3db4979a8e171b7aa7504c1d737029defdb03a6095a38 \ - --hash=sha256:3cfdab178a4add5483e26a9bb1c16d8018ccf39b4be7a3aea6c3979e6828f2ee \ - --hash=sha256:3d00e18e7bbf47e332ab63903d18bae31efc701b1d8cca0382b97784a621fc44 \ - --hash=sha256:401ea6e7af9e7852ed818f64714b579c1935482049670847ca3bd7ba45dc63fb \ - --hash=sha256:420b19411959eec115063229536788e6b32d0a7fa907d6b940317919120d702d \ - --hash=sha256:4253da1b4456b633651a8d59eb1dc7a8a8fa38241014dd7c217b353e547ae394 \ - --hash=sha256:4346a693c08b1d0cfc0e3325bfb0ecd4322fb1a6904d68cf416f8da5e981b234 \ - --hash=sha256:478650a70a750d75d5add401606c77f77069c32e4ba2c9131dc6cee566962ca0 \ - --hash=sha256:48920bf6fe83eb2226756ac623fa54940487154eb18f80889d5735cf234965c0 \ - --hash=sha256:49bd5feb59b0bf3cbf6ebcf4352e371c95b9da9bacd4449f8b64d0ad2c10a26e \ - --hash=sha256:4c440122e1ea68b1f8b44a631ebf49c39180f6869b1da22d76e8a724208ec6e9 \ - --hash=sha256:4ebebb410bc517e1d284c52a123e82704b21e4e7e26a21ebecf7439d0647b8a3 \ - --hash=sha256:527e28a5e751d9e11369b9c5f9ab35c748eb9c109101920c7deb40d6eadf8d03 \ - --hash=sha256:54c963ce6404e52255b737e8a06d356fc762d59096ae566203a67cf2b7d050f2 \ - --hash=sha256:562d24ca7797c1af8852994950c2e623a907b201fc4b0ed29e92af173d3828ca \ - --hash=sha256:5780a29823e1d2bec69b7a104ead4195a43f3e97782efaedbf1f79a0157af715 \ - --hash=sha256:58ca5dc0a0ef99f2801ec0574214c978e9574055bc783830bbb6e7433218609f \ - --hash=sha256:5a4ee37248dfac25107c758bda99d545ce73e60b44d2dd39e4a2bb9f2831e9f5 \ - --hash=sha256:5a54587f93f2e289f8faf25b35c997d4cc75cf677485ac6f50c985715989f99c \ - --hash=sha256:5b81980668800dd1c69faad8aea6e85a8cee0e13bcd3bba7671695ff16260293 \ - --hash=sha256:5c23fa4f6eccdd601949cb00f3988c01d64e671d8faba356397971077022e144 \ - --hash=sha256:5e68229977b2dea28e7061c0c0630a23f2f9f6e9c6fb38d77d3d6dbfe3768b74 \ - --hash=sha256:5f51a19dc52197a20218b05ec5336d0c6b3b09935f838724722032c8d45dc91a \ - --hash=sha256:606a86c1c3196f3738de39a67a7490bbd61cb31c0e0436070bd0c6a48170b38e \ - --hash=sha256:6083d10a846218502d664375b9448508d9fa580bd834567423156c6abfbe899d \ - --hash=sha256:608553f476fca509537e804c4a71f5eb166ce63b75141f89c2c686ce1aa36956 \ - --hash=sha256:63ea0cc840c66670183578c2630d138c0e944aeadfc33f25173ee240f5db780d \ - --hash=sha256:6753de11eef42f1c321b26d682957d92c7f7bbce6530f34bbe0f9291dd37cc6f \ - --hash=sha256:68b7e84ae8239a94f8d2c8f3f3a3a81bcde54805ec8f42a34de927d155688ec6 \ - --hash=sha256:69d647cf158eb6bc9c99503292abed1f2079a2de5859f06a403f8aee6417475d \ - --hash=sha256:6abb1f356fb865baeb6ebc3fadd843e9a96fbf49b9adcca55037f3cceccb7438 \ - --hash=sha256:6c06875a1d4a7537bef70f659b55c6b55b9a47ec3ba8f2db610350c2d9915e6e \ - --hash=sha256:6c57af4084c10cb3286688d65e4c654190ff5edcbc2411d08cdca0a8a44c59a1 \ - --hash=sha256:6c95450fce59f00c6d08eff6572ec2e736e5054c9450253afd5748f8416f2eb9 \ - --hash=sha256:6da562a20a49673fe365b05750e98d03bb2c5f8b8d03562b014c1abb3df739f1 \ - --hash=sha256:6e44bc2780516b3df986d6fe33103c7080cd9dcd5576fe3cb4b0f64309c8f22b \ - --hash=sha256:70ff1c16eb0eb5ee6bb12739292347f981a5ba764cc4df1bc2e69b0405d4ac3b \ - --hash=sha256:72982d9958a42f8132bf2d6b90214ed66477295ef1188731f98ae3511c6eeb5a \ - --hash=sha256:74892fe9f33d204860e782e0a2030bb39f9f0af1e7a24f7d5a5b632df311f655 \ - --hash=sha256:75e243abbb528c1a774390ed71e3f868a9f37b1373442e4bbadd401cfc505ff4 \ - --hash=sha256:7cdded069549b5eae3d5d9bb6c2e5bb4fe83f9b81863e2a193cd747bf197aebb \ - --hash=sha256:7faa47b56070b3dd6f4898ed28528843ab130d53266cb9948d9b1f3bb1a5c5e8 \ - --hash=sha256:7ffc43fe52618fcd7abc6ee0b46aea527db10da73305fcc6aaf9710ac7a33ec7 \ - --hash=sha256:815f143a91983ba3041bba066e492ae3c42de523fb1c699685a1abf3313b7d1b \ - --hash=sha256:826a295a039178479a325be1ae60eded1f0b10f7dda749df59e2440de8f61d64 \ - --hash=sha256:82cc5835997ec78afe293a192e385099355770a7db94b2fb1239d36b32796f1c \ - --hash=sha256:830c04a49998b5ed58c8b642c65b7b26419397f52392a64121ba9fd0e95e7f9f \ - --hash=sha256:83b62410bd36bb1178a7d563e2ee0cf21eb1c980c912ab99c2c78f06227f1731 \ - --hash=sha256:84b736e3b391601bc47b86da381c749c0f894e9191aaca9f31f30c2632206df3 \ - --hash=sha256:85f9e0e2724bbddf05de65e5fb03b73eb23e985b7df4259c1d19feb302eb8dc2 \ - --hash=sha256:8b3e9e29b8b07cc461b9ce7768db7693a93979d0dadf22046f6f3555ded2f516 \ - --hash=sha256:8b8788f114845c01f2b520e0b91ea58d143276cfc0483aa943e815f7b9555c15 \ - --hash=sha256:8cb9b6892b53bd6d11fa4cde3dbee020b1f0b6656be1fbaa1ec0d4324a7839db \ - --hash=sha256:8efc3f1563ed431882dd0dc0411b5f8ace1b1b89074981deaf6bd8af77dbe1bc \ - --hash=sha256:8f006866047c6ec4b627ec144b1e0bbc7427cb31fd7c08d19897d0ac9032af3d \ - --hash=sha256:91f9f7c151e772acebe489eaec96e96a2877202d7dd144e3f96b8676881715a0 \ - --hash=sha256:9491f594859b68052edebd69e05fb045055a713b57a67974e6c1553b4e503c39 \ - --hash=sha256:96720f2aeed3434bc48f4d52fbad64ecc820cfed88915d664780ed9ba09ede78 \ - --hash=sha256:96ae7ab5d8155fde927aa0864fbc8ba3cc4fde6d41ab0c7cea9d6012b4978603 \ - --hash=sha256:98820e1ceb25c6df7a80c4fd8efa59cb121f99bc7c4c1693ad94a2caff5b311d \ - --hash=sha256:993dfcbe75a85a3784abb5084f2c41b915767c90546fcc92803cffa28611baea \ - --hash=sha256:9a1d9b13e5e394e13e3c316f0d910d100b17681ff59797f30da1dba032061296 \ - --hash=sha256:9bb3e0d1345b9c0fe73673ea656375f38a78ec679c2edeae0c24800f04798a85 \ - --hash=sha256:9ce0f885239357379d92fd9a5fddbe20f0e30e0527c29ba69f8e99eeb1304a76 \ - --hash=sha256:9e0213f3f8a2674a6778be299aea1d6dc6dda015aab86f683bca6d78f81f27bb \ - --hash=sha256:9e726478d7a213847860219d74665a6892a643ac93b8f76580f6cf9ed39996b7 \ - --hash=sha256:a17864853f7c518ae7d4b368af98f427f9396805476af40af8698560f09d7d97 \ - --hash=sha256:a284c36b9c6616bf0a8aa4aabba668a0c75ba65ccf40a79868aeaa69ad996897 \ - --hash=sha256:a3ad0e3da22852533858663848608f3f24c0d35e5cde415a4903476f2b4c88ec \ - --hash=sha256:a5613a3a82c974227bde18f03409e30c467f8065cb56d822e3eb83708a5f223d \ - --hash=sha256:a565303d118ea3b94a4b6c076bf568069726be414e43b06d58f7070b076ce11d \ - --hash=sha256:a60773eb5fda796e6e6f76b9c152d270fe59f9788a51a6ff8ba44082d8548ae4 \ - --hash=sha256:a7cb4cd266bd85613367fb85a30cfbf6fe6349919e87e18ca8dba584951bfb8a \ - --hash=sha256:a864bdcacd8bff58bb4845304e031f821a3ec64b2b7259f2d409cd49c9e59ca3 \ - --hash=sha256:ac5a9cc079c67d75f4ddf343276031879eadbb333d1bb231cce297b8d7b9aae8 \ - --hash=sha256:ac68ebfa549cc623e0e9add2937526340c629ccf667b4da85b7ef5f99e70bbd9 \ - --hash=sha256:aff38231e3171c578b2c449a01afa44e9ff40844597a32873da102394f63d28e \ - --hash=sha256:b476cdb63df22da2b91837593380be3ddbe406f36c506c1c91d80e7196b66288 \ - --hash=sha256:b787efadba00f5da6fe89513bfbe3852d52ca3a448fdec165765cb3b44a80248 \ - --hash=sha256:b7eb3eab5c646d3de7dcb14a7c9caebace5249c5767da39e1761cb1576e521a3 \ - --hash=sha256:b8ea208b304587d47931b36481342d20336e0d338ab052f8b4305926482598d6 \ - --hash=sha256:bf1e75dc07a3850b53d1e5f75e04d3ae12afe56284be7821771eaa2466350c73 \ - --hash=sha256:bf91921009025e96ce57a03ced6d14604fc3baf0530351638e9504a55da6fa3b \ - --hash=sha256:c387c6bf91b4774e359a48a179e2872b8e8bf741e4fde06ba8d1665eb9a4760a \ - --hash=sha256:c38d1e9bc2073b0984d2099ea647fd7f6c0d8f83a1e14e0cd32926f16e4c44ce \ - --hash=sha256:c41b067eddcfa5ee6b1169c287605be7fb6b0ea22bba6474c5bb978a668def4f \ - --hash=sha256:c455829625df983f716cbaecbba77f2d1dc2e0e0ed1638c059cece15a279344b \ - --hash=sha256:c54036a518748b6c02e666f6d46c3817561998fb904c3be25b56fb4fe3dc5706 \ - --hash=sha256:c5c6d47a865147e0ae3322ce92e7fb52ba3169d94b447deda56897ea2aa6fac9 \ - --hash=sha256:c5e981a5ac8641381efe6f0029467500661616a530d27bc6eedfe45f840599f8 \ - --hash=sha256:c75191e3c8052045179646cb40e280800a4e0bdfda34d9c949c2f268d44e80e4 \ - --hash=sha256:c825661dfcf843119ab57cdcac0df7a48e168764c66917bc74f9a42ecb096da9 \ - --hash=sha256:c9bde7a960720c8b8e1b5ef7afaa0c9a2f3b55c44abd635b2b29dd066b298e3a \ - --hash=sha256:c9f45186390aee4d1f26f723c615b67df346766c3b16df000d84d6e374f06757 \ - --hash=sha256:d016dc857136c726958102c3b8a3986acdc65ace6fbf12cfdc09cc4bfa2935b2 \ - --hash=sha256:d08952c0f14eb56d9dad72a2e17773b5f709c55b28635822d18c4adf38680833 \ - --hash=sha256:d22a083497d2f7d06a57172c5b60ee66cedcf304fde5226d4dfdc94f6180f5b1 \ - --hash=sha256:d2478bd3b2ead3962a484fb802891be40d10049fb74f83e09cb4463fad023fea \ - --hash=sha256:d54625cbf4e6b60bf0639728cb8b4cb541e340f6d7cafae5806051a40ddf4c45 \ - --hash=sha256:d6100f877d2ed95f0856a3fde25334153add94bf2224c43f45f88e7039262aaa \ - --hash=sha256:d672f329ae504ee240eb39b6effb3318aa8e7e8924c0ce8eee5760b3fad98539 \ - --hash=sha256:d7229a99120c6c2792d96f4857c2648ce5530e93667a2c2388c5ef69a6b84775 \ - --hash=sha256:d74bcf1cdd8ac8267fb216473ce6b112efa07b163536288094541415084d131c \ - --hash=sha256:d788e2ded0c4c47efa4d73cfe59eaf975ee32f425219873d2cb3e3fbaa00f636 \ - --hash=sha256:d867cefea33acad8e33a3eb408cca7889a9cf999bd5433d962089d5a13b6e75f \ - --hash=sha256:d8a9316f4da85e937242642b537c6d55d7e9287dd38e5634732f8233932aff45 \ - --hash=sha256:d90254c8f609338c53ec180fcd4c4f9c16502e238e3fc88ca7fd4c2f38d445b8 \ - --hash=sha256:d9419f44e568f7fafcdc0b3b5c766a2364e705a9b34fb8a56b431e0d1f3f4258 \ - --hash=sha256:d95244906ed69d0f79f190893c65e336c15959003e21449256dc05c001b52ea2 \ - --hash=sha256:dc28949de1bb5f7f30a46f15d74ce7ac5aaa63e03c5de04d68f571c7423af834 \ - --hash=sha256:dc7f6aca0bdac5e6520c8b6769bda69315fe7cb57f69885f115bc8ca02d1d022 \ - --hash=sha256:deb99535e9bf0bea8e274c6413eb939a21be35a3f492678dba4d5b1f4d70f142 \ - --hash=sha256:e31786a947b136329bfdc458c82c06d4ec539b4a4436b7da4df4aafc9902ee80 \ - --hash=sha256:e3b9eaa99a6d8c9ace4cd303915947ef55088d4cd87c6676874f98c5c03aa040 \ - --hash=sha256:e46a37ea7fcf9ae01d71b2e5ece19f1565987f3e308394b829197cbefc061f92 \ - --hash=sha256:e4e8fa586df2208ef040684751345f10f503834a757c9a74ecd19c1a2f9b1ccd \ - --hash=sha256:e54dd1a66fa4bce0ccaf0db9dde336e49b3eec646dc4c1c0991279369d373a14 \ - --hash=sha256:e5f834965c2fe589837bac1002e07e25734ff70381903ccd95b3d649e22bfa40 \ - --hash=sha256:ec6c464cf45867f66a2273e2214d9199a8fbad5cb95ca0fd45f6a2fe1d9d2cf4 \ - --hash=sha256:f044cb1cf44012184715f46584658993b5fee9344d71c4b0c455a17a299730c0 \ - --hash=sha256:f0fde5e5100c735b2274ab898f0742a5dcde492796296cfbe7e0ad6a4cd1a396 \ - --hash=sha256:f1619a3cc174a7e3963dd34348e6fceb6e50db0ddeb0031bd7c73a58286454fa \ - --hash=sha256:f278e131afa96a3622cef9211c406ea2ad1b68eb06f8837cd443684a40e0ae50 \ - --hash=sha256:f2ce3d39fb4a9d674e6639dd5d3146b2e273475d2260f10163228d66fc04433d \ - --hash=sha256:f7496aed56b06325a1ad419c5bf23c6dd042558e874f71dd1b958f3e255f3053 \ - --hash=sha256:f8cd1283a9fe6c2065c807e9d5da81afe5e1e004caef39adc0d8ae86dd883698 \ - --hash=sha256:f9f91d3e8382900f3a68fa0ce94294479de9cd2de6bc0c70acd0f0dfd511836b \ - --hash=sha256:fd68c825548a611158230e2f9222e210ceb2e3391995c0aa5865cbdf3ab4bd49 \ - --hash=sha256:fded2e82ff082e5d8e017e2ddcc1411bd8cb83b8585097fc401ef574f756b888 \ - --hash=sha256:fec352b793cdc183cc9e7e0b6c10fd7bff38ec54ba44cc43599b9b56f7f3db2e \ - --hash=sha256:ffdd7ac514301d0a67f7c23b9f2b431ef909a3c3dd6c3766668d0a6f5900c94e +charset-normalizer==3.5.1 \ + --hash=sha256:00668ebb0609751758682eb0b5857e7c35b9f00e84dfdef062e103244ec94d45 \ + --hash=sha256:012a22b88a77ca2e59b98ac5889b0deb604147666032f45e6d6e217634d2550d \ + --hash=sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5 \ + --hash=sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b \ + --hash=sha256:0722590aabf9dc6a6c0343d523c05458fa2b5047dbe6302fd526bb570600753f \ + --hash=sha256:07ffd07412fc5d5e84cd8952acf9ff7e4ed7a708e69d1bada19d8ba91711353f \ + --hash=sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5 \ + --hash=sha256:0b2b1b3fa5670c127b246df1d0c059defd41f689a868a3b9d79df9b1cac42d22 \ + --hash=sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5 \ + --hash=sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac \ + --hash=sha256:13e3afe97712e8887cd516e960c63f0b93122971e5b5e4b2622fe7701771e838 \ + --hash=sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90 \ + --hash=sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626 \ + --hash=sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4 \ + --hash=sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369 \ + --hash=sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b \ + --hash=sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e \ + --hash=sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee \ + --hash=sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1 \ + --hash=sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102 \ + --hash=sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8 \ + --hash=sha256:29880d17a8eb0b5cfdfd8944b468322928059aa35f1f5fa8ff22b149ec0b42f8 \ + --hash=sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9 \ + --hash=sha256:2e9cf9253119d8e5d111f05d71626786fd3d6193817316eab1ca088cdb8593cf \ + --hash=sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0 \ + --hash=sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031 \ + --hash=sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e \ + --hash=sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235 \ + --hash=sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072 \ + --hash=sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb \ + --hash=sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c \ + --hash=sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950 \ + --hash=sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2 \ + --hash=sha256:366ec70f5547c640d3ce1985722490f23faf4eb5216a7eeba78277490e78dacb \ + --hash=sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e \ + --hash=sha256:3d27167433c0d5f18dc850f07d0b3816221984fecdc405d6c157a6f0b8f8e9e6 \ + --hash=sha256:3e5e1224c0a6a90e05843e07adfec669edebec17801c67072f51e59561d63c0b \ + --hash=sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2 \ + --hash=sha256:433c5a81eade63b47e522303bad236f59dba55ea6951746f5558355eeed8c75d \ + --hash=sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa \ + --hash=sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2 \ + --hash=sha256:494b70049a4d69aec6e8137c13af4cf8db8c9f9820a1392ac293b0dd2987a818 \ + --hash=sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032 \ + --hash=sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71 \ + --hash=sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96 \ + --hash=sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687 \ + --hash=sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8 \ + --hash=sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3 \ + --hash=sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61 \ + --hash=sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9 \ + --hash=sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1 \ + --hash=sha256:55261ac0d2941c42f196dd576f543d87a8ee03cd6f5e30dfb4d807b2e3b9121a \ + --hash=sha256:56490c595a28b1bb27dfc583e816152a9767721ef58b2c03b13f954d2f707420 \ + --hash=sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4 \ + --hash=sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65 \ + --hash=sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663 \ + --hash=sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f \ + --hash=sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591 \ + --hash=sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a \ + --hash=sha256:5ca0555312ae2fe82715cada7fac375530c2f3349e1eaa1bcb33d0283ac79a18 \ + --hash=sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e \ + --hash=sha256:5e2d0e146dcb57034f8b97dc58d2d512cb90aba253960ce449f695fec6a82c6f \ + --hash=sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7 \ + --hash=sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3 \ + --hash=sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c \ + --hash=sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3 \ + --hash=sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7 \ + --hash=sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96 \ + --hash=sha256:6ba32c4d2abf1d2fe7cf27d280f4cca5664233b0f885549c7761719eb977f486 \ + --hash=sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3 \ + --hash=sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6 \ + --hash=sha256:6e2912d4babbc65196ac13c2f53468dc57fb8b9c25ef913e8c59ddf7c6dc0e1b \ + --hash=sha256:6e5e4d73d588ca5ed09df1b7dcd1b203d1df3c542e3f50d126c947d432b10731 \ + --hash=sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959 \ + --hash=sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9 \ + --hash=sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf \ + --hash=sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8 \ + --hash=sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e \ + --hash=sha256:789b8982559ae28dad2356519f841655756cdcd96616410590ae0b17454ee64f \ + --hash=sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885 \ + --hash=sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0 \ + --hash=sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506 \ + --hash=sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2 \ + --hash=sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0 \ + --hash=sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e \ + --hash=sha256:85de3134b5379856e323ba37c19c9256d39425f7b76a63af52b09fb4664c2e8f \ + --hash=sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e \ + --hash=sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491 \ + --hash=sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a \ + --hash=sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20 \ + --hash=sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449 \ + --hash=sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af \ + --hash=sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c \ + --hash=sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712 \ + --hash=sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7 \ + --hash=sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a \ + --hash=sha256:94fbf1c0c6cc0d3d5e50f9a9313a8cdca90dd696d34b381cd1704f8c9e939f20 \ + --hash=sha256:950f23cb393f85543777b0433f082cddd25b51ab398eac7971146495679efe5f \ + --hash=sha256:96eefc178f8636b9c760c5829345307fd81cfae9ab1e80997dbddeb0f54ee9a3 \ + --hash=sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9 \ + --hash=sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e \ + --hash=sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5 \ + --hash=sha256:994e883d17c559cdfd38c84003c8b27d25424a1077272a17e7cd27bfe0bf57b2 \ + --hash=sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36 \ + --hash=sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263 \ + --hash=sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4 \ + --hash=sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11 \ + --hash=sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a \ + --hash=sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3 \ + --hash=sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375 \ + --hash=sha256:a545775cfe815855ea32d7c27731d79da358ef2055b4a25830231b1622dd18aa \ + --hash=sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d \ + --hash=sha256:a6d095662e73e74f0a49988e0593373e243e3a52e27bfeea0a859e88acf4a0f5 \ + --hash=sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99 \ + --hash=sha256:a951ad59cad9145664a730d3036b40b844e74d2d3683da40111463cd3a83845d \ + --hash=sha256:aa1099b956fb795e686d073568f6dc002a0bb89765ea6d5b055dd7d9bf1b116c \ + --hash=sha256:aa2bb0b37202dca27175591f761108b5d34096ade1191ffe4808bdf6b1571488 \ + --hash=sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6 \ + --hash=sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc \ + --hash=sha256:ac00177c4831ffa650f8609e4bdddd5fe09c03b1c0c47acece7e6ea20421598b \ + --hash=sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f \ + --hash=sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00 \ + --hash=sha256:ae31a1a1db2ee6cc2942fccaf695c934bc7f3db9f2133a3fef1f367cf1a4ab10 \ + --hash=sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598 \ + --hash=sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6 \ + --hash=sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962 \ + --hash=sha256:b54e7e13267d49ffbfe68e25b3cbd774dab38fa37238f71265e91b36146eb21c \ + --hash=sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08 \ + --hash=sha256:ba2f37ee79e6338845261a3c5b1784e5d1acdff2c0785b284f1b633033d136ab \ + --hash=sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573 \ + --hash=sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90 \ + --hash=sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5 \ + --hash=sha256:bd6c173f04743d483881bffa1478d5a4624475b8cd1d2194956a75548e191c18 \ + --hash=sha256:be47f99644b208bff7766314013f9acf57b056b04191d570d68ad14022cf5b1d \ + --hash=sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af \ + --hash=sha256:c1dcc36dcb96abc02236e182d17e0f71430152a6c2c7447421da2d2dc144edea \ + --hash=sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c \ + --hash=sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b \ + --hash=sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6 \ + --hash=sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8 \ + --hash=sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774 \ + --hash=sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004 \ + --hash=sha256:ce854f5f478050ade5a238731c4ca985a7d3b3cb53ff600a9b5c3b689b5f0a7a \ + --hash=sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a \ + --hash=sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2 \ + --hash=sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2 \ + --hash=sha256:d1ee1e296209fdce05b81b663250eefa02213a2da7b41bf26f7829b8ba3545aa \ + --hash=sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe \ + --hash=sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3 \ + --hash=sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc \ + --hash=sha256:e06efa066f7dbadbc84ebc126a97c452a6451dfcf589d89d788484949e1cf795 \ + --hash=sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d \ + --hash=sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc \ + --hash=sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893 \ + --hash=sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef \ + --hash=sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d \ + --hash=sha256:e9fbdce1e47394b09bc9f26ab117dfc8d6491977a11d86f592bb42c779db2fda \ + --hash=sha256:eb12fb2ba69ffa05f8695f61c69e591dc4b4a12ac3757ac8af8adb259bf56d17 \ + --hash=sha256:eda059b6bc8bc0812d626fd91a7ce01bf583df0a61296eff390fd94141a34e30 \ + --hash=sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7 \ + --hash=sha256:f298e218441525d3794428b4c8b8fb8662c6d3ea79925d4807ee6b9a96a3bca5 \ + --hash=sha256:f5542f9b941279d82d41eb0aa9f98eba36fe4df5c7086c651df7944935b37182 \ + --hash=sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f \ + --hash=sha256:f9b1e28d0e8dbfa858abdba91d6b547beaf2df1a59bec6da6faae7b96a4991a9 \ + --hash=sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada \ + --hash=sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876 \ + --hash=sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a \ + --hash=sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348 \ + --hash=sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3 \ + --hash=sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f \ + --hash=sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0 \ + --hash=sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f # via requests click==8.4.2 \ --hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \ @@ -1731,9 +1731,9 @@ google-crc32c==1.8.0 \ # via # google-cloud-storage # google-resumable-media -google-genai==2.17.0 \ - --hash=sha256:6b640a2390c82b4a240873eddb9f518c6d2c33244b2de16ed3d14526a6da7f57 \ - --hash=sha256:a4835563c60aee646c9c4b261c507aa4a624710d25017012d20dc65abf3d9a54 +google-genai==2.19.0 \ + --hash=sha256:36e0326dd886b52ef765be4c46042732b46b21f637abbe060e3db7c3de23974c \ + --hash=sha256:d8f4126643793a7de230c396bcd142d21c948c8bb57507580e152549a7a41d9d # via semantica (pyproject.toml) google-resumable-media==2.10.1 \ --hash=sha256:224975032ddb73f7ed9e2f0f4cc08ed1b06874c52d48cc8533e3eb72980b21a0 \ @@ -2191,6 +2191,10 @@ jsonlines==4.0.0 \ --hash=sha256:0c6d2c09117550c089995247f605ae4cf77dd1533041d366351f6f298822ea74 \ --hash=sha256:185b334ff2ca5a91362993f42e83588a360cf95ce4b71a73548502bda52a7c55 # via docling-ibm-models +jsonpatch==1.33 \ + --hash=sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade \ + --hash=sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c + # via langchain-core jsonpickle==4.1.2 \ --hash=sha256:7ffe34426bc797684dbf1dc84185558bd864cd25b1ff5fb01b7405e392d0a937 \ --hash=sha256:8afed18aa189fd81e2e833b426bb4af485594921f0b1d36c2001fc5637a2f210 @@ -2419,6 +2423,18 @@ kombu==5.6.2 \ --hash=sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55 \ --hash=sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93 # via celery +langchain-core==1.5.6 \ + --hash=sha256:b5f73bd9688c457b31ec73657a0ad56948f889fae27acee79286e9c285632ee6 \ + --hash=sha256:d6cf37bf695ecc22cddeb8461a684e353190b2ce430d99eb22bc11c0c7c00ea5 + # via semantica (pyproject.toml) +langchain-protocol==0.0.18 \ + --hash=sha256:70b53a86fbf9cedc863555effe44da192ab02d556ddbf2cf95b8873adcf41b5a \ + --hash=sha256:ec3e11782f1ed0c9db38e5a9ed01b0e7a0d3fba406faa8aef6594b73c56a63e6 + # via langchain-core +langsmith==0.11.0 \ + --hash=sha256:7339f90e6fd9a1a009445b5084a7a0e56a8b6f17305ee5d7e8c5e7582217854f \ + --hash=sha256:e87a3929915936c066b3fa3283ec3f3f0013e2ef7f98a443a7fbe3fab8e784a3 + # via langchain-core lark==1.3.1 \ --hash=sha256:b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905 \ --hash=sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12 @@ -2621,141 +2637,183 @@ loguru==0.7.3 \ # via # semantica (pyproject.toml) # fastembed -lxml==6.1.1 \ - --hash=sha256:05a82eb6e1530a64f26225b55cbd178113bd0b5af1c2b625f25e5296742c26d2 \ - --hash=sha256:07a4a68e286ee7a1ed7dfb8af83e615757c0ccfe9f18c6b4ea6771388d9ba8c9 \ - --hash=sha256:09dd5b7075dc2f7709654a46543ba1ea3c2e217b2ed8fbd413a8a945a0f40f60 \ - --hash=sha256:0b7e8a14c8634bf6f7a568634cb395305a6d964aeb5b7ee32248094bed3a7e2c \ - --hash=sha256:104c09bda8d2a562824c0e319d0768ce26a779b7601e0931d33b09b53c392ef7 \ - --hash=sha256:126c93f7f56f0eda92f6d8c619edc463a4f23d9252f1c9d0405a76f25fa9f11a \ - --hash=sha256:162af1091cd785f2f27e62d3547ae9bc58ec5c86dd314d67021fd02463708d83 \ - --hash=sha256:17e0e18d4ad8adbd0399291bc44845b69d9dd68439a3cdebdf35ff902ec05072 \ - --hash=sha256:18b73c339ae29b90fd2d06e58ebd555a751bde9cd6bbd36cc0281b9a2c94e9d8 \ - --hash=sha256:19607c6bbff2a44cf3fe8250abccd20942d3462473e0a721d01d379ed017e462 \ - --hash=sha256:19b7ab10b210b0b3ad7985d9ac4eb66ab09a90b20fe6e2f7ba55d01a234345d0 \ - --hash=sha256:1d4962d4c66bf830a7e59ed6cfc17d148149898a3aefa8ec6e59763e6e3ed085 \ - --hash=sha256:1db753c9115ec7100d073b744d17e25e88a8f90f5c39b2f5dd878149af59671f \ - --hash=sha256:1dde6131244bba38a17c745836ba190bc753fd73c9291666287fd0a3fa3dcf30 \ - --hash=sha256:25c6997a9a534e016695a0ba06b2f07945de682731ff01065b6d5a4474179da1 \ - --hash=sha256:26e6eda8d38c1fcab1090dd196ee87cbd13788e531937610e2589085de074e77 \ - --hash=sha256:27acc820660aaffa4f7c087f29120e12980f7779d56d8492d263170111284740 \ - --hash=sha256:2a0217714657e023ef4293500f65aa20fce6164c8fd6b08fa5bd4a859fb14b9b \ - --hash=sha256:2c8daa471358dc2d6fcf02165e80ec68f77871a286df95bc5cc3816153b0fd2c \ - --hash=sha256:30a89d3ac8faec007453fb541f3f46807eeec88edd5826f6e3fe001752a2c621 \ - --hash=sha256:31033dc34636ea6b7d5cc11b1ddbda78a14de858ba9d3e1ed4b69a3085bc521e \ - --hash=sha256:32ab449a5486f6c758e849bb86710d0e45edc24a04e250c01555f8f5653958f8 \ - --hash=sha256:3483644525531e1d5762b0c44a8e18b6efba321b6dcf8a8952de10b037618bca \ - --hash=sha256:34c2d737beabfe35baada43941ed519251e9a12e779031496bcd5d539fcfd730 \ - --hash=sha256:3779def59032b81e44a5f70096ef6bf2082f8d901937dca354474ba09782e245 \ - --hash=sha256:37a58976370f36d9329d118ad0b953c5aeb9119ac9c6a4e258942a225d0573a1 \ - --hash=sha256:3893c14c4b6ac5b2d54ba8cf03e99fe5104e592de491f19bd6b82756c09f8004 \ - --hash=sha256:3a12689be69a28ddaa0ab99a5a1137da2afd5f8f16df7b5680b66f616d3eda1d \ - --hash=sha256:3ab541146f1f6968c462d6c2ac495148e8cdba2f8347700b2141b6ec5a75bf52 \ - --hash=sha256:3abf332af33a74288675d936fe861fd4344da0dd6622193fbc4f2bfbb35536b5 \ - --hash=sha256:3fd9728a2735fda14f4e8235830c86b539e9661e849665bf926d3f867943b4bf \ - --hash=sha256:424aa57aca0897eb922aef34395bd1289b3b6f04e6bae20ea123c0c7e333cffc \ - --hash=sha256:441dd227fa0690eb9fc81edabc63cdcefc212bba99b906dcf6e32cc1a9d3e533 \ - --hash=sha256:469e3618338bd7ab5beb412d2439825479fcf0dab99e394ca563dbc4eaf6c834 \ - --hash=sha256:47402e62c52ff5988c1e8c6c63177f5708bccf48e366dea4e3dcf1e645e04947 \ - --hash=sha256:4f0dd2f01f9f8a89f565d000e03abcf0a13d692a346c8d22f628d49af098777a \ - --hash=sha256:53b7d2b7a10b1c35c0a5e21e9224accf60c1bbfba523990732e521b2b73adef2 \ - --hash=sha256:53c909b62a0532183542fed00c5a7218258c56292d409bc789886fe1cb04c438 \ - --hash=sha256:54a7f95e4de5fb94e2f9f4b9055c6ba33bf3d628fd77a1d647c5923caa2cdcdc \ - --hash=sha256:556e94a63c9b04716f8e4de2abb65775061f846e89331b6c5be79183a24f98ea \ - --hash=sha256:55b03549819867ea141c0202242c4816c82e52ec36e7e648db9d8da5a3dc3ed6 \ - --hash=sha256:581d4c8ae690a6609e64862dd6b7c2489635c2d13907fc2b20f2bc200ff1d21e \ - --hash=sha256:58bb955caba94e467d2a96da17660d2d704e0675894cba21ab8a775b8621fd1c \ - --hash=sha256:5b7328b46d49fc9477d91ae8f6d55340347d827b7734ba3ea33faae0efef1383 \ - --hash=sha256:5ba186ad207446c65d3bb3d3e0412b032b1d9f595e59861e2354798c5703d955 \ - --hash=sha256:5bec7d03d78d853597d6107854c2310ce3f761fd218fe9fe91d5101fcf6c2efe \ - --hash=sha256:5c6bf403fbb3b3e348a561a5f4f0b9961835657981c802a1df03653eef8a9074 \ - --hash=sha256:5f6994074ebae6ffb04447268e37dc16edc304f9859cf91acb86e0af6c1b395c \ - --hash=sha256:62aeb7e85b5d60320b9d77eef2e773994e2c0ce10121b277e0a19804e1654a5a \ - --hash=sha256:63876be28efefa04a1df615b46770e82042cce445cfdce55160522f57b231ccb \ - --hash=sha256:639f6c857d91d9be29bd7502348d6736dab168b54b5158cd899abf11684dc186 \ - --hash=sha256:640f97d43d867bcb9c75b3af013b64850756b746cb6bce8ace83b70da3abba9d \ - --hash=sha256:649dda677cf3bd6ac9ae14007ba0c824ded8ce5808b53fc7431d9140399118c1 \ - --hash=sha256:6540377fbd53fe1b629172288c464fb18db11ce1fa7dc15891da10aa9dcc3e7f \ - --hash=sha256:6689e828a94eee4f139408c337bb198e014724bb8a8c26d3cfac49d119ed69a6 \ - --hash=sha256:68a9198d0fc122d14bb76837de9aa80cf84caed990b5b237f532ed87d3706736 \ - --hash=sha256:6b1761fbf9ec984e2e9d9c589ef5f5fd684b7c19f92aadd567a26c5224958db6 \ - --hash=sha256:70cdfd80589d59e43e18005dd7244e8895e93db8ab6a620b7e23df5445a4e3d2 \ - --hash=sha256:70ef8a7e102a1508f8121aae5b0867abd663f72c14f0a9c937e6554cb4587b7b \ - --hash=sha256:73bc2086f141224ebddb7fc5c6a36ca58b31b94b561e1dfe8e073e3270fad1e7 \ - --hash=sha256:74a9717fd0d82effef5c2854f0d917231d5324b5a3eb7275c43ac9fa32f97a14 \ - --hash=sha256:752d3bbfe874715ccd0aec7f88d7fc623c0f1fd7aa7b3238a084e017bad2a009 \ - --hash=sha256:762ff394d5bd56da0cf034a23dcce4e13923f15321a2adfa2ac00201dc6d3fca \ - --hash=sha256:76447f65250ed2501ead1a1552f5ce8edff159a86f308348e6a9c4acb5e1f1b4 \ - --hash=sha256:766b010012d59470072c1816b5b6c69f1d243e5db36ea5968e94accf430a4635 \ - --hash=sha256:787b2496d0dbe8cd180984e8d29e3a6f76e7ea34db781cb3bd55e4ba1ef8b4ee \ - --hash=sha256:793033d6c5cdf33a573f910d9bea14ef8f5771820411d118da8e1182edb53d5e \ - --hash=sha256:7d47866cb32fb503450b6edc9df355d10dc49836af2e89901bd6ac6b0896d9d9 \ - --hash=sha256:7f7a92e8583f06b1fd49d01158143b8461cfcd135dcb10ec807270a3051bd603 \ - --hash=sha256:80c2dfadb855da477cf73373ad29a333535dedb9b12bad02c9814c8e2b43bf08 \ - --hash=sha256:83b6b30eb131da7a75b601f28c5d6971e6ed3e887919bf6b6a1ad3c2df289080 \ - --hash=sha256:86281fbdd6a8162756f8d603f37e3435bfa38043adb79c6dc6a2dfee065e7525 \ - --hash=sha256:86c89b9d55ebf820ad7c90bc533410f0d098054f293351f10603c0c46ff598f5 \ - --hash=sha256:876e1ff5930ed8bf295ec5ef9a8155e9b6b1876bbf1deed8b3a8069311875a8f \ - --hash=sha256:88136950da4d13c318bde414ce10219931937851327f44328f2df4d2c4614067 \ - --hash=sha256:88d8cb75b9d82858497a5393e3c63cfbf03035225e4b35a49ed7ccb151e4dc0e \ - --hash=sha256:8be8ad51249698103d24b0571df35a10990fbe93dd043b6c024172189485f5e3 \ - --hash=sha256:8d43ca737b20e106e4aebc42b2f3ae19f00ba63d7eb731698ee083d72d15646f \ - --hash=sha256:8dadbe5b217ff35b6a8d16610dd710219b59b76d13f0e3f0d9f36786206e4485 \ - --hash=sha256:9395002973c827b3ed67db77e6ec09f092919a587022174554096a269378fb13 \ - --hash=sha256:96f2ec43df44b1f76249ee0a615334f9b5b060e1c8bd90e706dad2d14d02f383 \ - --hash=sha256:98fc784c2c1440667aeedf8465bdfe10208acf0ead656a2c68627299f546b315 \ - --hash=sha256:9e36f163528fc50cbef305f02a5fd66d404edf7049cdaff211dbc2cba5a7013e \ - --hash=sha256:9eb9b5a968f6e0f6d640092a567e14529ff8cea2e29d00da6f78a79fa49f013c \ - --hash=sha256:9f76acfb5f68ba982635a53fd985a8044be98a35b43232c2a1ee235ffab3e1dd \ - --hash=sha256:a088f287f7d8275a33c07f2cac6c50b9319309a0200a39e7e75d80c707723099 \ - --hash=sha256:a10bd2fd62e8ce916ececb342f348f190724a098c1faa056fdfb2a22ad5e8660 \ - --hash=sha256:a4bbea04c97f6d78a48e3fbc1cb9116d2780b1b39e03a23f6eb9b603fd61f510 \ - --hash=sha256:aa366a1e55b8ebfe8ca8ddc3cfe75c8ebade181aeb0f661d0cb05986b647f72a \ - --hash=sha256:aa49e06d94aba782c6a02eecb7e507969e7e7a41b267f1b359bb35585f295d5b \ - --hash=sha256:aad9aa39483ed8ec44d6d2e59e5b98a0d80676ef0d92f44bfc374836111f62f5 \ - --hash=sha256:aae97dfdb60715c164419ac2532a76d013c3918a665eb6cb7288098b5f349aaf \ - --hash=sha256:abbefa31eee84842140f67acef1c828e28bba8bbf0c3bc6e5492a9af88152c28 \ - --hash=sha256:ac931cdc9442c1763b8a8f6cd62c0c938737eafc5be75eff88df55fc73bc0d00 \ - --hash=sha256:acd7d70b64c0aae0c7922cca83d288a16f5f6da523637697872253415269baef \ - --hash=sha256:add8cf6ddf9a65116119a28ece0f7886e30af27ba724a7594305f1d1b58a92a1 \ - --hash=sha256:aee395f5d0927f947758b4ec119fd5fc8ec71f07a1c5c52077b30b04c0fa6955 \ - --hash=sha256:b1b963fd8f5caa68e99dfae060d54de1fe9cba899b8718b44a00cdca53c3e590 \ - --hash=sha256:b2d444f2e66624d68e9c6b211e28a76e22fff5fcabcfff4deac18b529b7d4137 \ - --hash=sha256:b8d812c6011c08b8111a15e54dd990b8923692d80adf35488bee34026c35accf \ - --hash=sha256:ba96ae44888e0185281e937633a743ea90d5a196c6000f82565ebb0580012d40 \ - --hash=sha256:bdebcc8a75d38c7598dfb2c9ed852d7a9eb4a10d6e2d0764b919b802bf32ac88 \ - --hash=sha256:c07da4cebf6889f03ebac8d238f62318e29f495de0aa18a51ea14e61ae907e2e \ - --hash=sha256:c08e5c694306507275f2290073350c4f32e383db15213b2c69e7ff39c1193840 \ - --hash=sha256:c4f469aebd783bb741c2ecb2a681008fd26bfe5c16a9a72ed5467f834e810df2 \ - --hash=sha256:c5d7152ec39ca7c402d8fb9bad86140a15b9503bd0c54484e3f1bbe3dd37ceca \ - --hash=sha256:c674693f055fa2495de12292cb45e9944199d8eaef5a2dec45175c7c61cb73e3 \ - --hash=sha256:c6ed5141a5c7507cf3ee76bd363b0d6f801e3321adc35b5d825a23115faa5465 \ - --hash=sha256:c921ba5c51e4e9f63b8b00267d06566e1f63407408a0496da2d1d0bfc819c7fc \ - --hash=sha256:c9a4b821dc7055bf9e05ff5719e18ec501f75c0f0bbfabd573b277559780833d \ - --hash=sha256:c9f79d5325907f13e1be0b3e4dacc1049d1dffc4aeee3c995284bea5fe0fab7d \ - --hash=sha256:cd312b9692e831d2ffcad61eab31d91d4b4655a962e61de8fb410472cbcd37aa \ - --hash=sha256:cea3f4c1af79af13cdb2da0c028111d8f8522d4f22a000c82385535f24e5cf3a \ - --hash=sha256:cecdd5dfdc87b1fd87dbf81d4b037a544f47f4c744200a67013771682d67686a \ - --hash=sha256:cf9d57306d848218f3601fee7601fab1a327c942d56e2e97610583cb4dd74206 \ - --hash=sha256:d34bbf07dbc7ca5970671b1512e928991fb5e9d95365636c9b2d8b4f53af405e \ - --hash=sha256:d49514be2f28d895c38cf9d2b72d7b9a07d00314519f456c0b50b53cfcf4c785 \ - --hash=sha256:d680fbcb768404c601ecb43519ecd8461f6954cb11c06a78962f666832ccfca8 \ - --hash=sha256:db1d75f6617a49c1c01bc7023713e0ff59ab32c9579ae62a7674c0e34f3b0b0a \ - --hash=sha256:dcb292aa7fe485ceff7af4f92e46c5af397daec5dff64871a528f0fc47a3cc5b \ - --hash=sha256:e07c65f443c887bbcf31cc1771d932ecc192a5273943589b3c7572b749f1ffb2 \ - --hash=sha256:e902da4b04e6b52e5893900d4b8ab46068f75f3561f01bf1080957f9fd932ed6 \ - --hash=sha256:e9308ff8241c532df3f3e570f9a5aeed6c853f888512ba4b75638d7c11c95ef6 \ - --hash=sha256:eb7c9811bfaa8b1ed5ed319f5d370dfbcaa59d52ea64be2a5a85e18195930354 \ - --hash=sha256:ebe6af670449830d6d9b752c256a983291c766a1365ba5d5460048f9e33a7818 \ - --hash=sha256:ed21202aec73cda4d55d1ce57b389aadb90ffb044e6cd1080b8347efe1b1ec84 \ - --hash=sha256:efe0374196335f93b53269acd811b944f2e6bdc88e8894f214bd636455484909 \ - --hash=sha256:f64ec5397ea6a41fc1b4af0380d79b44a755b5531dcaccd9940fb260dca93038 \ - --hash=sha256:f6ac4ef4d82dff54670227a69c67782ae0b811b5cf6b17954f1e8f7502fc0d1d \ - --hash=sha256:f6f0ce10945fab9c4c06ce14e22af9059d1a87493a9af4501a5b0b9187e21cf2 \ - --hash=sha256:f8844cd288697c6425c9beba919302241e3278871dc6519515e72b04e987abcf \ - --hash=sha256:fe0306bd29505a9177aac19f1877174b0e7422c222a59f70b2cd41633448c3dc \ - --hash=sha256:ff3f333630ab480244a1bff72043e511a91eb22e7595dead8653ee5612dd8f3d \ - --hash=sha256:ffecec8eb889b58ba9be5b95fb1cc78e22ea8eedea38e8736a1568fe1979250e +lxml==6.1.2 \ + --hash=sha256:0349321a0537d4fdbebb2af06dd1b64676132c72e2ae250de8cdb58f8c43019c \ + --hash=sha256:04cf9e3f4ee9cab9d9ba05401bef8668840fa9620fcd4d8e85a2d2fd0b0fa960 \ + --hash=sha256:054175250531a5fb102d485743ff16412279c93add12385b3b1c3d7b16d8deaa \ + --hash=sha256:058c79e172926ef524fb3c7c6beea4b55e15886ac99cb0c139ecaac6b375f1e2 \ + --hash=sha256:0666943ee1576fa890a6dc6316ef42e8241b5dd56f67bc5475acb2ac298c6ca9 \ + --hash=sha256:074a88f70a7360a4a0c5be5d898062cd26f898c25b459efb1bdd43ae700c5a1a \ + --hash=sha256:08cd52e6487435c75f2da0a5b276beef7fed161681b93ab766e66b954f0c349a \ + --hash=sha256:08f0c9ed7cded07c5e798b17c9c25bbba5d0650c8ff0a7f65f84c634966f0f10 \ + --hash=sha256:093fbf547d0f3ca02705381f795a050fbb58988be4aac7f79f99f280c4082313 \ + --hash=sha256:0aa07065497f191ad26c4b587ce5dbb5a7105285a3789aafd0661750e8bac537 \ + --hash=sha256:1055241852f2b02068af4a625a5d32c087db193c12251928af2562ecd2239f18 \ + --hash=sha256:1133bd969f2bfcc6b0c0cf7cdf5f2631e62b23fa2471ee8bd44f6ab73554ee9a \ + --hash=sha256:11f529062255209a421ae4de5b1bb36b2f0a2e1a700745e675a4bf4084d13c00 \ + --hash=sha256:12acd337d2821cb8b9247dfe4b7aa2f2769a3df5ae8511b7e550df42b8f4d3c3 \ + --hash=sha256:12ecfea07d767f6accbf30b014e1c477b5eabb13eb4e8c748215efb52c0e314a \ + --hash=sha256:14879fa5eb2b793c040bbfcb62011aa3015c65d6c9875e063ea98ce2029d51fb \ + --hash=sha256:18467b0e9f7f0bc477df69e99829a59ae17fb37d34e5f68399371c7c67be9002 \ + --hash=sha256:1a2331da06dd55a8184985306eb2afd72d708283ce7e85d67bba77317b785060 \ + --hash=sha256:1c0173595dc1c25768f42681a1517dcfc74bb18a34695f127931cbd05f4dead6 \ + --hash=sha256:1c4c6dc1b2485aaa4adfb6ed754f90dddcb2b96a66bbebc9e1ac242b5ce5e818 \ + --hash=sha256:1d55a614d2f0457b1f7511c1b7bec0db0dcdd4af4d09d226829eb054c647527c \ + --hash=sha256:1e3c67b817867c484794d7fe0d73045d7d0c67460c78a0a1249a9e92266e6a0e \ + --hash=sha256:1edca8f4a92b94e873093df959f141d388f2141fcad0c47598442fb4730ef57a \ + --hash=sha256:1fcfe8481302e6dec07909914b8f3f9e1739ae1615209d4b9e7544325fb699c4 \ + --hash=sha256:20134744db7abcbd5232214e767814ef64e5ab57a5b7df93a2bd68b74ef0a6c0 \ + --hash=sha256:215bb3cc4be015ccac3c7d4f25eb7b941f857fe5b02c0e3504cca61f7fb12455 \ + --hash=sha256:2170d0a280c877b6e2dc6738217db947be35dd8cf09ca458b355aa1bab2a9e70 \ + --hash=sha256:2374235206ec83d4827ad219c93c0f7366b93626eab85392c0ee7c8026649376 \ + --hash=sha256:243ecef7cb7415766dd742336cd5b8361a84c6f297e2773c865b783724cbbe74 \ + --hash=sha256:261d98065326676d7253882db0198d0aa06748d7ee0443367acf10b148273f99 \ + --hash=sha256:26ff164c6629e5c4d11c9e55d5ea3d6eed0be2a420eee1f55cbce6e2c23e231a \ + --hash=sha256:2afd1688e372d8eafaa6f56c589399e0a87d086a0c110f6346b0b50f42e67e25 \ + --hash=sha256:2dcc69e307e0916c7a0b552212010938d02a664d29b6bda75ab2bc5fa487c861 \ + --hash=sha256:2e37fe49fe2d5aa40a2cb1cc8176673ad7de0d124e6f4a509d9318f5979c7871 \ + --hash=sha256:2f3194777c0d05945ac91d8594be25d2679d1d826e01e1fc90bae568ff3a547b \ + --hash=sha256:351318f5c0eb7fcab5b4fdb507c6f88fb2c4b5e67784c7e5911448c91fffb5d4 \ + --hash=sha256:351855814dec4ad55ca5f24d0f4b1cdaca7927fe48023a2965351845f3b60cff \ + --hash=sha256:3a698fad6f122a9b3e2dc2fb598c1de7329c74a67c7a334c9109a440de2508e5 \ + --hash=sha256:3be94d2464f19e42d8c39a299f356b12f2fd095c28793671eabfcd9db9c76987 \ + --hash=sha256:3e3b666f57a5d81562f38c766c762416b0f6eb58a00590546911514b48412abd \ + --hash=sha256:40366c23a938008a3bedfcfd80709b3a857c188b4d710b083e978ef5d2c1c715 \ + --hash=sha256:4303f904fb6c41b58dc70743b1d8a470aba6c9897427c48324cff1a95673ddb4 \ + --hash=sha256:442766b326d9892585a64e8c6c4b5ab81d0e6c0538c9f0fc11a84dc101a5d97f \ + --hash=sha256:446f1f92c137e0cbb97eb7e932e15315c11a7c86974f43f15e68c9707ac6a9f6 \ + --hash=sha256:4618b20f43dc98b49569b1dc822176140ea0f2598d672a6989187ba49bcbfec1 \ + --hash=sha256:4622c5616683faf63791b349e6c8dad7717412dc5f29f4febe7575f110609a86 \ + --hash=sha256:47c92dc5167de16e27ace8332454f12ba172dcab04f7a78a9eae14e2e41b6a41 \ + --hash=sha256:47e367dfe341521426692819803e260d0673899c0ff611f14af978d725e2c999 \ + --hash=sha256:48e912f37c99a297175ba955f55a47c0e1c834b506ef162e52a6e4fe276e6e45 \ + --hash=sha256:4a16457e330b7099aa5a8e8bfa5d53a33a1672a819fa656157e9e6dc433ac7a4 \ + --hash=sha256:4aced3284e0353c798b060fe2c175eb81410e99b9a7e2ae6951be5333732b111 \ + --hash=sha256:4b0fa7109b1d0bc1747d8241a0853e135eefb1c978685241b544c46937383efd \ + --hash=sha256:4bf14db2f0214003ec7f46c4300e2065668fc93e20448c1c95bac2e952072168 \ + --hash=sha256:4e220a9c297e5d36895d489a08c9a3f1f6193b6414e702c5fb751e4a3767f8d0 \ + --hash=sha256:4f4d2c36fd5997d30ff19c29fb93293401d0daaf87512297d47610e6883964b5 \ + --hash=sha256:5078ff51e6316c0f75ea8127c2cd24374747fb351f62fb93d1761f8ae5a04a40 \ + --hash=sha256:50ee0c360862f4152db835b456e38614f94b674bca2a47bc8de7171ee6ccbbb8 \ + --hash=sha256:522387e05cd015a81d1dc621fb167fb42b8f629ccd2e8b39de583828f165aae6 \ + --hash=sha256:5295205fd57510c19a0e46385b516119f3a781d45c2672159bce02949238981a \ + --hash=sha256:52f6d4dff133c9778a24e9a2cfc1608930b15869866171aacc5131b5a418a003 \ + --hash=sha256:57188e441ab24f906bd5a5c14eb55363ab51aa6c0de549f3dd320043721cc118 \ + --hash=sha256:575fef7f30048b744dffb3e4ff64a18cac7dba3fd26efdea5730ade9d1bdeb33 \ + --hash=sha256:5848f3de6a8de8a93cff9f068134393ff5fa69ac2a04399f7d49cd67c61c348c \ + --hash=sha256:5a096d6a5f96b776a5b020cb45c17c545effd2a3b6639e6fa97bc95537600923 \ + --hash=sha256:5c2bae42b3a09f977330a08f4a8fe72aec58c4bdb89069d3fe7272a71d885881 \ + --hash=sha256:5d78ba560f3dd404d87b1fcc89b2b382d638ea2998431a3b2e5cda0f3ba2da91 \ + --hash=sha256:604f4778632588d7c000e7e19430639dc12fca58b5b6e99edffba7631725ef0e \ + --hash=sha256:614d4c5a34556e369b86cfcc8d0cf71cd0759a3444a464a07a9427ab0f5e3a99 \ + --hash=sha256:6330cf0ce83f6273ad8ad99bdd25d6ebb3863912f9ac717f96bc8942706e0e26 \ + --hash=sha256:633ac039cb32366dd5935868e041e385875c017b8cd54ea56aeee3fe29ca5935 \ + --hash=sha256:6454d184d556eaf4cb3d6f69e405d21602d6fdcf08b8d57796824275986c6595 \ + --hash=sha256:648861c19b775b89ebefa14586f85090b10163367476d77f242c4131c835ce73 \ + --hash=sha256:65c32ddc5d0750129c7b119fb57d48192b76d334c21e6b690d19dfb06b34af79 \ + --hash=sha256:662432a6103e671d971e06e75ed146d9ff67f39d2c98c2f26613b6057f54eafc \ + --hash=sha256:678e35f1cbca98f55107511ee21a60568535c950f3c2371819bd64504c980d20 \ + --hash=sha256:69df1856cb6c065e5bfd23adcc7408bfa6dcf32b0018373a99b0769bd86e2256 \ + --hash=sha256:6c9cc4b6532abe154dbdebb42aaba8d52c852919591e45067f5b7d46a0405e88 \ + --hash=sha256:6cb0c87421946030b92b558be416852780a912454e3dcba0998e4497c9c588d5 \ + --hash=sha256:733dfb492ec3dfef8350a5cc896e90d202c5171e791e1609e77563751d69a15d \ + --hash=sha256:75530642d8471327e691ab9b0513a5f9c77f38871014ceda40f51bb51765c0a1 \ + --hash=sha256:7766e525282dd38fd89567311323e441996eb958e8e816d16b38f782e3aecd2a \ + --hash=sha256:785761d5123f222cd97f2263a510107226fe32ce7aa7824a90616a41c574ace1 \ + --hash=sha256:79b428c3242e63bdacf3b526a34e0b8b26583846fc597da84b8f0c3d5ea446b2 \ + --hash=sha256:7c444c3a6e8e75334879980eed96568f0e12064c8b1913424eac1805e976736b \ + --hash=sha256:7c482e87cc86bed78a50462560675bc2c348ef72c47596f9b933346d5a8e920e \ + --hash=sha256:7c534ed898413f439b048130011e99a4245ee13d62d431f6b4f7f2484d02a93a \ + --hash=sha256:7c687fd8e558c7d169f6f1987b696f37824d3a097f291bffd0ab4a2ea2307dfb \ + --hash=sha256:7d506bdba580ecb1a6ad2e2b5c49445e66d3e1f95894885739094393a1aad237 \ + --hash=sha256:7e81fc065ede5d58dd0bf0912025aee1bd04c52c2affd61fdb93226a97ce2fc6 \ + --hash=sha256:7f35ba7667004ecdafebbe08da7c9fa06ee6195275bb7ef7a29ee1901e69519c \ + --hash=sha256:7feb72424f19a893ae4f3373c7aae821b1aacb6076b708915c651f0683a97c49 \ + --hash=sha256:822d9397033edbe530a13bb1e0091c0e817536b6aba87a9b4ad626ed779ca0bd \ + --hash=sha256:827438bf6c8292d22a409bb7990d7cffce410f33e7664e46ca74d2ecc26975ef \ + --hash=sha256:83e7510a6dda8df41d1b68b783de2953b3feb55a11dcebf693201ebaa5cc0c4a \ + --hash=sha256:841630176c15fa5d3c5cd6f755435d3c5540a82e1dd2a7de1799401f92ee6d24 \ + --hash=sha256:84a2a46b93b789d8acb44cfcb3d967ce9dbe29884ddb93fbb1a33f0e0c8fcd86 \ + --hash=sha256:8512b3775d68994dd1d6d533161e0a214f2ad9c634659d34a99c98e86c6c3d68 \ + --hash=sha256:85690cfc8ed54c4292e36a08bcf984dde7957e653fd6d94f59184244bcc35843 \ + --hash=sha256:86d93dc3882c283e9aa2124d7d2b50c85579485216a2b3b7f91ba479e31a128f \ + --hash=sha256:87534cec6ea325435e4adf2326b0cf3110eee9a47abf73652eb155db639c08c6 \ + --hash=sha256:878e7c8ada8f92c52f13f35a2ab98ef0adf7fd0211d164fc2af589e4c3cfed63 \ + --hash=sha256:87e9673cd8a3445024fe38e7f91b55fa3428437eec9b7a7ff7d81979520c0d2d \ + --hash=sha256:8807998c1023d1e9d60e02500f90e85a0752dbc0b670989806bba87b82dd5b42 \ + --hash=sha256:8b68f2548259bb04e0b3d5df0c397abe8b0080f5e1ffe4019fb7a8bf01a9339e \ + --hash=sha256:8e613018a5ac66de7abaf1acaae0d7af37a5e1b9bf1ae190a1198b0fdb988ad8 \ + --hash=sha256:8ec111ff8067325f85c08aa9c2b26179ec0537bb89c003fde31127139f85f82d \ + --hash=sha256:8ffb17ec0a8bae18b6628ae40b0896eb264dd285e39a0faa864965c00933b64c \ + --hash=sha256:9031f5f01452681abf39fdd65f84a70cb01a7572a1bbf570042e826b1232d07b \ + --hash=sha256:9088da25ecd609965f838d89fda0465a905b48f4dd90331db9845518f2177372 \ + --hash=sha256:9221442682c27417f10fe11184ea4cce174b25ab52465570b1f3ee3f85f320fa \ + --hash=sha256:927f3e1d04dc0906265fc0416c13500363e42cd683bbb8d46911c79b73d26800 \ + --hash=sha256:92c2b366028ac01e90399e6d17734ce6e4f4aeddd8ba75fbaf80ea11d6c6d645 \ + --hash=sha256:94162456ed0a64fb1c06915df5bd06af4675ae3966d6048fcb73b0906e0e0222 \ + --hash=sha256:9429d2371d406344ed1da5b5686d9412e74137c07b0171278368ff704f470ed5 \ + --hash=sha256:9477e14217c212e6023c994a71a1a349db19b0e10fd5bf189666b281ae63b1fd \ + --hash=sha256:962c12b51d0b164f12569af225dea57568477e24a845b96eaccbef6c07e4cc03 \ + --hash=sha256:9b52ea73a37fc64aa3357ff8607801d46dd170506d3cf8253a91a1d91639d4f9 \ + --hash=sha256:9bdc2db9e04538f917bba0242920764dd740649d8df58700d6d687ead4429429 \ + --hash=sha256:a02164a8cd3e2dc028918e51af844c934c7a24a0b8f4064368360aa14ad1aac4 \ + --hash=sha256:a2b7fe53abced1fe8bd984a9ab3c8c98bc093ec4f9f543089a8817a493818208 \ + --hash=sha256:a5005c0c9e4d749a76a2ff8bd5918a8bb248df8e08e73a55654b9f79c9cd1e2b \ + --hash=sha256:a7fd1dd6faa3df9dcd8f1765237362cd885ca62cdf77a7c5f5ea383ae5b6048b \ + --hash=sha256:a8326e24ae6c3a6bfb03fa8b4793f9a5d804c125228aa067f652b0428e31b87c \ + --hash=sha256:aa224ecc613d411690aa650dbf01daafbe385cd6c67145e80bc5fc01b3a71469 \ + --hash=sha256:adbecbfe44a497c742792457b1c27300617967c18c3934d2416023eba8d8c553 \ + --hash=sha256:ae520f189895c5dd7eeb2b7a372d464da6f4a1ba1d0ecb741b1d4fe4c1f699ac \ + --hash=sha256:aea814342f6afd20d832937ff8b333cd6506428a39c0c4c70c2380aab1887bfb \ + --hash=sha256:aebcc6b184c935e1f7091c09124cfe5107b7c2253894ba23ad646828c17e4c3b \ + --hash=sha256:af6585a466cee2c5a524f7fffc591844bd604a29fdd9cade964f548512b5ef7e \ + --hash=sha256:b1c0d2dde8a50520efc51644587f0fc4810e3af7d3e029d7af0be93bf39e2b5c \ + --hash=sha256:b20440e578d269c5e8a722ab602ddd0f0cedb8b080006b3f936da9991a593d3b \ + --hash=sha256:b28842b30c4bc2e6afe137d98a5d2071a62589471e76d053bea55b0e53298af9 \ + --hash=sha256:b3ca02ef3b5920b88119c82eb6badfb2d082b1f681d528a856dcce17c8706da8 \ + --hash=sha256:b3db5497af55f7a557c95265dd3b91c75dc56364a7b59f258c45fa5576dce058 \ + --hash=sha256:b631174cd2e4d9f8a94ef17f911c6ded10ede93b5e7860dee7bbf85961d321e9 \ + --hash=sha256:b7233a987a101bdf79059014130262a01339094a0a709f175162542f33b55d4e \ + --hash=sha256:b97153ca609b434b712ddfb92cd6af101a7045a7724c542258bd4727a344472f \ + --hash=sha256:ba0dfead73be5be9ad0b7fbf9f31ff29c1b1eae858816dfc8d85099d6e4af0d6 \ + --hash=sha256:ba58574d710b82ead7cbedea01cac3e110bc3ef82d4731519b74a2c11f7cf5e9 \ + --hash=sha256:be365ce8d2d411cf2fb573747684b4fd470fa6224e0094d9d5a21155acc369d3 \ + --hash=sha256:be6f87cd224254a8f81324e34cc655508b83f1d70458a1a39857ad2aa9925852 \ + --hash=sha256:bfcbee8ffff4188f4c6d97eceeff36d8eb983cf838933cbc12ce5f5dd51476c6 \ + --hash=sha256:c0edde95e4b4278dcc0175eda06dc8aa2631ad9f83ae5dbdbc4f0925e200b0b0 \ + --hash=sha256:c20fa05d128c463209ef5323ebf33ee1cac6d87cdc3933fd789fd3c101017c8e \ + --hash=sha256:c470d192e27f97842a068cf12a1c1296b20ca716c56a9249715c6654bc192d19 \ + --hash=sha256:c67f3c1278f942e97d8665c2a690324aaea5137de16f056583a21f0ac706177f \ + --hash=sha256:cb0cf498efa3204621b3c5576f0accd80ad2ee85575f1cae5d2f98de32c8d9cc \ + --hash=sha256:cdd35422de747237f451e821766e2b6be3dd2c31955c1ecd7f17984c5b9bb62d \ + --hash=sha256:cde6b8db7d2e5135129eb5e74b7b44dd2053aa767cd5023541fccedddc262453 \ + --hash=sha256:ceafa5e0536c62a5cd9f65327fa0b57d6f0b0e3435daf2c98a78d0dde7ecbae1 \ + --hash=sha256:cfeac14425fc7a6fca7864b774d4ee63547926158f4a18c67d77b2c9a948acf1 \ + --hash=sha256:d0bfd719c254bbe60ea022cff0e6ffb799a6fa7d4d72852cebe0257957b32d68 \ + --hash=sha256:d117f39b28ab8a330a74abdbe61c2255b51973b238db25fd6c2448de1eb2a02d \ + --hash=sha256:d3e97ac4353cca3fbbfa829bc0c6a913771573d1c6d46932d4335c46f2b7796a \ + --hash=sha256:d50a44113fe6800dcc8a859332b823a4735b1e6ae1b0063882e4cca569ec3e29 \ + --hash=sha256:d858e718b94033ab4b67e4a58fe3114c65bae01ae2314a62fb39ae8897ed4324 \ + --hash=sha256:d86130d70a2557cdf825dffc56255f1f16b83a7bbeab677b4cd040c4c53d8c52 \ + --hash=sha256:da6a4f55f0e3308c07354b1ee239c5550afc212f81629a6067db505ace3b667a \ + --hash=sha256:dd7ea3fa47154b9fff90591b961e41b3718bd7fcd5bc2d9bb47e9845c8ace088 \ + --hash=sha256:e062f5ac1255dfa6c98e3e3863ec18bc79d0947d22d08921a3ca60cee40559fd \ + --hash=sha256:e17e2c30e27f56da5551e7a425888b45f013e940b99ab07d125a1c33f77a4605 \ + --hash=sha256:e7269cc410f3cdf84a66914fc0ef54b1618115c87fb4f9a59a05c5dfc23bece1 \ + --hash=sha256:e8b9a92652e75e7731309ea51db5dee892eef414ce70a6ec3441e5d36bf5189f \ + --hash=sha256:e8dc3d29f2ed2bbf24c205a86326d6681230ace55abfb3f9d5230f42078ad63d \ + --hash=sha256:e92e4419cad18d60b14bf18b82152fbae67f4b1128be7d73b172df275554f5d9 \ + --hash=sha256:ec8d09f460fdeb65f9ead9b75941e312def4bcbb23e1f951b7def061eb99501d \ + --hash=sha256:ee23f6599682bd4d48bb757c0633e78774eedfb65a7e52851f9ad182eeeb625e \ + --hash=sha256:ee7410c98222070fd717ad881ee2a80cc11826b7001b9a5a807155d8918bfc7a \ + --hash=sha256:ef0b8ba6e13597f681b2b4924ca9c4e8c88420bf0e21d9a9006c757f2fc39d1f \ + --hash=sha256:eff128ffdc093cc6317955934ad9751105d37ed8dbca3ff4ccd751af6be37185 \ + --hash=sha256:f16a407766bac51c65d605b06d900821751a79aa20e12185f273f14a17180e7b \ + --hash=sha256:f86e23ed610727a7f025ebbff788f22a7956d3f1b24a25bb1d9286fc7b7642b0 \ + --hash=sha256:f8b89b3be75a37509602b03f9cfa1a28298d4eed4625748148307aeb907901b7 \ + --hash=sha256:f93bc5e25992f5545709000d840c6cafdbd022781a7a0ed79d58a5633733a4e8 \ + --hash=sha256:fa813b0247d0543a563b993ac3dba6168eef59e3a61448432cf5453300c2412b \ + --hash=sha256:feda2ef68c339987dfb370af3a4b785dbc40f925723fe2365e68e43c2640f85a # via # semantica (pyproject.toml) # doclang @@ -4964,9 +5022,9 @@ pypdfium2==5.12.1 \ --hash=sha256:e10cbf41b21233ec5e20adfc170cf60edd77abead86a97dc708fff55a8a886c7 \ --hash=sha256:e5358d2ce4ebc5c899aab1df9ca5d215357244e9168aa443225d3c1e649c7eac # via docling-slim -pypickle==2.0.1 \ - --hash=sha256:0cc1ee65293e4dfa90f0db6435e8021c6a83346be98d0fee81aceb2dad1fb091 \ - --hash=sha256:894afd81d26443e8589d21361a3cc04bd9f5c1535aaa627c3bee1212b58bdf74 +pypickle==2.0.2 \ + --hash=sha256:d3307127314465fe3dc8f0162e11777d5e8284f3a29dc48b0f770d364a85d998 \ + --hash=sha256:d577e39cf501c7c80b1387f6d7dc885cf4efeba65f213df41226d1f24881b1e8 # via distfit pyshacl==0.40.1 \ --hash=sha256:011e3cf1a68b31747cb762ba3d755ae1bdcc464c8fad0dc212a9adc550719552 \ @@ -5441,6 +5499,10 @@ requests==2.34.2 \ # rapidocr # spacy # tiktoken +requests-toolbelt==1.0.0 \ + --hash=sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6 \ + --hash=sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06 + # via langsmith rfc3339-validator==0.1.4 \ --hash=sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b \ --hash=sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa @@ -6592,6 +6654,104 @@ urllib3==2.7.0 \ # pinecone-client # qdrant-client # requests +uuid-utils==0.17.0 \ + --hash=sha256:03815cea572c8a693cab5475b9d750cc161470961c7defa27e9286cad62f38f5 \ + --hash=sha256:04452640d8b6920c480c16e5afe91ff896d236e0c972830f9247e0898d38c803 \ + --hash=sha256:09a55b7a5ae764985cb46467496a1787678d0a1400356157a080ad95b1a36869 \ + --hash=sha256:0ab4a66e7a035ad6625cfc1fbdb34f5c2d25a80ae1ef4bfee458ea2036333c6d \ + --hash=sha256:0bc4c431ccd59c764080ceb43b126043325fe17861b87759d026a0cdd8423bb2 \ + --hash=sha256:0f3729e839209f3457d0d8b6a35a376fdf65577a5aecaf4cc3587d3305759ba6 \ + --hash=sha256:0fcca4e838af9ac9243b3358d7c14afa4dca286a87781124c272d6c4cad9c968 \ + --hash=sha256:1019476b6bdc047216ef7414be5babe0fa5ccfde977c0cac4fd6c75ddec66ff7 \ + --hash=sha256:14dc2f46abb1091260c0d203fcbdf4e045042cc07e49183fd3b255904b95eb70 \ + --hash=sha256:1776a80d16369999b21627028cc5dbce819be83e1e079fdd7a51b587d2916db9 \ + --hash=sha256:1edf2f8732e4ed95bd7b65f2658f4aa072efaaff321144f4e0d4bf6a22709263 \ + --hash=sha256:1fd6f0e8a162dc0e9255b6aebe3cd175e76c33202f1bf39da9e6294b93db0099 \ + --hash=sha256:21c79b61ff750abcf057163dd764ccb6196cde7a26cda1b31b45cd97769e03b3 \ + --hash=sha256:220b52746d99e11964badac3c0869016e0c24bafb70a7dd5c2c072a6be3da9cc \ + --hash=sha256:237722b6581bb5b4eb4cefbcbe5c6e2980a440aabe781fbe50ebf1cb71eee4cc \ + --hash=sha256:239d8a281fe10bae33205b5d43185834d556b18434e0a113b5dc1dfb2fd97e91 \ + --hash=sha256:29179ffb7b317239b6d6afb100d14c439c728770460718280b9c0a42d2561ec2 \ + --hash=sha256:2db386941cfdecdd0b5a8ceeed5cf7479c83d1730dcf64a48d43cfa018cc3310 \ + --hash=sha256:2dd4a21baaac9a88486f0dd166c5793feb101a0bb9f006f2c401657fff5a1343 \ + --hash=sha256:309a35f12d99dde19032bc2259cda6431c85eeac0879134dc777cc3087d7e1cb \ + --hash=sha256:3150d836290c88f1d26eb59c4db280d87417dd3bfaadd2889c77416c8f0ff6fa \ + --hash=sha256:32abaafc8e91928b3d9f4d82e42d2094041e38ad6bb964066faadff28e4162f1 \ + --hash=sha256:32df1944808877702ceea398c103881c09a679bb672a215e01c2a84231266bf9 \ + --hash=sha256:344f7c755e280ea0ba6aeb08022190d867a80000b1715cacded54fc4b5633607 \ + --hash=sha256:351462debd866f1f25e4d4f5c7fac89525b52151f0102a1bdfe94a999b046f5f \ + --hash=sha256:375cde148430d60a4a07c03abaa0774c4fddfdd90de99b4ba02f24088bc9d750 \ + --hash=sha256:387cf7437c94ddec08651a0f1081381299c7075bc48a6251d8922bf39973378a \ + --hash=sha256:3dac0ad0cd9a2818d1775215365a4e8c2f8ada215529dd26f3f8cceeb67a6988 \ + --hash=sha256:405233a5f625b3d995648f4647fa6befa4567cf3f74e1f6b9837e16f7310f0e0 \ + --hash=sha256:4134353bfe3026ddab8e886002dc52bc5a0ab04611aabb0eaae23c32e6e57f64 \ + --hash=sha256:42275ebd0e8e74e32cdbfb8bd88fc99576567d51d54a508020611fd8f4f463a0 \ + --hash=sha256:4441600447d340ae103a353f01dbcd22ff680e5ee1a22988efe8d7b791d8fdb3 \ + --hash=sha256:46a73cacdf512f473a81f65dbf84186e08cfe6e9118fa582b6c6b33a8288a30d \ + --hash=sha256:4bf4d9cd1e80e73922073b9b27c143bedeb109d65f94cd12712e2c87118f2b7d \ + --hash=sha256:4e2ac1c0b56f2c91b6f158e29ed96b1503223fe8aa6e79b1be1dc55bd8a5131c \ + --hash=sha256:52db0e471d3d2632d35445af352591f40a8f32959a412981d9f51e068bb9514b \ + --hash=sha256:53ce348ef4c6e98c02c19c522af01334fe94476ce9af0db8c4482f9f142ae9c1 \ + --hash=sha256:5641071337eb11d61a001ea08793bf72216f3241f0a433ed2764804b2a3e3cc7 \ + --hash=sha256:5670c52a438e21483ce715776144914a4e2a2a5c62d9dee15f8a3e90cf128ae6 \ + --hash=sha256:56aa6488b931246fae11924e4bd0e2b32677e63945eecb71c29e3c2ca0dc3131 \ + --hash=sha256:570db214f6d8507587a8faa968a3fe65e957daeb7bc48b27dc7f69bc3ecdd6f1 \ + --hash=sha256:58838921e377791ef22c64cc92141bfae030f43651ff9272f0f28a208a9e6a5a \ + --hash=sha256:589d9da7de8fa7f739bb970ac4632c9a268213117d634e1c4a58c1c1e821ca05 \ + --hash=sha256:5a4370089c8b2e42f1db51d76408c7fa8eaa2934bf854d17983d16179c07c098 \ + --hash=sha256:622cdde768300591ac79bfcd7bb3468e4b191b1105d5dbfe8d87c39d8f63dd46 \ + --hash=sha256:673d89cc434cc9b97a0b4cf61272f6fca70a81f64eb0afbface2a0d9f77f06cd \ + --hash=sha256:6a019a31bc4db89a0903a3e4f6b218571f3a6ff0ad4b3d3fe1c8f91a05ff6e3e \ + --hash=sha256:6c142bd0cb4dba31c10babe00d59f7ef6460f0ef55eaa9c1a9da270684af996a \ + --hash=sha256:6f29689a76fe7a49cbd629a794d0ec1eab48814e323a00a146a741b0195bde68 \ + --hash=sha256:75d7411e8eb9259764dd60310738540649057cda4509b4af14b36b7f663bfeb0 \ + --hash=sha256:793229621e1ad6cac55f015cfa9f4eff102accbc3da25d607b91c6b0bec167fb \ + --hash=sha256:7a49f47ac26df3e431c56b825c1bae8e6d3d591fdbb7438c227cc9845a7e3d73 \ + --hash=sha256:7b9044ce4acbf392d4b3a503fe377641f4deff82e6c341c36ef27af0dea76cdf \ + --hash=sha256:7c89359affecebe2e39e6a116d069b363c936511a9572b308402489a26957d89 \ + --hash=sha256:84ed3a2d5cd3ae6db87af20bfed3331116195ba4757ad7177fc8f12c1bbce2a9 \ + --hash=sha256:89a0980d49683c00539c59cd9f46b1908c538e6b5b0a48ad12187bb856d0f391 \ + --hash=sha256:8b72c2002202038666bf647f9a790906214c7c11cd0d6efef77b7d07bef3034a \ + --hash=sha256:8eb3e5caca8d3a6f72ea4cce024583f989f6f2e9186f98800213fff0176e8bcc \ + --hash=sha256:9082e709014946b1f6e96ae6ecd93652efca2d2a6a3ab67dbe151c8b4bf193a4 \ + --hash=sha256:9205068badf453d2f0821fd5d340389b4679992d7ff79d4f3e5608996dd1b287 \ + --hash=sha256:9472a8de37faf8bd216c628e0e68c8f6bef730d3ba0a5060f3b0fa460c992ac2 \ + --hash=sha256:967955620df45e6cffe2e9950cb9903cb455649396f896b26b04363a91a5054b \ + --hash=sha256:975c17da26c5b9d46c336b03c52a057ac28378d6f9d98b58d32a038589bb3912 \ + --hash=sha256:981cc10163988defea96e8d6c507df151eab8f483e7df9ae543d5a41a4be073b \ + --hash=sha256:98c88d3edd08e7245562e9815996dbc6f0bd4745e1c76462f24af5ae4e187dd1 \ + --hash=sha256:9a91c4814c7150a4d798da691b7804eacd78c4b84fb392a60fa0de21341861eb \ + --hash=sha256:9e311f908d2f842fca4c7dcebc4f10306b8089b204ef04cf6704b4332c9ff6ff \ + --hash=sha256:9e753e81457241e2200c56a898e268e8fa25796271af0489c608f24d8e631eed \ + --hash=sha256:a46bedc273b6f58f11dee816ff74999625ef8d007890f411b7a4975bf1c89330 \ + --hash=sha256:abb5667a36119019b3fa320c4d10c21ebccfcc87c8a739e6a0056cee7f48dde2 \ + --hash=sha256:b3131a82d0c7611f0aa480a6d36929e001a3f54ba0fc029a8118a5863cce513c \ + --hash=sha256:b5d11cccba076a32321ef1380dea956821f0b51794ef59df64e58fb1cd543aae \ + --hash=sha256:b6c5d2d71e1f17329150ad9427d27f4a3f29a01792e7ecdc64a98ac5368fc4d5 \ + --hash=sha256:b776c7fc8755c7de06dd5a22b47c40ae84f67d13277ebb233cc84933ba4dcbcd \ + --hash=sha256:c00d182e31034250690f417b9068b78eab423c10d76766664e82d9860c340479 \ + --hash=sha256:c351737e2e65497c7200ab4ffb8af97e9f48be6488309abdd265fe08d66ee92f \ + --hash=sha256:c4f845166b09acc65c5213a35551a7f81c17fa010ab467229b5813f79d17fe13 \ + --hash=sha256:c589f5023d471ce75dd2cce61acb25ed6347e562041588a1a366808f22d7176c \ + --hash=sha256:cee808b405e9095506f4e4e89924bec7ea77eac3129b6fe36eda04364b3b343b \ + --hash=sha256:d11a7bc1e02da8984d32e6de9e0826c6edac00eac17de270f372bf32f9a0af63 \ + --hash=sha256:d27c531edb8d1f38ca2eddaa1fa24913a460aeb721f2efd4ef42a124ce94e354 \ + --hash=sha256:d2d9a63a9e6f2416ace8c109043a9280d6b34f34bb2e5421903e149403db40a6 \ + --hash=sha256:d561a4c5747a1e6c7fa7c49a0292e78b4e8c456332caa084fc7abad8de828652 \ + --hash=sha256:d63010803d7c368963bbe6f7ec379593e76dd581d7db0f29118d88713c9e0354 \ + --hash=sha256:dd741c73440b328f937dc53b344ecadc46bc4f0cec0333a8f42b55f3468ce7ec \ + --hash=sha256:de1064663aa7c839286488a319d2b3b478ca5ab5b2091ade888ed0eeca11a98a \ + --hash=sha256:e252db239eb41c32248e096e0d170bce5896a4fd3405556362bc3dd83d912206 \ + --hash=sha256:e288a06cbbbcd01b44386e767985c9e21d2ad9bf59829aa7058d9a2a494804ab \ + --hash=sha256:e59b60a0a4cb7541480e02090d37dc2df3b72df4c2e776fff64ce3a4e3dd4637 \ + --hash=sha256:e671b2322ef09106ecb1ca0f4c398b134d5e2c1f80d7a4f3336847a3072c0e94 \ + --hash=sha256:e7b04935a79c03c41ad08d0a5f390aac968bfb561f1268897bc5b0f077971efd \ + --hash=sha256:f7e9b8728ba07a3cb2f29d5aa1a266c2664eb8ef0fd43afa34627c92f7fac8f0 \ + --hash=sha256:f9b093cb3b6c9d6233ef45a05cab064d2aa0a8cb3c5777084c9e20fcb77c2371 \ + --hash=sha256:fae8b282f0cb22a5de222999f7723f4e5ec04f6fcdf4aaef879b5b36625ae2b0 + # via + # langchain-core + # langsmith uvicorn==0.52.1 \ --hash=sha256:112ec661814189acbccd3f7b86460147cc065fc92c0821afa78918780e4354dd \ --hash=sha256:e4403f9d93188cf9d1088e9f40e3acd12630e2df8675316704379a7fc20fff6a @@ -7055,6 +7215,222 @@ xlsxwriter==3.2.9 \ --hash=sha256:254b1c37a368c444eac6e2f867405cc9e461b0ed97a3233b2ac1e574efb4140c \ --hash=sha256:9a5db42bc5dff014806c58a20b9eae7322a134abb6fce3c92c181bfb275ec5b3 # via python-pptx +xxhash==4.0.1 \ + --hash=sha256:0163b5d259de23ae9e07b7eabf435ce4704f6f205589a2b154e6af4be985ce1b \ + --hash=sha256:03600a8987849b2bef7be795a60a6052b635c63fa98b718b08ca5ee823691cfc \ + --hash=sha256:04f9a24de11a6647666d5302fd73d6a5224ce50ddc965fb0bb44cee736e6bd7c \ + --hash=sha256:06713a5aaf1d0905c5579416c020c02e42b3ceb931e86c7d3b7fb85403dee3f3 \ + --hash=sha256:06d7fbd609503c3be5e65cdb6bb2f040d6a98574404e2e1d5c60815c97fff4aa \ + --hash=sha256:0718ad66f4ded2411f8e62bdba549ee71e313a2d26ef5060ca3fdbf29897dd3c \ + --hash=sha256:08ed8da18cd4fd0a6a5d6a444852d8fbd0e565388a74a4937085451b5f1a312a \ + --hash=sha256:09f9feb118966cc6650e1806205d577eae7ca394aa6acf349a0b62a94bbeb329 \ + --hash=sha256:0ab851b45c70d4992be7cdeeee16f97a0b677408c758c4b1efb1cfe8030bfd37 \ + --hash=sha256:0b1082fd0f089ce9098ed77aad8b777b5d156f8ac601c69cab73811822b8ef07 \ + --hash=sha256:0b20a06454b34f1531fc677c54efe2ecdec691ef9224f7fa919bf2c1363f7ff1 \ + --hash=sha256:0b42a5a26607e4b2409fea174773a66f2dff9dfdbf2c1a851bb7b804e2c97535 \ + --hash=sha256:101aa300de6ceef3d9c77569706330d8921fc45dd82bceed2084f1e9f2557a24 \ + --hash=sha256:1216f7ba5683f17a89eb7dcb4bc50a0b743dfe1902278d7b3d0786f538118433 \ + --hash=sha256:1642907941ee4b75aacc3db688af52ea02ca2305ab22af7ee686ed726b332684 \ + --hash=sha256:168dd6b51725a222abc722832e56624d15a63fc2e8249021509c93f1063913f6 \ + --hash=sha256:1749f0688020209fe0d357ce1e1cd9ec9c6161ed0405ea949d24581c4c43fa91 \ + --hash=sha256:1b3cccf75eeb5b01639b2feadb042a8e07889293b7ca72fa2985e7dcb64763cf \ + --hash=sha256:1b50223d92df94d54e1a31469335a2c74b16692e6c1cb726f1e6949514458706 \ + --hash=sha256:1bc591533fc975614f7e13594daee76af96b8e1fbcf8de76c8773858fa9e7cea \ + --hash=sha256:1c2200b98a805351cb3142ae4e1fdcc9e91b5e20f5d30d4862b0b96f92558f4e \ + --hash=sha256:1c7c642a0f79c3e3cf2965475507574d3d1a50ec71060039d60cb87358667cb2 \ + --hash=sha256:1ee523f51718e41753f04f7102bb4dc55a18d2ea5cbaceef8ec7ca08571bd428 \ + --hash=sha256:1f3346c5c287ac3c7f38b20380f55e8768230e7252af59fabcf3b87ab21e4256 \ + --hash=sha256:2194bf96d5f3d4e0cb65deba370ec83dda3edfba42155f9384190ed5e51ea5e2 \ + --hash=sha256:237b8f63a2a0fcfb1ffc06e21dad23add44e6d354b2b014364a1d41e419a4dee \ + --hash=sha256:23a4376b4a3183cb50d4d2a3179f887a7773cc695eb2c908e551bec3221b8c60 \ + --hash=sha256:247ece770647c0aef080561fa996f9774b4dadce2d0c42eeb98229db7dcf820d \ + --hash=sha256:2696bbac613f6880fed60316c298bf3091d4f8eee3ae2e9466f70bb76204fb0c \ + --hash=sha256:26fe6238c2d5b11ed5063b9bf4eb290624b004fd074688da6bb079bd564f10d7 \ + --hash=sha256:2d52dc7c33c1b83082b707f6b7814dc76d2faaa2ea62bd9c5fab4b36f83c087f \ + --hash=sha256:2df3ca8757dc381e75e90a4d7995a6324f58a923c7145220a7b2c0231f66fddc \ + --hash=sha256:303121aab4b7f898058582d7962ea79d9e26e2379d7b6d8743f70f2671674481 \ + --hash=sha256:3088dadbffa33c29e0518578430a7dff2e901a212e487aefa5faaa0dc06dad34 \ + --hash=sha256:31d86f9e81f3e84e00131ac7c54caf5119ae4ddd82c09c31cff597c813ce1ee2 \ + --hash=sha256:3358097d333d40657569ec1121e21043dd7d0efa10aead1b50e8b4fa83077d7b \ + --hash=sha256:33e270d302c95ec426dfa0f5a4e16bff2ab8d7b8a46faa4746affb05e684ac77 \ + --hash=sha256:33fd538191f47071deef6b1f676535e2aa770f1fd150ae4cc75a34c9e930be3d \ + --hash=sha256:348c8f288dc961d6bbd1985c8152a3ed7a85c95df00e82320f0c5215d922a399 \ + --hash=sha256:349775ac30372b344d2338b2a168c0a1312a644194da25b8bec476d55761a128 \ + --hash=sha256:34ed93e20bfd98d722b902121643791eeb4b1641871e2dc63d0d4c2d93f187df \ + --hash=sha256:37f667dee0f867c42894b34e2a6fe26bf195c0ea4683d9d2b713db023f242c3a \ + --hash=sha256:3891efe3d7a531ce6da0a4a50a99dd41c75b8fd4ca19d73c86431b4db5c305f0 \ + --hash=sha256:38c3d22129a6958846a3098d68bc8e661704461c0be4793ae28836e4690c8478 \ + --hash=sha256:3c2445edafc300cc40feb6a25a8356a971c30cd0bf47b5349c2ad74c508343b1 \ + --hash=sha256:3f68fe400ceec235f3e4a4b02a28c2fd2d283584a193223c921dd4c48f1d0754 \ + --hash=sha256:3fb1d30d4b6d6e2c4a08e5ac6fffdb2b572d2cfcca15a5509cf4e7a1350f955c \ + --hash=sha256:41e579025a6e13a99e6d71e39c9cfc621a0dcdbbf19106325e145fa858f2d794 \ + --hash=sha256:421b94f3ba7067958d02e38960d987756347aa150df06df11aa68ae1af78c619 \ + --hash=sha256:427b62d62d4f967fbb10b82a3813e4875c2a6e7e7634739f17265b650c7f65a6 \ + --hash=sha256:436e11b4dd966afe5f7f665e4cc4c5485ffe3ceb42f25a22e1701d236abf1853 \ + --hash=sha256:43bcf2a871f28f16135545415cab3ec43904d4c80425a64598a9e6cebfb2b5ba \ + --hash=sha256:43e5f9169e73d0f0db33b5f6b8554bcce69ac278c966daf83d5eb4eb2f13829f \ + --hash=sha256:440c401e146ce64bdb3beb8ff0c84677b6f21307c28a34779071cecee5d4d70c \ + --hash=sha256:44ab12e8cd17d4f001769f00ad465208b4bcb897ed29e65f058f74466b57a98f \ + --hash=sha256:4528cf80ebbbf57d40edfb31521ae265daa6dd636d615b1cf0ac86209579e59d \ + --hash=sha256:45e88111ebe331de478ef8d4293efbe88f3cf8b863386c9a2357136b838e1af0 \ + --hash=sha256:4741d42d59e4e5fa1a86c17ab9c27dc8ea459c700d91b6742fdb9138d9a516cb \ + --hash=sha256:4751f1d7eecae6b2d2a773630f1a7248f125c9a92a456694d03c15bceffc9d68 \ + --hash=sha256:488ca5c5e28ef56ec4bbb12f835b3f1cbecc5f3510062e70117bc6594851932a \ + --hash=sha256:4972332c079d6aad69c4620a68d015a4ecb33141583f70d642cf9edf6a713763 \ + --hash=sha256:4a252fb862b0ae2590587e625f47a0e03da05cf0205e8830b67b6596c06038b1 \ + --hash=sha256:4a76345f5aceb4ec404918edf9c7f2b5507db864dc0d7455982009ac0890b57b \ + --hash=sha256:4af350bc3f329970c0e3a59af84a8a30998bf8a9167eb50cd48e59baaa1d7bec \ + --hash=sha256:4bbf3ff651e0f1a19beb5d0f48e0874a9bad2482a588c9d214c96ef1fff1cd9c \ + --hash=sha256:4e5141543c7f7fe3087500bbb4ac2845cb528a980aa91f8f1e661e2292ff4a5d \ + --hash=sha256:4f5e5c6df4b703afcbe9352d238a51efd97c3b91fdc3a2052e40fdacb1e7505f \ + --hash=sha256:515a822c73abbf6a0b7c70976d9662be342835c9d78b8dc7c023411f39c35dbc \ + --hash=sha256:554f87034635bcec47c5d72447bf3db7e02da1bf493a0ada010db28a76f891c6 \ + --hash=sha256:567cbc630302a46a8ecfd943b309ccf5372bb3718f1f3762d452df30f033bcf0 \ + --hash=sha256:57d7fa8f23908d173001c21a9e82bfc6ad997d1b6c270fb121812b7ed158891c \ + --hash=sha256:5adf927dca8c47fde7e683fe69efdd81bc865c4db1fb6bb00b391e2b6185207b \ + --hash=sha256:5b7875ac1a2edcb691f27642b8b94b904baa6bcecb7d79c72df2228ba8cb5c51 \ + --hash=sha256:5b7979f71d06ae45a769de0699900a246d8cb632db1e8bfdc79ec019063a503c \ + --hash=sha256:5c2d525a3afabcd8e3549d85fc7e111fde6bc302d06a1893fe73adb79823415e \ + --hash=sha256:5dc434c946012e6d8a72b10f970ea30755b718251dd7591dbfdabafd3bcb21bc \ + --hash=sha256:5f1ea31d61bcd2cd2f3ec4ca80a64187bbd7948f490b63cf0dcbc6e717b4c1e9 \ + --hash=sha256:62198213fc3e0c56e567894b318ba45834e007d065f84ba6dc9165d21546fc56 \ + --hash=sha256:63aa52659bc32bb9bd7cb5caf523b4d14429a477762cfac886132d687c1f80fc \ + --hash=sha256:649f2682c090cca1ac4037866381f3652eaacbd56e5178030f4ce1325b8f945b \ + --hash=sha256:67e57b834e07ed973cee7b6da1548ff28a56458d77696fd2a5f397f340694848 \ + --hash=sha256:684160b3c0a9b62c6f0de90f44e11dc5d8643dcfa18a5856b45fb1c47478bb71 \ + --hash=sha256:6a8c5ce76b94ba49f3be8a8f2611abc6564210702c72ac9e237ca2bebfd17794 \ + --hash=sha256:6a9f98af872355e0c02439e48583958eee00e60b928bb20476460d9d40cb7b4e \ + --hash=sha256:6c45258a37fc22721395c09927cb982d3e7a83607cab15be7e2416501bd3a330 \ + --hash=sha256:6cbf4e21ef0890804b5bb9ad25c48f9c127758d7f6c66bef374efcacc63c738a \ + --hash=sha256:6cf633df84d80a1668fcf61e330791dae46825e395549e7d34f376411e75088a \ + --hash=sha256:6efb8f21cc136c79b3e5bb747c8682d37916fb202cdbbc32182de5c4e47f821f \ + --hash=sha256:70129ebb8f20e1ac1da58b78ed381624bd689a43a9a7366560bd8fabea145105 \ + --hash=sha256:704381264b36a18b9c62ecbabe2e71d0fc58c77c129c15355c989b10bf05b6b0 \ + --hash=sha256:7236be540d6be9ce448d98b940dd26ddf70ca41012e8a14a53fd9354cefe4e8d \ + --hash=sha256:72f34834518157a75e7090f328ee7a16c70c804cfc7c694fa069cc888e9fc03e \ + --hash=sha256:74379a577a9f3b6afbdedf1b90e5c7764467051977f18a326d7d607336d743bd \ + --hash=sha256:74a164e8b63f1e9cf35c9a7809d082b033d1a00e7375d5d814415436e7867e57 \ + --hash=sha256:760de77279e9cf9c81d012ce0705cba13afccee9b09c480f17d778c8c5cefae8 \ + --hash=sha256:764b32d52d15b8b95ac8160e540772fa1adeb611fe40bffaeb42e7bf98279e44 \ + --hash=sha256:79a3203aadf39637869dfea1185227d8452844d78b837e54fb1117b4d34ba5c3 \ + --hash=sha256:7c343ee174d417a44d0c3355602c0cbbfa52a04d1bbbf1723378c7d2c8f60626 \ + --hash=sha256:7e27dbed5c4ba033919e4b4ed8dc14e029e91d14a93cd9f920d25277c7df6781 \ + --hash=sha256:81507a68ba84c55241fb61cce1469f473a5da4205fc8ef6f698e5948eea8dd88 \ + --hash=sha256:81664268dba92e037b740ecf37fa02f1cab4a391f93f28e35792b3341c60648f \ + --hash=sha256:839f58c5bd9989875be0fd28446dbf32cace2c2cd8bf2f6762acdc38a95cd1aa \ + --hash=sha256:83b8c2013edb5dc1f9e7268b6496130705bc48d79c86bb8817b3d210b81a5513 \ + --hash=sha256:84df5f8da574caadbc0cb1b8866ecc2368cc941f0cd05f677756c802f370dafa \ + --hash=sha256:8580aab306888224074c7edeec734de0c3c5ccde65b2da4e6c9a5e28f7c0a1bd \ + --hash=sha256:85bdd40cb505a11e0ca04191711266c5fd696ed786ae83849955e457774edc96 \ + --hash=sha256:85e402dab0f9acd3604539747c6fcc57dc188a18af6ab07eb8189351cd32466c \ + --hash=sha256:863f3d3b44110f7243e86cf994aa5c5d88f2348b6e84ab4402fadadfbf9f7da7 \ + --hash=sha256:86b2b12bec60c678ed8f5cca0258ad93a8928ebddb6ca7732f0875afe1451d1a \ + --hash=sha256:87aa309a93bd5ec13f14309a305ff4e9bf74c5363fc46c264c0a22edfd5b0670 \ + --hash=sha256:87cbdec1a7dd930079671a60b249f3ca4e773e6fbd0676e21e36fdc9dd0f3b00 \ + --hash=sha256:87da13df72c5612771cd905a8b121e0bfea62d7659b1c92198736eb722220e83 \ + --hash=sha256:88d87719fe6bddf117238b341c5db851f8e96ba68ad9832b450e4a43dc60b37f \ + --hash=sha256:8b4477edc03091f51f5309406d230851c23cf4822029e3bf40b8df53093fff1c \ + --hash=sha256:8b99ebaf9e816ac5069423b1367ee7e8078fbcebcf62545506bb0608d2f4f468 \ + --hash=sha256:8ba782ca3bf1e81492611152b9a0d5264971339e95e34d69de0ac2c926be496d \ + --hash=sha256:8bcba9456242ebf180a04d9443812fd85ffe6bd12bda464dd116fcece8886ff3 \ + --hash=sha256:8c9fe122444e129881afd1d4d1c7ac0d3ce2d91b68c2b40173b6025ff1c31f9a \ + --hash=sha256:8ec4777d92fd61a5c8fdeddab894fd65bea301a8092fb5419ec6472aa4d458d7 \ + --hash=sha256:90cb2a1c9cc503a054a19612b48ff6e8e47805f618bdb3224a07568aad03a37e \ + --hash=sha256:9283d9dd6b44acad35118e2976fc763a065509e4118debdb61916ec322ed17b9 \ + --hash=sha256:94ac8a6b8c47951173f0b67bf862bcb971bf24e493b9fbbdb0e010cbbc7d9f54 \ + --hash=sha256:96d8de55029d42251945531f6aa7590c32b48163c66a43bf29d8657d7446a377 \ + --hash=sha256:96dedccfb09a73a25751053a183159b88f4ee75f388df8166040c152ac0531c6 \ + --hash=sha256:9761ff4a0ffa583fe850731ad24fe82c88cccb7a2294727db0955f3279a4cb3f \ + --hash=sha256:97b455de3e8b1b0b1e4594cb61a468992563f03ca264062fbb0a66b393c01d90 \ + --hash=sha256:97b94fb29abf21f5f0bde15f7dbdd3a4aa2dc59f37026adc7b4bee8563b84375 \ + --hash=sha256:99054b838b74d8d3995ea0d410976ae967c46207ae22d6ddfc535e809197dab9 \ + --hash=sha256:99166cc98637e8bf550cda2aab07f4f1d5f899c45fbd721801aeabcc9d404824 \ + --hash=sha256:9a51b061d54cda8b83e62c44458bfbf0dabbef9b975dd9649952ba5076b9f349 \ + --hash=sha256:9b1dddc257279417d93c9e59420d49ef90aece90d7a01996db3aade74b0281b1 \ + --hash=sha256:9c3c4b9aa9a27196b921197f7daf9e6c1412739df06a99cfa6e923879362eff6 \ + --hash=sha256:a14578102a6081465aec9cf73c76c3cd3f79f0709bdb3b8ae7ab0b54c9d8b089 \ + --hash=sha256:a16a3fa6936e36bb1414d16a6bd012c9033e5161b68b426805b61d895392437d \ + --hash=sha256:a33de7633c948ab2dc144af370a66e7e7af29b425dcd0f7e4f59689fb9391b53 \ + --hash=sha256:a43418e1a90b4809a9caf64aeb8b0696e3e1f300a323acc1e6ee2f93ae319fcf \ + --hash=sha256:a4553d36cc0b7fce1f35ba8a94dfd775aa3ed12f5eab2dc3b46ac75a0706b0bb \ + --hash=sha256:a5b21b42a01a343096a1c018d35e9b7aec9c7065dda53ae8da071e37478b2cea \ + --hash=sha256:a65785e653573fcd1e33062760ab4c3c3440e8e910765018e4b6ed4ad07b54a0 \ + --hash=sha256:a6671a8f6ea4f2101ce11fab5023a2e59391cff249fc3928cecb69d971525fd5 \ + --hash=sha256:a69e8946e4902ea11fc1c557740cdbfe7d75c78fcc5e4324ff89a696a634357d \ + --hash=sha256:a6e3653df1a70b8ac4191216324242e4be2bca18c9a7c10934e1bd56dc7ca15e \ + --hash=sha256:a865d2d470220e659220fdb59d5b6c4422802d8d6098e1324bc4d12444798914 \ + --hash=sha256:a949b072ea59c6eca0811ccd9e95133cc50d2afda8d464b5b077c78f78efa269 \ + --hash=sha256:aa6ccc7f31018484d652cf52db020003433f3c9fa83189c028bd807d2adde503 \ + --hash=sha256:ac0f291ab6485bd71f33941f9b92771318332a05d505460b41e893a549caadc0 \ + --hash=sha256:acb31ecdd1a97fab5cd39a84ee9f515e727d319f796fec48703b8339b9998360 \ + --hash=sha256:acf52474b2494ef66dc7e0fb6d5e2b50c18313039ad4d275fbf9f9907c804bc5 \ + --hash=sha256:ad889d58361a26ba75f5d6a1a0da08ed4950ec4ac8a6da86e1c5ce1b95ccb43f \ + --hash=sha256:adbd48b30e3f82c89fb2b3e6a87cdd28d113b190a5ed0ee2dee286323ee9a621 \ + --hash=sha256:af05a3f650220a6c59fa0ad2410249f2d2470a05225807c378fb67458693f8df \ + --hash=sha256:b3662719007e059abde7eddacf8517142ba076ddc7b30c807260e57d28c3c191 \ + --hash=sha256:b3bece52127ac20044311ee73567f9f0893b5de64f9028aecc90cc740cfd525a \ + --hash=sha256:b4c8842fb19d78b5e8c2a52baf4c8357658cc56c62bc822b86ce0f942f28e286 \ + --hash=sha256:b659fad79c99b0238c7ad7e9d7dbf4eebfea9097c2dba65fa0a4d18a25b29a2f \ + --hash=sha256:b6c1f9c59bbe593f88a0aad30be4150f15bd57bd64efb95feeabcb8e563f1ecd \ + --hash=sha256:bdd16718b63aa3ebd68aabb79021a40e47c81374852d41a306b9453141bbcbee \ + --hash=sha256:bf430c587f447a554c53768ad76b9846fe7c5632180ef6f69c4fce8b0552fbd0 \ + --hash=sha256:bfed61996d618eb90d6eaae0178002e3466a28b06bfc557a7a3a7266378d8c5a \ + --hash=sha256:c09ada495567c9c9a8156c5ebcfb93be7fece0755062d738c972dcbecd0d84b5 \ + --hash=sha256:c0e6ccc2b19ec8a726b2e26062ac71ea63e15500d6bf85910e42481844fdffc1 \ + --hash=sha256:c101180495cb4ba3617b279a944345c53a5e73b0c150053d1fa8d8af32de9579 \ + --hash=sha256:c10b9206753b64aa791b35b201485477525b26fdec5bf86e8364c388a03e2592 \ + --hash=sha256:c3074db513c81f764053e3da079312ecf85a50d8350c71f4cc0105d9662a9e6c \ + --hash=sha256:c30dd1af66a820820398b26e0d74e7a9aa43cae705924f23ed828cd8e5c26c3d \ + --hash=sha256:c57963970d359a72262f7fe6be88f945e2334d4bc41462b7f08c37b0abf35ca6 \ + --hash=sha256:c6301d92545c591ad31c3e050aa40a5f8a4c16413f1f9e6f9322c6f0f9d2b736 \ + --hash=sha256:c6370189e8e66b7e608f533b939a9de092ddca6cce084ca0d3d414d2ed5b5d59 \ + --hash=sha256:c6fc415b5568bd9accc7187f1729a99707330c0a67a8b9f93c1149ed573ed75d \ + --hash=sha256:c7484fea54964edd417cc3a104d5180562514aa7c4e2a2bc26d776ef0c4cb4a1 \ + --hash=sha256:cba763d84b06bda2c38d5185dee76f1b9dfdc0789e96e476d9e10005526d0788 \ + --hash=sha256:cd878d32f5c6cbce9783f8d6897561fb772211edba9dde49d85672b88ed45276 \ + --hash=sha256:ce6d5cc94a50291d080259a126cbf1e9ba4ac861e6429d2f3cdbb1474f51945d \ + --hash=sha256:d0d24a4f3fb63852cd09af46ae4b7a4d00cc8b8615a046dca543786e728d1056 \ + --hash=sha256:d1e0d1ea6e44f51808a9e8469c8afdebcdf6fa23d1ea524a0303d57d23919712 \ + --hash=sha256:d54b8ae068af532c8cdf56abb9e09a60fbe7b10792444c9c27987bb6d3b450fa \ + --hash=sha256:d55bf4ef10eb09b8b6866790e083d26d087d84caa3cc0946ba87c3ca7ecaf7b7 \ + --hash=sha256:d9f3848ffaf010bdbabdbf4c25641fa258b6227ff27bc74a4d06edef521a4873 \ + --hash=sha256:da0264844a09b538c894e5eff25313d941deb4dedec2131b98418a71a3c9944e \ + --hash=sha256:da544672efd9ad76077928a3e6c5d894e52ce82d3bf14002db4a1bf17d1a36a2 \ + --hash=sha256:daade8936c4deaaf7b01561324ce438ba4f885d717e9adc62b4d67212ad7d7bd \ + --hash=sha256:dd649663ddeafbfd4734eb8abae921dd5baa1242f20bda54e8bc927369ccded4 \ + --hash=sha256:deca2a30d983d240b8375ec2ee0a4288e72042827fc61df2f7671f8467e4cb2f \ + --hash=sha256:e259bb7e1e2d8de6b35f430f5c7220b1c0ebf3962d1ba7ec7545980d5931edb8 \ + --hash=sha256:e3996ff9b6f99180357024336bf5749a8ad6476a9a2523e535c5212b995b12a2 \ + --hash=sha256:e3eba72f9bb84fe696516f4cbca68d3d74a376157e68bacddbb7f2516af61523 \ + --hash=sha256:e4296fcc790876a8b0f297edc83d3b088457b774d8f67b4636807f8a2ec69a79 \ + --hash=sha256:e53926e76131a74e79cc0b39fa712c227875f180afc68646bd1e1d8a17e60313 \ + --hash=sha256:e681a6fc7e4f715252b9b5acfb30536ec7dd1f75033a32dc617e6fa95af1a3fd \ + --hash=sha256:e71b34978e77868cbf2d18c5206a4603f9c644dd7181bec5643bd40141d3b8c5 \ + --hash=sha256:e8cda075b10bb3917b002c74a04f9e02b7d13b5bf732571404d51c52b11c7329 \ + --hash=sha256:e90b4bcf1d9eb1010fdaee7c9209fb667e74c0684f3ba17f9032bd7319da90c9 \ + --hash=sha256:e961093277ff9d42addb9dad5614dfb7800ccba07c245c39c8e9b4daa35d160c \ + --hash=sha256:e9701c073bd062fb6bf6be51b47186ad15f1e87feedf4ea07198e0333ec068dc \ + --hash=sha256:e998cb3685b92101ec5de0fb4d9485cf01e50bc418211955c55d98064664cf4c \ + --hash=sha256:ea5ecf800b45bdb34afe05a1d0dae1f8ea02a290e50636dccd399063f6b180f8 \ + --hash=sha256:ec1a470c6db94ac4589c203921e89ac1bc13e796a8b1784d8135e1893559cd3b \ + --hash=sha256:edccc2ec58435a580f96a48a3ccae8cd0a480824119165dd90108718ad81ae6e \ + --hash=sha256:f00330ac7e24769e2032203f2b01794d670916b0c1799fd261340f1af9499875 \ + --hash=sha256:f09ee747e2a5f876cc5ad56947734811828335e13b403dd8ea1e06d77a9dd48d \ + --hash=sha256:f18732adcc271741bd651c3e56fa519d8a237d2cccda01fe3afb226bf87f783b \ + --hash=sha256:f1b603d0686c99fa0879f104a74e7db58367634c6e50ba827bee9aa095e23205 \ + --hash=sha256:f33cf0baa91eccd2cb7b62bf00f10c2264ef578b71dd33a12962e71a36eb4d32 \ + --hash=sha256:f3e1a44af01b6692de0ec6caba5f0bf93ceb36896e02b7fc00952c6ea7ef39e1 \ + --hash=sha256:f484ed57bb3e4142f9d6439568658c38be5f94b702ba00a1ff32c69783b6c66d \ + --hash=sha256:f5d031f35962e5483a613214e61f09fe24ab523062c3646d592dc16c4a217451 \ + --hash=sha256:f6247f5e23ee94f2557ac9dab738a336f607c6ff476fcf66ca70c3aef5eee15a \ + --hash=sha256:f7db035447a0ac8959aa230c5d36545ecf9f547413eb1711c0ca6f0ba1418925 \ + --hash=sha256:f83295394d34e1287e5b30fcc496c13b92cf886a131f3dae5444e38da8757efb \ + --hash=sha256:fac4832b638000106207bc44e44b9616a6a416aaee56c62b01d61f3705e49f58 \ + --hash=sha256:fb59a0dd61fb2ad481c03fda399d78ce57dab6bb62c2c8fdb446a7ba4754b89a \ + --hash=sha256:fc737c05ca2d48e5dcdbbb249314df3fc6c2a0be6da8b0aa28e13d72afaad7cd \ + --hash=sha256:ff48915bf1871a1f19f74c11834c6329443d306cedc0c05fe7fe617810422a80 \ + --hash=sha256:ffa44b4c7c5d0ffa31356b4428659516c0e47647825c74079a296b3857b6d99d + # via langsmith yarl==1.24.5 \ --hash=sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36 \ --hash=sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331 \ @@ -7165,3 +7541,104 @@ zipp==4.1.0 \ --hash=sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f \ --hash=sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602 # via importlib-metadata +zstandard==0.25.0 \ + --hash=sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64 \ + --hash=sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a \ + --hash=sha256:05353cef599a7b0b98baca9b068dd36810c3ef0f42bf282583f438caf6ddcee3 \ + --hash=sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f \ + --hash=sha256:06acb75eebeedb77b69048031282737717a63e71e4ae3f77cc0c3b9508320df6 \ + --hash=sha256:07b527a69c1e1c8b5ab1ab14e2afe0675614a09182213f21a0717b62027b5936 \ + --hash=sha256:0bbc9a0c65ce0eea3c34a691e3c4b6889f5f3909ba4822ab385fab9057099431 \ + --hash=sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250 \ + --hash=sha256:106281ae350e494f4ac8a80470e66d1fe27e497052c8d9c3b95dc4cf1ade81aa \ + --hash=sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f \ + --hash=sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851 \ + --hash=sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3 \ + --hash=sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9 \ + --hash=sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6 \ + --hash=sha256:19796b39075201d51d5f5f790bf849221e58b48a39a5fc74837675d8bafc7362 \ + --hash=sha256:1cd5da4d8e8ee0e88be976c294db744773459d51bb32f707a0f166e5ad5c8649 \ + --hash=sha256:1f3689581a72eaba9131b1d9bdbfe520ccd169999219b41000ede2fca5c1bfdb \ + --hash=sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5 \ + --hash=sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439 \ + --hash=sha256:22a06c5df3751bb7dc67406f5374734ccee8ed37fc5981bf1ad7041831fa1137 \ + --hash=sha256:22a086cff1b6ceca18a8dd6096ec631e430e93a8e70a9ca5efa7561a00f826fa \ + --hash=sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd \ + --hash=sha256:25f8f3cd45087d089aef5ba3848cd9efe3ad41163d3400862fb42f81a3a46701 \ + --hash=sha256:2b6bd67528ee8b5c5f10255735abc21aa106931f0dbaf297c7be0c886353c3d0 \ + --hash=sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043 \ + --hash=sha256:3756b3e9da9b83da1796f8809dd57cb024f838b9eeafde28f3cb472012797ac1 \ + --hash=sha256:37daddd452c0ffb65da00620afb8e17abd4adaae6ce6310702841760c2c26860 \ + --hash=sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611 \ + --hash=sha256:3b870ce5a02d4b22286cf4944c628e0f0881b11b3f14667c1d62185a99e04f53 \ + --hash=sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b \ + --hash=sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088 \ + --hash=sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e \ + --hash=sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa \ + --hash=sha256:4b14abacf83dfb5c25eb4e4a79520de9e7e205f72c9ee7702f91233ae57d33a2 \ + --hash=sha256:4b6d83057e713ff235a12e73916b6d356e3084fd3d14ced499d84240f3eecee0 \ + --hash=sha256:4d441506e9b372386a5271c64125f72d5df6d2a8e8a2a45a0ae09b03cb781ef7 \ + --hash=sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf \ + --hash=sha256:51526324f1b23229001eb3735bc8c94f9c578b1bd9e867a0a646a3b17109f388 \ + --hash=sha256:53e08b2445a6bc241261fea89d065536f00a581f02535f8122eba42db9375530 \ + --hash=sha256:53f94448fe5b10ee75d246497168e5825135d54325458c4bfffbaafabcc0a577 \ + --hash=sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902 \ + --hash=sha256:5f1ad7bf88535edcf30038f6919abe087f606f62c00a87d7e33e7fc57cb69fcc \ + --hash=sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98 \ + --hash=sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a \ + --hash=sha256:6c0e5a65158a7946e7a7affa6418878ef97ab66636f13353b8502d7ea03c8097 \ + --hash=sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea \ + --hash=sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09 \ + --hash=sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb \ + --hash=sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7 \ + --hash=sha256:75ffc32a569fb049499e63ce68c743155477610532da1eb38e7f24bf7cd29e74 \ + --hash=sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b \ + --hash=sha256:78228d8a6a1c177a96b94f7e2e8d012c55f9c760761980da16ae7546a15a8e9b \ + --hash=sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b \ + --hash=sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91 \ + --hash=sha256:81dad8d145d8fd981b2962b686b2241d3a1ea07733e76a2f15435dfb7fb60150 \ + --hash=sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049 \ + --hash=sha256:89c4b48479a43f820b749df49cd7ba2dbc2b1b78560ecb5ab52985574fd40b27 \ + --hash=sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a \ + --hash=sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00 \ + --hash=sha256:9174f4ed06f790a6869b41cba05b43eeb9a35f8993c4422ab853b705e8112bbd \ + --hash=sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072 \ + --hash=sha256:933b65d7680ea337180733cf9e87293cc5500cc0eb3fc8769f4d3c88d724ec5c \ + --hash=sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c \ + --hash=sha256:98750a309eb2f020da61e727de7d7ba3c57c97cf6213f6f6277bb7fb42a8e065 \ + --hash=sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512 \ + --hash=sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1 \ + --hash=sha256:a3f79487c687b1fc69f19e487cd949bf3aae653d181dfb5fde3bf6d18894706f \ + --hash=sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2 \ + --hash=sha256:a51ff14f8017338e2f2e5dab738ce1ec3b5a851f23b18c1ae1359b1eecbee6df \ + --hash=sha256:a5a419712cf88862a45a23def0ae063686db3d324cec7edbe40509d1a79a0aab \ + --hash=sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7 \ + --hash=sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b \ + --hash=sha256:ab85470ab54c2cb96e176f40342d9ed41e58ca5733be6a893b730e7af9c40550 \ + --hash=sha256:b9af1fe743828123e12b41dd8091eca1074d0c1569cc42e6e1eee98027f2bbd0 \ + --hash=sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea \ + --hash=sha256:bfd06b1c5584b657a2892a6014c2f4c20e0db0208c159148fa78c65f7e0b0277 \ + --hash=sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2 \ + --hash=sha256:c2ba942c94e0691467ab901fc51b6f2085ff48f2eea77b1a48240f011e8247c7 \ + --hash=sha256:c8e167d5adf59476fa3e37bee730890e389410c354771a62e3c076c86f9f7778 \ + --hash=sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859 \ + --hash=sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d \ + --hash=sha256:d8c56bb4e6c795fc77d74d8e8b80846e1fb8292fc0b5060cd8131d522974b751 \ + --hash=sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12 \ + --hash=sha256:daab68faadb847063d0c56f361a289c4f268706b598afbf9ad113cbe5c38b6b2 \ + --hash=sha256:e05ab82ea7753354bb054b92e2f288afb750e6b439ff6ca78af52939ebbc476d \ + --hash=sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0 \ + --hash=sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3 \ + --hash=sha256:e59fdc271772f6686e01e1b3b74537259800f57e24280be3f29c8a0deb1904dd \ + --hash=sha256:e7360eae90809efd19b886e59a09dad07da4ca9ba096752e61a2e03c8aca188e \ + --hash=sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f \ + --hash=sha256:ea9d54cc3d8064260114a0bbf3479fc4a98b21dffc89b3459edd506b69262f6e \ + --hash=sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94 \ + --hash=sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708 \ + --hash=sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313 \ + --hash=sha256:f5aeea11ded7320a84dcdd62a3d95b5186834224a9e55b92ccae35d21a8b63d4 \ + --hash=sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c \ + --hash=sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344 \ + --hash=sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551 \ + --hash=sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01 + # via langsmith diff --git a/semantica/change_management/change_management_usage.md b/semantica/change_management/change_management_usage.md index 0b5c161d..d7dde3ae 100644 --- a/semantica/change_management/change_management_usage.md +++ b/semantica/change_management/change_management_usage.md @@ -1039,6 +1039,6 @@ manager = TemporalVersionManager(storage_path="large_data.db") ## Support For questions or issues: -- GitHub Issues: https://github.com/Hawksight-AI/semantica/issues +- GitHub Issues: https://github.com/semantica-agi/semantica/issues - Documentation: https://semantica.readthedocs.io - Community: https://discord.gg/sV34vps5hH diff --git a/semantica/change_management/version_storage.py b/semantica/change_management/version_storage.py index c272622b..e9d75e2a 100644 --- a/semantica/change_management/version_storage.py +++ b/semantica/change_management/version_storage.py @@ -31,6 +31,7 @@ import hashlib import json import sqlite3 import threading +import warnings from abc import ABC, abstractmethod from datetime import datetime from pathlib import Path @@ -62,6 +63,12 @@ def create_graph_snapshot_record( """ Creates a standardized snapshot metadata record for a named graph. + .. deprecated:: + ``create_graph_snapshot_record()`` is deprecated and will be removed in + a future major version. It has no callers inside Semantica; build the + record inline and checksum it with + :func:`semantica.change_management.compute_checksum` instead. + Args: version_id: Unique identifier for this snapshot graph_uri: The underlying named graph URI in the triplet store @@ -69,6 +76,13 @@ def create_graph_snapshot_record( description: Purpose or context of the snapshot metadata: Additional tags or pipeline context """ + warnings.warn( + "create_graph_snapshot_record() is deprecated and will be removed in a " + "future major version. Build the snapshot record inline and use " + "semantica.change_management.compute_checksum() instead.", + DeprecationWarning, + stacklevel=2, + ) record = { "label": version_id, diff --git a/semantica/cli.py b/semantica/cli.py index 403971e0..4e6b8cd9 100644 --- a/semantica/cli.py +++ b/semantica/cli.py @@ -773,10 +773,22 @@ def changelog(cli_ctx: CLIContext, local_json: bool) -> None: _run_with_error_handling(_action) +class _DeepEmbeddingFailure(Exception): + """A deep-probe failure from doctor's embedding checks. + + Marks failures that happened AFTER the backend imported cleanly โ€” model + load, probe, or runtime problems โ€” so the check's hint can point at the + real remediation instead of `pip install`. + """ + + @main.command() @click.option("--json", "local_json", is_flag=True, default=False) +@click.option("--deep-embeddings", "deep_embeddings", is_flag=True, default=False, + help="Also instantiate the local embedding backends and embed a probe " + "text (catches backends that import cleanly but cannot load).") @click.pass_obj -def doctor(cli_ctx: CLIContext, local_json: bool) -> None: +def doctor(cli_ctx: CLIContext, local_json: bool, deep_embeddings: bool) -> None: """Run a health check on all Semantica components and backends.""" import importlib.metadata cli_ctx = _require_ctx(cli_ctx) @@ -787,6 +799,16 @@ def doctor(cli_ctx: CLIContext, local_json: bool) -> None: try: note = fn() return label, "ok", note, None + except _DeepEmbeddingFailure as exc: + # A deep-probe failure means the package IMPORTED fine: the pip + # hint would be the wrong remediation for what is actually a + # runtime/model-load problem (broken torch, failed model + # download, missing shared libs). + return label, "fail", str(exc), ( + "runtime/model-load failure โ€” reinstalling the package usually " + "does not help; check the warnings above (torch install, model " + "download, disk space)" + ) except Exception as exc: return label, "fail", str(exc), hint @@ -827,6 +849,50 @@ def doctor(cli_ctx: CLIContext, local_json: bool) -> None: return f"{backend} importable" checks.append(_check("Vector store", _vector, hint="pip install semantica[vectorstore-โ€ฆ]")) + # Embedding backends (#994): `doctor` used to report all green while + # every local embedding backend was non-functional โ€” import success + # says nothing about model loading. Default checks stay cheap + # (import + version); --deep-embeddings (or + # SEMANTICA_DOCTOR_DEEP_EMBEDDINGS=1) instantiates the backend through + # TextEmbedder and embeds a probe, which is the only level that + # catches a backend that imports cleanly but cannot actually load. + deep = deep_embeddings or os.environ.get("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", "").strip().lower() in ("1", "true", "yes", "on") + + def _embedding_backend(method: str) -> str: + if method == "sentence_transformers": + import sentence_transformers # noqa: F401 + note = f"importable ({importlib.metadata.version('sentence-transformers')})" + else: + import fastembed # noqa: F401 + note = f"importable ({importlib.metadata.version('fastembed')})" + if not deep: + return note + try: + from .embeddings import TextEmbedder + embedder = TextEmbedder(method=method) + if embedder.model is None and embedder.fastembed_model is None: + raise RuntimeError( + "model failed to load โ€” the hash fallback is active " + "(see warnings above); embedding quality is degraded" + ) + probe = embedder.embed_text("semantica doctor embedding probe") + except _DeepEmbeddingFailure: + raise + except Exception as exc: + raise _DeepEmbeddingFailure(str(exc)) from exc + return f"{note}; deep probe ok ({len(probe)}-dim)" + + checks.append(_check( + "Embeddings (sentence-transformers)", + lambda: _embedding_backend("sentence_transformers"), + hint="pip install sentence-transformers", + )) + checks.append(_check( + "Embeddings (fastembed)", + lambda: _embedding_backend("fastembed"), + hint="pip install fastembed", + )) + # LLM provider keys for provider, var in [("OpenAI", "OPENAI_API_KEY"), ("Anthropic", "ANTHROPIC_API_KEY"), ("Groq", "GROQ_API_KEY")]: @@ -1666,6 +1732,93 @@ def embed(ctx: click.Context) -> None: click.echo(ctx.get_help()) +def _json_default(obj) -> object: + """JSON serialiser that converts NumPy scalars/arrays to native Python types. + + Falls back to ``str()`` for everything else so the writer never crashes on + unexpected types (e.g. ``datetime``, custom domain objects). + """ + try: + import numpy as np # local import โ€” only needed when result contains numpy + if isinstance(obj, np.ndarray): + return obj.tolist() + if isinstance(obj, np.generic): + return obj.item() + except ImportError: + pass + return str(obj) + + +def _write_result_output(out_path: Path, result) -> None: + """Serialize a structured CLI result (dict or list) for ``--output``. + + Domain commands like ``deduplicate`` and ``ontology align`` produce dicts + and lists, not numeric matrices โ€” routing them through the embeddings + writer rejected their shapes and extensions (.csv is documented for + deduplicate). JSON-family formats serialize anything; CSV serializes a + list of dicts (or a single dict as one row). + + Accepted extensions: .json, .jsonl, .csv (no-extension and .txt are + rejected so the path reported to the caller always matches the file + actually created, consistent with every other --output in the CLI). + """ + import json as _json + + suffix = out_path.suffix.lower() + + # โ”€โ”€ JSON โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + if suffix == ".json": + with open(out_path, "w", encoding="utf-8") as fh: + _json.dump(result, fh, indent=2, default=_json_default) + return + + # โ”€โ”€ JSON Lines โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + # Every record must occupy exactly one line. Wrap a bare dict in a list + # so callers never need to know whether their result is singular or plural. + if suffix == ".jsonl": + items = result if isinstance(result, list) else [result] + with open(out_path, "w", encoding="utf-8") as fh: + for item in items: + fh.write(_json.dumps(item, default=_json_default) + "\n") + return + + # โ”€โ”€ CSV โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + if suffix == ".csv": + import pandas as pd + + rows = result if isinstance(result, list) else [result] + if not rows: + raise click.ClickException( + "No results to write โ€” output file not created." + ) + # Normalise numpy scalars/arrays to Python natives so to_csv() does + # not fall back to repr() strings for array-valued cells. + def _normalise(row): + if not isinstance(row, dict): + return row + out = {} + for k, v in row.items(): + try: + import numpy as np + if isinstance(v, np.ndarray): + v = v.tolist() + elif isinstance(v, np.generic): + v = v.item() + except ImportError: + pass + out[k] = v + return out + + pd.DataFrame([_normalise(r) for r in rows]).to_csv(out_path, index=False) + return + + # โ”€โ”€ unsupported โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + display = suffix if suffix else "(no extension)" + raise click.ClickException( + f"Unsupported output format '{display}'. Use .json, .jsonl, or .csv" + ) + + @embed.command("generate") @click.argument("input_path") @click.option("--model", @@ -1708,7 +1861,43 @@ def embed_generate(cli_ctx: CLIContext, input_path: str, model: str, except ImportError as exc: raise click.ClickException(f"Embeddings module not available: {exc}") from exc if output: - Path(output).write_text(json.dumps(result, default=str), encoding="utf-8") + output_path = Path(output) + suffix = output_path.suffix.lower() + try: + import numpy as np + import pandas as pd + arr = np.asarray(result) + if arr.ndim == 1: + arr = arr[np.newaxis, :] + if arr.ndim != 2: + raise click.ClickException( + f"embed generate --output expects a 1-D or 2-D array, " + f"got {arr.ndim}-D (shape {arr.shape})" + ) + rows = [list(row) for row in arr] + if suffix == ".parquet": + # Schema: single 'embedding' column (list[float] per row). + # embed index detects vector columns via + # isinstance(df[c].iloc[0], (list, np.ndarray)). + df = pd.DataFrame({"embedding": rows}) + df.to_parquet(output_path, index=False) + elif suffix in (".json", ".jsonl"): + df = pd.DataFrame({"embedding": rows}) + df.to_json( + output_path, + orient="records", + lines=(suffix == ".jsonl"), + ) + else: + raise click.ClickException( + f"Unsupported output format '{suffix}'. " + "Use .parquet, .json, or .jsonl" + ) + except ImportError as exc: + raise click.ClickException( + f"Missing dependency for --output: {exc}. " + "Install pyarrow with: pip install pyarrow" + ) from exc _ok(cli_ctx, f"Wrote {output}") elif _is_json(cli_ctx, local_json): _jecho(result if isinstance(result, dict) else {"status": "ok"}) @@ -1930,7 +2119,7 @@ def deduplicate( except ImportError as exc: raise click.ClickException(f"Deduplication module not available: {exc}") from exc if output: - Path(output).write_text(json.dumps(result, default=str), encoding="utf-8") + _write_result_output(Path(output), result) _ok(cli_ctx, f"Wrote {output}") elif _is_json(cli_ctx, local_json): _jecho(result if isinstance(result, (dict, list)) else {"result": str(result)}) @@ -3054,7 +3243,7 @@ def ontology_align(cli_ctx: CLIContext, source: str, target: str, strategy: str, except ImportError as exc: raise click.ClickException(f"Ontology module not available: {exc}") from exc if output: - Path(output).write_text(json.dumps(result, default=str), encoding="utf-8") + _write_result_output(Path(output), result) _ok(cli_ctx, f"Wrote {output}") elif _is_json(cli_ctx, local_json): _jecho(result if isinstance(result, dict) else {"alignments": str(result)}) diff --git a/semantica/context/_markdown_filesystem.py b/semantica/context/_markdown_filesystem.py new file mode 100644 index 00000000..70872f85 --- /dev/null +++ b/semantica/context/_markdown_filesystem.py @@ -0,0 +1,32 @@ +"""Filesystem safety helpers for human-editable Markdown persistence.""" + +import os +import stat +from pathlib import Path +from typing import Optional + + +def is_filesystem_link(path: Path) -> bool: + """Return whether *path* is a symlink, junction, or Windows reparse point.""" + if path.is_symlink(): + return True + + isjunction = getattr(os.path, "isjunction", None) + if isjunction is not None and isjunction(path): + return True + + try: + attributes = getattr(os.lstat(path), "st_file_attributes", 0) + except (FileNotFoundError, NotADirectoryError): + return False + + reparse_point = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + return bool(attributes & reparse_point) + + +def find_filesystem_link(path: Path) -> Optional[Path]: + """Return the first linked component in *path*, including its ancestors.""" + for candidate in (path, *path.parents): + if is_filesystem_link(candidate): + return candidate + return None diff --git a/semantica/context/agent_memory.py b/semantica/context/agent_memory.py index 38dd122c..def79ee3 100644 --- a/semantica/context/agent_memory.py +++ b/semantica/context/agent_memory.py @@ -77,6 +77,7 @@ import yaml from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker from ..utils.types import EntityDict, RelationshipDict +from ._markdown_filesystem import find_filesystem_link class _UniqueKeySafeLoader(yaml.SafeLoader): @@ -1742,9 +1743,10 @@ class AgentMemory: @staticmethod def _write_markdown_file(file_path: Path, document: str) -> None: """Atomically replace a Markdown file without following output symlinks.""" - if file_path.is_symlink(): + if find_filesystem_link(file_path) is not None: raise ValueError( - f"Refusing to overwrite Markdown symbolic link: {file_path}" + "Refusing to overwrite Markdown symbolic link or junction: " + f"{file_path}" ) temporary_path = None @@ -1867,7 +1869,8 @@ class AgentMemory: if "\n" not in data and "\r" not in data: candidate = Path(data) try: - candidate_exists = candidate.exists() + candidate_is_link = find_filesystem_link(candidate) is not None + candidate_exists = candidate_is_link or candidate.exists() except OSError as exc: error_message = ( "Failed to inspect possible Markdown import " @@ -1909,62 +1912,71 @@ class AgentMemory: return memories def _read_markdown_file_content(self, file_path: Path) -> str: - if file_path.is_symlink(): - raise ValueError(f"Symlink Markdown import paths are rejected: {file_path}") + if find_filesystem_link(file_path) is not None: + raise ValueError( + "Symlink Markdown import paths are rejected; symbolic links and " + f"junctions are unsafe: {file_path}" + ) flags = os.O_RDONLY - if hasattr(os, "O_NOFOLLOW"): - # On POSIX, O_NOFOLLOW makes os.open() fail with ELOOP if the - # final path component is a symlink, atomically closing the TOCTOU - # window between the is_symlink() check above and the open call. - # On Windows, O_NOFOLLOW is not available; the is_symlink() pre-check - # above is the only symlink defense and remains vulnerable to a narrow - # race. The fstat()/S_ISREG guard below still rejects special files - # (FIFOs, devices) on both platforms. - flags |= os.O_NOFOLLOW + nofollow_flag = getattr(os, "O_NOFOLLOW", 0) + flags |= nofollow_flag try: fd = os.open(str(file_path), flags) except OSError as exc: - if exc.errno == getattr(errno, "ELOOP", None): + if ( + (nofollow_flag and exc.errno == errno.ELOOP) + or find_filesystem_link(file_path) is not None + ): raise ValueError( - f"Symlink Markdown import paths are rejected: {file_path}" + "Symlink Markdown import paths are rejected; symbolic links " + f"and junctions are unsafe: {file_path}" ) from exc raise try: - stat_res = os.fstat(fd) - if not stat.S_ISREG(stat_res.st_mode): + if find_filesystem_link(file_path) is not None: + raise ValueError( + "Symlink Markdown import paths are rejected; symbolic links " + f"and junctions are unsafe: {file_path}" + ) + if not stat.S_ISREG(os.fstat(fd).st_mode): raise ValueError( f"Markdown import path is not a regular file: {file_path}" ) - with open(fd, "r", encoding="utf-8", closefd=True) as f: - return f.read() - except Exception: - try: + with os.fdopen(fd, mode="r", encoding="utf-8") as source: + fd = -1 + return source.read() + finally: + if fd >= 0: os.close(fd) - except OSError: - pass - raise def _read_markdown_path(self, path: Path) -> List[Tuple[str, str]]: - if path.is_symlink(): - raise ValueError(f"Symlink Markdown import paths are rejected: {path}") + if find_filesystem_link(path) is not None: + raise ValueError( + "Symlink Markdown import paths are rejected; symbolic links and " + f"junctions are unsafe: {path}" + ) if not path.exists(): raise FileNotFoundError(f"Markdown import path does not exist: {path}") if path.is_dir(): - file_paths = sorted( - ( - file_path - for file_path in path.iterdir() - if file_path.is_file() - and not file_path.is_symlink() - and file_path.suffix.lower() in self._MARKDOWN_EXTENSIONS - ), - key=lambda file_path: (file_path.name.casefold(), file_path.name), - ) + file_paths = [] + for file_path in path.iterdir(): + if file_path.suffix.lower() not in self._MARKDOWN_EXTENSIONS: + continue + if find_filesystem_link(file_path) is not None: + continue + if file_path.is_file(): + file_paths.append(file_path) + if find_filesystem_link(path) is not None: + raise ValueError( + "Symlink Markdown import paths are rejected; symbolic links " + f"and junctions are unsafe: {path}" + ) + file_paths.sort(key=lambda item: (item.name.casefold(), item.name)) elif path.is_file(): file_paths = [path] else: diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index ac3f134f..62050953 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -105,27 +105,82 @@ Production Use Cases: - Business: Workflow decisions, policy compliance, audit trails """ +import copy +import errno +import hashlib +import itertools +import json +import os +import re +import shutil +import stat +import tempfile +import threading +import uuid from collections import defaultdict, deque from dataclasses import dataclass, field -from datetime import datetime, timezone -import json -import threading -import itertools +from datetime import date, datetime, timezone +from pathlib import Path from typing import Any, Dict, List, Optional, Set, Tuple, Union -import uuid +import yaml + +from ..utils.helpers import classify_path_distance from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker -from ..utils.helpers import classify_path_distance from ..utils.skos import is_skos_hierarchy_edge, validate_skos_hierarchy +from ._markdown_filesystem import find_filesystem_link from .entity_linker import EntityLinker + +class _UniqueKeySafeLoader(yaml.SafeLoader): + """Safe YAML loader that rejects ambiguous duplicate mapping keys.""" + + +def _construct_unique_mapping( + loader: _UniqueKeySafeLoader, node: yaml.MappingNode, deep: bool = False +) -> Dict[Any, Any]: + loader.flatten_mapping(node) + mapping = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + try: + duplicate = key in mapping + except TypeError as exc: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + "found an unhashable key", + key_node.start_mark, + ) from exc + if duplicate: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + f"found duplicate key {key!r}", + key_node.start_mark, + ) + mapping[key] = loader.construct_object(value_node, deep=deep) + return mapping + + +_UniqueKeySafeLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_unique_mapping +) + + # Optional imports for advanced features try: from ..kg import ( - GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector, - PathFinder, NodeEmbedder, SimilarityCalculator, LinkPredictor, - ConnectivityAnalyzer + CentralityCalculator, + CommunityDetector, + ConnectivityAnalyzer, + GraphAnalyzer, + GraphBuilder, + LinkPredictor, + NodeEmbedder, + PathFinder, + SimilarityCalculator, ) KG_AVAILABLE = True except ImportError: @@ -438,6 +493,23 @@ _ATTRS_MISSING = object() #: entities and timestamps. _CAUSAL_EDGE_TYPES = ("CAUSED", "INFLUENCED", "PRECEDENT_FOR") +# Causal edges circulate under two vocabularies: this module's canonical +# spellings above, and the present-tense spellings CausalChainAnalyzer also +# accepts ("causes", "influences", "leads_to", "supports"). The present-tense +# forms normalize onto the canonical types for storage; traversal accepts +# both vocabularies so an edge recorded either way is never invisible. +_CAUSAL_EDGE_ALIASES = { + "CAUSES": "CAUSED", + "CAUSED": "CAUSED", + "INFLUENCES": "INFLUENCED", + "INFLUENCED": "INFLUENCED", + "PRECEDES": "PRECEDENT_FOR", + "PRECEDENT_FOR": "PRECEDENT_FOR", +} +_CAUSAL_TRAVERSAL_TYPES = frozenset(_CAUSAL_EDGE_ALIASES) | { + "LEADS_TO", "LEAD_TO", "SUPPORTS", "SUPPORT", +} + class ContextGraph: """ @@ -453,6 +525,12 @@ class ContextGraph: Perfect for building intelligent AI agents that can learn from decisions! """ + _MARKDOWN_FORMAT = "semantica-context-graph" + _MARKDOWN_VERSION = 1 + _MARKDOWN_MANIFEST = "graph.md" + _MARKDOWN_NODES_DIRECTORY = "nodes" + _MARKDOWN_EXTENSIONS = frozenset({".md", ".markdown"}) + def __init__(self, config: Optional[Dict[str, Any]] = None, **kwargs): """ Initialize context graph with optional advanced features. @@ -821,6 +899,11 @@ class ContextGraph: return node.properties.update(attributes) node.metadata.update(attributes) + # Keep derived decision indexes consistent when a decision node is + # mutated so that category / entity / temporal lookups reflect the + # new property values without requiring a full graph reload. + if (getattr(node, "node_type", None) or "").lower() == "decision": + self._sync_decision_from_node(node_id) if getattr(self, "mutation_callback", None) and not getattr( self, "_suspend_mutation_callback", False @@ -1084,14 +1167,21 @@ class ContextGraph: ) ) - def save_to_file(self, path: str) -> None: + def save_to_file( + self, path: Union[str, Path], format: str = "json" + ) -> None: """ - Save context graph to file (JSON format). + Save the context graph in JSON or Markdown format. Args: - path: File path to save to + path: JSON file path or Markdown export directory + format: Persistence format (``json`` or ``markdown``) """ - import json + normalized_format = self._normalize_persistence_format(format) + if normalized_format == "markdown": + self._save_markdown_directory(Path(path)) + self.logger.info(f"Saved context graph Markdown to {path}") + return with self._lock: @@ -1118,15 +1208,31 @@ class ContextGraph: self.logger.info(f"Saved context graph to {path}") - def load_from_file(self, path: str) -> None: + def load_from_file( + self, path: Union[str, Path], format: str = "json" + ) -> None: """ - Load context graph from file (JSON format). + Load the context graph from JSON or Markdown. Args: - path: File path to load from + path: JSON file path or Markdown export directory + format: Persistence format (``json`` or ``markdown``) """ - import json - import os + normalized_format = self._normalize_persistence_format(format) + if normalized_format == "markdown": + markdown_path = Path(path) + linked_component = find_filesystem_link(markdown_path) + if linked_component is not None: + raise ValueError( + "Refusing to import Markdown symbolic link or junction: " + f"{linked_component}" + ) + if not markdown_path.exists(): + self.logger.warning(f"File not found: {path}") + return + self._load_markdown_directory(markdown_path) + self.logger.info(f"Loaded context graph Markdown from {path}") + return if not os.path.exists(path): self.logger.warning(f"File not found: {path}") @@ -1156,6 +1262,7 @@ class ContextGraph: self.edge_type_index.clear() self._linked_graphs.clear() self._unresolved_links.clear() + self._analytics_cache.clear() # Deletion metadata belongs to the graph being replaced; keeping it # would make entities in the loaded graph read as already retracted. self._retractions.clear() @@ -1189,8 +1296,810 @@ class ContextGraph: if link_id: self._unresolved_links[link_id] = link_meta + # Rebuild all derived decision indexes from the freshly-loaded + # nodes so that find_precedents_by_scenario, find_similar_decisions, + # and all decision analytics work correctly after a reload. + # _rebuild_decision_indexes() unconditionally clears the old indexes + # first, so repeated load_from_file calls never accumulate stale + # entries from a previous file. + self._rebuild_decision_indexes() + self.logger.info(f"Loaded context graph from {path}") + @staticmethod + def _normalize_persistence_format(format: str) -> str: + if not isinstance(format, str) or not format.strip(): + raise ValueError("Context graph persistence format must be a string.") + normalized = format.strip().lower() + if normalized not in {"json", "markdown"}: + raise ValueError( + f"Unsupported context graph persistence format: {format!r}. " + "Expected 'json' or 'markdown'." + ) + return normalized + + def _save_markdown_directory(self, destination: Path) -> None: + if not destination.name: + raise ValueError("Markdown export destination cannot be a filesystem root.") + linked_component = find_filesystem_link(destination) + if linked_component is not None: + raise ValueError( + "Refusing to replace Markdown symbolic link or junction: " + f"{linked_component}" + ) + if destination.exists() and not destination.is_dir(): + raise ValueError( + f"Markdown export destination is not a directory: {destination}" + ) + if destination.exists() and any(destination.iterdir()): + try: + self._parse_markdown_directory( + destination, require_canonical_layout=True + ) + except (FileNotFoundError, ValueError): + is_managed = False + else: + is_managed = True + if not is_managed: + raise ValueError( + "Refusing to replace a non-empty directory that is not a " + f"managed ContextGraph export: {destination}" + ) + + manifest_document, node_documents = self._markdown_documents() + destination.parent.mkdir(parents=True, exist_ok=True) + linked_component = find_filesystem_link(destination.parent) + if linked_component is not None: + raise ValueError( + "Refusing to export through Markdown symbolic link or junction: " + f"{linked_component}" + ) + staging_path = Path( + tempfile.mkdtemp( + dir=str(destination.parent), prefix=f".{destination.name}.staging-" + ) + ) + backup_path: Optional[Path] = None + try: + nodes_path = staging_path / self._MARKDOWN_NODES_DIRECTORY + nodes_path.mkdir() + self._write_staged_markdown( + staging_path / self._MARKDOWN_MANIFEST, manifest_document + ) + for filename, document in node_documents: + self._write_staged_markdown(nodes_path / filename, document) + + if destination.exists(): + backup_path = destination.parent / ( + f".{destination.name}.backup-{uuid.uuid4().hex}" + ) + os.replace(destination, backup_path) + try: + os.replace(staging_path, destination) + staging_path = None + except BaseException as publish_error: + if backup_path is not None and not destination.exists(): + try: + os.replace(backup_path, destination) + except BaseException as restore_error: + self.logger.error( + "Failed to restore previous ContextGraph Markdown " + "export from %s after publish failure; preserving " + "the original publish error", + backup_path, + exc_info=( + type(restore_error), + restore_error, + restore_error.__traceback__, + ), + ) + add_note = getattr(publish_error, "add_note", None) + if add_note is not None: + add_note( + "Restoring the previous ContextGraph Markdown " + f"export also failed: {restore_error}" + ) + else: + backup_path = None + raise + + if backup_path is not None: + shutil.rmtree(backup_path) + backup_path = None + finally: + if staging_path is not None: + shutil.rmtree(staging_path, ignore_errors=True) + if backup_path is not None and backup_path.exists(): + self.logger.warning( + "ContextGraph export left backup directory %s", backup_path + ) + + @staticmethod + def _write_staged_markdown(path: Path, document: str) -> None: + with path.open("x", encoding="utf-8") as output: + output.write(document) + output.flush() + os.fsync(output.fileno()) + + def _markdown_documents(self) -> Tuple[str, List[Tuple[str, str]]]: + with self._lock: + nodes = [ + { + "id": node.node_id, + "type": node.node_type, + "properties": copy.deepcopy(node.properties), + "metadata": copy.deepcopy(node.metadata), + "valid_from": node.valid_from, + "valid_until": node.valid_until, + "content": node.content, + } + for node in self.nodes.values() + ] + edges = [ + { + "id": edge.edge_id, + "family_id": edge.family_id or edge.edge_id, + "source": edge.source_id, + "target": edge.target_id, + "type": edge.edge_type, + "weight": edge.weight, + "metadata": copy.deepcopy(edge.metadata), + "valid_from": edge.valid_from, + "valid_until": edge.valid_until, + } + for edge in self.edges + ] + graph_id = self.graph_id + links_by_id = { + link_id: copy.deepcopy(link) + for link_id, link in self._unresolved_links.items() + } + for link_id, ( + other_graph, + source_node_id, + target_node_id, + ) in self._linked_graphs.items(): + links_by_id[link_id] = { + "link_id": link_id, + "source_node_id": source_node_id, + "target_node_id": target_node_id, + "other_graph_id": other_graph.graph_id, + } + + edges.sort( + key=lambda edge: ( + str(edge["id"]), + str(edge["source"]), + str(edge["target"]), + str(edge["type"]), + ) + ) + seen_edge_ids: Set[str] = set() + duplicate_edge_ids: Set[str] = set() + for edge in edges: + edge_id = edge["id"] + if not isinstance(edge_id, str) or not edge_id.strip(): + raise ValueError( + "Cannot export ContextGraph: every edge must have a string ID." + ) + if edge_id in seen_edge_ids: + duplicate_edge_ids.add(edge_id) + seen_edge_ids.add(edge_id) + if duplicate_edge_ids: + duplicates = ", ".join( + repr(edge_id) for edge_id in sorted(duplicate_edge_ids) + ) + raise ValueError( + f"Cannot export ContextGraph: duplicate edge ID(s): {duplicates}." + ) + links = sorted( + links_by_id.values(), key=lambda link: str(link.get("link_id", "")) + ) + manifest = { + "format": self._MARKDOWN_FORMAT, + "version": self._MARKDOWN_VERSION, + "graph_id": graph_id, + "edges": edges, + "links": links, + } + manifest_body = ( + "# Context Graph\n\n" + "Graph relationships are stored in frontmatter. Node content is in " + "the `nodes` directory.\n" + ) + manifest_document = self._render_markdown_document( + manifest, manifest_body, "graph manifest" + ) + + node_documents = [] + filenames = set() + for node in sorted(nodes, key=lambda item: str(item["id"])): + filename = self._node_markdown_filename(node["id"]) + normalized_filename = filename.casefold() + if normalized_filename in filenames: + raise ValueError( + f"Cannot export ContextGraph: duplicate filename {filename!r}." + ) + filenames.add(normalized_filename) + content = node.pop("content") + node_documents.append( + ( + filename, + self._render_markdown_document( + node, content, f"node {node['id']!r}" + ), + ) + ) + return manifest_document, node_documents + + @staticmethod + def _node_markdown_filename(node_id: Any) -> str: + if not isinstance(node_id, str) or not node_id.strip(): + raise ValueError("Cannot export a ContextGraph node without a string ID.") + slug = re.sub(r"[^A-Za-z0-9._-]+", "-", node_id) + slug = re.sub(r"-+", "-", slug).strip("._-")[:80].rstrip("._-") + slug = slug or "node" + digest = hashlib.sha256(node_id.encode("utf-8")).hexdigest()[:12] + return f"{slug}--{digest}.md" + + @classmethod + def _render_markdown_document( + cls, frontmatter: Dict[str, Any], body: str, source: str + ) -> str: + if not isinstance(body, str): + raise ValueError(f"Cannot export {source}: Markdown body must be a string.") + canonical = cls._canonical_markdown_value(frontmatter, source) + try: + yaml_text = yaml.safe_dump( + canonical, + sort_keys=False, + allow_unicode=True, + default_flow_style=False, + ) + except yaml.YAMLError as exc: + raise ValueError( + f"Cannot export {source}: metadata is not YAML serializable." + ) from exc + return f"---\n{yaml_text}---\n\n{body}" + + @classmethod + def _canonical_markdown_value( + cls, value: Any, source: str, ancestors: Optional[Set[int]] = None + ) -> Any: + ancestors = set() if ancestors is None else ancestors + if isinstance(value, dict): + if any(not isinstance(key, str) for key in value): + raise ValueError( + f"Invalid Markdown metadata in {source}: " + "mapping keys must be strings." + ) + identity = id(value) + if identity in ancestors: + raise ValueError( + f"Invalid Markdown metadata in {source}: " + "values cannot contain cycles." + ) + ancestors.add(identity) + try: + return { + key: cls._canonical_markdown_value( + value[key], source, ancestors + ) + for key in sorted(value) + } + finally: + ancestors.remove(identity) + if isinstance(value, list): + identity = id(value) + if identity in ancestors: + raise ValueError( + f"Invalid Markdown metadata in {source}: " + "values cannot contain cycles." + ) + ancestors.add(identity) + try: + return [ + cls._canonical_markdown_value(item, source, ancestors) + for item in value + ] + finally: + ancestors.remove(identity) + if isinstance(value, tuple) or isinstance(value, set): + raise ValueError( + f"Invalid Markdown metadata in {source}: " + "tuples and sets are not supported." + ) + return value + + def _load_markdown_directory(self, source: Path) -> None: + parsed_state = self._parse_markdown_directory(source) + graph_id, nodes_by_id, edges, unresolved_links = parsed_state + + adjacency: Dict[str, List[ContextEdge]] = defaultdict(list) + edge_index: Dict[str, ContextEdge] = {} + node_type_index: Dict[str, Set[str]] = defaultdict(set) + edge_type_index: Dict[str, List[ContextEdge]] = defaultdict(list) + for node in nodes_by_id.values(): + node_type_index[node.node_type].add(node.node_id) + for edge in edges: + edge_index[edge.edge_id] = edge + adjacency[edge.source_id].append(edge) + edge_type_index[edge.edge_type].append(edge) + + with self._lock: + self.graph_id = graph_id + self.nodes.clear() + self.nodes.update(nodes_by_id) + self.edges.clear() + self.edges.extend(edges) + self._edge_index.clear() + self._edge_index.update(edge_index) + self._adjacency.clear() + self._adjacency.update(adjacency) + self.node_type_index.clear() + self.node_type_index.update(node_type_index) + self.edge_type_index.clear() + self.edge_type_index.update(edge_type_index) + self._linked_graphs.clear() + self._unresolved_links.clear() + self._unresolved_links.update(unresolved_links) + self._analytics_cache.clear() + self._retractions.clear() + self._tombstones.clear() + # Rebuild derived decision indexes from the freshly-loaded nodes. + self._rebuild_decision_indexes() + + if self.mutation_callback and not self._suspend_mutation_callback: + mutation_events = [ + ("ADD_NODE", node.node_id, node.to_dict()) + for node in nodes_by_id.values() + ] + mutation_events.extend( + ("ADD_EDGE", edge.edge_id, edge.to_dict()) for edge in edges + ) + for operation, entity_id, payload in mutation_events: + try: + self.mutation_callback(operation, entity_id, payload) + except Exception as exc: + self.logger.warning( + "Audit trail callback failed for Markdown graph load " + "%s %s: %s", + operation, + entity_id, + exc, + ) + + def _parse_markdown_directory( + self, source: Path, require_canonical_layout: bool = False + ) -> Tuple[ + str, + Dict[str, ContextNode], + List[ContextEdge], + Dict[str, Dict[str, str]], + ]: + manifest_document, node_documents = self._read_markdown_directory( + source, require_canonical_layout=require_canonical_layout + ) + manifest, _ = self._parse_markdown_document( + manifest_document, str(source / self._MARKDOWN_MANIFEST) + ) + graph_id, edges, links = self._parse_markdown_manifest(manifest, source) + + nodes_by_id: Dict[str, ContextNode] = {} + for node_source, document in node_documents: + frontmatter, body = self._parse_markdown_document(document, node_source) + node = self._parse_markdown_node(frontmatter, body, node_source) + if node.node_id in nodes_by_id: + raise ValueError( + f"Duplicate Markdown node ID {node.node_id!r} in {node_source}." + ) + node_filename = Path(node_source).name + if ( + require_canonical_layout + and node_filename != self._node_markdown_filename(node.node_id) + ): + raise ValueError( + "Invalid managed ContextGraph export: node file " + f"{node_filename!r} is not the canonical filename " + f"for node {node.node_id!r}." + ) + nodes_by_id[node.node_id] = node + + missing_endpoints = sorted( + { + endpoint + for edge in edges + for endpoint in (edge.source_id, edge.target_id) + if endpoint not in nodes_by_id + } + ) + if missing_endpoints and require_canonical_layout: + missing = ", ".join(repr(endpoint) for endpoint in missing_endpoints) + raise ValueError( + "Invalid managed ContextGraph export: edge endpoint(s) " + f"{missing} do not have node files." + ) + for endpoint in missing_endpoints: + nodes_by_id[endpoint] = ContextNode(endpoint, "entity", endpoint) + + hierarchy_edges = [ + { + "source": edge.source_id, + "target": edge.target_id, + "type": edge.edge_type, + } + for edge in edges + if is_skos_hierarchy_edge(edge.to_dict()) + ] + if hierarchy_edges: + validate_skos_hierarchy(hierarchy_edges, []) + + unresolved_links = {} + for link in links: + link_id = link["link_id"] + if link_id in unresolved_links: + raise ValueError( + f"Duplicate cross-graph link ID {link_id!r} in graph manifest." + ) + if link["source_node_id"] not in nodes_by_id: + raise ValueError( + f"Cross-graph link {link_id!r} references missing source node " + f"{link['source_node_id']!r}." + ) + unresolved_links[link_id] = link + return graph_id, nodes_by_id, edges, unresolved_links + + def _read_markdown_directory( + self, source: Path, require_canonical_layout: bool = False + ) -> Tuple[str, List[Tuple[str, str]]]: + linked_component = find_filesystem_link(source) + if linked_component is not None: + raise ValueError( + "Refusing to import Markdown symbolic link or junction: " + f"{linked_component}" + ) + if not source.exists(): + raise FileNotFoundError( + f"ContextGraph Markdown import path does not exist: {source}" + ) + if not source.is_dir(): + raise ValueError( + f"ContextGraph Markdown import path is not a directory: {source}" + ) + + if require_canonical_layout: + expected_entries = { + self._MARKDOWN_MANIFEST, + self._MARKDOWN_NODES_DIRECTORY, + } + actual_entries = {path.name for path in source.iterdir()} + if actual_entries != expected_entries: + unexpected = sorted(actual_entries - expected_entries) + missing = sorted(expected_entries - actual_entries) + details = [] + if unexpected: + details.append(f"unexpected entries: {unexpected!r}") + if missing: + details.append(f"missing entries: {missing!r}") + raise ValueError( + "Invalid managed ContextGraph export layout (" + + "; ".join(details) + + ")." + ) + + manifest_path = source / self._MARKDOWN_MANIFEST + manifest_document = self._read_markdown_file(manifest_path) + nodes_path = source / self._MARKDOWN_NODES_DIRECTORY + linked_component = find_filesystem_link(nodes_path) + if linked_component is not None: + raise ValueError( + "Refusing to import Markdown symbolic link or junction: " + f"{linked_component}" + ) + if not nodes_path.is_dir(): + raise ValueError( + f"ContextGraph Markdown nodes directory is missing: {nodes_path}" + ) + + node_paths = [] + for path in nodes_path.iterdir(): + linked_component = find_filesystem_link(path) + if linked_component is not None: + raise ValueError( + "Refusing to import Markdown symbolic link or junction: " + f"{linked_component}" + ) + if path.suffix.lower() not in self._MARKDOWN_EXTENSIONS: + if require_canonical_layout: + raise ValueError( + "Invalid managed ContextGraph export: unexpected node " + f"entry {path.name!r}." + ) + continue + if not path.is_file(): + raise ValueError(f"Markdown node path is not a regular file: {path}") + node_paths.append(path) + linked_component = find_filesystem_link(nodes_path) + if linked_component is not None: + raise ValueError( + "Refusing to import Markdown symbolic link or junction: " + f"{linked_component}" + ) + node_paths.sort(key=lambda path: (path.name.casefold(), path.name)) + return manifest_document, [ + (str(path), self._read_markdown_file(path)) for path in node_paths + ] + + @staticmethod + def _read_markdown_file(path: Path) -> str: + linked_component = find_filesystem_link(path) + if linked_component is not None: + raise ValueError( + "Refusing to import Markdown symbolic link or junction: " + f"{linked_component}" + ) + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags) + except OSError as exc: + linked_component = find_filesystem_link(path) + if exc.errno == errno.ELOOP or linked_component is not None: + raise ValueError( + "Refusing to import Markdown symbolic link or junction: " + f"{linked_component or path}" + ) from exc + if exc.errno == errno.ENOENT: + raise FileNotFoundError(f"Markdown file is missing: {path}") from exc + raise OSError( + exc.errno, + f"Failed to read Markdown file {path}: {exc.strerror or str(exc)}", + exc.filename or str(path), + ) from exc + + try: + linked_component = find_filesystem_link(path) + if linked_component is not None: + raise ValueError( + "Refusing to import Markdown symbolic link or junction: " + f"{linked_component}" + ) + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise ValueError(f"Markdown path is not a regular file: {path}") + with os.fdopen(descriptor, "r", encoding="utf-8") as input_file: + descriptor = None + return input_file.read() + finally: + if descriptor is not None: + os.close(descriptor) + + @staticmethod + def _parse_markdown_document( + document: str, source: str + ) -> Tuple[Dict[str, Any], str]: + lines = document.splitlines(keepends=True) + if not lines or lines[0].rstrip("\r\n") != "---": + raise ValueError( + f"Invalid Markdown frontmatter in {source}: " + "document must start with '---'." + ) + closing_index = next( + ( + index + for index, line in enumerate(lines[1:], start=1) + if line.rstrip("\r\n") == "---" + ), + None, + ) + if closing_index is None: + raise ValueError( + f"Invalid Markdown frontmatter in {source}: missing closing '---'." + ) + try: + loaded = yaml.load( + "".join(lines[1:closing_index]), Loader=_UniqueKeySafeLoader + ) + except yaml.YAMLError as exc: + raise ValueError( + f"Invalid Markdown frontmatter in {source}: {exc}" + ) from exc + frontmatter = {} if loaded is None else loaded + if not isinstance(frontmatter, dict): + raise ValueError( + f"Invalid Markdown frontmatter in {source}: expected a YAML mapping." + ) + if any(not isinstance(key, str) for key in frontmatter): + raise ValueError( + f"Invalid Markdown frontmatter in {source}: " + "field names must be strings." + ) + + body = "".join(lines[closing_index + 1 :]) + if body.startswith("\r\n"): + body = body[2:] + elif body.startswith("\n"): + body = body[1:] + return frontmatter, body + + def _parse_markdown_manifest( + self, manifest: Dict[str, Any], source: Path + ) -> Tuple[str, List[ContextEdge], List[Dict[str, str]]]: + manifest_source = str(source / self._MARKDOWN_MANIFEST) + if manifest.get("format") != self._MARKDOWN_FORMAT: + raise ValueError( + f"Invalid ContextGraph Markdown manifest in {manifest_source}: " + f"'format' must be {self._MARKDOWN_FORMAT!r}." + ) + version = manifest.get("version") + if isinstance(version, bool) or version != self._MARKDOWN_VERSION: + raise ValueError( + f"Unsupported ContextGraph Markdown version {version!r} in " + f"{manifest_source}; expected {self._MARKDOWN_VERSION}." + ) + graph_id = self._required_markdown_string( + manifest.get("graph_id"), "graph_id", manifest_source + ) + + raw_edges = manifest.get("edges", []) + if not isinstance(raw_edges, list): + raise ValueError( + f"Invalid ContextGraph Markdown manifest in {manifest_source}: " + "'edges' must be a list." + ) + edges = [] + edge_ids = set() + for index, raw_edge in enumerate(raw_edges): + edge_source = f"{manifest_source} edge[{index}]" + edge = self._parse_markdown_edge(raw_edge, edge_source) + if edge.edge_id in edge_ids: + raise ValueError( + f"Duplicate Markdown edge ID {edge.edge_id!r} in {edge_source}." + ) + edge_ids.add(edge.edge_id) + edges.append(edge) + + raw_links = manifest.get("links", []) + if not isinstance(raw_links, list): + raise ValueError( + f"Invalid ContextGraph Markdown manifest in {manifest_source}: " + "'links' must be a list." + ) + links = [ + self._parse_markdown_link(link, f"{manifest_source} link[{index}]") + for index, link in enumerate(raw_links) + ] + return graph_id, edges, links + + def _parse_markdown_node( + self, frontmatter: Dict[str, Any], body: str, source: str + ) -> ContextNode: + node_id = self._required_markdown_string(frontmatter.get("id"), "id", source) + node_type = self._required_markdown_string( + frontmatter.get("type"), "type", source + ) + properties = self._markdown_mapping( + frontmatter.get("properties", {}), "properties", source + ) + metadata = self._markdown_mapping( + frontmatter.get("metadata", {}), "metadata", source + ) + return ContextNode( + node_id=node_id, + node_type=node_type, + content=body, + properties=properties, + metadata=metadata, + valid_from=self._markdown_temporal_value( + frontmatter.get("valid_from"), "valid_from", source + ), + valid_until=self._markdown_temporal_value( + frontmatter.get("valid_until"), "valid_until", source + ), + ) + + def _parse_markdown_edge(self, raw_edge: Any, source: str) -> ContextEdge: + if not isinstance(raw_edge, dict): + raise ValueError(f"Invalid Markdown edge in {source}: expected a mapping.") + edge_id = self._required_markdown_string(raw_edge.get("id"), "id", source) + family_value = raw_edge.get("family_id", raw_edge.get("familyId", edge_id)) + family_id = self._required_markdown_string(family_value, "family_id", source) + source_id = self._required_markdown_string( + raw_edge.get("source"), "source", source + ) + target_id = self._required_markdown_string( + raw_edge.get("target"), "target", source + ) + edge_type = self._required_markdown_string(raw_edge.get("type"), "type", source) + weight = raw_edge.get("weight", 1.0) + if isinstance(weight, bool) or not isinstance(weight, (int, float)): + raise ValueError( + f"Invalid Markdown edge in {source}: 'weight' must be a number." + ) + metadata = self._markdown_mapping( + raw_edge.get("metadata", {}), "metadata", source + ) + return ContextEdge( + edge_id=edge_id, + family_id=family_id, + source_id=source_id, + target_id=target_id, + edge_type=edge_type, + weight=float(weight), + metadata=metadata, + valid_from=self._markdown_temporal_value( + raw_edge.get("valid_from"), "valid_from", source + ), + valid_until=self._markdown_temporal_value( + raw_edge.get("valid_until"), "valid_until", source + ), + ) + + def _parse_markdown_link(self, raw_link: Any, source: str) -> Dict[str, str]: + if not isinstance(raw_link, dict): + raise ValueError( + f"Invalid cross-graph link in {source}: expected a mapping." + ) + return { + field_name: self._required_markdown_string( + raw_link.get(field_name), field_name, source + ) + for field_name in ( + "link_id", + "source_node_id", + "target_node_id", + "other_graph_id", + ) + } + + @staticmethod + def _required_markdown_string(value: Any, field_name: str, source: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError( + f"Invalid Markdown frontmatter in {source}: " + f"'{field_name}' must be a non-empty string." + ) + return value + + @classmethod + def _markdown_mapping( + cls, value: Any, field_name: str, source: str + ) -> Dict[str, Any]: + if value is None: + return {} + if not isinstance(value, dict): + raise ValueError( + f"Invalid Markdown frontmatter in {source}: " + f"'{field_name}' must be a mapping." + ) + canonical = cls._canonical_markdown_value(value, source) + return dict(canonical) + + @staticmethod + def _markdown_temporal_value( + value: Any, field_name: str, source: str + ) -> Optional[str]: + if value is None: + return None + if isinstance(value, datetime): + return _normalize_temporal_input(value) + if isinstance(value, date): + return value.isoformat() + if isinstance(value, str) and value.strip(): + try: + _normalize_temporal_input(value) + except ValueError as exc: + raise ValueError( + f"Invalid Markdown frontmatter in {source}: " + f"'{field_name}' must be a valid ISO-8601 string." + ) from exc + return value + raise ValueError( + f"Invalid Markdown frontmatter in {source}: " + f"'{field_name}' must be a valid ISO-8601 string." + ) + + def find_node(self, node_id: str) -> Optional[Dict[str, Any]]: """Return a dict representation of the node identified by *node_id*. @@ -1931,6 +2840,12 @@ class ContextGraph: self._unresolved_links.clear() self._retractions.clear() self._tombstones.clear() + # Reset derived decision indexes so that decision queries against + # a cleared graph return empty results rather than stale data. + self._decisions = {} + self._decision_index = defaultdict(set) + self._entity_index = defaultdict(set) + self._temporal_index = [] self.logger.debug("Graph state fully cleared.") # --- Internal Helpers --- @@ -2399,9 +3314,10 @@ class ContextGraph: def _load_conversation(self, file_path: str) -> Dict[str, Any]: """Load conversation from file.""" - from ..utils.helpers import read_json_file from pathlib import Path + from ..utils.helpers import read_json_file + return read_json_file(Path(file_path)) def to_dict(self) -> Dict[str, Any]: @@ -2587,6 +3503,9 @@ class ContextGraph: ) self._add_internal_edge(edge) + # Rebuild derived decision indexes from the now-populated node store. + self._rebuild_decision_indexes() + def state_at(self, timestamp: Union[str, int, float, datetime]) -> Dict[str, Any]: """Return a serializable snapshot of graph state valid at the given time.""" at_time = self._normalize_timestamp(timestamp) @@ -2745,9 +3664,15 @@ class ContextGraph: target_decision_id: Target decision ID relationship_type: Type of relationship (CAUSED, INFLUENCED, PRECEDENT_FOR) """ - valid_types = ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"] - if relationship_type not in valid_types: - raise ValueError(f"Relationship type must be one of: {valid_types}") + # Normalize so callers may use either vocabulary's spelling + # ("causes" from CausalChainAnalyzer, or "CAUSED" from this module's + # canonical constant); the stored form is always canonical. Invalid + # inputs keep raising ValueError rather than AttributeError. + if not isinstance(relationship_type, str): + raise ValueError(f"Relationship type must be one of: {_CAUSAL_EDGE_TYPES}") + relationship_type = _CAUSAL_EDGE_ALIASES.get(relationship_type.strip().upper()) + if relationship_type is None: + raise ValueError(f"Relationship type must be one of: {_CAUSAL_EDGE_TYPES}") # Check if decisions exist - if not, skip adding relationship if source_decision_id not in self.nodes or target_decision_id not in self.nodes: @@ -2839,11 +3764,11 @@ class ContextGraph: # Find connected decisions for edge in self.edges: if direction == "upstream": - if edge.target_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]: + if edge.target_id == current_id and edge.edge_type.upper() in _CAUSAL_TRAVERSAL_TYPES: if edge.source_id not in visited and depth < max_depth: queue.append((edge.source_id, depth + 1)) else: # downstream - if edge.source_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]: + if edge.source_id == current_id and edge.edge_type.upper() in _CAUSAL_TRAVERSAL_TYPES: if edge.target_id not in visited and depth < max_depth: queue.append((edge.target_id, depth + 1)) @@ -2866,10 +3791,13 @@ class ContextGraph: Returns: List of precedent decisions """ - # Find decisions connected via PRECEDENT_FOR relationships + # Find decisions connected via PRECEDENT_FOR relationships, accepting + # the analyzer vocabulary's "precedes" spelling as well (issue #1184). precedent_ids = [] for edge in self.edges: - if edge.target_id == decision_id and edge.edge_type == "PRECEDENT_FOR": + if edge.target_id == decision_id and edge.edge_type.upper() in { + "PRECEDENT_FOR", "PRECEDES", + }: precedent_ids.append(edge.source_id) # Convert to Decision objects @@ -3204,7 +4132,7 @@ class ContextGraph: """ import uuid from datetime import datetime - + # Input validation if not isinstance(category, str) or not category.strip(): raise ValueError("Category must be a non-empty string") @@ -3430,8 +4358,13 @@ class ContextGraph: # Explicit causal relationships recorded via add_causal_relationship() are # ground truth and always count as direct influence, in either direction. - for edge_type in _CAUSAL_EDGE_TYPES: - for edge in self.edge_type_index.get(edge_type, []): + # The index is keyed by the raw edge_type string ("causes" and "CAUSED" + # are separate keys), so filter by normalized type instead of iterating + # a fixed spelling list. + for edge_type, edges in self.edge_type_index.items(): + if edge_type.upper() not in _CAUSAL_TRAVERSAL_TYPES: + continue + for edge in edges: if edge.source_id == decision_id and edge.target_id in self._decisions: direct_influence.add(edge.target_id) elif edge.target_id == decision_id and edge.source_id in self._decisions: @@ -3577,8 +4510,12 @@ class ContextGraph: # record_decision() (e.g. a graph restored via from_dict), so only # causes with a known decision record are kept. incoming_causal_edges = defaultdict(list) - for edge_type in _CAUSAL_EDGE_TYPES: - for edge in self.edge_type_index.get(edge_type, []): + # The index is keyed by the raw edge_type string ("causes" and + # "CAUSED" are separate keys), so filter by normalized type. + for edge_type, edges in self.edge_type_index.items(): + if edge_type.upper() not in _CAUSAL_TRAVERSAL_TYPES: + continue + for edge in edges: if edge.source_id in self._decisions: incoming_causal_edges[edge.target_id].append(edge) @@ -3794,6 +4731,7 @@ class ContextGraph: scenario=decision["scenario"], decision_maker=decision.get("decision_maker", ""), reasoning=decision["reasoning"], + recorded_at=decision.get("recorded_at", ""), **safe_metadata, **extra_properties, ) @@ -3875,20 +4813,288 @@ class ContextGraph: return False return True - def _calculate_decision_content_similarity(self, scenario: str, decision: Dict[str, Any]) -> float: - """Calculate content similarity between scenario and decision.""" + # โ”€โ”€ decision-index helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + # Protected set of node properties whose values are *core* decision fields + # so that we can distinguish them from user-supplied metadata when + # rebuilding the in-memory indexes from a persisted node. + _DECISION_CORE_FIELDS: frozenset = frozenset({ + "id", "category", "scenario", "reasoning", "outcome", "confidence", + "entities", "decision_maker", "timestamp", "recorded_at", + "valid_from", "valid_until", "content", + }) + + def _rebuild_decision_indexes(self) -> None: + """Rebuild all derived decision indexes from the current node store. + + This method is the single authoritative rebuild path. It must be + called (under the graph lock) after any operation that wholesale + replaces ``self.nodes`` โ€” namely ``load_from_file`` (JSON and Markdown + paths) and ``from_dict``. + + Contract: + - Unconditionally clears ``_decisions``, ``_decision_index``, + ``_entity_index``, and ``_temporal_index`` before rebuilding so that + repeated calls never accumulate stale entries. + - Derives ``_decisions[node_id]["metadata"]`` from the full set of + node properties, excluding the protected core fields, so that + user-supplied metadata survives the round-trip. + - Runs under ``self._lock`` when called from load paths; callers that + already hold the lock must invoke ``_rebuild_decision_indexes`` + inside the lock block. + """ + # Always start fresh so repeated loads don't accumulate stale entries. + self._decisions: Dict[str, Any] = {} + self._decision_index: Dict[str, set] = defaultdict(set) + self._entity_index: Dict[str, set] = defaultdict(set) + self._temporal_index: List[Tuple[str, float]] = [] + + for node in self.nodes.values(): + if (getattr(node, "node_type", None) or "").lower() != "decision": + continue + + # Merge metadata and properties; properties win on collision. + meta: Dict[str, Any] = {} + meta.update(getattr(node, "metadata", {}) or {}) + meta.update(getattr(node, "properties", {}) or {}) + + # Timestamp: keep whatever was stored (float epoch or ISO string). + # The temporal index uses it for sorting; downstream code handles + # both types via _normalize_timestamp. + raw_ts = meta.get("timestamp", 0.0) + try: + sort_ts = float(raw_ts) + except (TypeError, ValueError): + sort_ts = 0.0 + + # Entities may be stored as a list in meta or inferred from + # outgoing "involves" edges if the list field is absent/empty. + # _add_decision_to_graph creates entity nodes connected via + # "involves" edges; it does NOT store the list as a node property. + entities = meta.get("entities") or [] + if not isinstance(entities, list): + entities = [] + if not entities: + # Recover entity list from "involves" edges on this decision node + for edge in self._adjacency.get(node.node_id, []): + if edge.edge_type == "involves": + entities.append(edge.target_id) + + # Everything that isn't a core field is user-supplied metadata. + extra_meta = { + k: v + for k, v in meta.items() + if k not in self._DECISION_CORE_FIELDS + } + + decision: Dict[str, Any] = { + "id": node.node_id, + "category": meta.get("category", ""), + "scenario": meta.get("scenario", getattr(node, "content", "") or ""), + "reasoning": meta.get("reasoning", ""), + "outcome": meta.get("outcome", ""), + "confidence": float(meta.get("confidence", 0.0) or 0.0), + "entities": entities, + "decision_maker": meta.get("decision_maker"), + "timestamp": raw_ts, + "recorded_at": meta.get("recorded_at", ""), + "valid_from": getattr(node, "valid_from", None), + "valid_until": getattr(node, "valid_until", None), + # Preserve all non-core node properties as decision metadata so + # that user-supplied fields survive a save โ†’ load round-trip. + "metadata": extra_meta, + } + + self._decisions[node.node_id] = decision + + category = decision["category"] + if category: + self._decision_index[category].add(node.node_id) + + for entity in entities: + self._entity_index[entity].add(node.node_id) + + self._temporal_index.append((node.node_id, sort_ts)) + + self._temporal_index.sort(key=lambda x: x[1], reverse=True) + + def _sync_decision_from_node(self, node_id: str) -> None: + """Synchronise a single decision index entry from the node store. + + Called after ``add_node_attribute`` mutates a decision node so that + ``_decisions`` and the derived indexes stay consistent without + requiring a full rebuild of all decisions. + """ + node = self.nodes.get(node_id) + if node is None: + return + if (getattr(node, "node_type", None) or "").lower() != "decision": + return + + if not hasattr(self, "_decisions"): + # Indexes don't exist yet โ€” a full rebuild is safer. + self._rebuild_decision_indexes() + return + + # Remove stale index entries for this decision ID. + old = self._decisions.get(node_id) + if old: + old_cat = old.get("category", "") + if old_cat and node_id in self._decision_index.get(old_cat, set()): + self._decision_index[old_cat].discard(node_id) + for ent in old.get("entities", []): + self._entity_index[ent].discard(node_id) + self._temporal_index = [ + (nid, ts) for nid, ts in self._temporal_index if nid != node_id + ] + + # Rebuild the entry for this node and re-insert index entries. + meta: Dict[str, Any] = {} + meta.update(getattr(node, "metadata", {}) or {}) + meta.update(getattr(node, "properties", {}) or {}) + + raw_ts = meta.get("timestamp", 0.0) try: - # Simple word-based similarity + sort_ts = float(raw_ts) + except (TypeError, ValueError): + sort_ts = 0.0 + + entities = meta.get("entities") or [] + if not isinstance(entities, list): + entities = [] + if not entities: + # Recover entity list from "involves" edges + for edge in self._adjacency.get(node_id, []): + if edge.edge_type == "involves": + entities.append(edge.target_id) + + extra_meta = { + k: v for k, v in meta.items() if k not in self._DECISION_CORE_FIELDS + } + + decision: Dict[str, Any] = { + "id": node_id, + "category": meta.get("category", ""), + "scenario": meta.get("scenario", getattr(node, "content", "") or ""), + "reasoning": meta.get("reasoning", ""), + "outcome": meta.get("outcome", ""), + "confidence": float(meta.get("confidence", 0.0) or 0.0), + "entities": entities, + "decision_maker": meta.get("decision_maker"), + "timestamp": raw_ts, + "recorded_at": meta.get("recorded_at", ""), + "valid_from": getattr(node, "valid_from", None), + "valid_until": getattr(node, "valid_until", None), + "metadata": extra_meta, + } + + self._decisions[node_id] = decision + if decision["category"]: + self._decision_index[decision["category"]].add(node_id) + for ent in entities: + self._entity_index[ent].add(node_id) + self._temporal_index.append((node_id, sort_ts)) + self._temporal_index.sort(key=lambda x: x[1], reverse=True) + + @staticmethod + def _char_bigrams(text: str) -> set: + """Character bigrams over whitespace-stripped text (CJK fallback). + + Strips whitespace so CJK characters without word-separating spaces are + treated as a contiguous character sequence rather than a single token. + """ + chars = "".join(text.lower().split()) + return {chars[i:i + 2] for i in range(len(chars) - 1)} + + @staticmethod + def _looks_cjk(text: str) -> bool: + """True if text contains CJK/Japanese/Korean script characters. + + Used to gate the character-bigram similarity fallback so it only + activates for scripts where whitespace tokenisation doesn't work. + """ + for ch in text: + code = ord(ch) + if ( + 0x4E00 <= code <= 0x9FFF # CJK Unified Ideographs + or 0x3400 <= code <= 0x4DBF # CJK Extension A + or 0x3040 <= code <= 0x30FF # Hiragana + Katakana + or 0xAC00 <= code <= 0xD7A3 # Hangul Syllables + or 0x1100 <= code <= 0x11FF # Hangul Jamo + ): + return True + return False + + def _calculate_decision_content_similarity(self, scenario: str, decision: Dict[str, Any]) -> float: + """Calculate content similarity between scenario and decision. + + Uses word-level Jaccard for space-separated languages. For text where + whitespace tokenisation is unreliable (CJK/Japanese/Korean scripts, or + a query with no whitespace at all) a character-bigram Jaccard is + computed over the *stripped* character sequences instead. + + The bigram fallback only activates when whitespace tokenisation would + not help โ€” i.e. the query is CJK-like or has at most one whitespace + token โ€” so it never contributes for ordinary multi-word English + queries, where incidental bigram overlap between unrelated sentences + would otherwise inflate scores. + + The bigram side uses *Jaccard* (|AโˆฉB|/|AโˆชB|), not the overlap + coefficient, so a 2-character query whose single bigram happens to + appear anywhere in a long document does not silently receive a score of + 1.0. A minimum bigram set size of 3 is required before the bigram + signal contributes; this prevents 1- and 2-character English queries + from polluting results while still allowing 3-character CJK phrases (2 + bigrams) to match. + """ + try: + decision_text = ( + f"{decision['scenario']} {decision['reasoning']} " + f"{' '.join(decision['entities'])}" + ) + + # --- word-level Jaccard (primary metric for Latin/space-delimited) --- scenario_words = set(scenario.lower().split()) - decision_text = f"{decision['scenario']} {decision['reasoning']} {' '.join(decision['entities'])}" decision_words = set(decision_text.lower().split()) - - intersection = scenario_words.intersection(decision_words) - union = scenario_words.union(decision_words) - - return len(intersection) / len(union) if union else 0.0 - - except Exception as e: + word_union = scenario_words | decision_words + word_sim = ( + len(scenario_words & decision_words) / len(word_union) + if word_union + else 0.0 + ) + + # --- character-bigram Jaccard (CJK / very-short-query fallback) --- + # Only used when whitespace tokenisation can't do the job: CJK-like + # scripts, or a query that is a single whitespace token (no spaces + # to split on). Ordinary multi-word English queries rely on + # word_sim alone, so incidental bigram overlap between unrelated + # sentences can never inflate their score. + bigram_sim = 0.0 + needs_bigram_fallback = ( + self._looks_cjk(scenario) or len(scenario.split()) <= 1 + ) + if needs_bigram_fallback: + scenario_bigrams = self._char_bigrams(scenario) + decision_bigrams = self._char_bigrams(decision_text) + + # Require at least 3 bigrams in the query before the bigram + # signal is used. A 2-char query produces only 1 bigram; that + # single bigram is far too likely to appear as a substring of + # any English word and would produce a spuriously high overlap + # coefficient. 3 bigrams correspond to a 4-char stripped query + # (e.g. two CJK characters produce 1 bigram each โ†’ need โ‰ฅ3 + # chars stripped). + if len(scenario_bigrams) >= 3 and decision_bigrams: + bigram_union = scenario_bigrams | decision_bigrams + bigram_sim = ( + len(scenario_bigrams & decision_bigrams) / len(bigram_union) + if bigram_union + else 0.0 + ) + + return max(word_sim, bigram_sim) + + except Exception: self.logger.exception("Content similarity calculation failed") return 0.0 diff --git a/semantica/context/graph_schema.py b/semantica/context/graph_schema.py index 6b46e642..9a34fb89 100644 --- a/semantica/context/graph_schema.py +++ b/semantica/context/graph_schema.py @@ -6,6 +6,7 @@ including node labels, relationship types, and indexes for graph databases. """ import json +import warnings from typing import Dict, Any, List from ..graph_store import GraphStore @@ -460,11 +461,25 @@ def drop_decision_schema(graph_store: GraphStore) -> None: """ Drop decision tracking schema (for cleanup/testing). + .. deprecated:: + ``drop_decision_schema()`` is deprecated and will be removed in a future + major version. It has no callers inside Semantica; issue the DROP + CONSTRAINT / DROP INDEX / DETACH DELETE statements directly against your + :class:`~semantica.graph_store.GraphStore` instead. + Args: graph_store: Graph database instance """ logger = get_logger(__name__) + warnings.warn( + "drop_decision_schema() is deprecated and will be removed in a future " + "major version. Issue the DROP CONSTRAINT / DROP INDEX / DETACH DELETE " + "statements directly against your GraphStore instead.", + DeprecationWarning, + stacklevel=2, + ) + try: # Drop constraints constraints = [ diff --git a/semantica/deduplication/duplicate_detector.py b/semantica/deduplication/duplicate_detector.py index c7a29d49..e5b91c11 100644 --- a/semantica/deduplication/duplicate_detector.py +++ b/semantica/deduplication/duplicate_detector.py @@ -278,8 +278,12 @@ class DuplicateDetector: for i, (entity1, entity2, score) in enumerate(similarities): candidate = self._create_duplicate_candidate(entity1, entity2, score) - # Filter by confidence threshold - if candidate.confidence >= self.confidence_threshold: + # Filter by confidence threshold; type mismatches are excluded + # structurally so no threshold value can admit them. + if ( + candidate.confidence >= self.confidence_threshold + and "type_mismatch" not in candidate.reasons + ): candidates.append(candidate) remaining = total_similarities - (i + 1) @@ -624,8 +628,12 @@ class DuplicateDetector: new_entity, existing_entity, similarity.score ) - # Filter by confidence threshold - if candidate.confidence >= self.confidence_threshold: + # Filter by confidence threshold; type mismatches are + # excluded structurally regardless of the threshold. + if ( + candidate.confidence >= self.confidence_threshold + and "type_mismatch" not in candidate.reasons + ): candidates.append(candidate) processed += 1 @@ -723,7 +731,9 @@ class DuplicateDetector: if key == "name": return getattr(entity, "text", default) if key == "type": - return getattr(entity, "label", default) + # Entity objects store the type on .type; extraction entities + # may expose .label. Missing .label never means "no type". + return getattr(entity, "type", default) or getattr(entity, "label", default) if key == "properties": # Check metadata for properties metadata = getattr(entity, "metadata", {}) @@ -757,6 +767,25 @@ class DuplicateDetector: reasons = [] confidence = similarity_score + # Check entity type mismatch first: two entities with different + # explicit types are not duplicates, whatever their similarity. + entity_type1 = self._get_entity_value(entity1, "type") + entity_type2 = self._get_entity_value(entity2, "type") + if entity_type1 and entity_type2 and entity_type1 != entity_type2: + return DuplicateCandidate( + entity1=entity1, + entity2=entity2, + similarity_score=similarity_score, + confidence=0.0, + reasons=["type_mismatch"], + metadata={ + "name_match": False, + "common_properties": 0, + "type_match": False, + "type_mismatch": True, + }, + ) + # Check for exact name match (strong indicator) name1 = str(self._get_entity_value(entity1, "name", "")).lower().strip() name2 = str(self._get_entity_value(entity2, "name", "")).lower().strip() @@ -779,9 +808,8 @@ class DuplicateDetector: # Boost confidence for each matching property confidence += 0.05 * prop_matches - # Check entity type match - entity_type1 = self._get_entity_value(entity1, "type") - entity_type2 = self._get_entity_value(entity2, "type") + # Check entity type match (only boosts when types are equal; mismatch + # is handled above) if entity_type1 and entity_type2 and entity_type1 == entity_type2: reasons.append("same_type") confidence += 0.05 diff --git a/semantica/deduplication/methods.py b/semantica/deduplication/methods.py index e51a1d23..0c12159b 100644 --- a/semantica/deduplication/methods.py +++ b/semantica/deduplication/methods.py @@ -147,9 +147,12 @@ def calculate_similarity( >>> result = calculate_similarity(entity1, entity2, method="levenshtein") >>> print(f"Similarity: {result.score:.2f}") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-referential wrappers. + # _multi_factor_similarity is registered under "multi_factor" and calls back + # into calculate_similarity(method="multi_factor"), creating indirect + # infinite recursion. The identity guard short-circuits that loop. custom_method = method_registry.get("similarity", method) - if custom_method: + if custom_method and custom_method is not calculate_similarity: return custom_method(entity1, entity2, **kwargs) # Use default SimilarityCalculator @@ -235,9 +238,11 @@ def detect_duplicates( >>> duplicates = detect_duplicates(entities, method="pairwise", similarity_threshold=0.8) >>> print(f"Found {len(duplicates)} duplicate candidates") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-referential wrappers. + # _pairwise_detection is registered under "pairwise" and calls back into + # detect_duplicates(method="pairwise"), creating indirect infinite recursion. custom_method = method_registry.get("detection", method) - if custom_method: + if custom_method and custom_method is not detect_duplicates: return custom_method( entities, similarity_threshold=similarity_threshold, **kwargs ) @@ -282,9 +287,10 @@ def dedup_triplets( List of duplicate relationship piars (rel1, rel2). """ - # Check for custom method in registry (but not ourself) + # Check for custom method in registry (but not ourself โ€” identity guard + # consistent with the other dispatch functions in this module). custom_method = method_registry.get("detection", "triplets") - if custom_method and custom_method.__name__ != "dedup_triplets": + if custom_method and custom_method is not dedup_triplets: return custom_method(relationships, mode=mode, threshold=threshold, **kwargs) detector = DuplicateDetector(**kwargs) @@ -328,9 +334,11 @@ def merge_entities( >>> operations = merge_entities(duplicate_entities, method="keep_most_complete") >>> print(f"Performed {len(operations)} merge operations") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-referential registration. + # merge_entities is now registered directly under its default method name; + # the identity guard prevents a direct recursion loop. custom_method = method_registry.get("merging", method) - if custom_method: + if custom_method and custom_method is not merge_entities: return custom_method( entities, preserve_provenance=preserve_provenance, **kwargs ) @@ -374,9 +382,12 @@ def build_clusters( >>> result = build_clusters(entities, method="graph_based", similarity_threshold=0.8) >>> print(f"Found {len(result.clusters)} clusters") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-referential wrappers. + # _graph_based_clustering is registered under "graph_based" and calls back + # into build_clusters(method="graph_based"), creating indirect infinite + # recursion. custom_method = method_registry.get("clustering", method) - if custom_method: + if custom_method and custom_method is not build_clusters: return custom_method( entities, similarity_threshold=similarity_threshold, **kwargs ) @@ -546,25 +557,12 @@ def list_available_methods(task: Optional[str] = None) -> Dict[str, List[str]]: return result -# Register default methods with registry -def _multi_factor_similarity(e1, e2, **kw): - return calculate_similarity(e1, e2, method="multi_factor", **kw) - - -def _pairwise_detection(entities, **kw): - return detect_duplicates(entities, method="pairwise", **kw) - - -def _keep_most_complete_merging(entities, **kw): - return merge_entities(entities, method="keep_most_complete", **kw) - - -def _graph_based_clustering(entities, **kw): - return build_clusters(entities, method="graph_based", **kw) - - -method_registry.register("similarity", "multi_factor", _multi_factor_similarity) -method_registry.register("detection", "pairwise", _pairwise_detection) -method_registry.register("merging", "keep_most_complete", _keep_most_complete_merging) -method_registry.register("clustering", "graph_based", _graph_based_clustering) +# Register default methods with registry. +# The public dispatch functions are registered directly so the identity guard +# in each function short-circuits the self-reference rather than going through +# an intermediate wrapper that re-enters the same dispatch path. +method_registry.register("similarity", "multi_factor", calculate_similarity) +method_registry.register("detection", "pairwise", detect_duplicates) +method_registry.register("merging", "keep_most_complete", merge_entities) +method_registry.register("clustering", "graph_based", build_clusters) method_registry.register("detection", "triplets", dedup_triplets) diff --git a/semantica/embeddings/embeddings_provenance.py b/semantica/embeddings/embeddings_provenance.py index 03d7adbd..886fd8c1 100644 --- a/semantica/embeddings/embeddings_provenance.py +++ b/semantica/embeddings/embeddings_provenance.py @@ -69,6 +69,15 @@ class EmbeddingGeneratorWithProvenance: return embeddings def __getattr__(self, name): + # __getattr__ only runs when normal lookup fails. Accessing + # self._generator by attribute syntax HERE would re-enter + # __getattr__ for ever when _generator itself is missing โ€” the shape + # pickle/copy protocol probes hit when __init__ never completed + # (#994's RecursionError family). Fail fast on private probes. + if name.startswith("_"): + raise AttributeError( + f"{type(self).__name__!r} object has no attribute {name!r}" + ) return getattr(self._generator, name) diff --git a/semantica/embeddings/methods.py b/semantica/embeddings/methods.py index 4128ffd8..907e8b6c 100644 --- a/semantica/embeddings/methods.py +++ b/semantica/embeddings/methods.py @@ -117,9 +117,9 @@ def generate_embeddings( >>> emb = generate_embeddings("Hello world", method="default") >>> embs = generate_embeddings(["text1", "text2"], method="text") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("generation", method) - if custom_method: + if custom_method and custom_method is not generate_embeddings: fallback = kwargs.pop("fallback_on_custom_error", False) result = call_custom_method( logger, method, custom_method, data, data_type=data_type, fallback_on_custom_error=fallback, **kwargs @@ -165,9 +165,9 @@ def embed_text( >>> emb = embed_text("Hello world", method="sentence_transformers") >>> embs = embed_text(["text1", "text2"], method="sentence_transformers") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("text", method) - if custom_method: + if custom_method and custom_method is not embed_text: fallback = kwargs.pop("fallback_on_custom_error", False) result = call_custom_method( logger, method, custom_method, text, fallback_on_custom_error=fallback, **kwargs @@ -225,9 +225,9 @@ def calculate_similarity( >>> similarity = calculate_similarity(emb1, emb2, method="cosine") >>> print(f"Similarity: {similarity:.3f}") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("similarity", method) - if custom_method: + if custom_method and custom_method is not calculate_similarity: fallback = kwargs.pop("fallback_on_custom_error", False) result = call_custom_method( logger, method, custom_method, embedding1, embedding2, fallback_on_custom_error=fallback, **kwargs @@ -272,9 +272,9 @@ def pool_embeddings( >>> pooled = pool_embeddings(embeddings, method="mean") >>> attention_pooled = pool_embeddings(embeddings, method="attention") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("pooling", method) - if custom_method: + if custom_method and custom_method is not pool_embeddings: fallback = kwargs.pop("fallback_on_custom_error", False) result = call_custom_method( logger, method, custom_method, embeddings, fallback_on_custom_error=fallback, **kwargs diff --git a/semantica/explorer/dependencies.py b/semantica/explorer/dependencies.py index 7a97faeb..862ce03f 100644 --- a/semantica/explorer/dependencies.py +++ b/semantica/explorer/dependencies.py @@ -2,8 +2,9 @@ Semantica Explorer : FastAPI Dependencies Provides ``Depends()``-compatible callables for injecting the -current ``GraphSession`` and ``ConnectionManager`` into route handlers, -and for enforcing API-key authentication on protected routes. +current ``GraphSession`` into route handlers, and for enforcing API-key +authentication on protected routes. WebSocket manager access is handled +directly via ``app.state.ws_manager``. """ import hmac @@ -14,7 +15,6 @@ from fastapi import Request, HTTPException, Security, status from fastapi.security.api_key import APIKeyHeader from .session import GraphSession -from .ws import ConnectionManager _api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False) @@ -80,13 +80,3 @@ def get_session(request: Request) -> GraphSession: detail="GraphSession not initialized." ) return request.app.state.session - - -def get_ws_manager(request: Request) -> ConnectionManager: - """Retrieve the ConnectionManager stored on ``app.state``.""" - if not hasattr(request.app.state, "ws_manager") or request.app.state.ws_manager is None: - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="WebSocket manager not initialized.", - ) - return request.app.state.ws_manager diff --git a/semantica/explorer/routes/graph.py b/semantica/explorer/routes/graph.py index bdd29741..d620bc15 100644 --- a/semantica/explorer/routes/graph.py +++ b/semantica/explorer/routes/graph.py @@ -78,66 +78,6 @@ def _parse_bbox(raw_bbox: Optional[str]) -> Optional[tuple[float, float, float, return min_x, min_y, max_x, max_y -def _coerce_embedding_vector(value: object) -> Optional[List[float]]: - if isinstance(value, dict): - # Probe keys in priority order: generic first, then framework-specific. - # Must stay aligned with the top-level keys in _extract_node_embeddings. - for key in ("embedding", "embeddings", "vector", "values", "node2vec", "semantic"): - nested = _coerce_embedding_vector(value.get(key)) - if nested is not None: - return nested - return None - - if not isinstance(value, (list, tuple)): - return None - - vector: List[float] = [] - for item in value: - try: - vector.append(float(item)) - except (TypeError, ValueError): - return None - - return vector if vector else None - - -def _extract_node_embeddings(graph_dict: dict) -> dict[str, List[float]]: - """Extract embeddings from graph dictionary.""" - # Top-level keys to probe on each entity (and its metadata/properties dicts). - # Priority: generic names first, then KG-extras-specific names. - # Must stay aligned with the inner probe list in _coerce_embedding_vector. - embedding_keys = ( - "embedding", - "embeddings", - "vector", - "node_embedding", - "node2vec_embedding", - "semantic_embedding", - "reasoning_embedding", - ) - - embeddings: dict[str, List[float]] = {} - for entity in graph_dict.get("entities") or graph_dict.get("nodes") or []: - if not isinstance(entity, dict): - continue - node_id = entity.get("id") or entity.get("node_id") - if not node_id: - continue - - metadata = entity.get("metadata") if isinstance(entity.get("metadata"), dict) else {} - properties = entity.get("properties") if isinstance(entity.get("properties"), dict) else {} - - for key in embedding_keys: - vector = _coerce_embedding_vector( - entity.get(key, metadata.get(key, properties.get(key))) - ) - if vector is not None: - embeddings[str(node_id)] = vector - break - - return embeddings - - def _get_cached_embeddings(session: GraphSession) -> dict[str, List[float]]: """Get embeddings from session cache for optimal performance.""" return session.get_cached_embeddings() diff --git a/semantica/explorer/routes/ontology.py b/semantica/explorer/routes/ontology.py index b50eb206..81877e50 100644 --- a/semantica/explorer/routes/ontology.py +++ b/semantica/explorer/routes/ontology.py @@ -456,10 +456,6 @@ class DraftResponse(BaseModel): updated_at: str -class ProposalState(BaseModel): - state: Literal["draft", "proposed", "approved", "published", "rejected"] - - class ProposalRequest(BaseModel): draft_id: str ontology_uri: str diff --git a/semantica/explorer/schemas.py b/semantica/explorer/schemas.py index 13f2ac3f..808bceed 100644 --- a/semantica/explorer/schemas.py +++ b/semantica/explorer/schemas.py @@ -8,11 +8,6 @@ from typing import Any, Dict, List, Literal, Optional, Tuple from pydantic import BaseModel, Field, field_validator -class ErrorResponse(BaseModel): - detail: str - status_code: int = 500 - - class NodeResponse(BaseModel): id: str type: str @@ -187,12 +182,6 @@ class ComplianceResponse(BaseModel): violations: List[Dict[str, Any]] = Field(default_factory=list) -class TemporalSnapshotResponse(BaseModel): - timestamp: str - active_nodes: List[NodeResponse] - active_node_count: int - - class TemporalDiffResponse(BaseModel): from_time: str to_time: str @@ -256,13 +245,6 @@ class ExportRequest(BaseModel): node_ids: Optional[List[str]] = None -class ExportResponse(BaseModel): - format: str - content_type: str - filename: str - size_bytes: int = 0 - - class ImportResponse(BaseModel): status: str = "success" message: str = "Import successful" @@ -272,11 +254,6 @@ class ImportResponse(BaseModel): edges_imported: Optional[int] = None -class StandardMessageResponse(BaseModel): - status: str - message: str - - class AnnotationCreate(BaseModel): node_id: str content: str diff --git a/semantica/export/json_exporter.py b/semantica/export/json_exporter.py index 0c3b492c..e31c39d8 100644 --- a/semantica/export/json_exporter.py +++ b/semantica/export/json_exporter.py @@ -28,12 +28,35 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Union from ..utils.exceptions import ProcessingError, ValidationError -from ..utils.helpers import ensure_directory, utc_now_iso, write_json_file +from ..utils.helpers import ensure_directory, hash_data, utc_now_iso, write_json_file from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker from .rdf_exporter import SEMANTICA_NS, mint_entity_iri, mint_relationship_iri +def _content_iri(prefix: str, payload: Any) -> str: + """Mint a document IRI from what was exported, not when. + + Minting from ``utc_now_iso()`` gave every export of the same graph a new + identity a few microseconds apart, so re-exporting an unchanged graph was + never idempotent and merging exports duplicated every node (#1147). This + mirrors ``mint_entity_iri`` (#1109): identical content hashes to the same + IRI, and any change to the content changes it too. ``default=str`` keeps + the hash defined for values ``json.dumps`` would otherwise reject, such as + ``datetime`` objects a caller may have left in the graph. + + Args: + prefix: IRI prefix the digest is appended to + payload: JSON-serializable value whose content determines the digest + + Returns: + A stable IRI of the form ``{prefix}{16-hex-char digest}`` + """ + canonical = json.dumps(payload, sort_keys=True, default=str) + digest = hash_data(canonical)[:16] + return f"{prefix}{digest}" + + def _is_jsonld_document(data: Dict[str, Any]) -> bool: """ Report whether a dictionary is already a JSON-LD document. @@ -230,7 +253,10 @@ class JSONExporter: - statistics: Statistics dictionary (optional) file_path: Output JSON file path format: Export format - 'json' or 'json-ld' (default: self.format) - **options: Additional options passed to conversion methods + **options: Additional options passed to conversion methods: + - graph_uri: Caller-supplied IRI for the graph node when + format='json-ld', overriding the default content-derived + IRI (see #1147) Example: >>> kg = { @@ -401,7 +427,9 @@ class JSONExporter: data: Data to convert (dict, list, or any value) include_metadata: Whether to include metadata (default: True) include_provenance: Whether to include provenance (default: True) - **options: Additional options passed to knowledge graph conversion + **options: Additional options passed to knowledge graph conversion: + - document_uri: Caller-supplied IRI for the document node, + overriding the default content-derived IRI (see #1147) Returns: Dictionary in JSON-LD format with @context, @graph/@value, and metadata @@ -451,13 +479,17 @@ class JSONExporter: # Add metadata and provenance if requested if include_metadata: - self._attach_document_metadata(jsonld, include_provenance) + self._attach_document_metadata( + jsonld, include_provenance, options.get("document_uri") + ) return jsonld @staticmethod def _attach_document_metadata( - jsonld: Dict[str, Any], include_provenance: bool + jsonld: Dict[str, Any], + include_provenance: bool, + document_uri: Optional[str] = None, ) -> None: """ Attach the export's own metadata without naming the graph. @@ -473,6 +505,9 @@ class JSONExporter: Args: jsonld: Document being built, modified in place include_provenance: Whether to record how and when it was exported + document_uri: Caller-supplied IRI for the document node. Falls back + to a content-derived IRI (#1147) so re-exporting unchanged data + is idempotent instead of minting a new identity every time. """ # A caller may hand us a document that is deliberately a named graph. # That name is theirs to keep, but our own statements must not end up @@ -483,7 +518,10 @@ class JSONExporter: # Do not overwrite an identifier the payload already carries: the # knowledge-graph conversion names its own document node. if "@id" not in jsonld or payload_is_named_graph: - document["@id"] = f"https://semantica.dev/data/{utc_now_iso()}" + content = {key: value for key, value in jsonld.items() if key != "@context"} + document["@id"] = document_uri or _content_iri( + "https://semantica.dev/data/", content + ) if include_provenance: document["semantica:exportedAt"] = utc_now_iso() document["semantica:format"] = "json-ld" @@ -553,7 +591,9 @@ class JSONExporter: - entities: List of entity dictionaries - relationships: List of relationship dictionaries - metadata: Metadata dictionary (optional) - **options: Additional options (unused) + **options: Additional options: + - graph_uri: Caller-supplied IRI for the graph node, + overriding the default content-derived IRI (see #1147) Returns: Dictionary in JSON-LD format with @context, @id, @type, and graph data @@ -566,7 +606,11 @@ class JSONExporter: "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdfs": "http://www.w3.org/2000/01/rdf-schema#", }, - "@id": f"https://semantica.dev/graph/{utc_now_iso()}", + # Minted from the graph's own content rather than the wall clock + # (#1147): re-exporting an unchanged graph must produce the same + # subject, or merging repeated exports duplicates every node. + "@id": options.get("graph_uri") + or _content_iri("https://semantica.dev/graph/", kg), "@type": "semantica:KnowledgeGraph", } diff --git a/semantica/export/rdf_exporter.py b/semantica/export/rdf_exporter.py index 71d12f71..7a290b0c 100644 --- a/semantica/export/rdf_exporter.py +++ b/semantica/export/rdf_exporter.py @@ -29,9 +29,12 @@ Author: Semantica Contributors License: MIT """ +import re from pathlib import Path from decimal import Decimal, InvalidOperation +from html import escape as xml_escape from typing import Any, Dict, List, Optional, Set, Union +from urllib.parse import quote, urlsplit from ..utils.exceptions import ProcessingError, ValidationError from ..utils.helpers import ensure_directory, hash_data @@ -104,7 +107,11 @@ def normalize_confidence(value: Any) -> Optional[str]: # "1e100000000" is eleven characters that expand to a hundred million, and # the export path continues past validation errors, so a single malformed # field could exhaust memory. Nothing near this magnitude is a confidence. - if not -MAX_CONFIDENCE_EXPONENT <= decimal_value.adjusted() <= MAX_CONFIDENCE_EXPONENT: + if ( + not -MAX_CONFIDENCE_EXPONENT + <= decimal_value.adjusted() + <= MAX_CONFIDENCE_EXPONENT + ): return None # `str(Decimal("0.00001"))` gives "0.00001", but a float that has already @@ -139,6 +146,294 @@ def mint_relationship_iri(index: int, source: Any, target: Any) -> str: return f"{SEMANTICA_NS}rel_{index}_{digest}" +#: The metadata keys Semantica itself produces, and the terms they are written +#: as. GraphBuilder.build_graph writes the first five, create_snapshot writes +#: snapshot_time, and load_from_neo4j writes source / uri / database. These are +#: Semantica's own vocabulary, so they are minted in the declared namespace and +#: declared in semantica-ns.ttl. +#: +#: A key the caller supplied is a different matter. Which namespace an +#: arbitrary metadata key belongs in is issue #1146, and until that is settled +#: the exporter refuses to guess: it warns and skips, and a caller who already +#: knows the answer passes ``metadata_terms``. +#: +#: The map is key -> term rather than key -> namespace because two of the keys +#: cannot keep their own name. ``source`` on a graph loaded from Neo4j is the +#: system it came from, while sem:source is already the ObjectProperty holding +#: the subject of a reified relationship; reusing it would put a string where +#: an entity belongs. +DEFAULT_METADATA_TERMS: Dict[str, str] = { + "num_entities": f"{SEMANTICA_NS}numEntities", + "num_relationships": f"{SEMANTICA_NS}numRelationships", + "temporal_enabled": f"{SEMANTICA_NS}temporalEnabled", + "entity_resolution_applied": f"{SEMANTICA_NS}entityResolutionApplied", + "timestamp": f"{SEMANTICA_NS}builtAt", + "snapshot_time": f"{SEMANTICA_NS}snapshotAt", + "source": f"{SEMANTICA_NS}sourceSystem", + "uri": f"{SEMANTICA_NS}sourceUri", + "database": f"{SEMANTICA_NS}sourceDatabase", +} + +#: Terms whose value is a node rather than a string. Everything else stays a +#: literal: a metadata value that merely looks like a URL is not thereby a +#: reference to one. +IRI_VALUED_METADATA_TERMS: Set[str] = {f"{SEMANTICA_NS}sourceUri"} + +_XSD_NS = "http://www.w3.org/2001/XMLSchema#" + + +def _escape_literal(value: str) -> str: + """Escape a string for a Turtle or N-Triples quoted literal.""" + return ( + value.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t") + ) + + +def _escape_temporal_literal(value: Any) -> str: + """Escape a temporal bound for a Turtle ``dateTimeStamp`` literal. + + Bounds are normally strings, but callers may hand us a ``datetime`` or + ``None``. ``_escape_literal`` is str-only, so stringify non-str values + first instead of calling ``.replace()`` on them; ``None`` yields an empty + bound rather than crashing. Datetimes must use ISO 8601 so the + ``xsd:dateTimeStamp`` ``T`` separator is preserved โ€” ``str()`` yields a + space ("00:00:00+00:00"), which is a lexically invalid timestamp. + """ + if value is None: + return "" + if isinstance(value, str): + return _escape_literal(value) + if hasattr(value, "isoformat"): + return value.isoformat() + return str(value) + + +#: Turtle/N-Triples IRIREF grammar excludes these unescaped between `<` and +#: `>`: control characters, space, and <>"{}|^`\. An IRI-valued metadata +#: value (currently only sem:sourceUri, from the caller-controlled "uri" +#: metadata key) is written as `<{value}>` with no other quoting, so a value +#: containing one of these characters โ€” a ">" followed by a full triple, for +#: instance โ€” closes the IRIREF early and lets the rest of the string be +#: parsed as further RDF statements. This is the same shape of defect the +#: entity/relationship IRIs were hardened against; that hardening resolves +#: prefixes as well, which a metadata value never needs, so this stays a +#: narrower, dedicated guard rather than reusing _as_turtle_iri. +_IRI_REF_UNSAFE_RE = re.compile(r'[\x00-\x20<>"{}|^`\\]') + + +def _safe_iri_ref(value: str) -> str: + """Percent-encode the characters an IRIREF may not contain unescaped.""" + return _IRI_REF_UNSAFE_RE.sub(lambda m: quote(m.group(0), safe=""), value) + + +def _escape_xml(value: str) -> str: + """Escape a string for either XML element text or an attribute value. + + The quotes matter. This helper feeds `rdf:about`, `rdf:resource` and + `xmlns:` attribute values, which are delimited by double quotes, so a value + carrying one would close the attribute early and produce a document that + does not parse. Escaping them in element text as well is harmless and + means one helper cannot be used in the wrong place. + """ + return ( + value.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) + + +def _is_ncname(value: str) -> bool: + """Whether a string can be an XML NCName, which is what RDF/XML requires. + + Checked over the ASCII range rather than the full XML production: the + grammar also admits combining characters and extenders, so this is + deliberately conservative. It refuses names it could have accepted, and it + never accepts one that would produce a document a parser rejects. The + earlier check tested only that the first character was not a digit, which + let through every other way a local name can fail to be a name. + """ + if not value: + return False + if not (value[0].isascii() and (value[0].isalpha() or value[0] == "_")): + return False + return all(c.isascii() and (c.isalnum() or c in "._-") for c in value[1:]) + + +def _split_iri(iri: str) -> Optional[tuple]: + """Split an IRI into (namespace, local name) for RDF/XML's QName syntax. + + Returns None when no split yields a usable local name. RDF/XML is the only + serialization here that cannot write an arbitrary predicate IRI, so this is + the one place a term can be unrepresentable, and the caller reports it + rather than dropping it quietly. + """ + for sep in ("#", "/"): + index = iri.rfind(sep) + if index != -1 and index + 1 < len(iri): + local = iri[index + 1 :] + if _is_ncname(local): + return iri[: index + 1], local + return None + + +def _metadata_statements( + metadata: Any, + terms: Dict[str, str], + logger: Any, +) -> List[tuple]: + """Resolve a metadata mapping to a list of (term IRI, value) pairs. + + A key with no term is skipped and reported. Silence is the defect this + fixes, so an unmapped key must be louder than a mapped one, not quieter. + """ + if not isinstance(metadata, dict): + return [] + + statements: List[tuple] = [] + for key, value in metadata.items(): + term = terms.get(key) + if term is None: + logger.warning( + "Metadata key %r has no term and was not exported. Which " + "namespace a caller-supplied key belongs in is issue #1146; " + "pass metadata_terms={%r: ''} to export it now.", + key, + key, + ) + continue + if value is None: + continue + if isinstance(value, (dict, list, tuple, set)): + logger.warning( + "Metadata key %r holds a %s, which has no modelled RDF shape " + "yet, and was not exported.", + key, + type(value).__name__, + ) + continue + statements.append((term, value)) + return statements + + +def _resolve_metadata_terms(overrides: Optional[Dict[str, str]]) -> Dict[str, str]: + if not overrides: + return DEFAULT_METADATA_TERMS + return {**DEFAULT_METADATA_TERMS, **overrides} + + +def _typed_literal_parts(term: str, value: Any) -> tuple: + """Return (kind, lexical, datatype) for one metadata value. + + kind is "iri" or "literal". The lexical form and datatype are chosen once, + here, so that the four serializers cannot disagree about them the way they + disagreed about confidence in #1100. + """ + if term in IRI_VALUED_METADATA_TERMS and isinstance(value, str): + return "iri", value, None + if isinstance(value, bool): + return "literal", "true" if value else "false", f"{_XSD_NS}boolean" + if isinstance(value, int): + return "literal", str(value), f"{_XSD_NS}integer" + if isinstance(value, float): + # xsd:double, not xsd:decimal. `repr(1e-05)` is "1e-05" and + # `repr(float("nan"))` is "nan", and xsd:decimal admits neither the + # exponent form nor the special values, so typing a float as decimal + # produced lexicals a strict parser rejects. A Python float is an IEEE + # 754 double; xsd:double has legal lexicals for all of them, and it is + # also the honest claim, since nothing that arrived as a float was ever + # exact. `normalize_confidence` keeps xsd:decimal for confidence + # deliberately: that is a bounded score where exactness is meaningful + # and NaN is not a confidence at all. + if value != value: + lexical = "NaN" + elif value == float("inf"): + lexical = "INF" + elif value == float("-inf"): + lexical = "-INF" + else: + lexical = repr(value) + return "literal", lexical, f"{_XSD_NS}double" + return "literal", str(value), None + + +def _turtle_object(term: str, value: Any) -> str: + kind, lexical, datatype = _typed_literal_parts(term, value) + if kind == "iri": + return f"<{_safe_iri_ref(lexical)}>" + if datatype is None: + return f'"{_escape_literal(lexical)}"' + return f'"{lexical}"^^<{datatype}>' + + +def _turtle_metadata_clauses(statements: List[tuple]) -> List[str]: + return [f"<{term}> {_turtle_object(term, value)}" for term, value in statements] + + +def _ntriples_metadata_lines(subject: str, statements: List[tuple]) -> List[str]: + return [ + f"<{subject}> <{term}> {_turtle_object(term, value)} ." + for term, value in statements + ] + + +def _rdfxml_metadata_lines( + statements: List[tuple], indent: str, logger: Any = None +) -> List[str]: + """RDF/XML needs a QName, so an unprefixed term declares its own prefix. + + A term with no QName form has no RDF/XML representation at all, and this is + the only serialization with that restriction. Skipping it quietly would + reintroduce, in one format, exactly the silent metadata loss this module + was changed to stop, so it is reported and the other three formats still + carry the statement in full. + """ + lines: List[str] = [] + for position, (term, value) in enumerate(statements): + split = _split_iri(term) + if split is None: + if logger is not None: + logger.warning( + "Term %r has no QName form, so it cannot be written in " + "RDF/XML and was omitted from that serialization only. " + "Turtle, N-Triples and JSON-LD carry it in full.", + term, + ) + continue + namespace, local = split + kind, lexical, datatype = _typed_literal_parts(term, value) + prefix = f"md{position}" + opening = f'{indent}<{prefix}:{local} xmlns:{prefix}="{_escape_xml(namespace)}"' + if kind == "iri": + lines.append(f'{opening} rdf:resource="{_escape_xml(lexical)}"/>') + continue + if datatype is not None: + opening += f' rdf:datatype="{_escape_xml(datatype)}"' + lines.append(f"{opening}>{_escape_xml(lexical)}") + return lines + + +def _jsonld_metadata_entries(statements: List[tuple]) -> Dict[str, Any]: + """Absolute IRIs as keys, and explicit @value/@type rather than JSON's own + types: JSON's number is xsd:double, which would make the JSON-LD export + disagree with the other three about the datatype of an integer.""" + entries: Dict[str, Any] = {} + for term, value in statements: + kind, lexical, datatype = _typed_literal_parts(term, value) + if kind == "iri": + entries[term] = {"@id": lexical} + elif datatype is None: + entries[term] = lexical + else: + entries[term] = {"@value": lexical, "@type": datatype} + return entries + + class NamespaceManager: """ RDF namespace management engine. @@ -395,6 +690,66 @@ class RDFSerializer: # OWL-Time namespace URI _OWL_TIME_NS = "http://www.w3.org/2006/time#" + _SEMANTICA_NS = "https://semantica.dev/ns#" + + # Matches an already-valid percent-escape so it can be passed through + # unchanged instead of being re-encoded into e.g. %2520. + _PERCENT_ESCAPE_RE = re.compile(r"%[0-9A-Fa-f]{2}") + + @classmethod + def _quote_preserving_escapes(cls, value: str, safe: str) -> str: + """quote() that leaves existing valid %XX escapes untouched. + + Blanket-quoting an absolute IRI double-encodes any percent-escape it + already carries (%20 -> %2520), which changes the identity of every + previously-valid IRI containing one. Only the spans between existing + valid escapes are quoted; a bare '%' that isn't part of a valid + escape (e.g. "%zz") still gets encoded to %25, keeping the malformed + case handled. + """ + parts = [] + pos = 0 + for match in cls._PERCENT_ESCAPE_RE.finditer(value): + parts.append(quote(value[pos : match.start()], safe=safe)) + parts.append(match.group(0)) + pos = match.end() + parts.append(quote(value[pos:], safe=safe)) + return "".join(parts) + + def _as_turtle_iri( + self, value: Any, namespaces: Optional[Dict[str, str]] = None + ) -> str: + """Return an absolute, safely encoded IRI for a Turtle resource.""" + value = str(value) + try: + parsed = urlsplit(value) + except ValueError: + parsed = urlsplit("") + if parsed.scheme: + prefix, separator, local_name = value.partition(":") + # Built-in namespaces (semantica:, rdf:, rdfs:, owl:, ...) must + # always be resolvable, not only when the caller passes no + # namespaces of its own โ€” otherwise a value like "semantica:Foo" + # resolves fine with no @context but stops resolving the moment + # any @context is present, since callers pass extract_namespaces() + # (context-only) here without merging in the built-ins. + effective_namespaces = { + **self.namespace_manager.namespaces, + **(namespaces or {}), + } + namespace = effective_namespaces.get(prefix) + if namespace and separator: + return self._quote_preserving_escapes( + namespace + local_name, safe=":/?#[]@!$&'()*+,;=" + ) + # A scheme with at least two characters is an absolute IRI, + # including opaque forms such as mailto:foo and isbn:0451450523. + # Keep one-character schemes as the existing Windows drive-path case. + if len(prefix) >= 2: + return self._quote_preserving_escapes( + value, safe=":/?#[]@!$&'()*+,;=" + ) + return self._SEMANTICA_NS + quote(value, safe="") # Design decision โ€” TemporalBound.OPEN in RDF: # OWL-Time has no standard predicate for "no known end date." We use @@ -434,6 +789,8 @@ class RDFSerializer: """ include_temporal: bool = options.pop("include_temporal", False) time_axis: str = options.pop("time_axis", "valid") + metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None)) + graph_uri: Optional[str] = options.pop("graph_uri", None) lines = [] @@ -467,18 +824,32 @@ class RDFSerializer: text = entity.get("text") or entity.get("label", "") confidence = normalize_confidence(entity.get("confidence", 1.0)) - lines.append(f"<{entity_id}> a <{entity_type}> ;") + clauses = [ + f"a <{self._as_turtle_iri(entity_type, merged_namespaces)}>", + f'semantica:text "{_escape_literal(text)}"', + ] if confidence is None: self.logger.warning( f"Entity {entity_id} has a confidence that is not a number " f"({entity.get('confidence')!r}), so no confidence is written" ) - lines.append(f' semantica:text "{text}" .') else: - lines.append(f' semantica:text "{text}" ;') - lines.append( - f' semantica:confidence "{confidence}"^^<{CONFIDENCE_DATATYPE}> .' + clauses.append( + f'semantica:confidence "{confidence}"^^<{CONFIDENCE_DATATYPE}>' ) + clauses.extend( + _turtle_metadata_clauses( + _metadata_statements( + entity.get("metadata"), metadata_terms, self.logger + ) + ) + ) + + entity_iri = self._as_turtle_iri(entity_id, merged_namespaces) + lines.append(f"<{entity_iri}> {clauses[0]} ;") + for clause in clauses[1:-1]: + lines.append(f" {clause} ;") + lines.append(f" {clauses[-1]} .") lines.append("") # Convert relationships to RDF triplets @@ -488,10 +859,16 @@ class RDFSerializer: target_id = rel.get("target_id") or rel.get("target") rel_type = rel.get("type", DEFAULT_RELATION_TYPE) - lines.append(f"<{source_id}> <{rel_type}> <{target_id}> .") + lines.append( + f"<{self._as_turtle_iri(source_id, merged_namespaces)}> " + f"<{self._as_turtle_iri(rel_type, merged_namespaces)}> " + f"<{self._as_turtle_iri(target_id, merged_namespaces)}> ." + ) if include_temporal: - owl_lines = self._owl_time_triples_for_rel(rel, idx, time_axis) + owl_lines = self._owl_time_triples_for_rel( + rel, idx, time_axis, merged_namespaces + ) if owl_lines: # The interval hangs off the relationship's own IRI, and a # relationship written as a single triple has no such node @@ -501,11 +878,36 @@ class RDFSerializer: # export, and every term is declared in the vocabulary. lines.extend( self._reified_relationship_triples( - rel, idx, source_id, target_id, rel_type + rel, idx, source_id, target_id, rel_type, merged_namespaces ) ) lines.extend(owl_lines) + # Graph-level metadata needs a subject, and this serializer has never + # minted a document node. Rather than invent one here, it is written + # only when the caller names the graph; issue #1147 is where the + # default subject comes from once that lands. + graph_clauses = ( + _turtle_metadata_clauses( + _metadata_statements( + rdf_data.get("metadata"), metadata_terms, self.logger + ) + ) + if graph_uri + else [] + ) + if graph_clauses: + graph_iri = self._as_turtle_iri(graph_uri, merged_namespaces) + lines.append("") + lines.append( + f"<{graph_iri}> {graph_clauses[0]} " + + (";" if len(graph_clauses) > 1 else ".") + ) + for clause in graph_clauses[1:-1]: + lines.append(f" {clause} ;") + if len(graph_clauses) > 1: + lines.append(f" {graph_clauses[-1]} .") + return "\n".join(lines) def _reified_relationship_triples( @@ -515,6 +917,7 @@ class RDFSerializer: source_id: str, target_id: str, rel_type: str, + namespaces: Optional[Dict[str, str]] = None, ) -> List[str]: """ Emit the reified relationship node that OWL-Time triples hang off. @@ -523,7 +926,11 @@ class RDFSerializer: to, using the same sem:Relationship shape the JSON-LD export already writes, so the two serializations describe relationships the same way. """ - rel_id = rel.get("id") or mint_relationship_iri(idx, source_id or "", target_id or "") + rel_id = self._as_turtle_iri( + rel.get("id") + or mint_relationship_iri(idx, source_id or "", target_id or ""), + namespaces, + ) # The full predicate, not its local name. Truncating to the fragment # made https://a.example/ns#employs and https://b.example/ns#employs the @@ -537,17 +944,25 @@ class RDFSerializer: .replace("\r", "\\r") ) - predicates = [f"a semantica:Relationship"] + predicates = ["a semantica:Relationship"] if source_id: - predicates.append(f"semantica:source <{source_id}>") + predicates.append( + f"semantica:source <{self._as_turtle_iri(source_id, namespaces)}>" + ) if target_id: - predicates.append(f"semantica:target <{target_id}>") + predicates.append( + f"semantica:target <{self._as_turtle_iri(target_id, namespaces)}>" + ) predicates.append(f'semantica:type "{escaped}"') return ["", f"<{rel_id}> " + " ;\n ".join(predicates) + " ."] def _owl_time_triples_for_rel( - self, rel: Dict[str, Any], idx: int, time_axis: str + self, + rel: Dict[str, Any], + idx: int, + time_axis: str, + namespaces: Optional[Dict[str, str]] = None, ) -> List[str]: """ Emit OWL-Time Turtle triples for a relationship that carries temporal metadata. @@ -562,7 +977,7 @@ class RDFSerializer: def _is_open(v: Any) -> bool: if v is None: return False - if hasattr(v, "value"): # TemporalBound enum + if hasattr(v, "value"): # TemporalBound enum return v.value == _OPEN_SENTINEL return str(v).strip().upper() == _OPEN_SENTINEL @@ -579,7 +994,10 @@ class RDFSerializer: # deterministic IRI. source_id = rel.get("source_id") or rel.get("source") or "" target_id = rel.get("target_id") or rel.get("target") or "" - rel_base_id = rel.get("id") or mint_relationship_iri(idx, source_id, target_id) + rel_base_id = self._as_turtle_iri( + rel.get("id") or mint_relationship_iri(idx, source_id, target_id), + namespaces, + ) lines = [""] # blank separator for axis_name, from_val, until_val in axes: @@ -594,22 +1012,22 @@ class RDFSerializer: lines.append(f" time:hasBeginning <{begin_id}> ;") if _is_open(until_val): - lines.append( - ' semantica:openEndedInterval "true"^^xsd:boolean .' - ) + lines.append(' semantica:openEndedInterval "true"^^xsd:boolean .') elif until_val is not None: end_id = f"{rel_base_id}__{axis_name}_end" lines.append(f" time:hasEnd <{end_id}> .") lines.append(f"<{end_id}> a time:Instant ;") lines.append( - f' time:inXSDDateTimeStamp "{until_val}"^^xsd:dateTimeStamp .' + f' time:inXSDDateTimeStamp "{_escape_temporal_literal(until_val)}"^^xsd:dateTimeStamp .' ) else: - lines[-1] = lines[-1].rstrip(" ;") + " ." # close interval without hasEnd + lines[-1] = ( + lines[-1].rstrip(" ;") + " ." + ) # close interval without hasEnd lines.append(f"<{begin_id}> a time:Instant ;") lines.append( - f' time:inXSDDateTimeStamp "{from_val}"^^xsd:dateTimeStamp .' + f' time:inXSDDateTimeStamp "{_escape_temporal_literal(from_val)}"^^xsd:dateTimeStamp .' ) lines.append("") @@ -638,12 +1056,17 @@ class RDFSerializer: ... } >>> rdfxml = serializer.serialize_to_rdfxml(rdf_data) """ + metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None)) + graph_uri: Optional[str] = options.pop("graph_uri", None) + lines = [''] lines.append('') lines.append("") + namespaces = self.namespace_manager.extract_namespaces(rdf_data) + # Convert entities to RDF/XML entities = rdf_data.get("entities", []) for entity in entities: @@ -653,13 +1076,22 @@ class RDFSerializer: entity_text = entity.get("text", "") entity_id = mint_entity_iri(entity_text) - entity_type = entity.get("type", DEFAULT_ENTITY_TYPE) + entity_type = entity.get("type") or DEFAULT_ENTITY_TYPE text = entity.get("text") or entity.get("label", "") confidence = normalize_confidence(entity.get("confidence", 1.0)) # RDF/XML syntax: rdf:Description with rdf:about - lines.append(f' ') - lines.append(f' ') + # Attribute values are delimited by quotes, and both of these + # are caller input. Element text is left alone deliberately: that + # is #1098, and it is being fixed on its own path. + entity_iri = xml_escape( + self._as_turtle_iri(entity_id, namespaces), quote=True + ) + entity_type_iri = xml_escape( + self._as_turtle_iri(entity_type, namespaces), quote=True + ) + lines.append(f' ') + lines.append(f' ') lines.append(f" {text}") if confidence is None: self.logger.warning( @@ -671,6 +1103,15 @@ class RDFSerializer: f' ' f"{confidence}" ) + lines.extend( + _rdfxml_metadata_lines( + _metadata_statements( + entity.get("metadata"), metadata_terms, self.logger + ), + " ", + self.logger, + ) + ) lines.append(" ") lines.append("") @@ -679,11 +1120,39 @@ class RDFSerializer: for rel in relationships: source_id = rel.get("source_id") or rel.get("source") target_id = rel.get("target_id") or rel.get("target") - rel_type = rel.get("type", "semantica:related_to") + # RDF/XML predicates are emitted as QNames, unlike resource + # attributes which use the shared absolute-IRI normalizer. + rel_type = rel.get("type") or "semantica:related_to" # Relationship as property on source entity - lines.append(f' ') - lines.append(f' <{rel_type} rdf:resource="{target_id}"/>') + source_iri = xml_escape( + self._as_turtle_iri(source_id, namespaces), quote=True + ) + target_iri = xml_escape( + self._as_turtle_iri(target_id, namespaces), quote=True + ) + lines.append(f' ') + lines.append(f' <{rel_type} rdf:resource="{target_iri}"/>') + lines.append(" ") + lines.append("") + + graph_lines = ( + _rdfxml_metadata_lines( + _metadata_statements( + rdf_data.get("metadata"), metadata_terms, self.logger + ), + " ", + self.logger, + ) + if graph_uri + else [] + ) + if graph_lines: + graph_iri = xml_escape( + self._as_turtle_iri(graph_uri, namespaces), quote=True + ) + lines.append(f' ') + lines.extend(graph_lines) lines.append(" ") lines.append("") @@ -717,6 +1186,9 @@ class RDFSerializer: """ import json + metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None)) + graph_uri: Optional[str] = options.pop("graph_uri", None) + # Initialize JSON-LD structure with context jsonld = { "@context": { @@ -761,6 +1233,13 @@ class RDFSerializer: "@value": confidence, "@type": CONFIDENCE_DATATYPE, } + node.update( + _jsonld_metadata_entries( + _metadata_statements( + entity.get("metadata"), metadata_terms, self.logger + ) + ) + ) jsonld["@graph"].append(node) # Convert relationships to JSON-LD @@ -785,6 +1264,18 @@ class RDFSerializer: } ) + graph_entries = ( + _jsonld_metadata_entries( + _metadata_statements( + rdf_data.get("metadata"), metadata_terms, self.logger + ) + ) + if graph_uri + else {} + ) + if graph_entries: + jsonld["@graph"].append({"@id": graph_uri, **graph_entries}) + return json.dumps(jsonld, indent=2, ensure_ascii=False) def serialize_to_ntriples(self, rdf_data: Dict[str, Any], **options) -> str: @@ -801,22 +1292,17 @@ class RDFSerializer: Returns: String containing N-Triples serialization """ + metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None)) + graph_uri: Optional[str] = options.pop("graph_uri", None) + lines = [] + namespaces = self.namespace_manager.extract_namespaces(rdf_data) + def expand_uri(uri: str) -> str: if not uri: return "" - if uri.startswith("http"): - return f"<{uri}>" - if uri.startswith("semantica:"): - return f"" - if uri.startswith("rdf:"): - return f"" - if uri.startswith("rdfs:"): - return f"" - if ":" in uri: - return f"<{uri}>" - return f"" + return f"<{self._as_turtle_iri(uri, namespaces)}>" # Convert entities entities = rdf_data.get("entities", []) @@ -830,7 +1316,7 @@ class RDFSerializer: subject = expand_uri(entity_id) # Type triple - entity_type = entity.get("type", "semantica:Entity") + entity_type = entity.get("type") or DEFAULT_ENTITY_TYPE lines.append( f"{subject} {expand_uri(entity_type)} ." ) @@ -838,7 +1324,7 @@ class RDFSerializer: # Text property text = entity.get("text") or entity.get("label", "") if text: - safe_text = text.replace('"', '\\"').replace("\n", "\\n") + safe_text = _escape_literal(text) lines.append( f'{subject} {expand_uri("semantica:text")} "{safe_text}" .' ) @@ -859,18 +1345,37 @@ class RDFSerializer: f'"{confidence}"^^<{CONFIDENCE_DATATYPE}> .' ) + lines.extend( + _ntriples_metadata_lines( + subject.strip("<>"), + _metadata_statements( + entity.get("metadata"), metadata_terms, self.logger + ), + ) + ) + # Convert relationships relationships = rdf_data.get("relationships", []) for rel in relationships: source_id = rel.get("source_id") or rel.get("source") target_id = rel.get("target_id") or rel.get("target") - rel_type = rel.get("type", "semantica:related_to") + rel_type = rel.get("type") or DEFAULT_RELATION_TYPE if source_id and target_id: lines.append( f"{expand_uri(source_id)} {expand_uri(rel_type)} {expand_uri(target_id)} ." ) + if graph_uri: + lines.extend( + _ntriples_metadata_lines( + graph_uri, + _metadata_statements( + rdf_data.get("metadata"), metadata_terms, self.logger + ), + ) + ) + return "\n".join(lines) diff --git a/semantica/ingest/db_ingestor.py b/semantica/ingest/db_ingestor.py index 351238d3..0d656dee 100644 --- a/semantica/ingest/db_ingestor.py +++ b/semantica/ingest/db_ingestor.py @@ -815,6 +815,8 @@ class DBIngestor: engine = connector.connect(connection_string) try: + from sqlalchemy import text + with engine.connect() as conn: # Execute query with parameters (parameterized queries for safety) result = conn.execute(text(query), params) diff --git a/semantica/ingest/ontology_ingestor.py b/semantica/ingest/ontology_ingestor.py index 3b8638f9..0d63a770 100644 --- a/semantica/ingest/ontology_ingestor.py +++ b/semantica/ingest/ontology_ingestor.py @@ -40,7 +40,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Union import rdflib -from rdflib import RDF, RDFS, OWL, Graph +from rdflib import RDF, RDFS, OWL, Dataset, Graph from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger @@ -106,15 +106,24 @@ class OntologyIngestor: raise ValidationError(f"File not found: {file_path}") self.progress.update_tracking(tracking_id, message="Parsing RDF graph...") - g = Graph() - + # `Dataset`, not `Graph`: a JSON-LD document with a top-level `@id` *and* + # `@graph` places its terms in a NAMED graph. `Graph.parse()` loads only the + # default graph and discards the rest without an error, so every class and + # property in such a document was dropped while the load reported success. + # Same migration #757 made for JenaStore; the ingest path was not covered by it. + # `default_union=True` makes the Dataset itself present triples from every + # graph as one merged view (it is an rdflib.Graph subclass, so it satisfies + # _convert_to_dict()'s Graph-typed contract directly) instead of copying every + # quad into a second in-memory Graph. + ds = Dataset(default_union=True) + # Use provided format or let rdflib guess based on extension parse_kwargs = kwargs.copy() if format: parse_kwargs['format'] = format - + try: - g.parse(file_path, **parse_kwargs) + ds.parse(file_path, **parse_kwargs) except Exception as e: # Fallback: try to guess format from extension if not provided and initial parse failed if not format: @@ -130,12 +139,14 @@ class OntologyIngestor: guessed_fmt = fmt_map.get(ext) if guessed_fmt: self.logger.info(f"Retrying with guessed format: {guessed_fmt}") - g.parse(file_path, format=guessed_fmt, **kwargs) + ds.parse(file_path, format=guessed_fmt, **kwargs) else: raise e else: raise e + g = ds + self.progress.update_tracking(tracking_id, message="Converting to internal format...") # Determine format for metadata diff --git a/semantica/kg/temporal_query.py b/semantica/kg/temporal_query.py index f5bc227a..c335c95c 100644 --- a/semantica/kg/temporal_query.py +++ b/semantica/kg/temporal_query.py @@ -510,6 +510,9 @@ class TemporalGraphQuery: - "count": Number of relationships - "diversity": Number of unique relationship types - "stability": Relationship duration/stability measure + (mean valid-time duration in seconds across + relationships that have both ``valid_from`` and + ``valid_until`` set) **options: Additional analysis options (unused) Returns: @@ -582,14 +585,17 @@ class TemporalGraphQuery: result["diversity"] = len(rel_types) if "stability" in metrics: - # Calculate stability based on relationship duration + # Stability is the mean duration (in seconds) that relationships + # remain valid. Relationships without a bounded validity interval + # (missing/open ``valid_from`` or ``valid_until``) are skipped, and + # non-positive intervals are clamped to zero. durations = [] for rel in relationships: valid_from = self._parse_time(rel.get("valid_from")) valid_until = self._parse_time(rel.get("valid_until")) if valid_from and valid_until: - # Simplified duration calculation - durations.append(1) # Placeholder + duration_seconds = (valid_until - valid_from).total_seconds() + durations.append(max(0.0, duration_seconds)) result["stability"] = sum(durations) / len(durations) if durations else 0 return result diff --git a/semantica/mcp_server/__init__.py b/semantica/mcp_server/__init__.py index 19fe0bf4..05b9d322 100644 --- a/semantica/mcp_server/__init__.py +++ b/semantica/mcp_server/__init__.py @@ -62,6 +62,13 @@ logging.basicConfig(stream=sys.stderr, level=_log_level, format="%(asctime)s [semantica-mcp] %(levelname)s %(message)s") log = logging.getLogger("semantica.mcp_server") +# MCP stdio framing IS stdout: a progress bar or other console renderer writing +# to stdout would interleave with the JSON-RPC stream and hang every client +# (observed 2026-08-20: export_graph over MCP timed out at 300s while the same +# call returned in <1s directly). Force the progress trackers off for this +# process โ€” stdout is not a console here. +os.environ["SEMANTICA_DISABLE_PROGRESS"] = "1" + # โ”€โ”€ lazy graph session โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ _graph: Any = None @@ -74,7 +81,7 @@ def _get_graph(): kg_path = os.environ.get("SEMANTICA_KG_PATH") if kg_path and os.path.exists(kg_path): try: - _graph.load(kg_path) + _graph.load_from_file(kg_path) log.info("Loaded graph from %s", kg_path) except Exception as exc: log.warning("Could not load graph from %s: %s", kg_path, exc) @@ -86,30 +93,57 @@ def _get_graph(): # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• def _tool_extract_entities(args: dict) -> dict: - """Extract named entities from text.""" + """Extract named entities from text. + + Optional ``model`` (spaCy pipeline, e.g. ``zh_core_web_sm`` for Chinese) + and ``language`` allow non-English NER; defaults to the Semantica English + pipeline when omitted. ``method`` defaults to ``ml`` (spaCy); other + options are ``huggingface``, ``llm``, ``pattern``. + """ text = args.get("text", "") if not text: return {"error": "text is required"} from semantica.semantic_extract import NamedEntityRecognizer - entities = NamedEntityRecognizer().extract_entities(text) + init_kwargs = {} + for k in ("model", "language", "confidence_threshold"): + if args.get(k) is not None: + init_kwargs[k] = args[k] + method = args.get("method", "ml") + ner = NamedEntityRecognizer(methods=[method], **init_kwargs) + entities = ner.extract_entities(text) return { "entities": [ - {"label": getattr(e, "label", str(e)), - "type": getattr(e, "type", None), - "start": getattr(e, "start", None), - "end": getattr(e, "end", None)} + {"text": getattr(e, "text", ""), + "label": getattr(e, "label", ""), + "type": getattr(e, "label", None), + "start": getattr(e, "start_char", getattr(e, "start", None)), + "end": getattr(e, "end_char", getattr(e, "end", None)), + "confidence": getattr(e, "confidence", 1.0)} for e in (entities or []) ] } def _tool_extract_relations(args: dict) -> dict: - """Extract relations and triplets from text.""" + """Extract relations and triplets from text. + + Optional ``model``/``language`` enable non-English extraction. + ``method`` defaults to ``pattern``; ``dependency`` uses spaCy syntactic + parsing (requires a spaCy model, e.g. ``zh_core_web_sm``). + """ text = args.get("text", "") if not text: return {"error": "text is required"} - from semantica.semantic_extract import RelationExtractor, TripletExtractor - relations = RelationExtractor().extract_relations(text) + from semantica.semantic_extract import NamedEntityRecognizer, RelationExtractor, TripletExtractor + rel_kwargs = {} + ner_kwargs = {} + for k in ("model", "language"): + if args.get(k) is not None: + rel_kwargs[k] = args[k] + ner_kwargs[k] = args[k] + method = args.get("method", "pattern") + entities = NamedEntityRecognizer(methods=["ml"], **ner_kwargs).extract_entities(text) or [] + relations = RelationExtractor(method=method, **rel_kwargs).extract_relations(text, entities) triplets = TripletExtractor().extract_triplets(text) return { "relations": [ @@ -156,10 +190,12 @@ def _tool_query_decisions(args: dict) -> dict: graph = _get_graph() try: if query: - results = graph.find_similar_decisions(query, max_results=limit) + results = graph.find_similar_decisions(query, max_results=limit, min_similarity=0.05) elif category: nodes = graph.find_nodes(node_type="decision") - results = [n for n in nodes if n.get("category") == category][:limit] + results = [n for n in nodes + if n.get("category") == category + or n.get("metadata", {}).get("category") == category][:limit] else: results = graph.find_nodes(node_type="decision")[:limit] return {"decisions": results if isinstance(results, list) else list(results)} @@ -175,7 +211,9 @@ def _tool_find_precedents(args: dict) -> dict: max_results = int(args.get("max_results", 5)) graph = _get_graph() try: - precedents = graph.find_similar_decisions(scenario, max_results=max_results) + min_similarity = float(args.get("min_similarity", 0.05)) + precedents = graph.find_similar_decisions( + scenario, max_results=max_results, min_similarity=min_similarity) return {"precedents": precedents if isinstance(precedents, list) else list(precedents)} except Exception as exc: return {"error": str(exc), "precedents": []} @@ -260,16 +298,28 @@ def _tool_get_graph_analytics(args: dict) -> dict: return {"error": str(exc)} +_EXPORT_GRAPH_FORMATS = ("turtle", "ttl", "nt", "xml", "json-ld", "json") + + def _tool_export_graph(args: dict) -> dict: """Export the current knowledge graph to a serialised format.""" fmt = args.get("format", "json-ld") + if fmt not in _EXPORT_GRAPH_FORMATS: + return { + "error": f"Unsupported format '{fmt}'. Supported: {', '.join(_EXPORT_GRAPH_FORMATS)}" + } graph = _get_graph() try: - from semantica.export import RDFExporter, JSONExporter + from semantica.export import RDFExporter + # The exporters consume the canonical kg dict, not the ContextGraph + # object (regression: the old code passed the object straight through, + # so every branch failed โ€” JSONExporter.export() with no file_path on + # the json branch, AttributeError on the RDF branches). + kg = graph.to_kg_dict() if fmt in ("turtle", "ttl", "nt", "xml", "json-ld"): - result = RDFExporter().export_to_rdf(graph, format=fmt) + result = RDFExporter().export_to_rdf(kg, format=fmt) else: - result = JSONExporter().export(graph) + result = json.dumps(kg, indent=2, ensure_ascii=False) return {"format": fmt, "data": result} except Exception as exc: return {"error": str(exc)} @@ -290,6 +340,160 @@ def _tool_get_graph_summary(args: dict) -> dict: return {"error": str(exc), "graph_ready": False} +def _tool_update_node(args: dict) -> dict: + """Update properties of an existing node and persist to SEMANTICA_KG_PATH. + + Common use: mark an action node's status (todo/doing/done) with an + optional note. The graph is mutated in-memory then saved back to the + file it was loaded from, so changes survive server restarts. + """ + node_id = args.get("node_id", "") + if not node_id: + return {"error": "node_id is required"} + properties = args.get("properties", {}) + if not isinstance(properties, dict) or not properties: + return {"error": "properties (non-empty object) is required"} + graph = _get_graph() + try: + if not graph.find_node(node_id): + return {"error": f"node '{node_id}' not found"} + graph.add_node_attribute(node_id, properties) + # Persist back to disk so the change survives restarts + kg_path = os.environ.get("SEMANTICA_KG_PATH") + if kg_path: + graph.save_to_file(kg_path) + persisted = True + else: + persisted = False + updated = graph.find_node(node_id) + return { + "status": "updated", + "node_id": node_id, + "properties": {k: (updated.get("metadata") or {}).get(k) for k in properties}, + "persisted": persisted, + } + except Exception as exc: + return {"error": str(exc)} + + +def _tool_delete_node(args: dict) -> dict: + """Archive a node (soft delete) and persist to SEMANTICA_KG_PATH. + + The node is kept in the graph for history but marked status='archived'. + Use to retire an action you no longer actively track. + """ + node_id = args.get("node_id", "") + if not node_id: + return {"error": "node_id is required"} + graph = _get_graph() + try: + if not graph.find_node(node_id): + return {"error": f"node '{node_id}' not found"} + graph.add_node_attribute(node_id, {"status": "archived"}) + kg_path = os.environ.get("SEMANTICA_KG_PATH") + if kg_path: + graph.save_to_file(kg_path) + return {"status": "archived", "node_id": node_id, "persisted": bool(kg_path)} + except Exception as exc: + return {"error": str(exc)} + + +def _tool_query_graph(args: dict) -> dict: + """Query the live knowledge graph: node detail, neighbours, or keyword search. + + mode: + - "node" : get one node by id (needs node_id) + - "neighbors": traverse up to `depth` hops from node_id (default depth=1) + - "search" : keyword search over node id+content (needs query) + """ + graph = _get_graph() + mode = args.get("mode", "neighbors") + try: + if mode == "node": + node_id = args.get("node_id", "") + if not node_id: + return {"error": "node_id is required"} + node = graph.find_node(node_id) + return {"node": node} + + if mode == "neighbors": + node_id = args.get("node_id", "") + if not node_id: + return {"error": "node_id is required"} + depth = int(args.get("depth", 1)) + rel_types = args.get("relationship_types") + if isinstance(rel_types, str): + rel_types = [rel_types] + rel_set = set(rel_types) if rel_types else None + limit = args.get("limit") + limit = int(limit) if limit is not None else None + depth = min(max(depth, 1), 5) + # Out-edges (multi-hop) via get_neighbors + nb = graph.get_neighbors( + node_id, hops=depth, relationship_types=rel_types, limit=limit, + ) + out = [ + {"id": n.get("id"), "type": n.get("type"), + "content": n.get("content"), + "relationship": n.get("relationship"), + "direction": "out", "hop": n.get("hop", 1)} + for n in (nb or []) + ] + # In-edges (1-hop): scan edges whose target == node_id. + # Deduplicate by source node so that multiple edges between the + # same pair of nodes (different edge types) produce one entry. + # Stop early once we have already collected `limit` inbound results + # (if a limit is set) to avoid scanning the full edge list. + inb = [] + seen_inbound = set() + for e in graph.find_edges(): + if e.get("target") != node_id: + continue + if rel_set is not None and e.get("type") not in rel_set: + continue + src_id = e.get("source") + if src_id in seen_inbound: + continue + seen_inbound.add(src_id) + src = graph.find_node(src_id) or {} + inb.append({"id": src_id, "type": src.get("type"), + "content": src.get("content"), + "relationship": e.get("type"), + "direction": "in", "hop": 1}) + # Early-exit: we already have `limit` inbound results; the + # combined list will be truncated to `limit` anyway. + if limit is not None and len(inb) >= limit: + break + neighbors = out + inb + # Apply final limit. Use ``is not None`` so limit=0 (zero results) + # is honoured correctly; ``if limit:`` would treat 0 as falsy. + if limit is not None: + neighbors = neighbors[:limit] + return {"node_id": node_id, "depth": depth, "neighbors": neighbors} + + if mode == "search": + q = (args.get("query") or "").lower() + if not q: + return {"error": "query is required"} + node_type = args.get("node_type") + limit = int(args.get("limit", 50)) + nodes = graph.find_nodes(node_type=node_type) if node_type else graph.find_nodes() + hits = [] + for n in nodes: + # Check limit BEFORE appending so limit=0 returns empty. + if len(hits) >= limit: + break + blob = f"{n.get('id','')} {n.get('content','')}".lower() + if q in blob: + hits.append({"id": n.get("id"), "type": n.get("type"), + "content": n.get("content")}) + return {"query": q, "results": hits, "total": len(hits)} + + return {"error": f"unknown mode '{mode}': use node|neighbors|search"} + except Exception as exc: + return {"error": str(exc)} + + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• # MCP protocol tables # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• @@ -301,7 +505,11 @@ TOOLS = [ "inputSchema": { "type": "object", "properties": { - "text": {"type": "string", "description": "Input text to extract entities from"} + "text": {"type": "string", "description": "Input text to extract entities from"}, + "model": {"type": "string", "description": "spaCy model name, e.g. 'zh_core_web_sm' for Chinese, 'en_core_web_sm' for English. Defaults to English pipeline."}, + "language": {"type": "string", "description": "Language code, e.g. 'zh', 'en'."}, + "method": {"type": "string", "description": "Extraction method: 'ml' (spaCy, default), 'huggingface', 'llm', 'pattern'."}, + "confidence_threshold": {"type": "number", "description": "Minimum confidence 0-1 (default 0.5)."} }, "required": ["text"], }, @@ -313,7 +521,10 @@ TOOLS = [ "inputSchema": { "type": "object", "properties": { - "text": {"type": "string", "description": "Input text to extract relations from"} + "text": {"type": "string", "description": "Input text to extract relations from"}, + "model": {"type": "string", "description": "spaCy model name for dependency parsing, e.g. 'zh_core_web_sm'."}, + "language": {"type": "string", "description": "Language code, e.g. 'zh'."}, + "method": {"type": "string", "description": "Extraction method: 'pattern' (default), 'dependency', 'cooccurrence', 'huggingface', 'llm'."} }, "required": ["text"], }, @@ -441,7 +652,7 @@ TOOLS = [ "properties": { "format": { "type": "string", - "enum": ["turtle", "ttl", "nt", "xml", "json-ld", "json"], + "enum": list(_EXPORT_GRAPH_FORMATS), "description": "Export format (default: json-ld)", } }, @@ -454,6 +665,48 @@ TOOLS = [ "inputSchema": {"type": "object", "properties": {}}, "_handler": _tool_get_graph_summary, }, + { + "name": "query_graph", + "description": "Query the live knowledge graph: get a node, traverse its neighbours (up to 5 hops), or keyword-search nodes by id+content.", + "inputSchema": { + "type": "object", + "properties": { + "mode": {"type": "string", "description": "node | neighbors | search (default: neighbors)"}, + "node_id": {"type": "string", "description": "Node id (required for node/neighbors mode)"}, + "depth": {"type": "integer", "description": "Hop depth for neighbors (1-5, default 1)"}, + "relationship_types": {"type": "array", "items": {"type": "string"}, "description": "Optional filter by edge type(s)"}, + "query": {"type": "string", "description": "Keyword for search mode (matched against node id+content)"}, + "node_type": {"type": "string", "description": "Optional node_type filter for search mode"}, + "limit": {"type": "integer", "description": "Max results for neighbors/search"} + }, + }, + "_handler": _tool_query_graph, + }, + { + "name": "update_node", + "description": "Update properties of an existing node (e.g. mark an action todo/doing/done with a note) and persist to SEMANTICA_KG_PATH.", + "inputSchema": { + "type": "object", + "properties": { + "node_id": {"type": "string", "description": "Node id to update"}, + "properties": {"type": "object", "description": "Property key-values to merge onto the node, e.g. {\"status\":\"done\",\"updated_at\":\"2026-08-13\",\"note\":\"...\"}"} + }, + "required": ["node_id", "properties"], + }, + "_handler": _tool_update_node, + }, + { + "name": "delete_node", + "description": "Archive a node (soft delete: marks status='archived', keeps it for history) and persist to SEMANTICA_KG_PATH. Use to retire an action you no longer track.", + "inputSchema": { + "type": "object", + "properties": { + "node_id": {"type": "string", "description": "Node id to delete"} + }, + "required": ["node_id"], + }, + "_handler": _tool_delete_node, + }, ] RESOURCES = [ diff --git a/semantica/normalize/number_normalizer.py b/semantica/normalize/number_normalizer.py index 29911cb9..0846465d 100644 --- a/semantica/normalize/number_normalizer.py +++ b/semantica/normalize/number_normalizer.py @@ -370,8 +370,12 @@ class UnitConverter: Raises: ValidationError: If units are incompatible or not in same category """ - from_unit = from_unit.lower() - to_unit = to_unit.lower() + # Normalize aliases before validating categories and looking up factors. + # The public API documents abbreviations such as ``kg`` and ``km``; + # validating those raw aliases against the canonical category lists + # incorrectly rejected otherwise supported conversions. + from_unit = self.normalize_unit(from_unit) + to_unit = self.normalize_unit(to_unit) # Validate units if not self.validate_units(from_unit, to_unit): @@ -401,8 +405,8 @@ class UnitConverter: Returns: bool: True if units are compatible (same category), False otherwise """ - from_unit = from_unit.lower() - to_unit = to_unit.lower() + from_unit = self.normalize_unit(from_unit) + to_unit = self.normalize_unit(to_unit) # Check if both units exist if ( @@ -498,6 +502,18 @@ class UnitConverter: "ml": "milliliter", "milliliter": "milliliter", "milliliters": "milliliter", + "ft": "foot", + "foot": "foot", + "feet": "foot", + "yd": "yard", + "yard": "yard", + "yards": "yard", + "mi": "mile", + "mile": "mile", + "miles": "mile", + "gal": "gallon", + "gallon": "gallon", + "gallons": "gallon", } return unit_map.get(unit_lower, unit_lower) diff --git a/semantica/ontology/__init__.py b/semantica/ontology/__init__.py index 98edff7a..27b1318a 100644 --- a/semantica/ontology/__init__.py +++ b/semantica/ontology/__init__.py @@ -159,6 +159,7 @@ from .ontology_validator import ( SHACLValidationReport, SHACLViolation, ValidationResult, + run_shacl_validation, validate_ontology, ) from .owl_generator import OWLGenerator @@ -192,6 +193,7 @@ __all__ = [ "PropertyShape", "SHACLValidationReport", "SHACLViolation", + "run_shacl_validation", # OWL/RDF generation "OWLGenerator", # Requirements and competency questions diff --git a/semantica/ontology/ontology_generator.py b/semantica/ontology/ontology_generator.py index 253fc852..73c73a5f 100644 --- a/semantica/ontology/ontology_generator.py +++ b/semantica/ontology/ontology_generator.py @@ -309,13 +309,15 @@ class OntologyGenerator: concepts[entity_type] = {"instances": [], "relationships": []} concepts[entity_type]["instances"].append(entity) + entity_aliases = self._build_entity_aliases(entities) + # Extract relationships normalized_relationships = [] for rel_item in relationships: # Normalize relationship to dictionary rel = None if isinstance(rel_item, dict): - rel = rel_item + rel = dict(rel_item) elif hasattr(rel_item, "subject") and hasattr(rel_item, "predicate") and hasattr(rel_item, "object"): # Handle Relation object (subject, predicate, object) rel = { @@ -356,26 +358,12 @@ class OntologyGenerator: if not rel: continue - rel_type = rel.get("type") or rel.get("relationship_type", "relatedTo") - source_type = rel.get("source_type") - target_type = rel.get("target_type") - - # Try to resolve source/target types if not provided - if not source_type or source_type == "Entity": - # Look up source in entities list to find its type - source_name = rel.get("source") - for ent in entities: - if ent.get("name") == source_name or ent.get("text") == source_name: - source_type = ent.get("type") or ent.get("entity_type") - break - - if not target_type or target_type == "Entity": - # Look up target in entities list to find its type - target_name = rel.get("target") - for ent in entities: - if ent.get("name") == target_name or ent.get("text") == target_name: - target_type = ent.get("type") or ent.get("entity_type") - break + source_type = self._resolve_relationship_endpoint_type( + rel, "source", entity_aliases + ) + target_type = self._resolve_relationship_endpoint_type( + rel, "target", entity_aliases + ) # Update rel with resolved types rel["source_type"] = source_type @@ -392,6 +380,60 @@ class OntologyGenerator: "relationships": normalized_relationships, } + @staticmethod + def _build_entity_aliases(entities: List[Dict[str, Any]]) -> Dict[str, set]: + """Build an unambiguous alias-to-type index for relationship endpoints.""" + aliases: Dict[str, set] = {} + for entity in entities: + entity_type = entity.get("type") or entity.get("entity_type") + if not entity_type: + continue + + for key in ("id", "entity_id", "name", "text", "label"): + if key not in entity or entity[key] is None or entity[key] == "": + continue + aliases.setdefault(str(entity[key]), set()).add(entity_type) + + return aliases + + @staticmethod + def _get_relationship_endpoint(rel: Dict[str, Any], endpoint: str) -> Any: + """Return an endpoint value from either ID or legacy relationship fields.""" + for key in (f"{endpoint}_id", endpoint): + if key not in rel: + continue + + value = rel[key] + if value is None or value == "": + continue + if isinstance(value, dict): + for alias_key in ("id", "entity_id", "name", "text", "label"): + if alias_key not in value: + continue + alias_value = value[alias_key] + if alias_value is not None and alias_value != "": + return alias_value + continue + return value + + return None + + def _resolve_relationship_endpoint_type( + self, rel: Dict[str, Any], endpoint: str, aliases: Dict[str, set] + ) -> Optional[str]: + """Resolve an endpoint type without treating missing fields as aliases.""" + explicit_type = rel.get(f"{endpoint}_type") + if explicit_type and explicit_type != "Entity": + return explicit_type + + endpoint_value = self._get_relationship_endpoint(rel, endpoint) + if endpoint_value is not None: + candidates = aliases.get(str(endpoint_value), set()) + if len(candidates) == 1: + return next(iter(candidates)) + + return explicit_type + def _stage2_yaml_to_definition( self, semantic_network: Dict[str, Any], **options ) -> Dict[str, Any]: @@ -840,7 +882,16 @@ class SHACLGenerator: """ self.logger = get_logger("ontology_shacl") self.progress_tracker = get_progress_tracker() - self.base_uri = base_uri.rstrip("/") + "/" + # Preserve an RDF namespace that already ends in `#` (the common + # convention for vocabularies): `...manufacturing#` must not become + # `...manufacturing#/`, or every generated URI lands in the wrong + # namespace and SHACL validation silently targets nothing. Matches + # the `#`-aware normalization in `generate()`. Slash-terminated + # bases are collapsed to a single trailing `/` so redundant runs + # (`.../ns////`) cannot leak a different namespace into emitted IRIs. + self.base_uri = ( + base_uri if base_uri.endswith("#") else base_uri.rstrip("/") + "/" + ) self.shapes_uri = shapes_uri or (self.base_uri + "shapes") self.include_inherited = include_inherited self.severity = severity diff --git a/semantica/ontology/ontology_validator.py b/semantica/ontology/ontology_validator.py index 5d9d10df..85adb0a8 100644 --- a/semantica/ontology/ontology_validator.py +++ b/semantica/ontology/ontology_validator.py @@ -145,14 +145,14 @@ class SHACLValidationReport: } -def _run_pyshacl( +def run_shacl_validation( data_graph_str: str, shacl_str: str, data_graph_format: str = "turtle", shacl_format: str = "turtle", ) -> SHACLValidationReport: """ - Run pyshacl validation and return a structured SHACLValidationReport. + Run pySHACL validation and return a structured SHACLValidationReport. Args: data_graph_str: Serialized data graph string. @@ -272,6 +272,21 @@ def _run_pyshacl( raw_report=results_text, ) + +def _run_pyshacl( + data_graph_str: str, + shacl_str: str, + data_graph_format: str = "turtle", + shacl_format: str = "turtle", +) -> SHACLValidationReport: + """Backward-compatible alias for :func:`run_shacl_validation`.""" + return run_shacl_validation( + data_graph_str, + shacl_str, + data_graph_format=data_graph_format, + shacl_format=shacl_format, + ) + @dataclass class ValidationResult: """Result of an ontology validation operation.""" diff --git a/semantica/ontology/property_generator.py b/semantica/ontology/property_generator.py index 680941b0..d43fd526 100644 --- a/semantica/ontology/property_generator.py +++ b/semantica/ontology/property_generator.py @@ -197,23 +197,22 @@ class PropertyGenerator: self, entities: List[Dict[str, Any]], classes: List[Dict[str, Any]], **options ) -> List[Dict[str, Any]]: """Infer data properties from entity attributes.""" - # Group entities by type - entity_types = defaultdict(list) + # Group entities by their inferred class so normalized class names remain + # aligned with the class definitions emitted by ClassInferrer. + class_entities = defaultdict(list) + class_lookup = self._build_class_type_lookup(classes) for entity in entities: entity_type = entity.get("type") or entity.get("entity_type", "Entity") - entity_types[entity_type].append(entity) + class_def = self._find_class_for_entity_type(entity_type, class_lookup) + if not class_def: + continue + class_name = class_def["name"] + class_entities[class_name].append(entity) # Extract data properties for each class properties = [] - for entity_type, type_entities in entity_types.items(): - # Find corresponding class - class_def = next( - (cls for cls in classes if cls["name"] == entity_type), None - ) - if not class_def: - continue - + for class_name, type_entities in class_entities.items(): # Extract data properties data_props = self._extract_data_properties(type_entities) @@ -233,7 +232,7 @@ class PropertyGenerator: else None, "label": normalized_name, "comment": f"Data property for {prop_name}", - "domain": [entity_type], + "domain": [class_name], "range": prop_type, "metadata": {"inferred_from": prop_name}, } @@ -242,6 +241,38 @@ class PropertyGenerator: return properties + def _build_class_type_lookup( + self, classes: List[Dict[str, Any]] + ) -> Dict[str, Dict[str, Any]]: + """Build a lookup for raw, normalized, and recorded source type names.""" + lookup: Dict[str, Dict[str, Any]] = {} + for class_def in classes: + class_name = class_def.get("name") + if class_name: + lookup.setdefault(str(class_name), class_def) + lookup.setdefault( + self.naming_conventions.normalize_class_name(str(class_name)), + class_def, + ) + + inferred_from = class_def.get("metadata", {}).get("inferred_from") + if inferred_from is not None: + lookup.setdefault(str(inferred_from), class_def) + lookup.setdefault( + self.naming_conventions.normalize_class_name(str(inferred_from)), + class_def, + ) + + return lookup + + def _find_class_for_entity_type( + self, entity_type: Any, class_lookup: Dict[str, Dict[str, Any]] + ) -> Optional[Dict[str, Any]]: + """Find a class using the precomputed type lookup.""" + raw_type = str(entity_type) + normalized_type = self.naming_conventions.normalize_class_name(raw_type) + return class_lookup.get(raw_type) or class_lookup.get(normalized_type) + def _extract_data_properties( self, entities: List[Dict[str, Any]] ) -> Dict[str, str]: diff --git a/semantica/ontology/vocabulary/semantica-ns.ttl b/semantica/ontology/vocabulary/semantica-ns.ttl index 42badf60..ffe36df5 100644 --- a/semantica/ontology/vocabulary/semantica-ns.ttl +++ b/semantica/ontology/vocabulary/semantica-ns.ttl @@ -135,6 +135,90 @@ JSONExporter.export_to_jsonld in export/json_exporter.py.""" ; rdfs:range xsd:string ; rdfs:isDefinedBy . +# โ”€โ”€ Metadata carried through from the graph builder โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# +# The keys GraphBuilder and the Neo4j loader write into "metadata". Declared +# here because the RDF serializers emit them (#1154); a caller-supplied key is +# not declared here and is not emitted, because which namespace it belongs in +# is #1146. + +sem:numEntities a owl:DatatypeProperty ; + rdfs:label "number of entities" ; + rdfs:comment """Count of entities in the graph as built, from +GraphBuilder.build_graph. A count of what was built, not a constraint on what +the graph contains: an export filtered after the fact will disagree with it.""" ; + rdfs:range xsd:integer ; + rdfs:isDefinedBy . + +sem:numRelationships a owl:DatatypeProperty ; + rdfs:label "number of relationships" ; + rdfs:comment "Count of relationships in the graph as built." ; + rdfs:range xsd:integer ; + rdfs:isDefinedBy . + +sem:temporalEnabled a owl:DatatypeProperty ; + rdfs:label "temporal enabled" ; + rdfs:comment """True when the builder was configured to track valid time. +False does not mean the graph is untimed; it means no temporal bounds were +recorded for it.""" ; + rdfs:range xsd:boolean ; + rdfs:isDefinedBy . + +sem:entityResolutionApplied a owl:DatatypeProperty ; + rdfs:label "entity resolution applied" ; + rdfs:comment """True when a resolver ran over the extracted entities, so a +consumer knows whether two nodes with the same surface text were ever +considered for merging.""" ; + rdfs:range xsd:boolean ; + rdfs:isDefinedBy . + +sem:builtAt a owl:DatatypeProperty ; + rdfs:label "built at" ; + rdfs:comment """When the graph was built, as GraphBuilder recorded it. + +The range is xsd:string, deliberately, and not xsd:dateTime. GraphBuilder +stamps with a timezone-naive datetime.now(), and #1114 is the demonstration of +what typing such a value as xsd:dateTime costs: a timezone-qualified SPARQL +filter over it raises an indeterminate comparison and silently drops the row. +#1121 swept the export and provenance modules to an explicit UTC offset and +deliberately left kg/ alone, because the context and vector-store modules +compare against naive values already on disk. Until that sweep reaches +GraphBuilder this value is a string that looks like a timestamp, and saying so +is more useful than a type that invites arithmetic it cannot support.""" ; + rdfs:range xsd:string ; + rdfs:isDefinedBy . + +sem:snapshotAt a owl:DatatypeProperty ; + rdfs:label "snapshot at" ; + rdfs:comment """The point in time a snapshot represents, from +GraphBuilder.create_snapshot. A string for the same reason as sem:builtAt.""" ; + rdfs:range xsd:string ; + rdfs:isDefinedBy . + +sem:sourceSystem a owl:DatatypeProperty ; + rdfs:label "source system" ; + rdfs:comment """The system a graph was loaded from, currently the literal +"neo4j" written by GraphBuilder.load_from_neo4j. + +Named sourceSystem rather than source because sem:source is already the +ObjectProperty carrying the subject of a reified relationship. The metadata key +is still "source"; the exporter maps the key to this term.""" ; + rdfs:range xsd:string ; + rdfs:isDefinedBy . + +sem:sourceUri a owl:ObjectProperty ; + rdfs:label "source URI" ; + rdfs:comment """The address of the system a graph was loaded from. The one +metadata term whose value is a node rather than a literal, because it names a +thing rather than describing one.""" ; + rdfs:isDefinedBy . + +sem:sourceDatabase a owl:DatatypeProperty ; + rdfs:label "source database" ; + rdfs:comment "The database within the source system a graph was loaded from." ; + rdfs:range xsd:string ; + rdfs:isDefinedBy . + # โ”€โ”€ Temporal term (OWL-Time export) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ sem:openEndedInterval a owl:DatatypeProperty ; diff --git a/semantica/pipeline/execution_engine.py b/semantica/pipeline/execution_engine.py index b3f3a0fe..63f382ea 100644 --- a/semantica/pipeline/execution_engine.py +++ b/semantica/pipeline/execution_engine.py @@ -379,7 +379,7 @@ class ExecutionEngine: data = delta_result - if step.handler: + if step.handler is not None: return step.handler(data, **step.config, **options) else: return data diff --git a/semantica/pipeline/pipeline_builder.py b/semantica/pipeline/pipeline_builder.py index d2044cbe..b38dc23c 100644 --- a/semantica/pipeline/pipeline_builder.py +++ b/semantica/pipeline/pipeline_builder.py @@ -131,13 +131,17 @@ class PipelineBuilder: delta_mode = config.pop("delta_mode", False) base_version_id = config.pop("base_version_id", None) target_version_id = config.pop("target_version_id", None) + dependencies = config.pop("dependencies", []) + handler = config.pop("handler", None) + if handler is None: + handler = self.step_registry.get(step_type) step = PipelineStep( name=step_name, step_type=step_type, config=config, - dependencies=config.get("dependencies", []), - handler=config.get("handler"), + dependencies=dependencies, + handler=handler, delta_mode = delta_mode, base_version_id=base_version_id, target_version_id=target_version_id, @@ -272,7 +276,19 @@ class PipelineBuilder: step_name = step_config.get("name") step_type = step_config.get("type") if step_name and step_type: - self.add_step(step_name, step_type, **step_config.get("config", {})) + step = self.add_step( + step_name, step_type, **step_config.get("config", {}) + ) + step.dependencies = list( + step_config.get("dependencies", step.dependencies) + ) + step.delta_mode = step_config.get("delta_mode", step.delta_mode) + step.base_version_id = step_config.get( + "base_version_id", step.base_version_id + ) + step.target_version_id = step_config.get( + "target_version_id", step.target_version_id + ) # Set parallelism if specified if "parallelism" in pipeline_config: @@ -398,14 +414,29 @@ class PipelineSerializer: Returns: Serialized pipeline + + Notes: + Step handlers are runtime callables and are intentionally omitted from + the serialized representation. They must be rebound after deserialization. """ + reserved_config_keys = { + "handler", + "dependencies", + "delta_mode", + "base_version_id", + "target_version_id", + } pipeline_data = { "name": pipeline.name, "steps": [ { "name": step.name, "type": step.step_type, - "config": step.config, + "config": { + key: value + for key, value in step.config.items() + if key not in reserved_config_keys + }, "dependencies": step.dependencies, "delta_mode": getattr(step, "delta_mode", False), "base_version_id": getattr(step, "base_version_id", None), @@ -445,6 +476,18 @@ class PipelineSerializer: else: pipeline_data = serialized_pipeline + # Runtime handlers are process-local and cannot be reconstructed safely + # from serialized data. Copy before sanitizing so dict inputs are not mutated. + pipeline_data = dict(pipeline_data) + sanitized_steps = [] + for step_data in pipeline_data.get("steps", []): + sanitized_step = dict(step_data) + step_config = dict(sanitized_step.get("config", {})) + step_config.pop("handler", None) + sanitized_step["config"] = step_config + sanitized_steps.append(sanitized_step) + pipeline_data["steps"] = sanitized_steps + # Reconstruct pipeline builder = PipelineBuilder(**self.config) pipeline = builder.build_pipeline(pipeline_data, **options) diff --git a/semantica/reasoning/__init__.py b/semantica/reasoning/__init__.py index 54665576..00a71576 100644 --- a/semantica/reasoning/__init__.py +++ b/semantica/reasoning/__init__.py @@ -8,6 +8,13 @@ and native Datalog evaluation. """ from .reasoner import Reasoner, InferenceResult, Rule, Fact, RuleType +from .reasoner import ( + Action, + AssertAction, + RetractAction, + CallAction, + EmitEventAction, +) from .graph_reasoner import GraphReasoner from .explanation_generator import ( Explanation, @@ -37,6 +44,12 @@ __all__ = [ "Rule", "Fact", "RuleType", + # Rule-driven actions + "Action", + "AssertAction", + "RetractAction", + "CallAction", + "EmitEventAction", # Rete engine "ReteEngine", "ReteNode", diff --git a/semantica/reasoning/reasoner.py b/semantica/reasoning/reasoner.py index b1fe9be5..cc16db40 100644 --- a/semantica/reasoning/reasoner.py +++ b/semantica/reasoning/reasoner.py @@ -7,13 +7,16 @@ supported by the Semantica framework. It serves as a facade for different reason import re import uuid +from collections.abc import Mapping, Sequence, Set as AbstractSet from dataclasses import dataclass, field +from datetime import datetime, timezone from enum import Enum -from typing import Any, Dict, List, Optional, Set, Tuple, Union, Callable +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker + class RuleType(Enum): """Rule types.""" IMPLICATION = "implication" @@ -21,6 +24,306 @@ class RuleType(Enum): CONSTRAINT = "constraint" TRANSFORMATION = "transformation" + +def _substitute_variables(template: str, bindings: Dict[str, str]) -> str: + """Substitute ``?var`` placeholders with their bound values, token-aware. + + A naive ``str.replace(f"?{var}", value)`` corrupts placeholders that share + a prefix -- e.g. binding ``?x`` would also rewrite the ``?x`` inside ``?xy``. + We replace every ``?word`` token in a single regex pass so that only whole + variable names are matched (``\\w+`` never partially matches a longer name), + leaving unbound placeholders untouched. + """ + if not bindings: + return template + + def _replace(match: "re.Match") -> str: + var_name = match.group(1) + # Preserve unbound placeholders verbatim. + return str(bindings[var_name]) if var_name in bindings else match.group(0) + + return re.sub(r"\?(\w+)", _replace, template) + + +def _canonicalize_activation_value( + value: Any, active_containers: Optional[Dict[int, int]] = None +) -> Tuple[Any, ...]: + """Convert nested activation data into a deterministic, hashable value.""" + if active_containers is None: + active_containers = {} + + value_type = (type(value).__module__, type(value).__qualname__) + is_mapping = isinstance(value, Mapping) + is_sequence = isinstance(value, Sequence) and not isinstance( + value, (str, bytes, bytearray) + ) + is_set = isinstance(value, AbstractSet) and not isinstance( + value, (str, bytes, bytearray) + ) + if not (is_mapping or is_sequence or is_set): + return ("scalar", value_type, repr(value)) + + object_id = id(value) + if object_id in active_containers: + return ("reference", active_containers[object_id]) + active_containers[object_id] = len(active_containers) + + try: + if is_mapping: + keyed_items = [ + ( + _canonicalize_activation_value(key, active_containers), + item, + ) + for key, item in value.items() + ] + keyed_items.sort(key=lambda entry: repr(entry[0])) + entries = tuple( + ( + key, + _canonicalize_activation_value(item, active_containers), + ) + for key, item in keyed_items + ) + return ("mapping", value_type, entries) + if is_sequence: + return ( + "sequence", + value_type, + tuple( + _canonicalize_activation_value(item, active_containers) + for item in value + ), + ) + items = tuple( + sorted( + ( + _canonicalize_activation_value(item, active_containers) + for item in value + ), + key=repr, + ) + ) + return ("set", value_type, items) + finally: + del active_containers[object_id] + + +def _make_activation_key( + rule_id: str, bindings: Dict[str, Any], fact_tokens: List[Any] +) -> Tuple[Any, ...]: + """Return a stable identity for one concrete rule activation.""" + canonical_bindings = tuple( + sorted( + (str(name), _canonicalize_activation_value(value)) + for name, value in bindings.items() + ) + ) + return ( + rule_id, + canonical_bindings, + tuple( + sorted( + (_canonicalize_activation_value(token) for token in fact_tokens), + key=repr, + ) + ), + ) + + +def _parse_fact(fact: str) -> Optional[Tuple[str, List[str]]]: + """Parse a ``Predicate(arg1, arg2, ...)`` fact string. + + Returns ``(predicate, [args])`` or ``None`` when the fact is not in the + canonical predicate form (e.g. a bare atom). Whitespace around args is + stripped and empty arg lists are supported (``Foo()`` -> ``("Foo", [])``). + """ + match = re.match(r"^\s*([^()\s]+)\s*\((.*)\)\s*$", fact) + if not match: + return None + predicate = match.group(1) + inner = match.group(2).strip() + if not inner: + return predicate, [] + args = [arg.strip() for arg in inner.split(",")] + return predicate, args + + +def _write_fact_to_graph(graph: Any, fact: str, *, retract: bool = False) -> None: + """Persist (or remove) a fact against a knowledge-graph-like target. + + Write-back follows an explicit, ordered protocol so that + ``AssertAction(write_back=True)`` never silently no-ops: + + 1. If the target exposes an explicit fact API (``add_fact`` / ``assert_fact`` + for asserts, ``remove_fact`` / ``retract_fact`` / ``discard_fact`` for + retracts), that is used verbatim. + 2. Otherwise, if the target looks like the canonical + :class:`~semantica.kg.knowledge_graph.KnowledgeGraph` (has ``entities`` + and ``relationships`` lists), the fact is translated into a node + (single-arg predicate) or relationship (two-arg predicate) and + added/removed accordingly. + 3. Any other target, or a fact that cannot be translated, raises + :class:`ValueError` so the failure surfaces instead of being swallowed. + """ + if retract: + for method_name in ("retract_fact", "remove_fact", "discard_fact"): + method = getattr(graph, method_name, None) + if callable(method): + method(fact) + return + else: + for method_name in ("add_fact", "assert_fact"): + method = getattr(graph, method_name, None) + if callable(method): + method(fact) + return + + entities = getattr(graph, "entities", None) + relationships = getattr(graph, "relationships", None) + if isinstance(entities, list) and isinstance(relationships, list): + parsed = _parse_fact(fact) + if parsed is None: + raise ValueError( + f"Cannot translate fact {fact!r} into graph node/relationship: " + "expected canonical Predicate(args) form." + ) + predicate, args = parsed + if len(args) == 1: + node = {"id": args[0], "type": predicate} + if retract: + _remove_matching( + entities, + lambda e: e.get("id") == args[0] and e.get("type") == predicate, + ) + elif node not in entities: + entities.append(node) + return + if len(args) == 2: + rel = {"source": args[0], "target": args[1], "type": predicate} + if retract: + _remove_matching( + relationships, + lambda r: r.get("source") == args[0] + and r.get("target") == args[1] + and r.get("type") == predicate, + ) + elif rel not in relationships: + relationships.append(rel) + return + raise ValueError( + f"Cannot write fact {fact!r} to graph: only unary (node) and binary " + "(relationship) predicates are supported by the default adapter." + ) + + raise ValueError( + f"knowledge_graph target {type(graph).__name__!r} does not expose a " + "supported write-back API (add_fact/assert_fact or entities/relationships)." + ) + + +def _remove_matching(items: List[Dict[str, Any]], predicate: Callable[[Dict[str, Any]], bool]) -> None: + """Remove in place every dict in ``items`` for which ``predicate`` is True.""" + items[:] = [item for item in items if not predicate(item)] + +class Action: + """Base class for an action fired when a rule matches. + + Actions turn the reasoner from a pure inference engine into a + production-rule system: when a rule's conditions match, its actions run + with the match's variable bindings, allowing side effects (asserting or + retracting facts, calling external tools, emitting events) rather than + only deriving a new fact. + + Subclasses implement :meth:`execute`, which receives the substituted + ``bindings`` and the owning ``reasoner`` and returns an optional + description of what happened (used for provenance / explanation). + """ + + def execute(self, bindings: Dict[str, str], reasoner: "Reasoner") -> Optional[str]: + raise NotImplementedError + + @staticmethod + def _substitute(template: str, bindings: Dict[str, str]) -> str: + return _substitute_variables(template, bindings) + + +@dataclass +class AssertAction(Action): + """Assert a new fact when the rule fires. + + ``fact`` may contain ``?var`` placeholders that are substituted with the + match bindings. If ``write_back`` is set and the reasoner exposes a + knowledge graph, the fact is also written there. + """ + + fact: str + write_back: bool = False + + def execute(self, bindings: Dict[str, str], reasoner: "Reasoner") -> Optional[str]: + concrete = self._substitute(self.fact, bindings) + reasoner.facts.add(concrete) + if self.write_back and getattr(reasoner, "knowledge_graph", None) is not None: + _write_fact_to_graph(reasoner.knowledge_graph, concrete) + return f"assert {concrete}" + + +@dataclass +class RetractAction(Action): + """Retract a fact when the rule fires (basic truth maintenance). + + If ``write_back`` is set and the reasoner exposes a knowledge graph, the + fact is also removed there using the graph's delete semantics (mirroring + :class:`AssertAction`'s write-back). + """ + + fact: str + write_back: bool = False + + def execute(self, bindings: Dict[str, str], reasoner: "Reasoner") -> Optional[str]: + concrete = self._substitute(self.fact, bindings) + reasoner.facts.discard(concrete) + if self.write_back and getattr(reasoner, "knowledge_graph", None) is not None: + _write_fact_to_graph(reasoner.knowledge_graph, concrete, retract=True) + return f"retract {concrete}" + + +@dataclass +class CallAction(Action): + """Call an external function/tool when the rule fires. + + Wraps an arbitrary callable, which is invoked as ``func(bindings, + reasoner)``. This is the structured replacement for the previously + unused ``Rule.handler`` callback. + """ + + func: Callable + name: str = "call" + + def execute(self, bindings: Dict[str, str], reasoner: "Reasoner") -> Optional[str]: + self.func(bindings, reasoner) + return f"call {self.name}" + + +@dataclass +class EmitEventAction(Action): + """Emit an event to the reasoner's registered event sink when fired. + + The event name may contain ``?var`` placeholders. Events are delivered to + any callable registered via :meth:`Reasoner.on_event`. + """ + + event: str + payload: Dict[str, Any] = field(default_factory=dict) + + def execute(self, bindings: Dict[str, str], reasoner: "Reasoner") -> Optional[str]: + concrete = self._substitute(self.event, bindings) + sink = getattr(reasoner, "_event_sink", None) + if callable(sink): + sink(concrete, {**self.payload, "bindings": dict(bindings)}) + return f"emit {concrete}" + + @dataclass class Rule: """Simplified rule definition.""" @@ -32,6 +335,7 @@ class Rule: confidence: float = 1.0 priority: int = 0 handler: Optional[Callable] = None + actions: List[Action] = field(default_factory=list) metadata: Dict[str, Any] = field(default_factory=dict) @dataclass @@ -79,7 +383,70 @@ class Reasoner: self.rules: List[Rule] = [] self.facts: Set[str] = set() self.rule_counter = 0 - + self._fired_activations: Set[Tuple[Any, ...]] = set() + + # Optional knowledge graph for AssertAction(write_back=True) targets. + self.knowledge_graph = kwargs.get("knowledge_graph") + # Optional event sink for EmitEventAction; register via on_event(). + self._event_sink: Optional[Callable] = None + # When True, action-induced fact changes are recorded for provenance + # via _record_action() -> self.action_log. + self.provenance: bool = bool(kwargs.get("provenance", False)) + self.action_log: List[Dict[str, Any]] = [] + + def on_event(self, sink: Callable) -> None: + """Register a callable ``sink(event_name, payload)`` for EmitEventAction.""" + self._event_sink = sink + + def _record_action( + self, rule: "Rule", action: "Action", description: Optional[str], bindings: Dict[str, str] + ) -> None: + """Record a fired action for provenance / explanation when enabled. + + Each entry is a structured dict carrying an ISO-8601 ``timestamp`` and a + parsed ``operation``/``fact`` split (when the description follows the + ``" "`` convention used by the built-in actions) so that + downstream consumers such as :class:`ExplanationGenerator` and the + provenance layer can reason about *what changed* without re-parsing the + free-text description. + """ + if not self.provenance or description is None: + return + operation, _, subject = description.partition(" ") + self.action_log.append( + { + "action_id": uuid.uuid4().hex[:8], + "rule_id": rule.rule_id, + "action": type(action).__name__, + "operation": operation or None, + "fact": subject or None, + "description": description, + "bindings": dict(bindings), + "confidence": rule.confidence, + "timestamp": datetime.now(timezone.utc).isoformat(), + } + ) + + def _fire_actions(self, rule: "Rule", bindings: Dict[str, str]) -> None: + """Run a fired rule's actions (and legacy handler) with match bindings. + + Backward compatible: a rule with an old-style ``handler`` but no + ``actions`` still has its handler invoked, so pre-existing rules keep + working while new rules use the structured Action layer. + """ + actions = list(rule.actions) + if rule.handler is not None: + actions.append(CallAction(rule.handler, name=f"handler:{rule.rule_id}")) + for action in actions: + try: + description = action.execute(bindings, self) + self._record_action(rule, action, description, bindings) + except Exception as exc: # noqa: BLE001 + self.logger.error( + f"Error executing action {type(action).__name__} " + f"for rule '{rule.rule_id}': {exc}" + ) + def add_rule(self, rule_def: Union[str, Rule]) -> Rule: """Add a rule to the reasoner. @@ -161,6 +528,20 @@ class Reasoner: Returns: List of inferred facts (conclusions) """ + return [result.conclusion for result in self.infer_with_results(facts, rules)] + + def infer_with_results( + self, + facts: Union[List[Any], Dict[str, Any]], + rules: Optional[List[Union[str, Rule]]] = None, + ) -> List[InferenceResult]: + """Infer new facts and return the full :class:`InferenceResult` objects. + + Unlike :meth:`infer_facts` (which returns only conclusion strings for + backward compatibility), this preserves each result's ``rule_used``, + ``premises`` and ``confidence`` so callers such as the provenance + wrapper can record real confidence values instead of ``None``. + """ tracking_id = self.progress_tracker.start_tracking( module="reasoning", submodule="Reasoner", @@ -180,17 +561,14 @@ class Reasoner: # Perform inference results = self.forward_chain() - - # Extract conclusions from results - inferred_facts = [result.conclusion for result in results] self.progress_tracker.stop_tracking( tracking_id, status="completed", - message=f"Inferred {len(inferred_facts)} new facts" + message=f"Inferred {len(results)} new facts" ) - return inferred_facts + return results except Exception as e: self.progress_tracker.stop_tracking( @@ -213,7 +591,15 @@ class Reasoner: new_facts_added = True max_iterations = self.config.get("max_iterations", 50) iteration = 0 - + # Activations (rule + concrete bindings + matched facts) whose actions + # have already fired. Actions are side-effecting and must + # fire exactly once per distinct match, decoupled from whether the + # rule's *conclusion* is new. This fixes two failure modes: + # * A valid binding whose conclusion is already known (or duplicated + # within a pass) previously never fired its actions. + # * A RetractAction that removes a premise of its own rule previously + # re-fired every pass, iterating to max_iterations. Recording the + # activation means it fires once and stops driving iterations. while new_facts_added and iteration < max_iterations: new_facts_added = False iteration += 1 @@ -236,7 +622,21 @@ class Reasoner: pass_results: Dict[str, InferenceResult] = {} for rule in self.rules: - for conclusion, matched_facts in self._match_rule(rule): + for conclusion, matched_facts, bindings in self._match_rule(rule): + # Fire this activation's actions exactly once, independent + # of the conclusion-dedup below. Keyed by rule id + the + # concrete bindings so distinct matches each fire, but a + # repeated match (same bindings across passes) does not. + if rule.actions or rule.handler is not None: + activation_key = _make_activation_key( + rule.rule_id, + bindings, + matched_facts, + ) + if activation_key not in self._fired_activations: + self._fired_activations.add(activation_key) + self._fire_actions(rule, bindings) + if conclusion in pass_results: # Another derivation of a conclusion already produced # earlier in this same pass: merge premises, dedup. @@ -372,14 +772,17 @@ class Reasoner: conclusion=conclusion_str.strip() ) - def _match_rule(self, rule: Rule) -> List[Tuple[str, List[str]]]: + def _match_rule(self, rule: Rule) -> List[Tuple[str, List[str], Dict[str, str]]]: """ Match rule conditions against facts and return instantiated conclusions - paired with the facts that satisfied each condition. + paired with the facts that satisfied each condition and the variable + bindings that produced them. Returns: - List of (conclusion, matched_facts) tuples, where matched_facts is - the ordered list of facts bound to this rule's conditions. + List of (conclusion, matched_facts, bindings) tuples, where + matched_facts is the ordered list of facts bound to this rule's + conditions and bindings maps variable name -> matched value (used + to fire the rule's actions). """ if not rule.conditions: return [] @@ -410,7 +813,7 @@ class Reasoner: results = [] for bindings, matched_facts in bindings_list: instantiated_conclusion = self._substitute(rule.conclusion, bindings) - results.append((instantiated_conclusion, matched_facts)) + results.append((instantiated_conclusion, matched_facts, bindings)) return results @@ -456,16 +859,18 @@ class Reasoner: def _substitute(self, pattern: str, bindings: Dict[str, str]) -> str: """Substitute variables in a pattern with bound values.""" - result = pattern - for var, value in bindings.items(): - result = result.replace(f"?{var}", value) - return result + return _substitute_variables(pattern, bindings) + def reset_action_history(self) -> None: + """Allow previously fired rule activations to execute their actions again.""" + self._fired_activations.clear() + def clear(self) -> None: - """Clear facts and rules.""" + """Clear facts, rules, and action activation history.""" self.facts.clear() self.rules.clear() self.rule_counter = 0 + self.reset_action_history() def reset(self) -> None: """Alias for clear().""" diff --git a/semantica/reasoning/reasoning_provenance.py b/semantica/reasoning/reasoning_provenance.py index 40d4418a..1c4b6506 100644 --- a/semantica/reasoning/reasoning_provenance.py +++ b/semantica/reasoning/reasoning_provenance.py @@ -13,9 +13,9 @@ Author: Semantica Contributors License: MIT """ -from typing import Any, Optional -from datetime import datetime import uuid +from datetime import datetime +from typing import Any, Optional class ReasoningEngineWithProvenance: @@ -28,10 +28,10 @@ class ReasoningEngineWithProvenance: is_automated: bool = True, **config, ): - from .reasoning_engine import ReasoningEngine + from .reasoner import Reasoner self.provenance = provenance - self._engine = ReasoningEngine(**config) + self._engine = Reasoner(provenance=provenance, **config) self._prov_manager = None self._agent_id = agent_id or self.__class__.__name__ self._is_automated = is_automated @@ -43,12 +43,27 @@ class ReasoningEngineWithProvenance: except ImportError: self.provenance = False - def infer(self, premises: Any, source: str = None, **kwargs): - """Perform inference with provenance tracking.""" + def infer(self, premises: Any, source: str = None, rules: Any = None): + """Perform inference with provenance tracking. + + Only the reasoner's real parameters (``premises`` and ``rules``) are + forwarded to the underlying engine; arbitrary keyword arguments are no + longer passed through (they previously reached + ``Reasoner.infer_facts`` -- which accepts only ``facts``/``rules`` -- + and raised ``TypeError``). + """ activity_started_at_time = datetime.utcnow().isoformat() - result = self._engine.infer(premises, **kwargs) + results = self._engine.infer_with_results(premises, rules) activity_ended_at_time = datetime.utcnow().isoformat() + # Aggregate confidence across the derived results (min = weakest link); + # None only when nothing was inferred. + confidence = ( + min(r.confidence for r in results) if results else None + ) + # Preserve the historical return shape: a list of conclusion strings. + inferred = [r.conclusion for r in results] + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"inference_{uuid.uuid4().hex[:8]}", @@ -61,11 +76,11 @@ class ReasoningEngineWithProvenance: activity_ended_at_time=activity_ended_at_time, metadata={ "premises_count": len(premises) if hasattr(premises, '__len__') else 1, - "confidence": getattr(result, 'confidence', None) + "confidence": confidence, } ) - return result + return inferred def __getattr__(self, name): return getattr(self._engine, name) diff --git a/semantica/reasoning/rete_engine.py b/semantica/reasoning/rete_engine.py index b6b79359..8fe9b7d3 100644 --- a/semantica/reasoning/rete_engine.py +++ b/semantica/reasoning/rete_engine.py @@ -33,14 +33,75 @@ Author: Semantica Contributors License: MIT """ -from collections import defaultdict +import re from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Set, Tuple -from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker -from .reasoner import Fact, Rule +from .reasoner import Fact, Rule, _make_activation_key + + +def _extract_bindings(condition: Any, fact: Fact) -> Dict[str, Any]: + """Extract ``?var`` bindings by matching a condition pattern against a fact. + + ``condition`` is the pattern stored on the alpha node (typically a string + like ``"Person(?x)"``); ``fact`` is the working-memory :class:`Fact`. The + fact's canonical string form (``Predicate(arg1, arg2, ...)``) is matched + against the pattern using the same ``?\\w+`` placeholder convention as the + Reasoner, so downstream actions receive real bindings (e.g. ``{"x": "John"}``) + instead of the empty dict that previously left ``?x`` placeholders + unsubstituted. + + Returns an empty dict when the condition is not a string pattern or does + not match -- callers treat that as "no bindings extracted". + """ + if not isinstance(condition, str): + return {} + + segments = re.split(r"(\?\w+)", condition) + seen_vars: Set[str] = set() + p_regex = "" + for seg in segments: + if seg.startswith("?"): + var_name = seg[1:] + if var_name in seen_vars: + p_regex += f"(?P={var_name})" + else: + p_regex += f"(?P<{var_name}>.+?)" + seen_vars.add(var_name) + else: + p_regex += re.escape(seg) + p_regex = f"^{p_regex}$" + + try: + match = re.match(p_regex, str(fact)) + except re.error: + return {} + if not match: + return {} + return {k: v for k, v in match.groupdict().items() if v is not None} + + +def _bindings_for_rule(rule: Rule, facts: List[Fact]) -> Dict[str, Any]: + """Merge ``?var`` bindings from matching a rule's conditions against facts. + + Each fact is matched against every condition of the rule; the first + condition that yields bindings for a fact contributes them. Bindings from + all facts are merged so multi-condition (joined) rules receive the full + variable environment. Later conflicting values do not overwrite earlier + ones, preserving the binding that a join already validated. + """ + bindings: Dict[str, Any] = {} + for fact in facts: + for condition in rule.conditions: + extracted = _extract_bindings(condition, fact) + if not extracted: + continue + for key, value in extracted.items(): + bindings.setdefault(key, value) + break + return bindings @dataclass @@ -58,7 +119,7 @@ class ReteNode: def __init__(self, node_id: str): self.node_id = node_id - self.children: List["ReteNode"] = [] + self.children: List[ReteNode] = [] class AlphaNode(ReteNode): @@ -151,6 +212,17 @@ class ReteEngine: self.facts: List[Fact] = [] self.fact_counter = 0 self.node_counter = 0 + self._executed_activations: Set[Tuple[Any, ...]] = set() + # Optional Reasoner used to fire rule-driven actions on match. When + # set, execute_matches() runs each matched rule's ``actions`` (and any + # legacy ``handler``) through the Reasoner's action machinery so that + # Rete-based matching benefits from the same production-rule behaviour + # as forward_chain(). Left None keeps the pure-matching mode. + self.reasoner: Optional[Any] = self.config.get("reasoner") + + def bind_reasoner(self, reasoner: Any) -> None: + """Attach a Reasoner so matched rules can fire their actions.""" + self.reasoner = reasoner def build_network(self, rules: List[Rule]) -> None: """ @@ -166,6 +238,7 @@ class ReteEngine: ) try: + self.reset_action_history() self.network.clear() self.progress_tracker.update_tracking( @@ -252,15 +325,24 @@ class ReteEngine: # Propagate to children for grandchild in child.children: if isinstance(grandchild, TerminalNode): + facts = [left_fact, fact] match = Match( rule=grandchild.rule, - facts=[left_fact, fact], + facts=facts, + bindings=_bindings_for_rule( + grandchild.rule, facts + ), confidence=1.0, ) grandchild.activate(match) elif isinstance(child, TerminalNode): # Direct activation - match = Match(rule=child.rule, facts=[fact], confidence=1.0) + match = Match( + rule=child.rule, + facts=[fact], + bindings=_bindings_for_rule(child.rule, [fact]), + confidence=1.0, + ) child.activate(match) def match_patterns(self, facts: Optional[List[Fact]] = None) -> List[Match]: @@ -276,7 +358,7 @@ class ReteEngine: tracking_id = self.progress_tracker.start_tracking( module="reasoning", submodule="ReteEngine", - message=f"Matching patterns using Rete algorithm", + message="Matching patterns using Rete algorithm", ) try: @@ -339,10 +421,28 @@ class ReteEngine: ) results = [] for match in matches: + # Conclusions are the pure inference result and remain + # independent from optional side-effect execution below. + results.append(match.rule.conclusion) try: - # Execute rule - result = match.rule.conclusion - results.append(result) + # Fire the rule's actions (and any legacy handler) through + # the bound Reasoner so Rete matching produces the same + # side effects / provenance as forward_chain(). Falls back + # to just recording the conclusion when no Reasoner is bound. + if self.reasoner is not None and ( + match.rule.actions or match.rule.handler is not None + ): + activation_key = _make_activation_key( + match.rule.rule_id, + match.bindings, + [ + (fact.fact_id, fact.predicate, fact.arguments) + for fact in match.facts + ], + ) + if activation_key not in self._executed_activations: + self._executed_activations.add(activation_key) + self.reasoner._fire_actions(match.rule, match.bindings) except Exception as e: self.logger.error(f"Error executing match: {e}") @@ -359,13 +459,16 @@ class ReteEngine: ) raise + def reset_action_history(self) -> None: + """Allow previously executed activations to fire their actions again.""" + self._executed_activations.clear() + def reset(self) -> None: - """Reset Rete engine.""" + """Reset Rete working memory and action activation history.""" self.facts.clear() + self.reset_action_history() for node in self.network.values(): - if isinstance(node, AlphaNode): - node.matches.clear() - elif isinstance(node, BetaNode): + if isinstance(node, AlphaNode) or isinstance(node, BetaNode): node.matches.clear() elif isinstance(node, TerminalNode): node.activations.clear() diff --git a/semantica/reasoning/sparql_reasoner.py b/semantica/reasoning/sparql_reasoner.py index 58e0c351..16be2a9f 100644 --- a/semantica/reasoning/sparql_reasoner.py +++ b/semantica/reasoning/sparql_reasoner.py @@ -84,6 +84,9 @@ class SPARQLReasoner: self.triplet_store = self.config.get("triplet_store") self.enable_inference = self.config.get("enable_inference", True) + # Reserved for query caching once a triplet-store execution path + # lands. execute_query() raises NotImplementedError until then, so + # the cache cannot be populated through any public path yet. self.query_cache: Dict[str, Any] = {} def expand_query(self, query: str, **options) -> str: @@ -330,79 +333,32 @@ class SPARQLReasoner: """ Execute SPARQL query with reasoning. + Not implemented: no triplet-store execution path exists yet, so the + query is refused loudly instead of returning an empty result set + that callers would read as "no matches" (issue #1083). + Args: query: SPARQL query string **options: Additional options - Returns: - Query results + Raises: + NotImplementedError: always, until a triplet-store execution + path lands. """ - tracking_id = self.progress_tracker.start_tracking( - module="reasoning", - submodule="SPARQLReasoner", - message="Executing SPARQL query with reasoning", + raise NotImplementedError( + "SPARQLReasoner.execute_query() is not implemented: no " + "triplet-store execution path exists yet. Returning an empty " + "result set would be misread as 'no matches', so the query " + "is refused instead." ) - try: - # Check cache - self.progress_tracker.update_tracking( - tracking_id, message="Checking query cache..." - ) - if query in self.query_cache: - self.logger.debug("Returning cached query result") - self.progress_tracker.stop_tracking( - tracking_id, - status="completed", - message="Returned cached query result", - ) - return self.query_cache[query] - - # Expand query - self.progress_tracker.update_tracking( - tracking_id, message="Expanding query with inference rules..." - ) - expanded_query = self.expand_query(query, **options) - - # Execute query (if triplet store available) - self.progress_tracker.update_tracking( - tracking_id, message="Executing query..." - ) - if self.triplet_store: - # This would call the triplet store's query method - # For now, return empty result - result = SPARQLQueryResult(bindings=[], variables=[]) - else: - # Mock result for testing - result = SPARQLQueryResult(bindings=[], variables=[]) - - # Infer additional results - if self.enable_inference: - self.progress_tracker.update_tracking( - tracking_id, message="Inferring additional results..." - ) - result = self.infer_results(result, **options) - - # Cache result - self.progress_tracker.update_tracking( - tracking_id, message="Caching query result..." - ) - self.query_cache[query] = result - - self.progress_tracker.stop_tracking( - tracking_id, - status="completed", - message=f"Query executed: {len(result.bindings)} results", - ) - return result - - except Exception as e: - self.progress_tracker.stop_tracking( - tracking_id, status="failed", message=str(e) - ) - raise - def clear_cache(self) -> None: - """Clear query cache.""" + """Clear query cache. + + Reserved for when a triplet-store execution path lands: until then, + ``execute_query()`` raises ``NotImplementedError`` and nothing can + populate the cache. + """ self.query_cache.clear() def add_inference_rule(self, rule_definition: str, **options) -> Rule: diff --git a/semantica/seed/seed_manager.py b/semantica/seed/seed_manager.py index 3e6ebbf8..1a31175e 100644 --- a/semantica/seed/seed_manager.py +++ b/semantica/seed/seed_manager.py @@ -482,8 +482,8 @@ class SeedDataManager: List of loaded data records as dictionaries Raises: - ProcessingError: If API request fails, response parsing fails, or - requests library is not available + ProcessingError: If the API request fails (connection error, + timeout, non-2xx status) or the response cannot be parsed Example: >>> records = manager.load_from_api( @@ -559,10 +559,6 @@ class SeedDataManager: self.logger.info(f"Loaded {len(records)} records from API: {full_url}") return records - except (ImportError, OSError): - raise ProcessingError( - "requests library not available. Install with: pip install requests" - ) except Exception as e: raise ProcessingError(f"Failed to load from API: {e}") from e diff --git a/semantica/semantic_extract/methods.py b/semantica/semantic_extract/methods.py index f0559dec..00482df1 100644 --- a/semantica/semantic_extract/methods.py +++ b/semantica/semantic_extract/methods.py @@ -140,6 +140,47 @@ _result_cache = ExtractionCache( if not config.get("cache_enabled", True): _result_cache.enabled = False +# Generation kwargs that affect provider output and must therefore be part of +# the cache key. This is the union of every generation-affecting parameter +# read across providers.py, including params picked up outside _add_if_set +# (e.g. AnthropicProvider's manual pass-through loop). Sensitive values +# (api_key, token, etc.) are already filtered out by +# ExtractionCache._generate_key, so they need not be excluded here. +_GENERATION_CACHE_KEYS = frozenset({ + "max_tokens", + "max_completion_tokens", + "temperature", + "top_p", + "top_k", + "seed", + "frequency_penalty", + "presence_penalty", + "stop", + "stop_sequences", # Anthropic/Gemini spelling of "stop" + "logit_bias", + "user", + "system", # Anthropic system prompt + "metadata", # Anthropic request metadata + "candidate_count", # Gemini + "repeat_penalty", # Ollama + "num_ctx", # Ollama + "context_window", # Ollama alias for num_ctx +}) + + +def _generation_cache_params(kwargs: dict) -> dict: + """Return the subset of *kwargs* that affects generation output. + + Only keys listed in ``_GENERATION_CACHE_KEYS`` are included so that + irrelevant or sensitive caller kwargs do not pollute the cache key. + Values that are ``None`` are omitted; a caller passing + ``temperature=None`` is equivalent to not passing it at all. + """ + return { + k: v for k, v in kwargs.items() + if k in _GENERATION_CACHE_KEYS and v is not None + } + # Try to import spaCy from ..utils.helpers import safe_import @@ -281,6 +322,8 @@ def find_best_match_index(text: str, candidates: List[str]) -> Tuple[int, float] Find the best matching candidate index and score. Uses hybrid similarity approach: Exact -> Synonym -> Substring -> Embeddings -> Vector -> Fuzzy. Optimized for batch processing to avoid redundant embedding calculations. + Embedding/vector similarity stages are best-effort; if they fail, matching falls back + to remaining strategies instead of raising. Returns: Tuple[int, float]: (best_candidate_index, best_score). Index is -1 if no candidates. @@ -404,10 +447,12 @@ def find_best_match_index(text: str, candidates: List[str]) -> Tuple[int, float] vector_idx = -1 if nlp and nlp.vocab.vectors.shape[0] > 0: + failing_candidate_idx = -1 try: doc = nlp(text) if doc.vector_norm: for i, candidate in enumerate(candidates): + failing_candidate_idx = i if not candidate: continue cand_doc = nlp(candidate) if cand_doc.vector_norm: @@ -416,7 +461,13 @@ def find_best_match_index(text: str, candidates: List[str]) -> Tuple[int, float] vector_score = score vector_idx = i except Exception: - pass + logger.debug( + "Vector similarity calculation failed at candidate index %s; continuing with fallback scoring.", + failing_candidate_idx if failing_candidate_idx >= 0 else "N/A", + exc_info=True, + ) + vector_score = 0.0 + vector_idx = -1 if vector_score > best_score: best_score = vector_score @@ -779,9 +830,11 @@ def extract_entities_huggingface( """ loader = HuggingFaceModelLoader(device=device) # Pass kwargs (like aggregation_strategy) to load_ner_model - model_obj = loader.load_ner_model(model, **kwargs) + loader_kwargs = { + key: value for key, value in kwargs.items() if key != "huggingface_model" + } + model_obj = loader.load_ner_model(model, **loader_kwargs) results = loader.extract_entities(model_obj, text) - entities = [] # Check if manual aggregation is needed (raw IOB tags detected) @@ -945,6 +998,7 @@ def extract_entities_llm( "max_text_length": max_text_length, "structured_output_mode": structured_output_mode, "entity_types": kwargs.get("entity_types"), + **_generation_cache_params(kwargs), } cached_result = _result_cache.get("entities", text, **cache_params) if cached_result is not None: @@ -1112,47 +1166,6 @@ Text to extract from: return [] -def _parse_entity_result(result: Any, provider: str, model: Optional[str]) -> List[Entity]: - """Helper to parse raw LLM result into Entity objects.""" - entities = [] - items = [] - - if isinstance(result, list): - items = result - elif isinstance(result, dict): - # Handle cases where LLM wraps the list in a key - for key in ["entities", "data", "results"]: - if key in result and isinstance(result[key], list): - items = result[key] - break - if not items and "text" in result: # Single object instead of list - items = [result] - - for item in items: - if not isinstance(item, dict): - continue - - text = item.get("text", "") - if not text: - continue - - entities.append( - Entity( - text=text, - label=item.get("label", "UNKNOWN"), - start_char=item.get("start", 0), - end_char=item.get("end", 0), - confidence=item.get("confidence", 0.9), - metadata={ - "provider": provider, - "model": model, - "extraction_method": "llm", - }, - ) - ) - return entities - - def _extract_entities_chunked( text: str, provider: str, @@ -1735,7 +1748,8 @@ def extract_relations_llm( "relation_types": kwargs.get("relation_types"), "extract_temporal_bounds": extract_temporal_bounds, # Include entities hash/str in cache key implicitly via **cache_params - "entities_hash": hash(tuple(sorted([e.text for e in entities]))) if entities else 0 + "entities_hash": hash(tuple(sorted([e.text for e in entities]))) if entities else 0, + **_generation_cache_params(kwargs), } cached_result = _result_cache.get("relations", text, **cache_params) if cached_result is not None: @@ -1935,13 +1949,10 @@ Entities found in text: {entities_str}""" "[methods.extract_relations_llm] Calling llm.generate_typed (%s/%s)...", provider, model, ) - # Only forward minimal, safe parameters to provider calls - call_kwargs = {} - if "temperature" in kwargs: - call_kwargs["temperature"] = kwargs["temperature"] - if "verbose" in kwargs: - call_kwargs["verbose"] = kwargs["verbose"] - + # Forward all caller-supplied generation kwargs so they reach + # generate_typed and the underlying provider API. max_retries is + # always set from the explicit parameter. + call_kwargs = kwargs.copy() call_kwargs["max_retries"] = max_retries # Select schema based on whether temporal extraction is requested @@ -2393,7 +2404,8 @@ def extract_triplets_llm( "triplet_types": kwargs.get("triplet_types"), # Include entities/relations hash in cache key implicitly via **cache_params "entities_hash": hash(tuple(sorted([e.text for e in entities]))) if entities else 0, - "relations_hash": hash(tuple(sorted([str(r) for r in relations]))) if relations else 0 + "relations_hash": hash(tuple(sorted([str(r) for r in relations]))) if relations else 0, + **_generation_cache_params(kwargs), } cached_result = _result_cache.get("triplets", text, **cache_params) if cached_result is not None: @@ -2547,48 +2559,6 @@ Text to extract from: return [] -def _parse_triplet_result(result: Any, provider: str, model: Optional[str]) -> List[Triplet]: - """Helper to parse raw LLM result into Triplet objects.""" - triplets = [] - items = [] - - if isinstance(result, list): - items = result - elif isinstance(result, dict): - for key in ["triplets", "data", "results"]: - if key in result and isinstance(result[key], list): - items = result[key] - break - if not items and "subject" in result: - items = [result] - - for item in items: - if not isinstance(item, dict): - continue - - subject = item.get("subject", "") - predicate = item.get("predicate", "") - obj = item.get("object", "") - - if not subject or not predicate or not obj: - continue - - triplets.append( - Triplet( - subject=str(subject), - predicate=str(predicate), - object=str(obj), - confidence=item.get("confidence", 0.9), - metadata={ - "provider": provider, - "model": model, - "extraction_method": "llm", - }, - ) - ) - return triplets - - def _extract_triplets_chunked( text: str, provider: str, diff --git a/semantica/triplet_store/methods.py b/semantica/triplet_store/methods.py index 12380aa8..dc999c20 100644 --- a/semantica/triplet_store/methods.py +++ b/semantica/triplet_store/methods.py @@ -101,7 +101,6 @@ from .triplet_store import TripletStore # Global store registry _global_stores: Dict[str, TripletStore] = {} _default_store_id: Optional[str] = None -_global_query_engine: Optional[QueryEngine] = None _global_bulk_loader: Optional[BulkLoader] = None @@ -131,25 +130,6 @@ def _get_store(store_id: Optional[str] = None) -> TripletStore: return _global_stores[target_id] -def _get_query_engine() -> QueryEngine: - """Get or create global QueryEngine instance.""" - global _global_query_engine - if _global_query_engine is None: - # We need a store backend for the engine, but QueryEngine in this module - # seems to be initialized with config in the old code. - # In the new code, TripletStore has its own query_engine. - # If we use this standalone function, we might need to rely on the store's engine. - # But let's keep a standalone one if needed, or better, delegate to store. - config = triplet_store_config.get_all() - # QueryEngine now expects a backend, but we can initialize it without one - # if we pass the backend at execution time? - # Checking QueryEngine implementation... it takes `store_backend` in __init__. - # So we can't easily have a global one without a store. - # We'll rely on the store's engine. - pass - return None # Deprecated use of global engine - - def _get_bulk_loader() -> BulkLoader: """Get or create global BulkLoader instance.""" global _global_bulk_loader diff --git a/semantica/triplet_store/oxigraph_store.py b/semantica/triplet_store/oxigraph_store.py index 5262c00a..4ba94f03 100644 --- a/semantica/triplet_store/oxigraph_store.py +++ b/semantica/triplet_store/oxigraph_store.py @@ -42,6 +42,10 @@ class OxigraphStore: ProcessingError: If the store cannot be opened. """ self.logger = get_logger("oxigraph_store") + # Accept storage_path as an alias for path (matches the convention used + # by other Semantica stores). Pop it so it isn't left in self.config. + if path is None and "storage_path" in config: + path = config.pop("storage_path") self.config = config self.path = path if path is not None else config.get("path") @@ -74,24 +78,65 @@ class OxigraphStore: ) from exc def add_triplet(self, triplet: Triplet, **options) -> Dict[str, Any]: - """Add one triplet to the default graph or ``options['graph']``.""" - return self.add_triplets([triplet], **options) + """Add one triplet to the default graph or ``options['graph']``. - def add_triplets(self, triplets: List[Triplet], **options) -> Dict[str, Any]: - """Add triplets in one native Oxigraph batch.""" + The write is committed to the store's in-memory state immediately. + pyoxigraph's background threads will persist it to disk shortly + afterward; call :meth:`flush` explicitly if you need a synchronous + durability guarantee before reopening or crashing. + """ try: graph_name = self._graph_name(options.get("graph")) - quads = [self._to_quad(triplet, graph_name) for triplet in triplets] - self.store.extend(quads) + self.store.extend([self._to_quad(triplet, graph_name)]) return { "success": True, - "triplets_loaded": len(triplets), + "triplets_loaded": 1, "graph": options.get("graph"), } except Exception as exc: self.logger.error(f"Oxigraph load failed: {exc}") raise ProcessingError(f"Oxigraph load failed: {exc}") from exc + def add_triplets(self, triplets: List[Triplet], **options) -> Dict[str, Any]: + """Add triplets in one native Oxigraph batch. + + The batch is written transactionally and then explicitly flushed to + disk before returning. This makes the full batch durable without + requiring a separate :meth:`flush` call. In-memory stores skip the + flush (there is nothing to sync). + + For high-volume imports the :class:`~.bulk_loader.BulkLoader` splits + work into chunks and calls this method once per chunk, so each chunk + lands as one atomic, durable unit. + """ + try: + graph_name = self._graph_name(options.get("graph")) + quads = [self._to_quad(triplet, graph_name) for triplet in triplets] + self.store.extend(quads) + except Exception as exc: + self.logger.error(f"Oxigraph load failed: {exc}") + raise ProcessingError(f"Oxigraph load failed: {exc}") from exc + + # Flush is kept outside the write try/except so that a flush I/O error + # does not produce a misleading "load failed" message when extend() + # already committed the batch successfully. + if self.path is not None: + try: + self.flush() + except OSError as exc: + self.logger.warning( + f"Oxigraph flush failed after successful write: {exc}" + ) + raise ProcessingError( + f"Oxigraph flush failed after successful write: {exc}" + ) from exc + + return { + "success": True, + "triplets_loaded": len(triplets), + "graph": options.get("graph"), + } + def bulk_load(self, triplets: List[Triplet], **options) -> Dict[str, Any]: """Load a batch of triplets using Oxigraph's native bulk operation.""" return self.add_triplets(triplets, **options) diff --git a/semantica/triplet_store/rdf4j_store.py b/semantica/triplet_store/rdf4j_store.py index 8dad6997..d788ab7b 100644 --- a/semantica/triplet_store/rdf4j_store.py +++ b/semantica/triplet_store/rdf4j_store.py @@ -28,7 +28,7 @@ License: MIT import re from typing import Any, Dict, List, Optional -from urllib.parse import urlparse +from urllib.parse import quote, urlparse import requests from rdflib import Graph, Literal @@ -67,7 +67,8 @@ class RDF4JStore: self.progress_tracker.enabled = True self.endpoint = endpoint.rstrip("/") - self.repository_id = config.get("repository_id", "default") + self.repository_id = repository_id or config.get("repository_id", "default") + self._encoded_repository_id = quote(self.repository_id, safe="") self.username = config.get("username") self.password = config.get("password") self.timeout = config.get("timeout", 30) @@ -79,7 +80,7 @@ class RDF4JStore: """Connect to RDF4J server.""" try: # Test connection - test_url = f"{self.endpoint}/repositories/{self.repository_id}" + test_url = f"{self.endpoint}/repositories/{self._encoded_repository_id}" response = requests.get( test_url, timeout=self.timeout, @@ -100,11 +101,11 @@ class RDF4JStore: def _get_sparql_endpoint(self) -> str: """Get SPARQL query endpoint.""" - return f"{self.endpoint}/repositories/{self.repository_id}" + return f"{self.endpoint}/repositories/{self._encoded_repository_id}" def _get_update_endpoint(self) -> str: """Get SPARQL Update endpoint.""" - return f"{self.endpoint}/repositories/{self.repository_id}/statements" + return f"{self.endpoint}/repositories/{self._encoded_repository_id}/statements" def _is_construct_query(self, query: str) -> bool: """ @@ -163,7 +164,7 @@ class RDF4JStore: """ # RDF4J transaction support transaction_url = ( - f"{self.endpoint}/repositories/{self.repository_id}/transactions" + f"{self.endpoint}/repositories/{self._encoded_repository_id}/transactions" ) try: diff --git a/semantica/utils/helpers.py b/semantica/utils/helpers.py index 05ece01f..48065855 100644 --- a/semantica/utils/helpers.py +++ b/semantica/utils/helpers.py @@ -680,6 +680,35 @@ _TRIPLET_KEYS = ("triplets",) # 'metadata' and 'count'. _CONTEXT_KEYS = ("metadata", "statistics", "count") +# Validation errors below interpolate caller-controlled keys. A pathological +# key (megabytes long) would otherwise size the exception string and, through +# the export wrappers that log the full exception, the log entry. The display +# keeps the offending key recognizable while bounding the message. +_MAX_KEY_DISPLAY = 64 + + +def _truncate_key(key: Any) -> str: + """Render a mapping key for an error message, bounded in length.""" + value = str(key) + if len(value) > _MAX_KEY_DISPLAY: + return value[:_MAX_KEY_DISPLAY] + "โ€ฆ" + return value + + +# Truncating each key bounds the per-key cost; capping the count of keys +# shown bounds the total, so a payload carrying many unknown keys cannot +# size the message (or the log entry that records it) either. +_MAX_KEYS_DISPLAY = 8 + + +def _truncate_key_list(keys: Iterable[Any]) -> str: + """Render keys for an error message, bounded in count and length.""" + rendered = [_truncate_key(key) for key in keys] + if len(rendered) <= _MAX_KEYS_DISPLAY: + return ", ".join(f"'{key}'" for key in rendered) + shown = ", ".join(f"'{key}'" for key in rendered[:_MAX_KEYS_DISPLAY]) + return f"{shown}, and {len(rendered) - _MAX_KEYS_DISPLAY} more" + def _require_recognized_keys( payload: Mapping, recognized_keys: Sequence[str], *, what: str @@ -702,7 +731,7 @@ def _require_recognized_keys( if not payload or any(key in payload for key in recognized_keys): return - supplied = ", ".join(f"'{key}'" for key in sorted(map(str, payload))) + supplied = _truncate_key_list(sorted(map(str, payload))) expected = ", ".join(f"'{key}'" for key in recognized_keys) raise ValidationError( f"{what} has no recognized key. Supplied: {supplied}. " @@ -753,7 +782,7 @@ def _require_nothing_dropped( if not dropped: return - named = ", ".join(f"'{key}'" for key in dropped) + named = _truncate_key_list(dropped) expected = ", ".join(f"'{key}'" for key in recognized_keys) raise ValidationError( f"{what} resolved to nothing, but {named} still holds records. " diff --git a/semantica/utils/progress_tracker.py b/semantica/utils/progress_tracker.py index febfb4b9..f27768d4 100644 --- a/semantica/utils/progress_tracker.py +++ b/semantica/utils/progress_tracker.py @@ -64,6 +64,29 @@ def _progress_disabled_from_env() -> bool: "on", ) + +def _progress_forced_from_env() -> bool: + """Return whether console progress is forced on despite a non-interactive stdout.""" + return os.getenv("SEMANTICA_FORCE_PROGRESS", "").strip().lower() in ( + "1", + "true", + "yes", + "on", + ) + + +def _stdout_is_tty() -> bool: + """Return whether stdout is an interactive terminal. + + Replacement streams do not always implement ``isatty`` and closed streams can + raise, so both cases are treated as non-interactive. + """ + try: + return bool(sys.stdout is not None and sys.stdout.isatty()) + except (AttributeError, ValueError): + return False + + # Try to import IPython for Jupyter support try: from IPython import get_ipython @@ -1040,18 +1063,24 @@ class ProgressTracker: # Create displays self.displays: List[ProgressDisplay] = [] + # Console output only suits an interactive stdout. When output is piped or + # redirected (scripts, CI logs) the progress bars and their escape + # sequences would otherwise drown the program's own output. + console_ok = _stdout_is_tty() or self.is_jupyter or _progress_forced_from_env() + # Always try Jupyter first if available, fallback to console if IPYTHON_AVAILABLE: # Try to detect Jupyter - if available, use it if self.is_jupyter and not self.disable_jupyter_progress: self.displays.append(JupyterProgressDisplay(use_emoji=use_emoji)) # Also add console as fallback for immediate feedback - self.displays.append( - ConsoleProgressDisplay( - use_emoji=use_emoji, update_interval=update_interval + if console_ok: + self.displays.append( + ConsoleProgressDisplay( + use_emoji=use_emoji, update_interval=update_interval + ) ) - ) - else: + elif console_ok: self.displays.append( ConsoleProgressDisplay( use_emoji=use_emoji, update_interval=update_interval diff --git a/semantica/vector_store/methods.py b/semantica/vector_store/methods.py index d74d763b..5a558b27 100644 --- a/semantica/vector_store/methods.py +++ b/semantica/vector_store/methods.py @@ -297,7 +297,11 @@ def create_index( config = vector_store_config.get_all() backend = config.get("default_backend", "faiss") dimension = config.get("dimension", 768) - indexer = VectorIndexer(backend=backend, dimension=dimension, **config) + # backend/dimension are already passed explicitly; drop them from the + # forwarded config so VectorIndexer(..., **remaining_config) doesn't + # receive duplicate keyword arguments. + remaining_config = {k: v for k, v in config.items() if k not in ("default_backend", "dimension")} + indexer = VectorIndexer(backend=backend, dimension=dimension, **remaining_config) return indexer.create_index(vectors, ids, **options) diff --git a/tests/conflicts/test_conflicts.py b/tests/conflicts/test_conflicts.py index 34dad4d2..cf2e5e11 100644 --- a/tests/conflicts/test_conflicts.py +++ b/tests/conflicts/test_conflicts.py @@ -238,6 +238,297 @@ class TestConflictsModule(unittest.TestCase): checklist = generator.export_investigation_checklist(guide, format="text") self.assertIn("INVESTIGATION GUIDE: c1", checklist) + def test_conflict_resolver_credibility_weighted(self): + """Test credibility-weighted resolution strategy (#865).""" + resolver = ConflictResolver() + + # Boost the credibility of "doc_trusted" so its value should win + # even though it only has one vote, vs. two lower-credibility votes. + # Use the validated setter rather than mutating the internal dict + # directly, so the test stays coupled to the public API surface. + resolver.source_tracker.set_source_credibility("doc_trusted", 1.0) + resolver.source_tracker.set_source_credibility("doc_flaky", 0.1) + + conflict = Conflict( + conflict_id="c_cred", + conflict_type=ConflictType.VALUE_CONFLICT, + entity_id="e1", + property_name="age", + conflicting_values=[30, 30, 40], + sources=[ + {"document": "doc_flaky", "confidence": 0.9}, + {"document": "doc_flaky", "confidence": 0.9}, + {"document": "doc_trusted", "confidence": 0.9}, + ], + ) + + result = resolver.resolve_conflict(conflict, strategy="credibility_weighted") + self.assertTrue(result.resolved) + self.assertEqual(result.resolved_value, 40) + self.assertEqual(result.resolution_strategy, "credibility_weighted") + self.assertGreater(result.confidence, 0.0) + + def test_conflict_resolver_first_seen(self): + """Test first-seen resolution strategy (#865).""" + resolver = ConflictResolver() + + conflict = Conflict( + conflict_id="c_first", + conflict_type=ConflictType.VALUE_CONFLICT, + entity_id="e1", + property_name="age", + conflicting_values=[30, 32], + sources=[ + {"document": "doc1", "confidence": 0.9}, + {"document": "doc2", "confidence": 0.9}, + ], + ) + + result = resolver.resolve_conflict(conflict, strategy="first_seen") + self.assertTrue(result.resolved) + self.assertEqual(result.resolved_value, 30) # first value in the list + self.assertEqual(result.resolution_strategy, "first_seen") + self.assertEqual(result.sources_used, ["doc1"]) + + def test_conflict_resolver_manual_review(self): + """Test manual-review resolution strategy flags without resolving (#865).""" + resolver = ConflictResolver() + + conflict = Conflict( + conflict_id="c_manual", + conflict_type=ConflictType.VALUE_CONFLICT, + entity_id="e1", + property_name="age", + conflicting_values=[30, 32], + sources=[{"document": "doc1"}, {"document": "doc2"}], + severity="high", + ) + + result = resolver.resolve_conflict(conflict, strategy="manual_review") + self.assertFalse(result.resolved) + self.assertEqual(result.resolution_strategy, "manual_review") + self.assertTrue(result.metadata.get("requires_manual_review")) + self.assertEqual(result.metadata.get("severity"), "high") + + def test_conflict_resolver_expert_review(self): + """Test expert-review resolution strategy flags without resolving (#865).""" + resolver = ConflictResolver() + + conflict = Conflict( + conflict_id="c_expert", + conflict_type=ConflictType.VALUE_CONFLICT, + entity_id="e1", + property_name="age", + conflicting_values=[30, 32], + sources=[{"document": "doc1"}, {"document": "doc2"}], + severity="critical", + ) + + result = resolver.resolve_conflict(conflict, strategy="expert_review") + self.assertFalse(result.resolved) + self.assertEqual(result.resolution_strategy, "expert_review") + self.assertTrue(result.metadata.get("requires_expert_review")) + self.assertEqual(result.metadata.get("severity"), "critical") + + def test_conflict_detector_relationship_conflicts(self): + """Test relationship conflict detection (#865).""" + detector = ConflictDetector() + + relationships = [ + { + "id": "r1", + "source_id": "e1", + "target_id": "e2", + "type": "works_at", + "source": "doc1", + }, + { + "id": "r1", + "source_id": "e1", + "target_id": "e2", + "type": "founded", + "source": "doc2", + }, + ] + + conflicts = detector.detect_relationship_conflicts(relationships) + self.assertEqual(len(conflicts), 1) + conflict = conflicts[0] + self.assertEqual(conflict.conflict_type, ConflictType.RELATIONSHIP_CONFLICT) + self.assertEqual(conflict.relationship_id, "r1") + self.assertEqual(conflict.property_name, "type") + self.assertIn("works_at", conflict.conflicting_values) + self.assertIn("founded", conflict.conflicting_values) + + def test_conflict_detector_relationship_conflicts_no_conflict(self): + """Relationships with a single occurrence should not raise conflicts (#865).""" + detector = ConflictDetector() + + relationships = [ + {"id": "r1", "source_id": "e1", "target_id": "e2", "type": "works_at"}, + ] + + conflicts = detector.detect_relationship_conflicts(relationships) + self.assertEqual(len(conflicts), 0) + + def test_conflict_detector_temporal_conflicts(self): + """Test temporal conflict detection (#865).""" + detector = ConflictDetector() + + entities = [ + {"id": "e1", "founded": "1998", "source": "doc1", "confidence": 0.9}, + {"id": "e1", "founded": "2004", "source": "doc2", "confidence": 0.8}, + ] + + conflicts = detector.detect_temporal_conflicts(entities) + self.assertEqual(len(conflicts), 1) + conflict = conflicts[0] + self.assertEqual(conflict.conflict_type, ConflictType.TEMPORAL_CONFLICT) + self.assertEqual(conflict.entity_id, "e1") + self.assertEqual(conflict.property_name, "founded") + self.assertIn("1998", conflict.conflicting_values) + self.assertIn("2004", conflict.conflicting_values) + + def test_conflict_detector_temporal_conflicts_no_conflict(self): + """Matching temporal values across sources should not raise conflicts (#865).""" + detector = ConflictDetector() + + entities = [ + {"id": "e1", "founded": "1998", "source": "doc1"}, + {"id": "e1", "founded": "1998", "source": "doc2"}, + ] + + conflicts = detector.detect_temporal_conflicts(entities) + self.assertEqual(len(conflicts), 0) + + def test_conflict_detector_logical_conflicts(self): + """Test logical conflict detection for incompatible entity types (#865).""" + detector = ConflictDetector() + + entities = [ + {"id": "e1", "type": "Person", "source": "doc1"}, + {"id": "e1", "type": "Organization", "source": "doc2"}, + ] + + conflicts = detector.detect_logical_conflicts(entities) + self.assertEqual(len(conflicts), 1) + conflict = conflicts[0] + self.assertEqual(conflict.conflict_type, ConflictType.LOGICAL_CONFLICT) + self.assertEqual(conflict.entity_id, "e1") + self.assertEqual(conflict.severity, "critical") + self.assertIn("Person", conflict.conflicting_values) + self.assertIn("Organization", conflict.conflicting_values) + + def test_conflict_detector_logical_conflicts_compatible_types(self): + """Compatible/unrelated types should not raise logical conflicts (#865).""" + detector = ConflictDetector() + + entities = [ + {"id": "e1", "type": "Person", "source": "doc1"}, + {"id": "e1", "type": "Employee", "source": "doc2"}, + ] + + conflicts = detector.detect_logical_conflicts(entities) + self.assertEqual(len(conflicts), 0) + + def test_conflict_analyzer_by_source_breakdown(self): + """Test the by_source breakdown of analyze_conflicts (#902).""" + analyzer = ConflictAnalyzer() + + conflicts = [ + Conflict( + conflict_id="c1", + conflict_type=ConflictType.VALUE_CONFLICT, + entity_id="e1", + property_name="age", + conflicting_values=[30, 32], + sources=[{"document": "doc1"}, {"document": "doc2"}], + severity="medium", + ), + Conflict( + conflict_id="c2", + conflict_type=ConflictType.TYPE_CONFLICT, + entity_id="e2", + property_name="type", + conflicting_values=["Person", "Org"], + sources=[{"document": "doc1"}, {"document": "doc3"}], + severity="critical", + ), + ] + + analysis = analyzer.analyze_conflicts(conflicts) + + self.assertIn("by_source", analysis) + by_source = analysis["by_source"] + + # doc1 appears in both conflicts, doc2 and doc3 in one each. + self.assertEqual(by_source["counts"]["doc1"], 2) + self.assertEqual(by_source["counts"]["doc2"], 1) + self.assertEqual(by_source["counts"]["doc3"], 1) + + top_sources = { + s["source"]: s["conflict_count"] for s in by_source["top_sources"] + } + self.assertEqual(top_sources["doc1"], 2) + + self.assertIn("doc1", by_source["details"]) + doc1_entries = by_source["details"]["doc1"] + self.assertEqual(len(doc1_entries), 2) + self.assertEqual({e["conflict_id"] for e in doc1_entries}, {"c1", "c2"}) + + def test_conflict_analyzer_analyze_trends(self): + """Test analyze_trends over deterministic, time-ordered data (#902).""" + analyzer = ConflictAnalyzer() + + def make_conflict(conflict_id, timestamp): + return Conflict( + conflict_id=conflict_id, + conflict_type=ConflictType.VALUE_CONFLICT, + entity_id="e1", + property_name="age", + conflicting_values=[30, 32], + sources=[{"document": "doc1", "metadata": {"timestamp": timestamp}}], + ) + + # January: 1 conflict. February: 3 conflicts (>10% increase -> "increasing"). + conflicts = [ + make_conflict("c1", "2023-01-05T00:00:00"), + make_conflict("c2", "2023-02-01T00:00:00"), + make_conflict("c3", "2023-02-10T00:00:00"), + make_conflict("c4", "2023-02-20T00:00:00"), + ] + + trends = analyzer.analyze_trends(conflicts) + + self.assertEqual(len(trends), 2) + self.assertEqual(trends[0]["period"], "2023-01") + self.assertEqual(trends[0]["conflict_count"], 1) + self.assertEqual(trends[1]["period"], "2023-02") + self.assertEqual(trends[1]["conflict_count"], 3) + self.assertEqual(trends[1]["trend"], "increasing") + self.assertEqual(trends[1]["trend_direction"], "up") + + def test_conflict_analyzer_analyze_trends_insufficient_data(self): + """Single-period data should report insufficient_data, not crash (#902).""" + analyzer = ConflictAnalyzer() + + conflict = Conflict( + conflict_id="c1", + conflict_type=ConflictType.VALUE_CONFLICT, + entity_id="e1", + property_name="age", + conflicting_values=[30, 32], + sources=[ + {"document": "doc1", "metadata": {"timestamp": "2023-01-05T00:00:00"}} + ], + ) + + trends = analyzer.analyze_trends([conflict]) + + self.assertEqual(len(trends), 1) + self.assertEqual(trends[0]["trend"], "insufficient_data") + self.assertEqual(trends[0]["conflict_count"], 1) + if __name__ == "__main__": - unittest.main() + unittest.main() \ No newline at end of file diff --git a/tests/context/test_agent_memory_markdown.py b/tests/context/test_agent_memory_markdown.py index bf1ca128..c1f71301 100644 --- a/tests/context/test_agent_memory_markdown.py +++ b/tests/context/test_agent_memory_markdown.py @@ -1,7 +1,11 @@ import errno +import os +import stat +import subprocess import sys from copy import deepcopy from datetime import datetime, timedelta, timezone +from pathlib import Path from unittest.mock import MagicMock, patch import pytest @@ -56,6 +60,25 @@ def markdown_document(frontmatter, body=""): return f"---\n{yaml_text}---\n\n{body}" +def _require_symlink_support(tmp_path): + """Skip the test if this environment cannot create symbolic links. + + Symlink creation can be unavailable even on POSIX (e.g. restricted + containers) and commonly requires elevated privilege or Developer Mode + on Windows. Probe actual capability instead of assuming based on + platform, so these tests still run wherever symlinks genuinely work. + """ + probe_target = tmp_path / ".symlink_probe_target" + probe_link = tmp_path / ".symlink_probe_link" + probe_target.write_text("", encoding="utf-8") + try: + probe_link.symlink_to(probe_target) + except OSError as exc: + pytest.skip(f"environment cannot create symbolic links: {exc}") + probe_link.unlink() + probe_target.unlink() + + def required_frontmatter(memory_id="mem_test", **overrides): frontmatter = { "id": memory_id, @@ -707,6 +730,151 @@ def test_markdown_string_path_inspection_errors_are_actionable(): assert exc_info.value.__cause__ is original_error +@pytest.mark.parametrize("use_string_path", [False, True]) +def test_markdown_import_rejects_symlinked_file(tmp_path, use_string_path): + _require_symlink_support(tmp_path) + outside = tmp_path / "outside.md" + outside.write_text( + markdown_document(required_frontmatter(), "Do not import"), + encoding="utf-8", + ) + source = tmp_path / "memory.md" + source.symlink_to(outside) + payload = str(source) if use_string_path else source + memory = AgentMemory() + + with pytest.raises(ValueError, match="symbolic link"): + memory.import_data(payload, format="markdown") + + assert memory.count() == 0 + + +@pytest.mark.parametrize("use_string_path", [False, True]) +def test_markdown_import_rejects_broken_symlink(tmp_path, use_string_path): + _require_symlink_support(tmp_path) + source = tmp_path / "missing-memory.md" + source.symlink_to(tmp_path / "missing-target.md") + payload = str(source) if use_string_path else source + memory = AgentMemory() + + with pytest.raises(ValueError, match="symbolic link"): + memory.import_data(payload, format="markdown") + + assert memory.count() == 0 + + +@pytest.mark.parametrize("use_string_path", [False, True]) +def test_markdown_import_rejects_symlinked_directory(tmp_path, use_string_path): + _require_symlink_support(tmp_path) + outside = tmp_path / "outside" + outside.mkdir() + (outside / "memory.md").write_text( + markdown_document(required_frontmatter(), "Do not import"), + encoding="utf-8", + ) + source = tmp_path / "memory-export" + source.symlink_to(outside, target_is_directory=True) + payload = str(source) if use_string_path else source + memory = AgentMemory() + + with pytest.raises(ValueError, match="symbolic link"): + memory.import_data(payload, format="markdown") + + assert memory.count() == 0 + + +def test_markdown_import_skips_symlinked_file_in_directory(tmp_path): + _require_symlink_support(tmp_path) + outside = tmp_path / "outside.md" + outside.write_text( + markdown_document(required_frontmatter(), "Do not import"), + encoding="utf-8", + ) + source = tmp_path / "memory-export" + source.mkdir() + (source / "memory.md").symlink_to(outside) + memory = AgentMemory() + + assert memory.import_data(source, format="markdown") == 0 + assert memory.count() == 0 + + +@pytest.mark.skipif(not hasattr(os, "O_NOFOLLOW"), reason="requires O_NOFOLLOW") +def test_markdown_import_does_not_follow_symlink_raced_before_open(tmp_path): + _require_symlink_support(tmp_path) + outside = tmp_path / "outside.md" + outside.write_text( + markdown_document(required_frontmatter(), "Do not import"), + encoding="utf-8", + ) + source = tmp_path / "memory.md" + source.symlink_to(outside) + + with patch.object(Path, "is_symlink", return_value=False): + with pytest.raises(ValueError, match="symbolic link"): + AgentMemory()._read_markdown_file_content(source) + + +def test_markdown_import_rejects_windows_junction(tmp_path, monkeypatch): + source = tmp_path / "junction" + source.mkdir() + (source / "memory.md").write_text("not read", encoding="utf-8") + + monkeypatch.setattr( + os.path, + "isjunction", + lambda candidate: Path(candidate) == source, + raising=False, + ) + + with pytest.raises(ValueError, match="junction"): + AgentMemory().import_data(source, format="markdown") + + +def test_markdown_import_rejects_windows_reparse_point_fallback(tmp_path, monkeypatch): + source = tmp_path / "reparse-point" + source.mkdir() + real_lstat = os.lstat + + class ReparseStat: + st_file_attributes = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + + monkeypatch.delattr(os.path, "isjunction", raising=False) + monkeypatch.setattr(Path, "is_symlink", lambda self: False) + monkeypatch.setattr( + os, + "lstat", + lambda candidate: ( + ReparseStat() if Path(candidate) == source else real_lstat(candidate) + ), + ) + + with pytest.raises(ValueError, match="junction"): + AgentMemory().import_data(source, format="markdown") + + +@pytest.mark.skipif(os.name != "nt", reason="requires Windows junctions") +def test_markdown_import_rejects_real_windows_junction(tmp_path): + outside = tmp_path / "outside" + outside.mkdir() + (outside / "memory.md").write_text("not read", encoding="utf-8") + source = tmp_path / "junction" + result = subprocess.run( + ["cmd.exe", "/c", "mklink", "/J", str(source), str(outside)], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + pytest.skip(f"could not create Windows junction: {result.stderr}") + + try: + with pytest.raises(ValueError, match="junction"): + AgentMemory().import_data(source, format="markdown") + finally: + os.rmdir(source) + + def test_legacy_dict_import_behavior_is_unchanged(): memory = AgentMemory() data = { @@ -751,6 +919,7 @@ def test_markdown_import_file_open_security_rejects_symlink(tmp_path): with pytest.raises(ValueError, match="Symlink Markdown import paths are rejected"): memory._read_markdown_file_content(symlink_file) + def test_markdown_import_public_api_rejects_symlink(tmp_path): """ import_data(..., format="markdown") must propagate the symlink rejection diff --git a/tests/context/test_context_graph_markdown.py b/tests/context/test_context_graph_markdown.py new file mode 100644 index 00000000..538421f3 --- /dev/null +++ b/tests/context/test_context_graph_markdown.py @@ -0,0 +1,713 @@ +import os +import stat +import subprocess +from pathlib import Path + +import pytest +import yaml + +import semantica.context.context_graph as context_graph_module +from semantica.change_management.managers import TemporalVersionManager +from semantica.context.context_graph import ContextEdge, ContextGraph, ContextNode + + +def _read_markdown(path: Path): + lines = path.read_text(encoding="utf-8").splitlines(keepends=True) + closing_index = next( + index + for index, line in enumerate(lines[1:], start=1) + if line.rstrip("\r\n") == "---" + ) + frontmatter = yaml.safe_load("".join(lines[1:closing_index])) or {} + body = "".join(lines[closing_index + 1 :]) + if body.startswith("\r\n"): + body = body[2:] + elif body.startswith("\n"): + body = body[1:] + return frontmatter, body + + +def _write_markdown(path: Path, frontmatter, body: str) -> None: + yaml_text = yaml.safe_dump( + frontmatter, + sort_keys=False, + allow_unicode=True, + default_flow_style=False, + ) + path.write_text(f"---\n{yaml_text}---\n\n{body}", encoding="utf-8") + + +def _node_file(export_path: Path, node_id: str) -> Path: + for path in (export_path / "nodes").glob("*.md"): + frontmatter, _ = _read_markdown(path) + if frontmatter.get("id") == node_id: + return path + raise AssertionError(f"No Markdown file found for node {node_id!r}") + + +def _normalized_state(graph: ContextGraph): + links = {link_id: dict(link) for link_id, link in graph._unresolved_links.items()} + for link_id, ( + other_graph, + source_node_id, + target_node_id, + ) in graph._linked_graphs.items(): + links[link_id] = { + "link_id": link_id, + "source_node_id": source_node_id, + "target_node_id": target_node_id, + "other_graph_id": other_graph.graph_id, + } + return { + "graph_id": graph.graph_id, + "nodes": { + node_id: { + "type": node.node_type, + "content": node.content, + "properties": node.properties, + "metadata": node.metadata, + "valid_from": node.valid_from, + "valid_until": node.valid_until, + } + for node_id, node in graph.nodes.items() + }, + "edges": sorted( + ( + { + "id": edge.edge_id, + "family_id": edge.family_id, + "source": edge.source_id, + "target": edge.target_id, + "type": edge.edge_type, + "weight": edge.weight, + "metadata": edge.metadata, + "valid_from": edge.valid_from, + "valid_until": edge.valid_until, + } + for edge in graph.edges + ), + key=lambda edge: edge["id"], + ), + "links": links, + } + + +def _sample_graph(): + graph = ContextGraph(advanced_analytics=False) + graph.graph_id = "graph-primary" + graph._add_internal_node( + ContextNode( + node_id="policy/\u6771\u4eac", + node_type="Policy", + content="# Retention\n\nKeep evidence.\n---\n", + properties={"priority": 2, "nested": {"owner": "governance"}}, + metadata={"source": "manual", "tags": ["retention", "legal"]}, + valid_from="2026-01-01T00:00:00+00:00", + valid_until="2027-01-01T00:00:00+00:00", + ) + ) + graph._add_internal_node( + ContextNode( + node_id="evidence-1", + node_type="Evidence", + content="Original source", + properties={"checksum": "abc123"}, + metadata={"classification": "internal"}, + ) + ) + graph._add_internal_edge( + ContextEdge( + edge_id="edge-supports", + family_id="family-supports", + source_id="evidence-1", + target_id="policy/\u6771\u4eac", + edge_type="SUPPORTS", + weight=0.75, + metadata={"confidence": 0.9}, + valid_from="2026-02-01", + valid_until="2026-12-31", + ) + ) + + other = ContextGraph(advanced_analytics=False) + other.graph_id = "graph-secondary" + other.add_node("source-page", "Page", "Source page") + link_id = graph.link_graph(other, "policy/\u6771\u4eac", "source-page") + return graph, other, link_id + + +def _directory_contents(path: Path): + return { + str(file_path.relative_to(path)): file_path.read_bytes() + for file_path in path.rglob("*") + if file_path.is_file() + } + + +def test_markdown_round_trip_preserves_complete_graph_state(tmp_path): + graph, other, link_id = _sample_graph() + export_path = tmp_path / "context-graph" + + graph.save_to_file(export_path, format="markdown") + + restored = ContextGraph(advanced_analytics=False) + restored.load_from_file(export_path, format="markdown") + + assert _normalized_state(restored) == _normalized_state(graph) + assert restored.resolve_links({other.graph_id: other}) == 1 + linked_graph, entry_node = restored.navigate_to(link_id) + assert linked_graph is other + assert entry_node == "source-page" + + +def test_markdown_manual_node_and_edge_edits_are_imported(tmp_path): + graph, _, _ = _sample_graph() + export_path = tmp_path / "context-graph" + graph.save_to_file(export_path, format="markdown") + + node_path = _node_file(export_path, "policy/\u6771\u4eac") + node_frontmatter, _ = _read_markdown(node_path) + node_frontmatter["metadata"]["reviewed"] = True + _write_markdown(node_path, node_frontmatter, "# Updated policy\n") + + manifest_path = export_path / "graph.md" + manifest, manifest_body = _read_markdown(manifest_path) + manifest["edges"][0]["type"] = "VERIFIES" + manifest["edges"][0]["weight"] = 1.0 + manifest["edges"][0]["metadata"]["reviewed_by"] = "human" + _write_markdown(manifest_path, manifest, manifest_body) + + restored = ContextGraph(advanced_analytics=False) + restored.load_from_file(export_path, format="markdown") + + node = restored.nodes["policy/\u6771\u4eac"] + assert node.content == "# Updated policy\n" + assert node.metadata["reviewed"] is True + edge = restored.edges[0] + assert edge.edge_type == "VERIFIES" + assert edge.weight == 1.0 + assert edge.metadata["reviewed_by"] == "human" + + +def test_markdown_export_is_deterministic_and_removes_stale_nodes(tmp_path): + graph = ContextGraph(advanced_analytics=False) + graph.graph_id = "deterministic-graph" + graph.add_node("kept", "Note", "Keep") + graph.add_node("removed", "Note", "Remove") + graph.add_edge("kept", "removed", "REFERENCES") + first = tmp_path / "first" + second = tmp_path / "second" + + graph.save_to_file(first, format="markdown") + graph.save_to_file(second, format="markdown") + assert _directory_contents(first) == _directory_contents(second) + + stale_path = _node_file(first, "removed") + graph.nodes.pop("removed") + graph.edges.clear() + graph.save_to_file(first, format="markdown") + + assert not stale_path.exists() + restored = ContextGraph(advanced_analytics=False) + restored.load_from_file(first, format="markdown") + assert set(restored.nodes) == {"kept"} + assert restored.edges == [] + + +def test_markdown_empty_graph_round_trip(tmp_path): + graph = ContextGraph(advanced_analytics=False) + graph.graph_id = "empty-graph" + export_path = tmp_path / "empty" + + graph.save_to_file(export_path, format="markdown") + restored = ContextGraph(advanced_analytics=False) + restored.load_from_file(export_path, format="markdown") + + assert restored.graph_id == "empty-graph" + assert restored.nodes == {} + assert restored.edges == [] + + +def test_markdown_export_rejects_duplicate_edge_ids_before_writing(tmp_path): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("source", "Note", "Source") + graph.add_node("target", "Note", "Target") + graph.add_edge("source", "target", "REFERENCES") + graph.edges.append(graph.edges[0]) + export_path = tmp_path / "graph" + + assert graph.edges[0].edge_id == graph.edges[1].edge_id + with pytest.raises(ValueError, match=r"duplicate edge ID.*[0-9a-f-]+"): + graph.save_to_file(export_path, format="markdown") + + assert not export_path.exists() + + +@pytest.mark.parametrize( + "corruption, expected_error", + [ + ("unsupported-version", "Unsupported ContextGraph Markdown version"), + ("duplicate-edge", "Duplicate Markdown edge ID"), + ("duplicate-node", "Duplicate Markdown node ID"), + ("cyclic-skos", "SKOS hierarchy contains a cycle"), + ], +) +def test_invalid_markdown_does_not_mutate_existing_graph( + tmp_path, corruption, expected_error +): + source, _, _ = _sample_graph() + export_path = tmp_path / corruption + source.save_to_file(export_path, format="markdown") + + manifest_path = export_path / "graph.md" + manifest, manifest_body = _read_markdown(manifest_path) + if corruption == "unsupported-version": + manifest["version"] = 2 + elif corruption == "duplicate-edge": + manifest["edges"].append(dict(manifest["edges"][0])) + elif corruption == "duplicate-node": + original = _node_file(export_path, "evidence-1") + (original.parent / "duplicate.md").write_bytes(original.read_bytes()) + elif corruption == "cyclic-skos": + manifest["edges"] = [ + { + "id": "broader-1", + "family_id": "broader-1", + "source": "evidence-1", + "target": "policy/\u6771\u4eac", + "type": "skos:broader", + "weight": 1.0, + "metadata": {}, + }, + { + "id": "broader-2", + "family_id": "broader-2", + "source": "policy/\u6771\u4eac", + "target": "evidence-1", + "type": "skos:broader", + "weight": 1.0, + "metadata": {}, + }, + ] + _write_markdown(manifest_path, manifest, manifest_body) + + target = ContextGraph(advanced_analytics=False) + target.add_node("sentinel", "Existing", "Do not replace") + before = _normalized_state(target) + + with pytest.raises(ValueError, match=expected_error): + target.load_from_file(export_path, format="markdown") + + assert _normalized_state(target) == before + + +def test_markdown_import_creates_json_compatible_stub_nodes_for_dangling_edges( + tmp_path, +): + source, _, _ = _sample_graph() + export_path = tmp_path / "dangling-edge" + source.save_to_file(export_path, format="markdown") + manifest_path = export_path / "graph.md" + manifest, manifest_body = _read_markdown(manifest_path) + manifest["edges"][0]["target"] = "missing-node" + _write_markdown(manifest_path, manifest, manifest_body) + + restored = ContextGraph(advanced_analytics=False) + restored.load_from_file(export_path, format="markdown") + + stub = restored.nodes["missing-node"] + assert stub.node_type == "entity" + assert stub.content == "missing-node" + assert any(edge.target_id == "missing-node" for edge in restored.edges) + + +@pytest.mark.parametrize( + ("location", "field_name"), + [("node", "valid_from"), ("edge", "valid_until")], +) +def test_invalid_markdown_temporal_value_does_not_mutate_existing_graph( + tmp_path, location, field_name +): + source, _, _ = _sample_graph() + export_path = tmp_path / location + source.save_to_file(export_path, format="markdown") + + if location == "node": + document_path = _node_file(export_path, "policy/\u6771\u4eac") + else: + document_path = export_path / "graph.md" + frontmatter, body = _read_markdown(document_path) + if location == "node": + frontmatter[field_name] = "not-a-date" + else: + frontmatter["edges"][0][field_name] = "not-a-date" + _write_markdown(document_path, frontmatter, body) + + target = ContextGraph(advanced_analytics=False) + target.add_node("sentinel", "Existing", "Do not replace") + before = _normalized_state(target) + + with pytest.raises(ValueError, match=rf"'{field_name}'.*valid ISO-8601"): + target.load_from_file(export_path, format="markdown") + + assert _normalized_state(target) == before + + +def test_duplicate_yaml_keys_are_rejected(tmp_path): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("node-1", "Note", "Body") + export_path = tmp_path / "graph" + graph.save_to_file(export_path, format="markdown") + node_path = _node_file(export_path, "node-1") + document = node_path.read_text(encoding="utf-8") + node_path.write_text(document.replace("id: node-1", "id: node-1\nid: duplicate")) + + with pytest.raises(ValueError, match="duplicate key 'id'"): + ContextGraph(advanced_analytics=False).load_from_file( + export_path, format="markdown" + ) + + +def test_markdown_export_refuses_unmanaged_nonempty_directory(tmp_path): + destination = tmp_path / "existing" + destination.mkdir() + marker = destination / "keep.txt" + marker.write_text("keep", encoding="utf-8") + + with pytest.raises(ValueError, match="not a managed ContextGraph export"): + ContextGraph(advanced_analytics=False).save_to_file( + destination, format="markdown" + ) + + assert marker.read_text(encoding="utf-8") == "keep" + + +def test_markdown_export_rejects_unrelated_graph_markdown_file(tmp_path): + destination = tmp_path / "existing" + destination.mkdir() + unrelated = destination / "graph.md" + _write_markdown(unrelated, {"title": "Unrelated notes"}, "Keep me") + + with pytest.raises(ValueError, match="not a managed ContextGraph export"): + ContextGraph(advanced_analytics=False).save_to_file( + destination, format="markdown" + ) + + assert unrelated.exists() + + +@pytest.mark.parametrize("extra_location", ["root", "nodes"]) +def test_markdown_export_refuses_managed_directory_with_untracked_files( + tmp_path, extra_location +): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("node-1", "Note", "Body") + destination = tmp_path / "existing" + graph.save_to_file(destination, format="markdown") + parent = destination if extra_location == "root" else destination / "nodes" + human_file = parent / "human-notes.txt" + human_file.write_text("do not delete", encoding="utf-8") + original_contents = _directory_contents(destination) + + with pytest.raises(ValueError, match="not a managed ContextGraph export"): + graph.save_to_file(destination, format="markdown") + + assert _directory_contents(destination) == original_contents + + +def test_markdown_export_refuses_noncanonical_node_layout(tmp_path): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("node-1", "Note", "Body") + destination = tmp_path / "existing" + graph.save_to_file(destination, format="markdown") + canonical = _node_file(destination, "node-1") + renamed = canonical.with_name("human-name.md") + canonical.rename(renamed) + original_contents = _directory_contents(destination) + + with pytest.raises(ValueError, match="not a managed ContextGraph export"): + graph.save_to_file(destination, format="markdown") + + assert _directory_contents(destination) == original_contents + + +def test_markdown_export_preserves_manifest_inspection_errors(tmp_path, monkeypatch): + graph = ContextGraph(advanced_analytics=False) + destination = tmp_path / "existing" + graph.save_to_file(destination, format="markdown") + real_reader = ContextGraph._read_markdown_file + + def fail_manifest_read(path): + if path == destination / "graph.md": + raise PermissionError("permission denied") + return real_reader(path) + + monkeypatch.setattr( + ContextGraph, "_read_markdown_file", staticmethod(fail_manifest_read) + ) + + with pytest.raises(PermissionError, match="permission denied"): + graph.save_to_file(destination, format="markdown") + + +def test_markdown_export_restores_previous_directory_when_publish_fails( + tmp_path, monkeypatch +): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("original", "Note", "Original") + destination = tmp_path / "graph" + graph.save_to_file(destination, format="markdown") + original_contents = _directory_contents(destination) + graph.add_node("new", "Note", "New") + + real_replace = context_graph_module.os.replace + + def fail_staged_publish(source, target): + if ".staging-" in Path(source).name and Path(target) == destination: + raise OSError("simulated publish failure") + return real_replace(source, target) + + monkeypatch.setattr(context_graph_module.os, "replace", fail_staged_publish) + + with pytest.raises(OSError, match="simulated publish failure"): + graph.save_to_file(destination, format="markdown") + + assert _directory_contents(destination) == original_contents + assert not list(tmp_path.glob(".graph.staging-*")) + assert not list(tmp_path.glob(".graph.backup-*")) + + +def test_markdown_export_preserves_publish_error_when_restore_fails( + tmp_path, monkeypatch, caplog +): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("original", "Note", "Original") + destination = tmp_path / "graph" + graph.save_to_file(destination, format="markdown") + original_contents = _directory_contents(destination) + graph.add_node("new", "Note", "New") + + real_replace = context_graph_module.os.replace + + def fail_publish_and_restore(source, target): + source_path = Path(source) + if ".staging-" in source_path.name and Path(target) == destination: + raise OSError("simulated publish failure") + if ".backup-" in source_path.name and Path(target) == destination: + raise PermissionError("simulated restore failure") + return real_replace(source, target) + + monkeypatch.setattr(context_graph_module.os, "replace", fail_publish_and_restore) + caplog.set_level("ERROR") + + with pytest.raises(OSError, match="simulated publish failure"): + graph.save_to_file(destination, format="markdown") + + assert "preserving the original publish error" in caplog.text + assert "simulated restore failure" in caplog.text + assert not destination.exists() + backup_paths = list(tmp_path.glob(".graph.backup-*")) + assert len(backup_paths) == 1 + assert _directory_contents(backup_paths[0]) == original_contents + assert not list(tmp_path.glob(".graph.staging-*")) + + +def test_markdown_load_rebuilds_indexes_and_emits_json_compatible_events(tmp_path): + source, _, _ = _sample_graph() + destination = tmp_path / "graph" + source.save_to_file(destination, format="markdown") + events = [] + target = ContextGraph( + advanced_analytics=False, + mutation_callback=lambda *event: events.append(event), + ) + target._retractions[("node", "stale")] = {"entity_id": "stale"} + target._tombstones[("edge", "stale")] = {"entity_id": "stale"} + + target.load_from_file(destination, format="markdown") + + assert target.node_type_index["Policy"] == {"policy/\u6771\u4eac"} + assert target.edge_type_index["SUPPORTS"][0].edge_id == "edge-supports" + assert target._edge_index["edge-supports"].edge_type == "SUPPORTS" + assert target._adjacency["evidence-1"][0].target_id == "policy/\u6771\u4eac" + assert target._retractions == {} + assert target._tombstones == {} + assert [event[0] for event in events] == ["ADD_NODE"] * len(target.nodes) + [ + "ADD_EDGE" + ] * len(target.edges) + assert {event[1] for event in events if event[0] == "ADD_NODE"} == set(target.nodes) + assert {event[1] for event in events if event[0] == "ADD_EDGE"} == { + edge.edge_id for edge in target.edges + } + + +def test_markdown_load_records_granular_change_manager_history(tmp_path): + source, _, _ = _sample_graph() + destination = tmp_path / "graph" + source.save_to_file(destination, format="markdown") + target = ContextGraph(advanced_analytics=False) + manager = TemporalVersionManager() + manager.attach_to_graph(target) + + target.load_from_file(destination, format="markdown") + + assert manager.get_node_history("evidence-1")[0]["operation"] == "ADD_NODE" + assert manager.get_node_history("edge-supports")[0]["operation"] == "ADD_EDGE" + + +def test_markdown_export_rejects_recursive_metadata(tmp_path): + recursive = {} + recursive["self"] = recursive + graph = ContextGraph(advanced_analytics=False) + graph._add_internal_node(ContextNode("node-1", "Note", "Body", metadata=recursive)) + + with pytest.raises(ValueError, match="values cannot contain cycles"): + graph.save_to_file(tmp_path / "graph", format="markdown") + + +def test_markdown_import_and_export_reject_symlinks(tmp_path): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("node-1", "Note", "Body") + real_export = tmp_path / "real" + graph.save_to_file(real_export, format="markdown") + + directory_link = tmp_path / "directory-link" + try: + directory_link.symlink_to(real_export, target_is_directory=True) + except (NotImplementedError, OSError): + pytest.skip("Symbolic links are not available on this platform") + + with pytest.raises(ValueError, match="symbolic link"): + ContextGraph(advanced_analytics=False).load_from_file( + directory_link, format="markdown" + ) + with pytest.raises(ValueError, match="symbolic link"): + graph.save_to_file(directory_link, format="markdown") + + manifest_path = real_export / "graph.md" + manifest_target = tmp_path / "manifest.md" + manifest_target.write_bytes(manifest_path.read_bytes()) + manifest_path.unlink() + manifest_path.symlink_to(manifest_target) + with pytest.raises(ValueError, match="symbolic link"): + ContextGraph(advanced_analytics=False).load_from_file( + real_export, format="markdown" + ) + + +def test_markdown_import_and_export_reject_windows_junctions(tmp_path, monkeypatch): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("node-1", "Note", "Body") + export_path = tmp_path / "junction" + graph.save_to_file(export_path, format="markdown") + + monkeypatch.setattr( + os.path, + "isjunction", + lambda candidate: Path(candidate) == export_path, + raising=False, + ) + + with pytest.raises(ValueError, match="junction"): + ContextGraph(advanced_analytics=False).load_from_file( + export_path, format="markdown" + ) + with pytest.raises(ValueError, match="junction"): + graph.save_to_file(export_path, format="markdown") + + +def test_markdown_import_rejects_windows_reparse_point_fallback(tmp_path, monkeypatch): + source = tmp_path / "reparse-point" + source.mkdir() + real_lstat = os.lstat + + class ReparseStat: + st_file_attributes = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + + monkeypatch.delattr(os.path, "isjunction", raising=False) + monkeypatch.setattr(Path, "is_symlink", lambda self: False) + monkeypatch.setattr( + os, + "lstat", + lambda candidate: ( + ReparseStat() if Path(candidate) == source else real_lstat(candidate) + ), + ) + + with pytest.raises(ValueError, match="junction"): + ContextGraph(advanced_analytics=False).load_from_file(source, format="markdown") + + +@pytest.mark.skipif(os.name != "nt", reason="requires Windows junctions") +def test_markdown_import_rejects_real_windows_junction(tmp_path): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("node-1", "Note", "Body") + outside = tmp_path / "outside" + graph.save_to_file(outside, format="markdown") + source = tmp_path / "junction" + result = subprocess.run( + ["cmd.exe", "/c", "mklink", "/J", str(source), str(outside)], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + pytest.skip(f"could not create Windows junction: {result.stderr}") + + try: + with pytest.raises(ValueError, match="junction"): + ContextGraph(advanced_analytics=False).load_from_file( + source, format="markdown" + ) + finally: + os.rmdir(source) + + +@pytest.mark.skipif( + not hasattr(context_graph_module.os, "O_NOFOLLOW"), + reason="O_NOFOLLOW is unavailable on this platform", +) +def test_markdown_import_nofollow_check_closes_symlink_race(tmp_path, monkeypatch): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("node-1", "Note", "Body") + export_path = tmp_path / "graph" + graph.save_to_file(export_path, format="markdown") + node_path = _node_file(export_path, "node-1") + target = tmp_path / "target.md" + target.write_bytes(node_path.read_bytes()) + node_path.unlink() + node_path.symlink_to(target) + + real_is_symlink = Path.is_symlink + + def miss_precheck(path): + if path == node_path: + return False + return real_is_symlink(path) + + monkeypatch.setattr(Path, "is_symlink", miss_precheck) + + with pytest.raises(ValueError, match="symbolic link"): + ContextGraph(advanced_analytics=False).load_from_file( + export_path, format="markdown" + ) + + +def test_json_remains_default_and_unknown_format_is_rejected(tmp_path): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("node-1", "Note", "Body") + json_path = tmp_path / "graph.json" + graph.save_to_file(json_path) + + restored = ContextGraph(advanced_analytics=False) + restored._analytics_cache["stale"] = {"value": True} + restored.load_from_file(json_path) + assert "node-1" in restored.nodes + assert restored._analytics_cache == {} + + before = _normalized_state(restored) + restored.load_from_file(tmp_path / "missing", format="markdown") + assert _normalized_state(restored) == before + + with pytest.raises(ValueError, match="Unsupported context graph"): + graph.save_to_file(tmp_path / "graph", format="html") diff --git a/tests/context/test_decision_causal_edge_regression.py b/tests/context/test_decision_causal_edge_regression.py index 91bade6d..eae7eae3 100644 --- a/tests/context/test_decision_causal_edge_regression.py +++ b/tests/context/test_decision_causal_edge_regression.py @@ -7,6 +7,8 @@ extraction found nothing, the chain came back empty even though an explicit ``CAUSED`` edge was stored in the graph. """ +import pytest + from semantica.context import ContextGraph from semantica.context.context_graph import ContextEdge @@ -323,3 +325,130 @@ def test_entity_based_inference_still_applies_without_explicit_edges(): hop["from"] == earlier and hop["to"] == later and hop["type"] == "influences" for hop in hops ) + + +def test_get_causal_chain_accepts_lowercase_causal_edge_types(): + """Issue #1184: edges recorded with the analyzer's lowercase vocabulary + must be traversed by get_causal_chain(). + + CausalChainAnalyzer documents causal types as lowercase ("causes", + "influences", ...) while get_causal_chain() matched only the uppercase + spellings, so an edge recorded as "causes" produced an empty audit + chain โ€” silent and in the dangerous direction. + """ + graph = ContextGraph(advanced_analytics=True) + cause = graph.record_decision( + category="a", scenario="upstream", reasoning="r", + outcome="x", confidence=0.9, + ) + effect = graph.record_decision( + category="b", scenario="downstream", reasoning="r", + outcome="y", confidence=0.9, + ) + graph.add_edge(cause, effect, "causes") + + chain = graph.get_causal_chain(effect, direction="upstream") + + assert [decision.decision_id for decision in chain] == [cause] + + +def test_add_causal_relationship_accepts_any_case_and_stores_canonical(): + """Issue #1184: add_causal_relationship() should accept either spelling + and store the canonical uppercase vocabulary.""" + graph = ContextGraph(advanced_analytics=True) + cause = graph.record_decision( + category="a", scenario="upstream", reasoning="r", + outcome="x", confidence=0.9, + ) + effect = graph.record_decision( + category="b", scenario="downstream", reasoning="r", + outcome="y", confidence=0.9, + ) + + graph.add_causal_relationship(cause, effect, relationship_type="causes") + + edges = [ + edge for edge in graph.edges + if edge.source_id == cause and edge.target_id == effect + ] + assert edges, "add_causal_relationship must store the edge" + assert edges[0].edge_type == "CAUSED" + + +def test_add_causal_relationship_rejects_non_string_with_value_error(): + """Invalid relationship types must keep raising ValueError (issue #1184 + follow-up): normalization must not turn them into AttributeError.""" + graph = ContextGraph(advanced_analytics=True) + cause = graph.record_decision( + category="a", scenario="upstream", reasoning="r", + outcome="x", confidence=0.9, + ) + effect = graph.record_decision( + category="b", scenario="downstream", reasoning="r", + outcome="y", confidence=0.9, + ) + + for bad_type in (None, 42, ["CAUSED"]): + with pytest.raises(ValueError): + graph.add_causal_relationship(cause, effect, relationship_type=bad_type) + + +def test_analyze_decision_influence_sees_lowercase_causal_edge(): + """Issue #1184 follow-up: influence analysis reads the same edge index as + the causal traversal, so lowercase edges must count as direct influence.""" + graph = ContextGraph(advanced_analytics=True) + cause = graph.record_decision( + category="a", scenario="upstream", reasoning="r", + outcome="x", confidence=0.9, + ) + effect = graph.record_decision( + category="b", scenario="downstream", reasoning="r", + outcome="y", confidence=0.9, + ) + graph.add_edge(cause, effect, "causes") + + impact = graph.analyze_decision_influence(cause) + + direct_ids = {entry["decision_id"] for entry in impact["direct_influence"]} + assert effect in direct_ids + + +def test_trace_decision_causality_sees_lowercase_causal_edge(): + """Issue #1184 follow-up: the trace must not return an empty audit chain + for a decision with an explicit lowercase upstream causal edge.""" + graph = ContextGraph(advanced_analytics=True) + cause = graph.record_decision( + category="a", scenario="upstream", reasoning="r", + outcome="x", confidence=0.9, + ) + effect = graph.record_decision( + category="b", scenario="downstream", reasoning="r", + outcome="y", confidence=0.9, + ) + graph.add_edge(cause, effect, "causes") + + trace = graph.trace_decision_causality(effect) + + assert any( + hop["from"] == cause and hop["to"] == effect + for chain in trace for hop in chain["hops"] + ), "lowercase causal edge must appear in the traced chain" + + +def test_find_precedents_sees_lowercase_precedent_edge(): + """Issue #1184 follow-up: precedent lookup must accept the analyzer's + spelling alongside the canonical PRECEDENT_FOR.""" + graph = ContextGraph(advanced_analytics=True) + precedent = graph.record_decision( + category="a", scenario="earlier", reasoning="r", + outcome="x", confidence=0.9, + ) + later = graph.record_decision( + category="b", scenario="later", reasoning="r", + outcome="y", confidence=0.9, + ) + graph.add_edge(precedent, later, "precedes") + + precedents = graph.find_precedents(later) + + assert [d.decision_id for d in precedents] == [precedent] diff --git a/tests/context/test_decision_persistence_pr967.py b/tests/context/test_decision_persistence_pr967.py new file mode 100644 index 00000000..dd96fed4 --- /dev/null +++ b/tests/context/test_decision_persistence_pr967.py @@ -0,0 +1,1041 @@ +""" +Regression tests for PR #967 โ€” Decision persistence / index consistency, +CJK similarity, query_graph limit, and MCP tool correctness. + +These tests cover every bug confirmed by the pre-PR investigation and every +new correctness issue introduced or left behind by the PR: + + 1. save โ†’ load โ†’ find_similar_decisions (core persistence invariant) + 2. save โ†’ load โ†’ metadata preservation + 3. save โ†’ load โ†’ all decision analytics callable + 4. Repeated load clears stale indexes (no ghost decisions) + 5. In-memory decisions cleared when loading into a graph that already has + decisions recorded in-memory + 6. Category filtering via find_nodes / query_decisions + 7. Decision index stays consistent after add_node_attribute mutation + 8. CJK similarity โ€” short CJK query matches relevant text + 9. Bigram spike regression โ€” 2-char English query must NOT produce 1.0 + 10. English similarity still works normally + 11. Empty / 1-char input safety + 12. query_graph limit=0 returns empty (not unlimited) + 13. query_graph limit=None returns all + 14. query_graph limit=1 caps combined results + 15. query_graph inbound-only topology + 16. query_graph outbound-only topology + 17. query_graph mixed inbound + outbound + 18. from_dict also rebuilds decision indexes + 19. MCP _get_graph loads from SEMANTICA_KG_PATH + 20. update_node smoke test + decision index sync + 21. delete_node soft-archive smoke test + 22. update_node / delete_node persistence after reload + 23. entity extraction returns surface text +""" + +import json +import os +import sys +import tempfile +import unittest +from collections import defaultdict +from unittest.mock import patch + +# Make sure the repo root is importable even when running from the tests dir. +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) + +from semantica.context.context_graph import ContextGraph + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _decision_graph() -> ContextGraph: + """Return a ContextGraph with three decisions pre-recorded.""" + g = ContextGraph(advanced_analytics=False) + g.record_decision( + category="loan_approval", + scenario="High-income applicant with perfect credit history", + reasoning="Credit score 800+, stable employment for 10 years", + outcome="approved", + confidence=0.95, + entities=["applicant_123", "bank_abc"], + decision_maker="underwriter", + metadata={"risk_tier": "low", "custom_flag": True}, + ) + g.record_decision( + category="loan_approval", + scenario="Self-employed applicant with variable income", + reasoning="Good credit but income variability poses moderate risk", + outcome="conditional_approval", + confidence=0.72, + entities=["applicant_456"], + decision_maker="underwriter", + ) + g.record_decision( + category="fraud_detection", + scenario="Unusual transaction pattern detected in account", + reasoning="Multiple small transactions in rapid succession across geographies", + outcome="flagged", + confidence=0.88, + entities=["account_789", "transaction_seq"], + decision_maker="fraud_engine", + ) + return g + + +# --------------------------------------------------------------------------- +# Part 1: Core persistence invariant โ€” save โ†’ load โ†’ find_similar_decisions +# --------------------------------------------------------------------------- + +class TestDecisionPersistenceRoundTrip(unittest.TestCase): + """save โ†’ load must produce decision-query-equivalent behaviour.""" + + def test_find_similar_decisions_after_reload(self): + """Core invariant: similarity search works after save/load.""" + g = _decision_graph() + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + + # Query that matches the loan_approval decisions + results = g2.find_similar_decisions( + "credit history approval", max_results=5, min_similarity=0.01 + ) + self.assertGreater(len(results), 0, "Expected at least one match after reload") + # Each result must be a dict with a decision key + self.assertIn("decision", results[0]) + finally: + os.unlink(path) + + def test_find_precedents_by_scenario_after_reload(self): + """find_precedents_by_scenario must not return [] after reload.""" + g = _decision_graph() + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + + results = g2.find_precedents_by_scenario( + "applicant credit history loan", + similarity_threshold=0.01, + ) + self.assertGreater(len(results), 0) + finally: + os.unlink(path) + + def test_decision_count_after_reload(self): + """_decisions must be populated for statistics calls after reload.""" + g = _decision_graph() + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + + stats = g2.get_decision_insights() + # Should not be the "No decisions" sentinel + self.assertNotEqual(stats, {"message": "No decisions recorded yet"}) + self.assertEqual(stats.get("total_decisions", 0), 3) + finally: + os.unlink(path) + + def test_decisions_dict_populated_after_reload(self): + """_decisions must exist and contain 3 entries after reload.""" + g = _decision_graph() + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + + self.assertTrue(hasattr(g2, "_decisions")) + self.assertEqual(len(g2._decisions), 3) + finally: + os.unlink(path) + + +# --------------------------------------------------------------------------- +# Part 2: Metadata preservation across save/load +# --------------------------------------------------------------------------- + +class TestDecisionMetadataPreservation(unittest.TestCase): + + def test_custom_metadata_survives_reload(self): + """User-supplied metadata must survive a save โ†’ load round-trip.""" + g = ContextGraph(advanced_analytics=False) + did = g.record_decision( + category="test", + scenario="Testing metadata preservation", + reasoning="Verifying that custom fields survive reload", + outcome="pass", + confidence=0.9, + metadata={"foo": "bar", "priority": 42}, + ) + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + + reloaded_decision = g2._decisions.get(did) + self.assertIsNotNone(reloaded_decision, "_decisions must contain the decision after reload") + # Metadata should contain the custom fields + meta = reloaded_decision.get("metadata", {}) + self.assertEqual(meta.get("foo"), "bar") + self.assertEqual(meta.get("priority"), 42) + finally: + os.unlink(path) + + def test_core_fields_preserved_after_reload(self): + """All core decision fields must survive a round-trip unchanged.""" + g = ContextGraph(advanced_analytics=False) + did = g.record_decision( + category="compliance", + scenario="Regulatory check for derivative trade", + reasoning="Trade complies with Dodd-Frank Section 732", + outcome="compliant", + confidence=0.85, + entities=["trader_X", "instrument_Y"], + decision_maker="compliance_engine", + ) + recorded_at_before = g._decisions[did]["recorded_at"] + self.assertTrue(recorded_at_before, "recorded_at must be set at record time") + + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + + dec = g2._decisions.get(did) + self.assertIsNotNone(dec) + self.assertEqual(dec["category"], "compliance") + self.assertIn("Regulatory check", dec["scenario"]) + self.assertEqual(dec["outcome"], "compliant") + self.assertAlmostEqual(dec["confidence"], 0.85, places=3) + self.assertIn("trader_X", dec["entities"]) + self.assertEqual(dec["decision_maker"], "compliance_engine") + self.assertEqual(dec["recorded_at"], recorded_at_before, + "recorded_at must survive a save -> load round trip") + finally: + os.unlink(path) + + +# --------------------------------------------------------------------------- +# Part 3: Repeated load clears stale indexes +# --------------------------------------------------------------------------- + +class TestRepeatedLoadClearsStaleIndexes(unittest.TestCase): + + def test_second_load_replaces_first(self): + """Loading file B into a graph that already loaded file A must leave + only B's decisions visible โ€” no ghost decisions from A.""" + g_a = ContextGraph(advanced_analytics=False) + g_a.record_decision( + category="cat_A", + scenario="Decision from file A", + reasoning="Reason A", + outcome="outcome_A", + confidence=0.9, + ) + + g_b = ContextGraph(advanced_analytics=False) + g_b.record_decision( + category="cat_B", + scenario="Decision from file B", + reasoning="Reason B", + outcome="outcome_B", + confidence=0.8, + ) + + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as fa, \ + tempfile.NamedTemporaryFile(suffix=".json", delete=False) as fb: + path_a, path_b = fa.name, fb.name + try: + g_a.save_to_file(path_a) + g_b.save_to_file(path_b) + + target = ContextGraph(advanced_analytics=False) + + # Load A + target.load_from_file(path_a) + self.assertEqual(len(target._decisions), 1) + cats_after_a = {d["category"] for d in target._decisions.values()} + self.assertIn("cat_A", cats_after_a) + + # Load B into same instance โ€” must replace A entirely + target.load_from_file(path_b) + self.assertEqual(len(target._decisions), 1, + "Stale cat_A decision must not persist after loading B") + cats_after_b = {d["category"] for d in target._decisions.values()} + self.assertIn("cat_B", cats_after_b) + self.assertNotIn("cat_A", cats_after_b) + finally: + os.unlink(path_a) + os.unlink(path_b) + + def test_load_into_graph_with_in_memory_decisions(self): + """Loading a file into a graph that already has in-memory decisions + must produce indexes that reflect ONLY the file's decisions.""" + g = ContextGraph(advanced_analytics=False) + # Record an in-memory decision first + g.record_decision( + category="in_memory", + scenario="Decision recorded before load", + reasoning="Testing stale index reset", + outcome="ok", + confidence=0.5, + ) + + # Now create a graph file with different content + g_file = ContextGraph(advanced_analytics=False) + g_file.record_decision( + category="from_file", + scenario="Decision loaded from file", + reasoning="This is what should survive", + outcome="loaded", + confidence=0.7, + ) + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g_file.save_to_file(path) + g.load_from_file(path) + + cats = {d["category"] for d in g._decisions.values()} + self.assertIn("from_file", cats) + self.assertNotIn("in_memory", cats, + "In-memory decision must be evicted after load_from_file") + finally: + os.unlink(path) + + +# --------------------------------------------------------------------------- +# Part 4: Category filtering +# --------------------------------------------------------------------------- + +class TestCategoryFiltering(unittest.TestCase): + + def test_decision_index_correct_after_reload(self): + """_decision_index must map categories correctly after reload.""" + g = _decision_graph() + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + + loan_ids = g2._decision_index.get("loan_approval", set()) + fraud_ids = g2._decision_index.get("fraud_detection", set()) + + self.assertEqual(len(loan_ids), 2, "Expected 2 loan_approval decisions") + self.assertEqual(len(fraud_ids), 1, "Expected 1 fraud_detection decision") + # No overlap + self.assertTrue(loan_ids.isdisjoint(fraud_ids)) + finally: + os.unlink(path) + + def test_find_nodes_category_in_metadata(self): + """find_nodes returns category inside 'metadata', not at top level.""" + g = ContextGraph(advanced_analytics=False) + g.record_decision( + category="risk_check", + scenario="Scenario", + reasoning="Reasoning", + outcome="pass", + confidence=0.9, + ) + nodes = g.find_nodes(node_type="decision") + self.assertGreater(len(nodes), 0) + # Category must be accessible via metadata key + found = any( + n.get("metadata", {}).get("category") == "risk_check" + for n in nodes + ) + self.assertTrue(found, "category must be in n['metadata']['category']") + # Must NOT be at top level (that is the bug that was fixed) + top_level = any(n.get("category") == "risk_check" for n in nodes) + self.assertFalse(top_level, "category must NOT appear at the top level of find_nodes result") + + +# --------------------------------------------------------------------------- +# Part 5: Decision index consistency after mutation +# --------------------------------------------------------------------------- + +class TestDecisionIndexMutationSync(unittest.TestCase): + + def test_add_node_attribute_syncs_decision_index(self): + """After add_node_attribute on a decision node, _decisions must reflect + the new values without requiring a reload.""" + g = ContextGraph(advanced_analytics=False) + did = g.record_decision( + category="original_cat", + scenario="Original scenario", + reasoning="Original reasoning", + outcome="original", + confidence=0.6, + ) + # Verify original state + self.assertIn(did, g._decision_index.get("original_cat", set())) + + # Mutate via add_node_attribute + g.add_node_attribute(did, {"confidence": 0.95, "custom_note": "reviewed"}) + + # _decisions must reflect updated confidence + updated = g._decisions.get(did) + self.assertIsNotNone(updated) + self.assertAlmostEqual(updated["confidence"], 0.95, places=3) + # custom_note should appear in metadata + self.assertEqual(updated["metadata"].get("custom_note"), "reviewed") + + def test_update_node_does_not_leave_stale_index(self): + """update_node (via add_node_attribute) must not break category lookup.""" + g = ContextGraph(advanced_analytics=False) + did = g.record_decision( + category="cat_original", + scenario="Some scenario", + reasoning="Some reasoning", + outcome="ok", + confidence=0.7, + ) + # The decision should be findable by category + results_before = g.find_precedents_by_scenario( + "Some scenario", similarity_threshold=0.01 + ) + self.assertGreater(len(results_before), 0) + + # Mutate some non-index fields + g.add_node_attribute(did, {"status": "reviewed", "reviewer": "alice"}) + + # Decision should still be findable after mutation + results_after = g.find_precedents_by_scenario( + "Some scenario", similarity_threshold=0.01 + ) + self.assertGreater(len(results_after), 0) + + +# --------------------------------------------------------------------------- +# Part 6: from_dict also rebuilds decision indexes +# --------------------------------------------------------------------------- + +class TestFromDictDecisionIndexes(unittest.TestCase): + + def test_from_dict_populates_decision_indexes(self): + """from_dict must rebuild _decisions, _decision_index, etc.""" + g = _decision_graph() + d = g.to_dict() + + g2 = ContextGraph(advanced_analytics=False) + g2.from_dict(d) + + self.assertTrue(hasattr(g2, "_decisions")) + self.assertEqual(len(g2._decisions), 3) + self.assertGreater(len(g2._decision_index), 0) + + def test_from_dict_repeated_call_clears_stale(self): + """Calling from_dict twice must not accumulate ghost entries.""" + g1 = ContextGraph(advanced_analytics=False) + g1.record_decision( + category="x", scenario="s", reasoning="r", outcome="o", confidence=0.5 + ) + g2 = ContextGraph(advanced_analytics=False) + g2.record_decision( + category="y", scenario="s2", reasoning="r2", outcome="o2", confidence=0.6 + ) + + target = ContextGraph(advanced_analytics=False) + target.from_dict(g1.to_dict()) + self.assertEqual(len(target._decisions), 1) + cats = {d["category"] for d in target._decisions.values()} + self.assertIn("x", cats) + + target.from_dict(g2.to_dict()) + self.assertEqual(len(target._decisions), 1) + cats2 = {d["category"] for d in target._decisions.values()} + self.assertIn("y", cats2) + self.assertNotIn("x", cats2) + + +# --------------------------------------------------------------------------- +# Part 7: CJK similarity +# --------------------------------------------------------------------------- + +class TestCJKSimilarity(unittest.TestCase): + + def _sim(self, scenario, decision_scenario, decision_reasoning="", entities=None): + """Helper: compute _calculate_decision_content_similarity directly.""" + g = ContextGraph(advanced_analytics=False) + decision = { + "scenario": decision_scenario, + "reasoning": decision_reasoning, + "entities": entities or [], + } + return g._calculate_decision_content_similarity(scenario, decision) + + def test_cjk_two_char_query_matches_relevant_text(self): + """A 2-character CJK query should match text containing those chars.""" + # ไธญๆ–‡ (Chinese text) โ€” 2 chars, produces 1 bigram: not enough for + # bigram signal. But a 3-char query should work. + # Use a 4-char CJK phrase (โ†’ 3 bigrams) to activate the bigram path. + query = "ไธญๆ–‡ๅฎกๆ‰น" # 4 CJK chars โ†’ 3 bigrams + doc_scenario = "ไธญๆ–‡ๅฎกๆ‰นๆต็จ‹ ่ดทๆฌพๅ†ณ็ญ–" + sim = self._sim(query, doc_scenario) + self.assertGreater(sim, 0.0, "CJK query must produce a non-zero similarity") + + def test_cjk_irrelevant_text_low_similarity(self): + """A CJK query must NOT produce high similarity with unrelated text.""" + query = "ไธญๆ–‡ๅฎกๆ‰น" + unrelated = "Python programming language feature request" + sim = self._sim(query, unrelated) + # Some accidental bigram overlap is possible with stripped chars, but + # should be significantly less than 1.0 + self.assertLess(sim, 0.5) + + def test_cjk_identical_text_high_similarity(self): + """Identical CJK text must produce similarity close to 1.0.""" + text = "ไธญๆ–‡ๅฎกๆ‰นๆต็จ‹ๅ†ณ็ญ–" # 8 chars โ†’ 7 bigrams + sim = self._sim(text, text) + self.assertGreater(sim, 0.9) + + +# --------------------------------------------------------------------------- +# Part 8: Bigram spike regression (2-char English query must NOT give 1.0) +# --------------------------------------------------------------------------- + +class TestBigramSpikeRegression(unittest.TestCase): + + def _sim(self, scenario, doc_scenario): + g = ContextGraph(advanced_analytics=False) + return g._calculate_decision_content_similarity( + scenario, {"scenario": doc_scenario, "reasoning": "", "entities": []} + ) + + def test_two_char_english_query_no_spike(self): + """A 2-char English query must NOT receive similarity 1.0 merely + because those chars appear as a substring in the document text.""" + # 'in' is a 2-char query โ†’ 1 bigram โ†’ below the 3-bigram threshold. + # Word-based Jaccard also gives 0.0 ('in' not a word in the doc). + sim = self._sim("in", "interest rate decision analysis") + self.assertLess(sim, 0.5, + "2-char English query 'in' must not spike to 1.0") + + def test_single_char_query_safe(self): + """A single-character query must return 0.0 without crashing.""" + sim = self._sim("a", "apple analysis algorithm") + self.assertEqual(sim, 0.0) + + def test_empty_query_safe(self): + """An empty query must return 0.0 without crashing.""" + sim = self._sim("", "some decision text here") + self.assertEqual(sim, 0.0) + + def test_normal_english_similarity_preserved(self): + """Normal English word overlap must still produce reasonable scores.""" + sim = self._sim( + "credit approval loan applicant", + "loan applicant credit history approval decision", + ) + self.assertGreater(sim, 0.3, "Normal English similarity must remain reasonable") + + def test_common_bigram_substring_below_threshold(self): + """2-char queries 'al', 'ba', 'at' must not produce similarity 1.0.""" + for q in ("al", "ba", "at", "re"): + sim = self._sim(q, "algorithm alignment base rate attention") + self.assertLess(sim, 0.5, + f"2-char query {q!r} must not produce high similarity") + + def test_unrelated_multiword_english_queries_score_zero(self): + """The bigram fallback must not activate for ordinary multi-word + English queries -- it exists only for CJK/single-token queries where + whitespace tokenisation can't help. Unrelated multi-word English + sentences must score 0.0, not a nonzero incidental bigram overlap.""" + sim = self._sim( + "employee vacation request approval process", + "Server infrastructure migration to cloud provider", + ) + self.assertEqual(sim, 0.0, + "Unrelated multi-word English queries must not " + "receive a nonzero score from bigram overlap") + + +# --------------------------------------------------------------------------- +# Part 9: query_graph limit semantics +# --------------------------------------------------------------------------- + +class TestQueryGraphLimitSemantics(unittest.TestCase): + """Tests for _tool_query_graph limit correctness.""" + + def _make_graph_and_patch(self): + """Build a simple graph and patch _get_graph to return it.""" + g = ContextGraph(advanced_analytics=False) + g.add_node("center", "hub", label="Center") + for i in range(5): + g.add_node(f"out_{i}", "spoke", label=f"Spoke {i}") + g.add_edge("center", f"out_{i}", "connects") + for i in range(3): + g.add_node(f"in_{i}", "feeder", label=f"Feeder {i}") + g.add_edge(f"in_{i}", "center", "feeds") + return g + + def test_limit_none_returns_all(self): + """limit=None must return all neighbours (outbound + inbound).""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph_and_patch() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({ + "mode": "neighbors", + "node_id": "center", + "depth": 1, + "limit": None, + }) + neighbors = result.get("neighbors", []) + self.assertGreaterEqual(len(neighbors), 5, "Should include all outbound") + + def test_limit_zero_returns_empty(self): + """limit=0 must return an empty neighbors list, not bypass the cap.""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph_and_patch() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({ + "mode": "neighbors", + "node_id": "center", + "depth": 1, + "limit": 0, + }) + neighbors = result.get("neighbors", []) + self.assertEqual(neighbors, [], + "limit=0 must produce an empty result, not bypass the cap") + + def test_limit_one_caps_result(self): + """limit=1 must return exactly 1 neighbour regardless of total.""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph_and_patch() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({ + "mode": "neighbors", + "node_id": "center", + "depth": 1, + "limit": 1, + }) + neighbors = result.get("neighbors", []) + self.assertEqual(len(neighbors), 1) + + def test_limit_larger_than_available(self): + """limit > total results must return all available without error.""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph_and_patch() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({ + "mode": "neighbors", + "node_id": "center", + "depth": 1, + "limit": 1000, + }) + self.assertNotIn("error", result) + neighbors = result.get("neighbors", []) + # center has 5 outbound + 3 inbound = 8 total + self.assertGreaterEqual(len(neighbors), 5) + + def test_outbound_only_topology(self): + """Nodes with only outbound edges must return outbound neighbours.""" + from semantica.mcp_server import _tool_query_graph + g = ContextGraph(advanced_analytics=False) + g.add_node("source", "hub") + g.add_node("dest1", "leaf") + g.add_node("dest2", "leaf") + g.add_edge("source", "dest1", "points_to") + g.add_edge("source", "dest2", "points_to") + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({ + "mode": "neighbors", + "node_id": "source", + "depth": 1, + }) + neighbors = result.get("neighbors", []) + directions = {n["direction"] for n in neighbors} + self.assertIn("out", directions) + + def test_inbound_only_topology(self): + """Nodes with only inbound edges must return inbound neighbours.""" + from semantica.mcp_server import _tool_query_graph + g = ContextGraph(advanced_analytics=False) + g.add_node("sink", "hub") + g.add_node("src1", "feeder") + g.add_node("src2", "feeder") + g.add_edge("src1", "sink", "feeds") + g.add_edge("src2", "sink", "feeds") + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({ + "mode": "neighbors", + "node_id": "sink", + "depth": 1, + }) + neighbors = result.get("neighbors", []) + directions = {n["direction"] for n in neighbors} + self.assertIn("in", directions) + self.assertNotIn("out", directions) + + def test_mixed_inbound_outbound(self): + """A node with both inbound and outbound edges returns both directions.""" + from semantica.mcp_server import _tool_query_graph + g = ContextGraph(advanced_analytics=False) + g.add_node("mid", "hub") + g.add_node("up", "parent") + g.add_node("down", "child") + g.add_edge("up", "mid", "parent_of") + g.add_edge("mid", "down", "child_of") + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({ + "mode": "neighbors", + "node_id": "mid", + "depth": 1, + }) + neighbors = result.get("neighbors", []) + directions = {n["direction"] for n in neighbors} + self.assertIn("in", directions) + self.assertIn("out", directions) + + +# --------------------------------------------------------------------------- +# Part 10: MCP _get_graph loads from SEMANTICA_KG_PATH +# --------------------------------------------------------------------------- + +class TestMCPGetGraphLoadsFromPath(unittest.TestCase): + + def test_get_graph_loads_kg_path(self): + """When SEMANTICA_KG_PATH is set, _get_graph must load it.""" + import semantica.mcp_server as mcp_mod + + g = ContextGraph(advanced_analytics=False) + g.add_node("kg_node_1", "entity", label="Loaded from file") + g.record_decision( + category="test_load", + scenario="Testing KG path load", + reasoning="Verifying MCP server auto-load", + outcome="verified", + confidence=0.99, + ) + + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + + # Reset the module-level _graph so _get_graph re-initialises + original_graph = mcp_mod._graph + mcp_mod._graph = None + try: + with patch.dict(os.environ, {"SEMANTICA_KG_PATH": path}): + loaded_graph = mcp_mod._get_graph() + self.assertTrue(loaded_graph.has_node("kg_node_1"), + "Graph must contain node from persisted file") + # Decision indexes must also be rebuilt + self.assertTrue( + hasattr(loaded_graph, "_decisions") and loaded_graph._decisions, + "Decision indexes must be rebuilt when loading from SEMANTICA_KG_PATH" + ) + finally: + mcp_mod._graph = original_graph + finally: + os.unlink(path) + + +# --------------------------------------------------------------------------- +# Part 11: update_node / delete_node smoke tests + decision sync +# --------------------------------------------------------------------------- + +class TestUpdateDeleteNodeMCP(unittest.TestCase): + + def _fresh_graph_with_decision(self): + g = ContextGraph(advanced_analytics=False) + g.add_node("task_1", "task", label="A task node") + did = g.record_decision( + category="project", + scenario="Scope definition for Q3", + reasoning="Requirements complete", + outcome="approved", + confidence=0.9, + ) + return g, did + + def test_update_node_returns_updated_properties(self): + """update_node must reflect new property values in its response.""" + from semantica.mcp_server import _tool_update_node + g, _ = self._fresh_graph_with_decision() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_update_node({ + "node_id": "task_1", + "properties": {"status": "done", "note": "completed by alice"}, + }) + self.assertEqual(result.get("status"), "updated") + self.assertEqual(result.get("node_id"), "task_1") + # Verify node actually updated in graph + node = g.find_node("task_1") + self.assertEqual((node.get("metadata") or {}).get("status"), "done") + + def test_update_node_nonexistent_returns_error(self): + """update_node on a nonexistent node must return an error dict.""" + from semantica.mcp_server import _tool_update_node + g, _ = self._fresh_graph_with_decision() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_update_node({ + "node_id": "does_not_exist", + "properties": {"status": "done"}, + }) + self.assertIn("error", result) + + def test_delete_node_soft_archives(self): + """delete_node must mark the node status='archived', not remove it.""" + from semantica.mcp_server import _tool_delete_node + g, _ = self._fresh_graph_with_decision() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_delete_node({"node_id": "task_1"}) + self.assertEqual(result.get("status"), "archived") + # Node must still exist + node = g.find_node("task_1") + self.assertIsNotNone(node, "Node must still exist after soft-delete") + self.assertEqual((node.get("metadata") or {}).get("status"), "archived") + + def test_delete_node_nonexistent_returns_error(self): + """delete_node on a nonexistent node must return an error dict.""" + from semantica.mcp_server import _tool_delete_node + g, _ = self._fresh_graph_with_decision() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_delete_node({"node_id": "ghost_id"}) + self.assertIn("error", result) + + def test_update_decision_node_syncs_index(self): + """update_node on a decision node must keep _decisions consistent.""" + from semantica.mcp_server import _tool_update_node + g, did = self._fresh_graph_with_decision() + with patch("semantica.mcp_server._get_graph", return_value=g): + _tool_update_node({ + "node_id": did, + "properties": {"status": "reviewed", "reviewer": "bob"}, + }) + # _decisions must reflect the new metadata + dec = g._decisions.get(did) + self.assertIsNotNone(dec) + self.assertEqual(dec["metadata"].get("reviewer"), "bob") + + def test_update_delete_persist_after_reload(self): + """Changes made by update_node / delete_node must survive save โ†’ load.""" + from semantica.mcp_server import _tool_update_node, _tool_delete_node + g, _ = self._fresh_graph_with_decision() + with patch("semantica.mcp_server._get_graph", return_value=g): + _tool_update_node({"node_id": "task_1", + "properties": {"status": "done"}}) + _tool_delete_node.__wrapped__ = None # noop; we call the real fn below + + # Manually save and reload + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + node = g2.find_node("task_1") + self.assertIsNotNone(node) + self.assertEqual((node.get("metadata") or {}).get("status"), "done") + finally: + os.unlink(path) + + +# --------------------------------------------------------------------------- +# Part 12: MCP entity extraction surface text +# --------------------------------------------------------------------------- + +class TestEntityExtractionSurfaceText(unittest.TestCase): + + def test_extract_entities_returns_text_field(self): + """extract_entities must include a 'text' key with the surface form.""" + from semantica.mcp_server import _tool_extract_entities + + # Minimal smoke test: verify the response shape regardless of whether + # spaCy models are available. If no entities are found we skip the + # assertion on content but still verify no crash and no missing key + # structure. + try: + result = _tool_extract_entities({"text": "Apple announced new iPhone"}) + except Exception as exc: + self.skipTest(f"NER dependency unavailable: {exc}") + + if "error" in result: + # spaCy model not installed in this environment โ€” acceptable skip + self.skipTest(f"NER not available: {result['error']}") + + entities = result.get("entities", []) + for ent in entities: + self.assertIn("text", ent, + "Each entity must have a 'text' key with the surface form") + self.assertIn("label", ent, + "Each entity must have a 'label' key (NER category)") + self.assertIn("start", ent) + self.assertIn("end", ent) + + def test_extract_entities_missing_text_returns_error(self): + """extract_entities with no text must return an error dict.""" + from semantica.mcp_server import _tool_extract_entities + result = _tool_extract_entities({}) + self.assertIn("error", result) + + def test_extract_relations_missing_text_returns_error(self): + """extract_relations with no text must return an error dict.""" + from semantica.mcp_server import _tool_extract_relations + result = _tool_extract_relations({}) + self.assertIn("error", result) + + def test_extract_relations_with_text_does_not_raise(self): + """extract_relations must not raise TypeError for missing `entities` + (RelationExtractor.extract_relations requires an `entities` arg; + the tool must supply one, e.g. by running NER first).""" + from semantica.mcp_server import _tool_extract_relations + + try: + result = _tool_extract_relations({"text": "Apple announced new iPhone"}) + except Exception as exc: + self.fail(f"extract_relations raised unexpectedly: {exc!r}") + + self.assertNotIn("error", result, + "extract_relations should not error on valid text input") + self.assertIn("relations", result) + self.assertIn("triplets", result) + + +# --------------------------------------------------------------------------- +# Part 13: query_graph node / search modes +# --------------------------------------------------------------------------- + +class TestQueryGraphNodeAndSearch(unittest.TestCase): + + def _make_graph(self): + g = ContextGraph(advanced_analytics=False) + g.add_node("alpha", "concept", label="Alpha Concept") + g.add_node("beta", "concept", label="Beta Concept") + g.add_edge("alpha", "beta", "relates_to") + return g + + def test_node_mode_existing(self): + """node mode must return the node dict for an existing id.""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({"mode": "node", "node_id": "alpha"}) + self.assertIn("node", result) + self.assertIsNotNone(result["node"]) + + def test_node_mode_missing_id(self): + """node mode with no node_id must return an error.""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({"mode": "node"}) + self.assertIn("error", result) + + def test_search_mode_finds_matching(self): + """search mode must return nodes whose id or content contains the query.""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({"mode": "search", "query": "alpha"}) + hits = result.get("results", []) + self.assertGreater(len(hits), 0) + ids = [h["id"] for h in hits] + self.assertIn("alpha", ids) + + def test_search_mode_limit_respected(self): + """search mode must respect the limit parameter.""" + from semantica.mcp_server import _tool_query_graph + g = ContextGraph(advanced_analytics=False) + for i in range(20): + g.add_node(f"item_{i}", "thing", label=f"item {i}") + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({"mode": "search", "query": "item", "limit": 3}) + self.assertLessEqual(len(result.get("results", [])), 3) + + def test_search_mode_limit_zero_returns_empty(self): + """search mode with limit=0 must return no results.""" + from semantica.mcp_server import _tool_query_graph + g = ContextGraph(advanced_analytics=False) + for i in range(5): + g.add_node(f"alpha_{i}", "thing", label=f"alpha item {i}") + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({"mode": "search", "query": "alpha", "limit": 0}) + hits = result.get("results", []) + self.assertEqual(hits, [], f"limit=0 must return empty, got {len(hits)} results") + + def test_unknown_mode_returns_error(self): + """An unknown mode string must return an error.""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({"mode": "invalid_mode"}) + self.assertIn("error", result) + + def test_inbound_no_duplicates_when_multiple_edges(self): + """Multiple edges between same sourceโ†’target must produce only one + inbound entry for the source node.""" + from semantica.mcp_server import _tool_query_graph + g = ContextGraph(advanced_analytics=False) + g.add_node("hub", "center") + g.add_node("src", "node") + g.add_edge("src", "hub", "type_A") + g.add_edge("src", "hub", "type_B") + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({"mode": "neighbors", "node_id": "hub", "depth": 1}) + in_ids = [n["id"] for n in result.get("neighbors", []) if n.get("direction") == "in"] + self.assertEqual(in_ids.count("src"), 1, + "src must appear exactly once even with two edges") + + +# --------------------------------------------------------------------------- +# Part 14: clear() resets decision indexes +# --------------------------------------------------------------------------- + +class TestClearResetsDecisionIndexes(unittest.TestCase): + + def test_clear_removes_decision_indexes(self): + """clear() must reset _decisions so that decision queries return empty.""" + g = ContextGraph(advanced_analytics=False) + g.record_decision( + category="test", scenario="s", reasoning="r", outcome="o", confidence=0.9 + ) + self.assertTrue(hasattr(g, "_decisions")) + self.assertEqual(len(g._decisions), 1) + + g.clear() + + # After clear, _decisions must be empty + self.assertEqual(len(getattr(g, "_decisions", {})), 0, + "_decisions must be empty after clear()") + # find_similar_decisions must return empty + results = g.find_similar_decisions("s", min_similarity=0.01) + self.assertEqual(results, [], + "find_similar_decisions must return [] after clear()") + + def test_clear_then_record_works(self): + """clear() followed by record_decision must work correctly.""" + g = ContextGraph(advanced_analytics=False) + g.record_decision(category="old", scenario="s", reasoning="r", outcome="o", confidence=0.9) + g.clear() + did = g.record_decision( + category="new", scenario="fresh decision", reasoning="fresh", + outcome="ok", confidence=0.8 + ) + self.assertEqual(len(g._decisions), 1) + self.assertIn(did, g._decisions) + self.assertEqual(g._decisions[did]["category"], "new") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/deduplication/test_deduplication.py b/tests/deduplication/test_deduplication.py index dd351642..8dc7b7c2 100644 --- a/tests/deduplication/test_deduplication.py +++ b/tests/deduplication/test_deduplication.py @@ -12,6 +12,7 @@ from semantica.deduplication.cluster_builder import ClusterBuilder from semantica.deduplication.registry import MethodRegistry from semantica.deduplication.config import DeduplicationConfig from semantica.deduplication.methods import get_deduplication_method +from semantica.utils.types import Entity from semantica.utils.progress_tracker import ConsoleProgressDisplay class TestDeduplication(unittest.TestCase): @@ -87,6 +88,85 @@ class TestDeduplication(unittest.TestCase): # One group should have at least 2 entities (the Apple ones) apple_group = next((g for g in groups if len(g.entities) >= 2), None) self.assertIsNotNone(apple_group) + + def test_different_types_are_never_duplicates(self): + """Entities with different non-empty types must not merge (issue #1137).""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.4 + ) + entities = [ + {"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"}, + {"id": "e2", "type": "Organization", "name": "Acme", "text": "Acme"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertEqual( + duplicates, [], + "Person 'Alice' and Organization 'Acme' must not be duplicate candidates", + ) + # GraphBuilder with merge_entities=True must keep both entities + from semantica.kg import GraphBuilder + graph = GraphBuilder(merge_entities=True).build( + {"entities": entities, "relationships": []} + ) + self.assertEqual(len(graph["entities"]), 2) + + def test_same_type_same_name_still_merges(self): + """Type guard must not break legitimate dedup of same-type entities.""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.4 + ) + entities = [ + {"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"}, + {"id": "e2", "type": "Person", "name": "Alice", "text": "Alice"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertTrue( + duplicates, "Same-type same-name entities must still be detected as duplicates" + ) + + def test_untyped_same_name_still_merges(self): + """Entities with no type must retain previous behavior (merge on similarity).""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.4 + ) + entities = [ + {"id": "x1", "name": "Apple"}, + {"id": "x2", "name": "Apple"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertTrue( + duplicates, "Untyped same-name entities must still be detected as duplicates" + ) + + def test_entity_objects_different_types_not_duplicates(self): + """Entity objects expose their type via .type, not .label (issue #1137).""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.0 + ) + entities = [ + Entity(id="e1", text="Alice", type="Person"), + Entity(id="e2", text="Acme", type="Organization"), + ] + duplicates = detector.detect_duplicates(entities) + self.assertEqual( + duplicates, [], + "Entity objects with different types must never be detected as duplicates", + ) + + def test_zero_threshold_still_excludes_type_mismatch(self): + """Type mismatch must be excluded structurally, not just by confidence 0.""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.0 + ) + entities = [ + {"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"}, + {"id": "e2", "type": "Organization", "name": "Acme", "text": "Acme"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertEqual( + duplicates, [], + "Different-type candidates must be excluded even with confidence_threshold=0.0", + ) def test_entity_merger(self): """Test entity merging.""" diff --git a/tests/explorer/test_decision_route_timestamp.py b/tests/explorer/test_decision_route_timestamp.py new file mode 100644 index 00000000..4b568619 --- /dev/null +++ b/tests/explorer/test_decision_route_timestamp.py @@ -0,0 +1,237 @@ +"""Regression tests for the /api/decisions 422 bug. + +``ContextGraph.record_decision()`` stores ``timestamp`` as a POSIX float +(``datetime.now().timestamp()``). ``DecisionResponse.timestamp`` is typed +``Optional[str]``. Without the ``_normalize_timestamp`` field-validator on +``DecisionResponse`` the raw float fails Pydantic validation and every decision +endpoint returns 422. + +The validator lives on ``DecisionResponse`` in ``semantica/explorer/schemas.py`` +and converts float/int epochs to ISO-8601 strings via +``datetime.fromtimestamp(value, tz=timezone.utc).isoformat()``. + +``_node_to_decision()`` must pass the raw stored value through unchanged so the +validator can do its job. A route-level ``str()`` cast would pre-empt the +validator and produce raw numeric strings instead of ISO-8601, breaking the API +contract and all callers that call ``datetime.fromisoformat()`` on the result. + +Each test below is written so that it *fails* when the route-level ``str()`` +cast is present (i.e. it would have caught the regression introduced by the +incorrect fix). +""" + +import math +from datetime import datetime + +import pytest +from fastapi.testclient import TestClient +from pydantic import ValidationError + +from semantica.context.context_graph import ContextGraph +from semantica.explorer.app import create_app +from semantica.explorer.routes.decisions import _node_to_decision +from semantica.explorer.schemas import DecisionResponse +from semantica.explorer.session import GraphSession + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _decision_node(timestamp): + """Minimal node dict as returned by the graph session layer.""" + return { + "id": "d-test", + "type": "decision", + "properties": { + "category": "loan_underwriting", + "scenario": "A-7291 review", + "reasoning": "DTI within policy", + "outcome": "approved", + "confidence": 0.94, + "timestamp": timestamp, + }, + } + + +def _recorded_client(): + """TestClient backed by a graph built with real record_decision() calls. + + This is the path that was broken in production: record_decision() stores + timestamp as a float epoch, which must come out the other side as an + ISO-8601 string, not a raw numeric string. + """ + graph = ContextGraph(advanced_analytics=False) + graph.record_decision( + category="credit_application", + scenario="Personal loan, $85k income, 31% DTI", + reasoning="Income meets threshold; employment stable", + outcome="proceed_to_underwriting", + confidence=0.88, + entities=["applicant_A7291"], + ) + return TestClient(create_app(session=GraphSession(graph))) + + +# --------------------------------------------------------------------------- +# Unit tests: _node_to_decision() โ†’ DecisionResponse +# +# Each assertion must fail when the route contains the incorrect str() cast: +# timestamp=None if ... is None else str(properties.get("timestamp")) +# because that cast turns floats into numeric strings such as "1786513069.69" +# rather than ISO-8601 strings such as "2026-08-12T05:37:49+00:00". +# --------------------------------------------------------------------------- + +def test_float_timestamp_becomes_iso8601(): + """A POSIX-float epoch must be normalised to an ISO-8601 string. + + Fails with the str() cast because str(1786513069.694965) == + '1786513069.694965', which is not a valid isoformat string. + """ + decision = _node_to_decision(_decision_node(timestamp=1786513069.694965)) + + assert isinstance(decision.timestamp, str) + # Must parse as a valid ISO-8601 datetime โ€” this is the key assertion that + # the incorrect str() cast breaks. + parsed = datetime.fromisoformat(decision.timestamp) + # Round-trip: parsed timestamp must be within 1 s of the original epoch. + assert abs(parsed.timestamp() - 1786513069.694965) < 1.0 + + +def test_int_timestamp_becomes_iso8601(): + """An integer epoch (no sub-second component) must also become ISO-8601. + + Fails with the str() cast because str(1786513069) == '1786513069'. + """ + decision = _node_to_decision(_decision_node(timestamp=1786513069)) + + assert isinstance(decision.timestamp, str) + parsed = datetime.fromisoformat(decision.timestamp) + assert abs(parsed.timestamp() - 1786513069) < 1.0 + + +def test_none_timestamp_stays_none(): + """A stored None must remain None, not become the string 'None'.""" + decision = _node_to_decision(_decision_node(timestamp=None)) + + assert decision.timestamp is None + + +def test_missing_timestamp_key_stays_none(): + """A node without a timestamp key at all must not raise and must be None.""" + node = { + "id": "d-no-ts", + "type": "decision", + "properties": {"category": "x", "outcome": "y"}, + } + + decision = _node_to_decision(node) + + assert decision.timestamp is None + + +def test_iso_string_passes_through_unchanged(): + """An already-ISO-8601 string must be returned verbatim.""" + iso = "2026-08-12T10:04:20+00:00" + decision = _node_to_decision(_decision_node(timestamp=iso)) + + assert decision.timestamp == iso + + +def test_nan_timestamp_raises_validation_error(): + """NaN must be rejected by the validator, not silently accepted. + + With the str() cast, str(nan) == 'nan' bypasses the validator's finiteness + check and is silently accepted โ€” this test would pass the incorrect version + of the code if it expected 'nan', but it correctly expects a ValidationError. + """ + with pytest.raises(ValidationError): + _node_to_decision(_decision_node(timestamp=float("nan"))) + + +def test_inf_timestamp_raises_validation_error(): + """Positive infinity must be rejected, not silently accepted as 'inf'.""" + with pytest.raises(ValidationError): + _node_to_decision(_decision_node(timestamp=float("inf"))) + + +def test_negative_inf_timestamp_raises_validation_error(): + """Negative infinity must be rejected, not silently accepted as '-inf'.""" + with pytest.raises(ValidationError): + _node_to_decision(_decision_node(timestamp=float("-inf"))) + + +def test_out_of_range_epoch_raises_validation_error(): + """A millisecond epoch accidentally passed as seconds must be rejected. + + With the str() cast, str(1723600000000) is silently accepted as a string. + The validator correctly raises ValidationError for out-of-range epochs. + """ + with pytest.raises(ValidationError): + _node_to_decision(_decision_node(timestamp=1723600000000)) + + +# --------------------------------------------------------------------------- +# Integration tests: full HTTP path through TestClient +# +# These exercise the complete production path: +# record_decision() โ†’ float stored in graph โ†’ HTTP GET โ†’ JSON response +# +# They are the definitive check: if the route emits numeric strings instead of +# ISO-8601 the fromisoformat() assertion below fails immediately. +# --------------------------------------------------------------------------- + +def test_list_decisions_float_timestamp_serialised_as_iso8601(): + """GET /api/decisions must return ISO-8601 timestamps for all decisions. + + This is the exact production failure path. record_decision() stores + timestamp as a float; the endpoint must return an ISO-8601 string, not a + raw numeric string like '1786513069.69'. + """ + with _recorded_client() as client: + response = client.get("/api/decisions") + + assert response.status_code == 200 + payload = response.json() + assert len(payload) >= 1 + + for item in payload: + ts = item["timestamp"] + assert isinstance(ts, str), f"timestamp must be str, got {type(ts)}" + # This is the line that fails when the str() cast is present: + datetime.fromisoformat(ts) + + +def test_get_decision_float_timestamp_serialised_as_iso8601(): + """GET /api/decisions/{id} must return an ISO-8601 timestamp.""" + with _recorded_client() as client: + decision_id = client.get("/api/decisions").json()[0]["decision_id"] + response = client.get(f"/api/decisions/{decision_id}") + + assert response.status_code == 200 + ts = response.json()["timestamp"] + assert isinstance(ts, str) + datetime.fromisoformat(ts) + + +def test_get_precedents_float_timestamp_serialised_as_iso8601(): + """GET /api/decisions/{id}/precedents must return ISO-8601 timestamps.""" + graph = ContextGraph(advanced_analytics=False) + for i in range(3): + graph.record_decision( + category="risk", + scenario=f"loan assessment scenario {i}", + reasoning="standard criteria", + outcome="approved", + confidence=0.9, + ) + + with TestClient(create_app(session=GraphSession(graph))) as client: + decision_id = client.get("/api/decisions").json()[0]["decision_id"] + response = client.get(f"/api/decisions/{decision_id}/precedents") + + assert response.status_code == 200 + for item in response.json(): + ts = item["timestamp"] + assert isinstance(ts, str) + datetime.fromisoformat(ts) diff --git a/tests/export/test_jsonld_default_graph.py b/tests/export/test_jsonld_default_graph.py index c66b9a1a..e9f8dc09 100644 --- a/tests/export/test_jsonld_default_graph.py +++ b/tests/export/test_jsonld_default_graph.py @@ -160,7 +160,7 @@ def test_a_url_valued_context_is_not_thrown_away(tmp_path): context = json.loads(path.read_text())["@context"] flattened = context if isinstance(context, list) else [context] - assert "https://schema.org/" in flattened, ( + assert any(entry == "https://schema.org/" for entry in flattened), ( "the caller's context was replaced by Semantica's defaults, " "which silently changes how every term expands" ) diff --git a/tests/export/test_jsonld_document_iri.py b/tests/export/test_jsonld_document_iri.py new file mode 100644 index 00000000..34b2366e --- /dev/null +++ b/tests/export/test_jsonld_document_iri.py @@ -0,0 +1,134 @@ +"""The document IRI of a JSON-LD export must depend on content, not the clock +(issue #1147). + +``_convert_kg_to_jsonld`` minted the graph's ``@id`` from ``utc_now_iso()``, +and the generic ``_attach_document_metadata`` path did the same for a plain +document ``@id``. Exporting an unchanged graph therefore produced a new +subject every time: three exports of one one-entity graph merged into 3 +``semantica:KnowledgeGraph`` nodes and 15 triples for what should have been a +single graph. Neither identifier resolves and the timestamp is already +recorded correctly in ``semantica:exportedAt``, so the fix mints the IRI from +the exported content instead (mirroring ``mint_entity_iri``, #1109), with an +optional caller-supplied override for callers who already name their graphs. +""" + +import json + +from rdflib import RDF, Graph, URIRef + +from semantica.export.json_exporter import JSONExporter + +KG = { + "entities": [{"id": "https://example.org/e1", "text": "Acme Corp", "type": "ORG"}], + "relationships": [], +} + +OTHER_KG = { + "entities": [ + {"id": "https://example.org/e1", "text": "Acme Corp Renamed", "type": "ORG"} + ], + "relationships": [], +} + + +def _export(kg, tmp_path, name="out.jsonld", **options): + path = tmp_path / name + JSONExporter().export_knowledge_graph(kg, path, format="json-ld", **options) + return path + + +def test_reexporting_an_unchanged_graph_is_idempotent(tmp_path): + """The whole point of an identifier: same content, same @id.""" + first = json.loads(_export(KG, tmp_path, "a.jsonld").read_text()) + second = json.loads(_export(KG, tmp_path, "b.jsonld").read_text()) + + assert first["@id"] == second["@id"] + + +def test_a_changed_graph_gets_a_different_id(tmp_path): + unchanged = json.loads(_export(KG, tmp_path, "a.jsonld").read_text()) + changed = json.loads(_export(OTHER_KG, tmp_path, "b.jsonld").read_text()) + + assert unchanged["@id"] != changed["@id"] + + +def test_merging_repeated_exports_yields_one_graph_node(tmp_path): + """Regression for the exact repro in #1147: churn no longer multiplies nodes.""" + merged = Graph() + for i in range(3): + path = _export(KG, tmp_path, f"churn{i}.jsonld") + merged.parse(str(path), format="json-ld") + + # Exactly one subject typed as a KnowledgeGraph, regardless of how many + # times the unchanged graph was exported and merged. + kg_nodes = set( + merged.subjects(RDF.type, URIRef("https://semantica.dev/ns#KnowledgeGraph")) + ) + assert len(kg_nodes) == 1 + + entity_nodes = set( + merged.subjects(RDF.type, URIRef("https://semantica.dev/vocab/ORG")) + ) + assert len(entity_nodes) == 1 + + +def test_exported_at_still_varies_between_exports(tmp_path): + """Identity is now content-derived, but provenance still records each run.""" + first = json.loads(_export(KG, tmp_path, "a.jsonld").read_text()) + second = json.loads(_export(KG, tmp_path, "b.jsonld").read_text()) + + assert first["@id"] == second["@id"] + assert first["semantica:exportedAt"] != second["semantica:exportedAt"] + + +def test_caller_supplied_graph_uri_is_honored(tmp_path): + path = _export(KG, tmp_path, graph_uri="https://example.org/my-graph") + document = json.loads(path.read_text()) + + assert document["@id"] == "https://example.org/my-graph" + + +def _document_node_id(document): + """The generic (non-knowledge-graph) path hangs its own @id off a member + of @graph rather than the top level, to avoid re-creating the named-graph + bug fixed by #1145. Find that member and return its @id.""" + for node in document["@graph"]: + if "semantica:exportedAt" in node: + return node["@id"] + raise AssertionError(f"no document metadata node in @graph: {document}") + + +def test_caller_supplied_document_uri_is_honored_for_a_generic_export(tmp_path): + payload = {"note": "no entities or relationships here"} + path = tmp_path / "generic.jsonld" + JSONExporter().export( + payload, path, format="json-ld", document_uri="https://example.org/my-doc" + ) + document = json.loads(path.read_text()) + + assert _document_node_id(document) == "https://example.org/my-doc" + + +def test_generic_document_id_is_also_content_derived(tmp_path): + """The non-knowledge-graph path (_attach_document_metadata) gets the same fix.""" + payload = {"note": "plain data, no @id of its own"} + + first = tmp_path / "a.jsonld" + second = tmp_path / "b.jsonld" + JSONExporter().export(payload, first, format="json-ld") + JSONExporter().export(dict(payload), second, format="json-ld") + + first_id = _document_node_id(json.loads(first.read_text())) + second_id = _document_node_id(json.loads(second.read_text())) + assert first_id == second_id + + +def test_document_id_still_differs_for_different_generic_payloads(tmp_path): + a = tmp_path / "a.jsonld" + b = tmp_path / "b.jsonld" + JSONExporter().export({"note": "one"}, a, format="json-ld") + JSONExporter().export({"note": "two"}, b, format="json-ld") + + a_id = _document_node_id(json.loads(a.read_text())) + b_id = _document_node_id(json.loads(b.read_text())) + assert a_id != b_id diff --git a/tests/export/test_metadata_passthrough.py b/tests/export/test_metadata_passthrough.py new file mode 100644 index 00000000..ff4c3ddc --- /dev/null +++ b/tests/export/test_metadata_passthrough.py @@ -0,0 +1,401 @@ +"""Metadata must survive serialization (issue #1154). + +``convert_kg_to_rdf`` copies ``metadata`` into the RDF-ready dictionary at +rdf_exporter.py:302, and no serializer has ever read it back out. Turtle, +N-Triples, RDF/XML and RDFExporter's JSON-LD all write the entity's id, type, +text and confidence, and none of them writes a single metadata statement, so an +entity keeps its confidence score and loses what produced it: the source +document, the page, the extractor, the reviewer. JSONExporter's json-ld path +keeps all of them, which is how the same knowledge graph exported two ways came +to carry ten triples of user data through one exporter and none through the +other. + +The keys Semantica itself produces (GraphBuilder writes num_entities, +num_relationships, temporal_enabled, timestamp and entity_resolution_applied; +the Neo4j loader writes source, uri and database) are Semantica's own +vocabulary, so they are minted in the declared namespace and declared in +semantica-ns.ttl. Keys the caller supplied are not: which namespace those +belong in is issue #1146, and until that is settled the exporter refuses to +guess rather than inventing an IRI, warns, and takes an explicit +``metadata_terms`` mapping from any caller who already knows the answer. +""" + +import json + +import pytest +from rdflib import Graph, Literal, URIRef +from rdflib.namespace import XSD + +from semantica.export.rdf_exporter import ( + DEFAULT_METADATA_TERMS, + RDFSerializer, + SEMANTICA_NS, + mint_entity_iri, +) + +ENTITY_IRI = "https://example.org/e1" + +# The provenance fields the issue names, plus one key Semantica itself writes. +GRAPH_WITH_METADATA = { + "entities": [ + { + "id": ENTITY_IRI, + "type": "https://example.org/Org", + "text": "Acme Corp", + "confidence": 0.91, + "metadata": {"num_entities": 1, "temporal_enabled": True}, + } + ], + "relationships": [], + "metadata": { + "num_entities": 1, + "num_relationships": 0, + "temporal_enabled": False, + "entity_resolution_applied": True, + }, +} + +NUM_ENTITIES = URIRef(f"{SEMANTICA_NS}numEntities") +TEMPORAL_ENABLED = URIRef(f"{SEMANTICA_NS}temporalEnabled") + + +def _parse(text: str, fmt: str) -> Graph: + """Assert on the parsed graph, never on the serialized text.""" + g = Graph() + g.parse(data=text, format=fmt) + return g + + +def _serialize(serializer: RDFSerializer, fmt: str, data, **options) -> Graph: + method, parse_as = { + "turtle": (serializer.serialize_to_turtle, "turtle"), + "ntriples": (serializer.serialize_to_ntriples, "nt"), + "rdfxml": (serializer.serialize_to_rdfxml, "xml"), + "jsonld": (serializer.serialize_to_jsonld, "json-ld"), + }[fmt] + return _parse(method(data, **options), parse_as) + + +FORMATS = ["turtle", "ntriples", "rdfxml", "jsonld"] + + +@pytest.mark.parametrize("fmt", FORMATS) +def test_entity_metadata_reaches_every_serialization(fmt): + """The headline defect: the statement is absent from all four formats.""" + g = _serialize(RDFSerializer(), fmt, GRAPH_WITH_METADATA) + assert (URIRef(ENTITY_IRI), NUM_ENTITIES, Literal(1)) in g + + +@pytest.mark.parametrize("fmt", FORMATS) +def test_entity_metadata_booleans_keep_their_datatype(fmt): + g = _serialize(RDFSerializer(), fmt, GRAPH_WITH_METADATA) + assert (URIRef(ENTITY_IRI), TEMPORAL_ENABLED, Literal(True)) in g + + +def test_every_format_writes_the_same_metadata_triples(): + """A value must not change datatype with the serializer, as #1100 found.""" + per_format = {} + for fmt in FORMATS: + g = _serialize(RDFSerializer(), fmt, GRAPH_WITH_METADATA) + per_format[fmt] = { + (p, o) + for s, p, o in g + if str(p).startswith(SEMANTICA_NS) and "numEntities" in str(p) + } + assert len(set(map(frozenset, per_format.values()))) == 1, per_format + + +def test_graph_metadata_needs_a_subject_the_caller_named(): + """Graph-level metadata hangs off graph_uri; #1147 owns the default.""" + doc = URIRef("https://example.org/graph/1") + g = _serialize( + RDFSerializer(), + "turtle", + GRAPH_WITH_METADATA, + graph_uri=str(doc), + ) + assert (doc, NUM_ENTITIES, Literal(1)) in g + assert (doc, URIRef(f"{SEMANTICA_NS}entityResolutionApplied"), Literal(True)) in g + + +def test_graph_metadata_is_not_invented_without_a_subject(): + g = _serialize(RDFSerializer(), "turtle", GRAPH_WITH_METADATA) + assert not list(g.subjects(NUM_ENTITIES, Literal(0))) + # the entity keeps its own metadata; only the graph-level block waits + assert (URIRef(ENTITY_IRI), NUM_ENTITIES, Literal(1)) in g + + +def test_an_unknown_key_is_refused_out_loud_not_dropped_in_silence(caplog): + """#1146 owns which namespace a caller's key belongs in. Until then: warn.""" + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"reviewed_by": "fabio"}} + ], + "relationships": [], + } + with caplog.at_level("WARNING"): + g = _serialize(RDFSerializer(), "turtle", data) + assert not any("reviewed_by" in str(p) for p in g.predicates()) + assert any("reviewed_by" in r.getMessage() for r in caplog.records) + assert any("1146" in r.getMessage() for r in caplog.records) + + +@pytest.mark.parametrize("fmt", FORMATS) +def test_a_caller_who_knows_the_answer_can_supply_the_term(fmt): + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"reviewed_by": "fabio"}} + ], + "relationships": [], + } + terms = {"reviewed_by": "http://purl.org/dc/terms/creator"} + g = _serialize(RDFSerializer(), fmt, data, metadata_terms=terms) + assert ( + URIRef(ENTITY_IRI), + URIRef("http://purl.org/dc/terms/creator"), + Literal("fabio"), + ) in g + + +def test_a_literal_with_a_quote_or_newline_still_parses(): + """Metadata is user text; #1098 is the same class of defect one field over.""" + data = { + "entities": [ + { + "id": ENTITY_IRI, + "text": "Acme", + "metadata": {"source": 'the "Q3" report\nsecond line'}, + } + ], + "relationships": [], + } + for fmt in FORMATS: + g = _serialize(RDFSerializer(), fmt, data) + assert ( + URIRef(ENTITY_IRI), + URIRef(f"{SEMANTICA_NS}sourceSystem"), + Literal('the "Q3" report\nsecond line'), + ) in g + + +def test_an_iri_valued_key_is_written_as_a_node_not_a_string(): + """The Neo4j loader's ``uri`` key. Note the term is sem:sourceUri, not + sem:uri: the key names a field, the term names a relation.""" + data = { + "entities": [ + { + "id": ENTITY_IRI, + "text": "Acme", + "metadata": {"uri": "https://example.org/db"}, + } + ], + "relationships": [], + } + g = _serialize(RDFSerializer(), "turtle", data) + assert ( + URIRef(ENTITY_IRI), + URIRef(f"{SEMANTICA_NS}sourceUri"), + URIRef("https://example.org/db"), + ) in g + + +def test_output_is_unchanged_when_no_metadata_is_present(): + plain = { + "entities": [ + {"id": ENTITY_IRI, "type": "https://example.org/Org", "text": "Acme"} + ], + "relationships": [ + {"source_id": ENTITY_IRI, "target_id": "https://example.org/e2"} + ], + } + serializer = RDFSerializer() + assert serializer.serialize_to_turtle(plain) == serializer.serialize_to_turtle( + plain + ) + g = _parse(serializer.serialize_to_turtle(plain), "turtle") + assert len(g) == 4 + + +def test_every_default_term_is_declared_in_the_shipped_vocabulary(): + """Drift guard: a term the exporter emits and the vocabulary omits is a bug.""" + from semantica.ontology.vocabulary import vocabulary_path + + vocab = Graph() + vocab.parse(vocabulary_path(), format="turtle") + declared = {str(s) for s in vocab.subjects()} + missing = sorted(set(DEFAULT_METADATA_TERMS.values()) - declared) + assert not missing, f"emitted but undeclared: {missing}" + + +def test_jsonld_metadata_survives_a_real_jsonld_processor(): + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"num_entities": 3}} + ], + "relationships": [], + } + raw = RDFSerializer().serialize_to_jsonld(data) + json.loads(raw) # must be valid JSON before it can be valid JSON-LD + g = _parse(raw, "json-ld") + assert (URIRef(ENTITY_IRI), NUM_ENTITIES, Literal(3)) in g + + +# --- Findings from the Qodo review of PR #1165 ----------------------------- + + +@pytest.mark.parametrize("fmt", FORMATS) +@pytest.mark.parametrize( + "value", [1e-05, 1e300, 0.1, -0.0, float("nan"), float("inf"), float("-inf")] +) +def test_a_float_metadata_value_is_a_double_and_keeps_a_legal_lexical(fmt, value): + """`repr()` of a float is not an xsd:decimal lexical. + + `repr(1e-05)` is "1e-05" and `repr(float("nan"))` is "nan", neither of which + xsd:decimal admits, so typing a float as decimal produced RDF a strict + parser rejects. A Python float is an IEEE 754 double, xsd:double has legal + lexicals for the exponent form and for the three special values, and saying + double is also the honest claim: nothing here was ever exact. + """ + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"num_entities": value}} + ], + "relationships": [], + } + g = _serialize(RDFSerializer(), fmt, data) + objects = list(g.objects(URIRef(ENTITY_IRI), NUM_ENTITIES)) + assert len(objects) == 1, f"{fmt}: {objects}" + (written,) = objects + assert written.datatype == XSD.double, written.datatype + parsed = written.toPython() + if value != value: # NaN + assert parsed != parsed + else: + assert parsed == value + + +def test_every_format_agrees_on_a_float_metadata_value(): + per_format = {} + for fmt in FORMATS: + g = _serialize( + RDFSerializer(), + fmt, + { + "entities": [ + {"id": ENTITY_IRI, "text": "A", "metadata": {"num_entities": 1e-05}} + ], + "relationships": [], + }, + ) + per_format[fmt] = {(p, o) for s, p, o in g if p == NUM_ENTITIES} + assert len(set(map(frozenset, per_format.values()))) == 1, per_format + + +def test_a_term_rdfxml_cannot_name_is_refused_out_loud(caplog): + """RDF/XML needs a QName, and the PR's whole point is no silent drops. + + A term whose local part is not an XML NCName has no RDF/XML form at all. + Skipping it quietly reintroduces, in one format, exactly the loss this + change exists to stop. + """ + unnameable = "http://example.org/ns/123" + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"reviewed_by": "fabio"}} + ], + "relationships": [], + } + with caplog.at_level("WARNING"): + xml = RDFSerializer().serialize_to_rdfxml( + data, metadata_terms={"reviewed_by": unnameable} + ) + _parse(xml, "xml") # must still be well-formed + messages = " ".join(r.getMessage() for r in caplog.records) + assert unnameable in messages + assert "RDF/XML" in messages + + +@pytest.mark.parametrize("fmt", ["turtle", "ntriples", "jsonld"]) +def test_the_other_formats_still_carry_a_term_rdfxml_cannot_name(fmt): + """Only RDF/XML has the QName restriction; the rest write the full IRI.""" + unnameable = "http://example.org/ns/123" + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"reviewed_by": "fabio"}} + ], + "relationships": [], + } + g = _serialize( + RDFSerializer(), fmt, data, metadata_terms={"reviewed_by": unnameable} + ) + assert (URIRef(ENTITY_IRI), URIRef(unnameable), Literal("fabio")) in g + + +def test_a_quote_in_an_attribute_value_cannot_break_the_document(): + """`_escape_xml` feeds attribute values, which are delimited by quotes. + + Escaping only &, < and > leaves a caller-supplied value able to close the + attribute early and produce XML that does not parse. + """ + data = { + "entities": [ + { + "id": 'https://example.org/e"1', + "text": "Acme", + "metadata": {"uri": 'https://example.org/db"x'}, + } + ], + "relationships": [], + } + xml = RDFSerializer().serialize_to_rdfxml(data) + from xml.dom.minidom import parseString + + parseString(xml) # well-formedness is the assertion + + +# --- Finding from review of PR #1165 ---------------------------------------- + + +@pytest.mark.parametrize("fmt", ["turtle", "ntriples"]) +def test_an_iri_valued_metadata_value_cannot_inject_a_second_triple(fmt): + """`sem:sourceUri` (the "uri" key) is the one metadata term written as a + node, ``<{value}>``, with no other quoting. Turtle/N-Triples IRIREFs + exclude '>' (among other characters) unescaped, so a value shaped like + `` .

    `` closed the reference early and let the + rest of the string be parsed as an unrelated, attacker-chosen triple. + """ + payload = ( + "https://evil.example/x> . " + " ' โ€” cover the control-character half of + the grammar, not only the delimiter characters. + """ + payload = "https://evil.example/x\ninjected line\ttabbed" + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"uri": payload}} + ], + "relationships": [], + } + g = _serialize(RDFSerializer(), fmt, data) + assert len(g) == 4 diff --git a/tests/export/test_owl_time_reachability.py b/tests/export/test_owl_time_reachability.py index 92473390..e83339f7 100644 --- a/tests/export/test_owl_time_reachability.py +++ b/tests/export/test_owl_time_reachability.py @@ -187,3 +187,59 @@ def test_the_reified_type_matches_the_direct_triples_predicate(): assert set(graph.objects(node, URIRef(NS + "type"))) == {Literal(EMPLOYS)} assert (URIRef(E1), URIRef(EMPLOYS), URIRef(E2)) in graph + + +# โ”€โ”€ Qodo review: temporal bounds are str-only at the escape helper โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +def test_datetime_bounds_do_not_crash_the_turtle_export(): + """_escape_literal is str-only; datetime bounds must be stringified, not + run through .replace(). Regression for Qodo high-priority finding #2 on + PR #1221. + + Also asserts the lexical form: xsd:dateTimeStamp requires an ISO 8601 "T" + separator (e.g. 2024-01-01T00:00:00+00:00). plain str() emits a space + ("2024-01-01 00:00:00+00:00"), which is format-invalid; isoformat() fixes + it. Regression for the maintainer review on PR #1221.""" + from datetime import datetime, timezone + + kg = { + "entities": [dict(e) for e in KG["entities"]], + "relationships": [ + { + "source_id": E1, + "target_id": E2, + "type": EMPLOYS, + "valid_from": datetime(2024, 1, 1, tzinfo=timezone.utc), + "valid_until": datetime(2025, 1, 1, tzinfo=timezone.utc), + } + ], + } + turtle = RDFSerializer().serialize_to_turtle(kg, include_temporal=True) + graph = Graph() + graph.parse(data=turtle, format="turtle") + stamps = { + str(o) for o in graph.objects(None, URIRef(TIME + "inXSDDateTimeStamp")) + } + assert len(stamps) == 2, stamps + assert "2024-01-01T00:00:00+00:00" in stamps, stamps + assert "2025-01-01T00:00:00+00:00" in stamps, stamps + + +def test_end_only_interval_does_not_crash_the_turtle_export(): + """A valid_until bound with no valid_from passes None as from_val; it must + not be handed to the str-only escaper. Regression for Qodo finding #2.""" + kg = { + "entities": [dict(e) for e in KG["entities"]], + "relationships": [ + { + "source_id": E1, + "target_id": E2, + "type": EMPLOYS, + "valid_until": "2025-01-01T00:00:00Z", + } + ], + } + turtle = RDFSerializer().serialize_to_turtle(kg, include_temporal=True) + graph = Graph() + graph.parse(data=turtle, format="turtle") + assert list(graph.subjects(RDF.type, URIRef(TIME + "Instant"))) diff --git a/tests/export/test_rdf_exporter_turtle_iris.py b/tests/export/test_rdf_exporter_turtle_iris.py new file mode 100644 index 00000000..bc197a08 --- /dev/null +++ b/tests/export/test_rdf_exporter_turtle_iris.py @@ -0,0 +1,233 @@ +"""Regression tests for valid Turtle IRI generation (issue #1099).""" + +from rdflib import RDF, Graph, URIRef + +from semantica.export import RDFExporter +from semantica.kg.graph_builder import GraphBuilder + + +def test_turtle_normalizes_graph_builder_default_identifiers(): + """Default GraphBuilder labels with spaces become stable absolute IRIs.""" + source = { + "entities": [ + { + "id": "Kochi, Kerala", + "name": "Kochi, Kerala", + "type": "LOCATION", + }, + {"id": "Jane Doe", "name": "Jane Doe", "type": "PERSON"}, + ], + "relationships": [ + { + "source": "Jane Doe", + "target": "Kochi, Kerala", + "type": "located_in", + }, + ], + } + graph_data = GraphBuilder(resolve_conflicts=False).build(sources=[source]) + + turtle = RDFExporter().export_to_rdf(graph_data, format="turtle") + parsed = Graph().parse(data=turtle, format="turtle") + assert "" not in turtle + assert "" not in turtle + + kochi = URIRef("https://semantica.dev/ns#Kochi%2C%20Kerala") + jane = URIRef("https://semantica.dev/ns#Jane%20Doe") + assert ( + kochi, + RDF.type, + URIRef("https://semantica.dev/ns#LOCATION"), + ) in parsed + jane_type = URIRef("https://semantica.dev/ns#PERSON") + assert (jane, RDF.type, jane_type) in parsed + assert ( + jane, + URIRef("https://semantica.dev/ns#located_in"), + kochi, + ) in parsed + + +def test_turtle_preserves_absolute_iris(): + """Already-valid absolute resource IRIs remain unchanged.""" + turtle = RDFExporter().export_to_rdf( + { + "entities": [ + { + "id": "https://example.org/entities/jane", + "text": "Jane", + "type": "urn:example:Person", + } + ], + "relationships": [], + }, + format="turtle", + ) + parsed = Graph().parse(data=turtle, format="turtle") + + assert ( + URIRef("https://example.org/entities/jane"), + RDF.type, + URIRef("urn:example:Person"), + ) in parsed + + +def test_turtle_preserves_opaque_absolute_iris_and_encodes_bad_percent_escapes(): + """Opaque schemes remain absolute and malformed percent escapes are encoded.""" + turtle = RDFExporter().export_to_rdf( + { + "entities": [ + {"id": "mailto:foo", "type": "isbn:0451450523"}, + {"id": "http://example.org/bad%zz", "type": "PERSON"}, + ], + "relationships": [], + }, + format="turtle", + ) + parsed = Graph().parse(data=turtle, format="turtle") + + assert ( + URIRef("mailto:foo"), + RDF.type, + URIRef("isbn:0451450523"), + ) in parsed + assert URIRef("http://example.org/bad%25zz") in parsed.all_nodes() + + +def test_turtle_normalizes_temporal_relationship_endpoints(): + """Temporal relationship metadata uses the same normalized resource IRIs.""" + turtle = RDFExporter().export_to_rdf( + { + "entities": [ + {"id": "Jane Doe", "type": "PERSON"}, + {"id": "Kochi, Kerala", "type": "LOCATION"}, + ], + "relationships": [ + { + "source": "Jane Doe", + "target": "Kochi, Kerala", + "type": "located_in", + "valid_from": "2024-01-01T00:00:00+00:00", + "valid_until": "2024-02-01T00:00:00+00:00", + } + ], + }, + format="turtle", + include_temporal=True, + ) + parsed = Graph().parse(data=turtle, format="turtle") + + assert ( + None, + URIRef("https://semantica.dev/ns#source"), + URIRef("https://semantica.dev/ns#Jane%20Doe"), + ) in parsed + assert ( + None, + URIRef("https://semantica.dev/ns#target"), + URIRef("https://semantica.dev/ns#Kochi%2C%20Kerala"), + ) in parsed + + +def test_turtle_expands_context_prefixes_and_mints_relative_values(): + """Context prefixes expand while bare values use the fallback namespace.""" + turtle = RDFExporter().export_to_rdf( + { + "@context": {"ex": "https://example.org/"}, + "entities": [{"id": "ORG", "type": "ex:Person"}], + "relationships": [], + }, + format="turtle", + ) + parsed = Graph().parse(data=turtle, format="turtle") + + assert ( + URIRef("https://semantica.dev/ns#ORG"), + RDF.type, + URIRef("https://example.org/Person"), + ) in parsed + + +def test_rdfxml_normalizes_resource_iris(): + """RDF/XML resource attributes use the same safe absolute IRIs.""" + data = { + "entities": [ + {"id": "Acme Corp", "type": "Person"}, + {"id": "mailto:foo", "type": "isbn:0451450523"}, + ], + "relationships": [ + {"source": "Jane Doe", "target": "Acme Corp", "type": "knows"} + ], + } + rdfxml = RDFExporter().export_to_rdf(data, format="rdfxml") + parsed = Graph().parse(data=rdfxml, format="xml") + + assert URIRef("https://semantica.dev/ns#Acme%20Corp") in parsed.all_nodes() + assert URIRef("mailto:foo") in parsed.all_nodes() + + +def test_ntriples_normalizes_resource_iris(): + """N-Triples resource IRIs reject neither spaces nor opaque schemes.""" + data = { + "entities": [ + {"id": "Acme Corp", "type": "Person"}, + {"id": "mailto:foo", "type": "isbn:0451450523"}, + ], + "relationships": [ + {"source": "Jane Doe", "target": "Acme Corp", "type": "knows"} + ], + } + ntriples = RDFExporter().export_to_rdf(data, format="ntriples") + parsed = Graph().parse(data=ntriples, format="nt") + + assert URIRef("https://semantica.dev/ns#Acme%20Corp") in parsed.all_nodes() + assert URIRef("mailto:foo") in parsed.all_nodes() + + +def test_turtle_preserves_existing_valid_percent_escapes(): + """A pre-encoded absolute IRI keeps its escape, instead of %20 -> %2520.""" + turtle = RDFExporter().export_to_rdf( + { + "entities": [ + { + "id": "https://example.org/entities/path%20name", + "type": "PERSON", + } + ], + "relationships": [], + }, + format="turtle", + ) + parsed = Graph().parse(data=turtle, format="turtle") + + assert ( + URIRef("https://example.org/entities/path%20name"), + RDF.type, + URIRef("https://semantica.dev/ns#PERSON"), + ) in parsed + assert "%2520" not in turtle + + +def test_ntriples_and_rdfxml_expand_builtin_prefixes_alongside_context(): + """A user @context must not shadow built-in prefixes like semantica:.""" + data = { + "@context": {"ex": "https://example.org/"}, + "entities": [{"id": "ORG", "type": "semantica:Entity"}], + "relationships": [], + } + + ntriples = RDFExporter().export_to_rdf(data, format="ntriples") + nt_parsed = Graph().parse(data=ntriples, format="nt") + assert ( + URIRef("https://semantica.dev/ns#ORG"), + RDF.type, + URIRef("https://semantica.dev/ns#Entity"), + ) in nt_parsed + + rdfxml = RDFExporter().export_to_rdf(data, format="rdfxml") + xml_parsed = Graph().parse(data=rdfxml, format="xml") + assert ( + URIRef("https://semantica.dev/ns#ORG"), + RDF.type, + URIRef("https://semantica.dev/ns#Entity"), + ) in xml_parsed diff --git a/tests/export/test_rdf_literal_escaping.py b/tests/export/test_rdf_literal_escaping.py new file mode 100644 index 00000000..8a3c2354 --- /dev/null +++ b/tests/export/test_rdf_literal_escaping.py @@ -0,0 +1,107 @@ +"""Regression tests for #1098: Turtle/N-Triples literal escaping.""" +import pytest + +from semantica.export.rdf_exporter import RDFExporter, RDFSerializer + + +@pytest.fixture +def serializer(): + return RDFSerializer() + + +class TestTurtleLiteralEscaping: + def test_quote_in_text_is_escaped(self, serializer): + kg = { + "entities": [{"id": "e1", "text": 'He said "hello"', "type": "ORG"}], + "relationships": [], + } + turtle = serializer.serialize_to_turtle(kg) + assert '"He said \\"hello\\""' in turtle + + def test_backslash_in_text_is_escaped(self, serializer): + kg = { + "entities": [{"id": "e1", "text": r"path\to\file", "type": "ORG"}], + "relationships": [], + } + turtle = serializer.serialize_to_turtle(kg) + assert r"path\\to\\file" in turtle + + def test_newline_in_text_is_escaped(self, serializer): + kg = { + "entities": [{"id": "e1", "text": "line1\nline2", "type": "ORG"}], + "relationships": [], + } + turtle = serializer.serialize_to_turtle(kg) + assert "line1\\nline2" in turtle + + def test_tab_in_text_is_escaped(self, serializer): + kg = { + "entities": [{"id": "e1", "text": "a\tb", "type": "ORG"}], + "relationships": [], + } + turtle = serializer.serialize_to_turtle(kg) + assert "a\\tb" in turtle + + def test_plain_text_unchanged(self, serializer): + kg = { + "entities": [{"id": "e1", "text": "Apple Inc.", "type": "ORG"}], + "relationships": [], + } + turtle = serializer.serialize_to_turtle(kg) + assert 'semantica:text "Apple Inc."' in turtle + + +class TestNTriplesLiteralEscaping: + def test_quote_in_text_is_escaped(self, serializer): + kg = { + "entities": [{"id": "e1", "text": 'He said "hello"', "type": "ORG"}], + "relationships": [], + } + ntriples = serializer.serialize_to_ntriples(kg) + assert '\\"hello\\"' in ntriples + + def test_backslash_in_text_is_escaped(self, serializer): + kg = { + "entities": [{"id": "e1", "text": r"path\to\file", "type": "ORG"}], + "relationships": [], + } + ntriples = serializer.serialize_to_ntriples(kg) + assert r"path\\to\\file" in ntriples + + def test_newline_in_text_is_escaped(self, serializer): + kg = { + "entities": [{"id": "e1", "text": "line1\nline2", "type": "ORG"}], + "relationships": [], + } + ntriples = serializer.serialize_to_ntriples(kg) + assert "line1\\nline2" in ntriples + + def test_tab_in_text_is_escaped(self, serializer): + kg = { + "entities": [{"id": "e1", "text": "a\tb", "type": "ORG"}], + "relationships": [], + } + ntriples = serializer.serialize_to_ntriples(kg) + assert "a\\tb" in ntriples + + +class TestOWLTimeLiteralEscaping: + """Timestamp literals in OWL-Time turtle output must also be escaped.""" + + def test_owl_time_timestamps_are_escaped(self): + exporter = RDFExporter() + kg = { + "entities": [], + "relationships": [ + { + "id": "r1", + "source_id": "a", + "target_id": "b", + "type": "works_for", + "valid_from": "2020-01-01T00:00:00Z", + "valid_until": None, + } + ], + } + turtle = exporter.export_to_rdf(kg, format="turtle", include_temporal=True) + assert 'time:inXSDDateTimeStamp "2020-01-01T00:00:00Z"' in turtle \ No newline at end of file diff --git a/tests/export/test_timestamp_timezones.py b/tests/export/test_timestamp_timezones.py index df34dd29..b97b658e 100644 --- a/tests/export/test_timestamp_timezones.py +++ b/tests/export/test_timestamp_timezones.py @@ -107,20 +107,31 @@ def test_exported_timestamp_survives_a_timezone_qualified_sparql_filter(): assert len(rows) == 1, "the export was dropped by a timezone-qualified filter" -def test_document_iri_carrying_an_offset_is_a_valid_iri(): - """The offset puts '+' and ':' in the @id; both are legal in a path.""" +def test_document_iri_is_a_valid_iri(): + """The graph @id must be a valid IRI regardless of how it is minted. + + Before #1147, this @id was minted from the offset-carrying timestamp + itself (``+00:00`` interpolated straight into the path), so this test + asserted the offset survived without breaking IRI validity. #1147 mints + the @id from the graph's content instead, so the timestamp no longer + appears here at all โ€” it stays in ``semantica:exportedAt`` (still + offset-aware, per ``test_jsonld_export_timestamp_is_offset_aware`` above). + What's left worth guarding is the general case: whatever the @id is + minted from, it has to be a valid IRI that round-trips through RDF. + """ rdflib = pytest.importorskip("rdflib") document_iri = JSONExporter()._convert_kg_to_jsonld(KG)["@id"] - assert "+00:00" in document_iri assert rdflib.term._is_valid_uri(document_iri) graph = rdflib.Graph() - graph.add(( - rdflib.URIRef(document_iri), - rdflib.RDF.type, - rdflib.URIRef("https://semantica.dev/ns#KnowledgeGraph"), - )) + graph.add( + ( + rdflib.URIRef(document_iri), + rdflib.RDF.type, + rdflib.URIRef("https://semantica.dev/ns#KnowledgeGraph"), + ) + ) reparsed = rdflib.Graph().parse(data=graph.serialize(format="nt"), format="nt") assert document_iri in {str(s) for s in reparsed.subjects()} diff --git a/tests/ingest/test_db_ingestor_query.py b/tests/ingest/test_db_ingestor_query.py new file mode 100644 index 00000000..4c2fcfa3 --- /dev/null +++ b/tests/ingest/test_db_ingestor_query.py @@ -0,0 +1,112 @@ +"""Query-execution coverage for DBIngestor and DataExporter. + +Regression tests for #1015: ``sqlalchemy.text`` was imported function-locally inside +``DatabaseConnector.connect()`` and ``DatabaseConnector.test_connection()``, but called +in ``DataExporter.export_table_data()`` and ``DBIngestor.execute_query()``, which never +imported it. Both raised ``NameError``, re-wrapped by their ``except Exception`` handlers +into a ``ProcessingError`` reading "Failed to execute query: name 'text' is not defined" +-- a message that looks like a database problem rather than a missing import. + +Nothing caught it because no test exercised either method; the only ``execute_query`` +references under ``tests/`` are Mock stand-ins for the unrelated graph-store method of +the same name. + +These tests run against a temporary SQLite database, so they need no external service. +""" + +import os +import tempfile +import unittest + +try: + from sqlalchemy import create_engine, text + + SQLALCHEMY_AVAILABLE = True +except ImportError: # pragma: no cover - exercised only where sqlalchemy is absent + SQLALCHEMY_AVAILABLE = False + +from semantica.ingest.db_ingestor import DataExporter, DBIngestor + + +@unittest.skipUnless(SQLALCHEMY_AVAILABLE, "sqlalchemy is required for these tests") +class TestDBIngestorQueryExecution(unittest.TestCase): + """Both query paths must survive the call that needed sqlalchemy.text -- see #1015.""" + + def setUp(self): + # Register each cleanup as soon as the resource exists: tearDown is not + # called when setUp raises partway through, but addCleanup callbacks are. + self._tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(self._tmpdir.cleanup) + self.db_path = os.path.join(self._tmpdir.name, "test.db") + self.connection_string = f"sqlite:///{self.db_path}" + self.engine = create_engine(self.connection_string) + self.addCleanup(self.engine.dispose) + + with self.engine.begin() as conn: + conn.execute(text("CREATE TABLE widgets (id INTEGER, name TEXT)")) + for row_id, name in [(1, "alpha"), (2, "beta"), (3, "gamma")]: + conn.execute( + text("INSERT INTO widgets VALUES (:id, :name)"), + {"id": row_id, "name": name}, + ) + + def test_execute_query_returns_rows(self): + """DBIngestor.execute_query -- the text() call that raised NameError.""" + rows = DBIngestor().execute_query( + self.connection_string, + "SELECT id, name FROM widgets ORDER BY id", + ) + self.assertEqual( + rows, + [ + {"id": 1, "name": "alpha"}, + {"id": 2, "name": "beta"}, + {"id": 3, "name": "gamma"}, + ], + ) + + def test_execute_query_binds_parameters(self): + """The params argument is passed alongside text(), so cover it explicitly.""" + rows = DBIngestor().execute_query( + self.connection_string, + "SELECT name FROM widgets WHERE id = :wanted", + wanted=2, + ) + self.assertEqual(rows, [{"name": "beta"}]) + + def test_export_table_data_with_limit(self): + """Exercises the main text() call; the COUNT(*) branch is skipped when limit is set.""" + result = DataExporter().export_table_data(self.engine, "widgets", limit=2) + + self.assertEqual(result.table_name, "widgets") + self.assertEqual(len(result.rows), 2) + self.assertEqual(result.row_count, 2) + self.assertEqual([c["name"] for c in result.columns], ["id", "name"]) + + def test_export_table_data_without_limit_counts_rows(self): + """Covers the second text() call. + + ``export_table_data`` only issues its ``SELECT COUNT(*)`` when no ``limit`` is + passed, so the test above never reaches that line. Without this case one of the + three call sites the bug touched would stay untested. + """ + result = DataExporter().export_table_data(self.engine, "widgets") + + self.assertEqual(len(result.rows), 3) + self.assertEqual(result.row_count, 3) + + def test_export_table_data_honors_where_and_order(self): + """The WHERE/ORDER BY clauses are interpolated before text() wraps the query.""" + result = DataExporter().export_table_data( + self.engine, + "widgets", + where="id >= 2", + order_by="id DESC", + ) + + self.assertEqual([r["name"] for r in result.rows], ["gamma", "beta"]) + self.assertEqual(result.row_count, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ingest/test_notebook_02.py b/tests/ingest/test_notebook_02.py index 0cbb6a52..f99a8b37 100644 --- a/tests/ingest/test_notebook_02.py +++ b/tests/ingest/test_notebook_02.py @@ -157,7 +157,7 @@ class TestNotebook02DataIngestion: repo_ingestor = RepoIngestor() with patch.object(repo_ingestor, 'ingest_repository') as mock_ingest: mock_ingest.return_value = {'name': 'semantica'} - repo_data = repo_ingestor.ingest_repository("https://github.com/Hawksight-AI/semantica.git") + repo_data = repo_ingestor.ingest_repository("https://github.com/semantica-agi/semantica.git") assert repo_data['name'] == 'semantica' def test_07_email_ingestion(self): diff --git a/tests/ingest/test_ontology_named_graph.py b/tests/ingest/test_ontology_named_graph.py new file mode 100644 index 00000000..baa8dea9 --- /dev/null +++ b/tests/ingest/test_ontology_named_graph.py @@ -0,0 +1,92 @@ +"""A JSON-LD ontology whose terms live in a named graph must not be silently dropped. + +A JSON-LD document with a top-level ``@id`` *and* ``@graph`` places its terms in a NAMED +graph. ``rdflib.Graph.parse()`` loads only the default graph and discards the rest without +raising, so every class and property in such a document disappeared while the load reported +success โ€” see issue #1129 for the reproduction through the public API. + +This is the same ``Graph`` -> ``Dataset`` migration #757 made for ``JenaStore`` (#756); the +ingest path was not covered by it. +""" + +from __future__ import annotations + +import json + +import pytest + +from semantica.ingest.ontology_ingestor import OntologyIngestor + +NAMED_GRAPH_ONTOLOGY = { + "@context": { + "ex": "https://example.org/ns#", + "owl": "http://www.w3.org/2002/07/owl#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + }, + "@id": "https://example.org/ns", + "@type": "owl:Ontology", + "@graph": [ + {"@id": "ex:Thing", "@type": "owl:Class", "rdfs:label": "Thing"}, + {"@id": "ex:Other", "@type": "owl:Class", "rdfs:label": "Other"}, + { + "@id": "ex:relatesTo", + "@type": "owl:ObjectProperty", + "rdfs:domain": {"@id": "ex:Thing"}, + "rdfs:range": {"@id": "ex:Other"}, + }, + ], +} + +DEFAULT_GRAPH_ONTOLOGY = { + "@context": NAMED_GRAPH_ONTOLOGY["@context"], + "@graph": [ + {"@id": "ex:Thing", "@type": "owl:Class", "rdfs:label": "Thing"}, + {"@id": "ex:Other", "@type": "owl:Class", "rdfs:label": "Other"}, + ], +} + + +def _write(tmp_path, name, document): + path = tmp_path / name + path.write_text(json.dumps(document), encoding="utf-8") + return path + + +def test_terms_in_a_named_graph_are_ingested(tmp_path): + """The regression: two classes and one object property, all inside the named graph.""" + path = _write(tmp_path, "named.jsonld", NAMED_GRAPH_ONTOLOGY) + + data = OntologyIngestor().ingest_ontology(path).data + + assert len(data["classes"]) == 2, ( + "classes inside a JSON-LD named graph were dropped; the ingestor is reading only " + "the default graph" + ) + assert len(data["properties"]) == 1 + assert {c["uri"] for c in data["classes"]} == { + "https://example.org/ns#Thing", + "https://example.org/ns#Other", + } + + +def test_terms_in_the_default_graph_still_work(tmp_path): + """Canary for the test above: a document *without* a top-level ``@id`` keeps its terms + in the default graph and always parsed correctly. If this stopped passing, the fix would + have traded one blind spot for another.""" + path = _write(tmp_path, "default.jsonld", DEFAULT_GRAPH_ONTOLOGY) + + data = OntologyIngestor().ingest_ontology(path).data + + assert len(data["classes"]) == 2 + + +@pytest.mark.parametrize("document", [NAMED_GRAPH_ONTOLOGY, DEFAULT_GRAPH_ONTOLOGY]) +def test_metadata_reports_what_was_actually_read(tmp_path, document): + """Whatever the shape of the document, the counts reported have to match the terms + returned โ€” a load that says it succeeded while returning nothing is what made #1129 + cost an afternoon to find.""" + path = _write(tmp_path, "any.jsonld", document) + + result = OntologyIngestor().ingest_ontology(path) + + assert result.data["classes"], "reported success with zero classes" diff --git a/tests/integrations/agno/test_load_urls_ssrf.py b/tests/integrations/agno/test_load_urls_ssrf.py new file mode 100644 index 00000000..28ad1cbe --- /dev/null +++ b/tests/integrations/agno/test_load_urls_ssrf.py @@ -0,0 +1,278 @@ +"""SSRF regression tests for AgnoKnowledgeGraph.load_urls(). + +Prior to the fix, load_urls() used urllib.request.urlopen with only a +scheme check โ€” private/loopback/link-local/metadata IPs were not blocked +and redirects were followed without re-validation. + +These tests exercise the real SSRF guard (no mock of request_with_ssrf_guard +itself) by patching at the socket.getaddrinfo level, confirming that +blocked addresses never reach the network layer. +""" + +from __future__ import annotations + +import socket +from unittest.mock import MagicMock, patch + +import pytest + +# conftest.py installs the full agno stub before this file is collected. +from integrations.agno.knowledge_graph import AgnoKnowledgeGraph + +from semantica.utils.exceptions import ValidationError + + +# --------------------------------------------------------------------------- +# Minimal fakes so AgnoKnowledgeGraph.__init__ succeeds without real imports. +# --------------------------------------------------------------------------- +class _FakeNER: + def extract_entities(self, text): + return [] + + +class _FakeRelExtractor: + def extract_relations(self, text, entities=None): + return [] + + +class _FakeGraphBuilder: + def build(self, sources): + pass + + +class _FakeContextGraph: + def find_nodes(self, label=None): + return [] + + def get_neighbors(self, node_id=None, hops=1): + return [] + + +def _make_kg() -> AgnoKnowledgeGraph: + return AgnoKnowledgeGraph( + graph_builder=_FakeGraphBuilder(), + ner_extractor=_FakeNER(), + relation_extractor=_FakeRelExtractor(), + context_graph=_FakeContextGraph(), + ) + + +def _public_getaddrinfo(host, *args, **kwargs): + """Stub that makes every hostname resolve to a public IP.""" + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0))] + + +# --------------------------------------------------------------------------- +# Tests: blocked addresses must never be fetched +# --------------------------------------------------------------------------- + +class TestLoadUrlsBlockedAddresses: + """load_urls() must silently skip (warn) any URL that fails the SSRF guard.""" + + @pytest.mark.parametrize("url", [ + "http://127.0.0.1/secret", + "http://127.0.0.1:9200/", # common internal service port + "http://0.0.0.0/", + "http://169.254.169.254/latest/meta-data/", + "http://169.254.169.254/computeMetadata/v1/", + "http://10.0.0.1/internal", + "http://10.255.255.255/", + "http://172.16.0.1/", + "http://172.31.255.255/", + "http://192.168.0.1/admin", + "http://192.168.100.200/", + "http://[::1]/ipv6-loopback", + "http://[fc00::1]/ipv6-ula", + "http://[fe80::1]/ipv6-link-local", + ]) + def test_blocked_ip_never_reaches_network(self, url): + """Blocked addresses must raise ValidationError inside the guard, + which load_urls() catches and logs โ€” _ingest_text must NOT be called.""" + kg = _make_kg() + with patch.object(kg, "_ingest_text") as mock_ingest: + kg.load_urls([url]) + mock_ingest.assert_not_called() + + def test_localhost_hostname_blocked(self): + kg = _make_kg() + with patch.object(kg, "_ingest_text") as mock_ingest: + kg.load_urls(["http://localhost/admin"]) + mock_ingest.assert_not_called() + + def test_localhost_subdomain_blocked(self): + kg = _make_kg() + with patch.object(kg, "_ingest_text") as mock_ingest: + kg.load_urls(["http://foo.localhost/"]) + mock_ingest.assert_not_called() + + def test_hostname_resolving_to_private_ip_blocked(self): + """A hostname that resolves to a private IP must be blocked even though + the URL string itself looks like a normal hostname.""" + def _internal_getaddrinfo(host, *a, **kw): + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.5", 0))] + + kg = _make_kg() + with patch("semantica.ingest.ssrf.socket.getaddrinfo", side_effect=_internal_getaddrinfo): + with patch.object(kg, "_ingest_text") as mock_ingest: + kg.load_urls(["http://internal.corp/secret"]) + mock_ingest.assert_not_called() + + def test_hostname_resolving_to_metadata_ip_blocked(self): + def _meta_getaddrinfo(host, *a, **kw): + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("169.254.169.254", 0))] + + kg = _make_kg() + with patch("semantica.ingest.ssrf.socket.getaddrinfo", side_effect=_meta_getaddrinfo): + with patch.object(kg, "_ingest_text") as mock_ingest: + kg.load_urls(["http://metadata.internal/v1/token"]) + mock_ingest.assert_not_called() + + +class TestLoadUrlsNonHttpSchemes: + """Non-HTTP(S) schemes must be rejected.""" + + @pytest.mark.parametrize("url", [ + "file:///etc/passwd", + "file://localhost/etc/shadow", + "ftp://example.com/file.txt", + "gopher://example.com/1", + "dict://example.com/", + "sftp://example.com/data", + ]) + def test_non_http_scheme_blocked(self, url): + kg = _make_kg() + with patch.object(kg, "_ingest_text") as mock_ingest: + kg.load_urls([url]) + mock_ingest.assert_not_called() + + +class TestLoadUrlsRedirects: + """Redirects to private/blocked addresses must be rejected.""" + + def test_redirect_to_loopback_blocked(self): + """A public first hop that redirects to loopback must be blocked.""" + redirect = MagicMock() + redirect.status_code = 302 + redirect.headers = {"Location": "http://127.0.0.1/secret"} + redirect.close = MagicMock() + + kg = _make_kg() + with patch( + "semantica.ingest.ssrf.socket.getaddrinfo", + return_value=[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0))], + ): + # Patch requests.Session so the first hop returns our redirect mock. + # The guard sees the 302, then validates the Location โ€” 127.0.0.1 is + # blocked without a second network call. + with patch("semantica.ingest.ssrf.requests.Session") as MockSession: + mock_session = MockSession.return_value + mock_session.adapters = {} + mock_session.headers = {} + mock_session.auth = None + mock_session.trust_env = True + mock_session.request.return_value = redirect + + with patch.object(kg, "_ingest_text") as mock_ingest: + kg.load_urls(["https://example.com/start"]) + mock_ingest.assert_not_called() + + def test_redirect_to_metadata_ip_blocked(self): + """Redirect to cloud metadata endpoint must be blocked.""" + redirect = MagicMock() + redirect.status_code = 301 + redirect.headers = {"Location": "http://169.254.169.254/latest/meta-data/"} + redirect.close = MagicMock() + + kg = _make_kg() + with patch( + "semantica.ingest.ssrf.socket.getaddrinfo", + return_value=[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0))], + ): + with patch("semantica.ingest.ssrf.requests.Session") as MockSession: + mock_session = MockSession.return_value + mock_session.adapters = {} + mock_session.headers = {} + mock_session.auth = None + mock_session.trust_env = True + mock_session.request.return_value = redirect + + with patch.object(kg, "_ingest_text") as mock_ingest: + kg.load_urls(["https://example.com/redirect-me"]) + mock_ingest.assert_not_called() + + +class TestLoadUrlsValidUrls: + """Valid public URLs must succeed and call _ingest_text.""" + + def test_valid_public_url_ingested(self): + """A URL resolving to a public IP must be fetched and ingested.""" + ok_response = MagicMock() + ok_response.status_code = 200 + ok_response.headers = {} + ok_response.text = "This is the document content." + + kg = _make_kg() + with patch("semantica.ingest.ssrf.socket.getaddrinfo", side_effect=_public_getaddrinfo): + with patch("semantica.ingest.ssrf.requests.Session") as MockSession: + mock_session = MockSession.return_value + mock_session.adapters = {} + mock_session.headers = {} + mock_session.auth = None + mock_session.trust_env = True + mock_session.request.return_value = ok_response + + with patch.object(kg, "_ingest_text") as mock_ingest: + kg.load_urls(["https://example.com/doc.txt"]) + + mock_ingest.assert_called_once_with( + "This is the document content.", source="https://example.com/doc.txt" + ) + + def test_multiple_urls_each_independently_validated(self): + """Each URL in the list is independently validated; one blocked URL + must not prevent valid subsequent URLs from being ingested.""" + ok_response = MagicMock() + ok_response.status_code = 200 + ok_response.headers = {} + ok_response.text = "Valid content." + + def _selective_getaddrinfo(host, *a, **kw): + if host == "internal.corp": + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.5", 0))] + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0))] + + kg = _make_kg() + with patch("semantica.ingest.ssrf.socket.getaddrinfo", side_effect=_selective_getaddrinfo): + with patch("semantica.ingest.ssrf.requests.Session") as MockSession: + mock_session = MockSession.return_value + mock_session.adapters = {} + mock_session.headers = {} + mock_session.auth = None + mock_session.trust_env = True + mock_session.request.return_value = ok_response + + with patch.object(kg, "_ingest_text") as mock_ingest: + kg.load_urls([ + "http://internal.corp/secret", # blocked + "https://example.com/public.txt", # allowed + ]) + + # Only the valid URL triggers ingestion + mock_ingest.assert_called_once_with("Valid content.", source="https://example.com/public.txt") + + def test_failed_fetch_does_not_raise(self): + """A network failure on a valid URL must log a warning, not raise.""" + import requests as _requests + + kg = _make_kg() + with patch("semantica.ingest.ssrf.socket.getaddrinfo", side_effect=_public_getaddrinfo): + with patch("semantica.ingest.ssrf.requests.Session") as MockSession: + mock_session = MockSession.return_value + mock_session.adapters = {} + mock_session.headers = {} + mock_session.auth = None + mock_session.trust_env = True + mock_session.request.side_effect = _requests.exceptions.ConnectionError("refused") + + # Must not raise; failure is logged and skipped + kg.load_urls(["https://example.com/unreachable"]) diff --git a/tests/integrations/langchain/test_degradation.py b/tests/integrations/langchain/test_degradation.py new file mode 100644 index 00000000..b50ce76c --- /dev/null +++ b/tests/integrations/langchain/test_degradation.py @@ -0,0 +1,92 @@ +""" +Graceful-degradation tests for the LangChain integration. + +Runs the adapters in a fresh subprocess with langchain-core hidden, so the +object-base path is proven even when this env has langchain-core installed. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from types import SimpleNamespace + +import pytest + +from integrations.langchain import ( + LANGCHAIN_AVAILABLE, + SemanticaDecisionTool, + SemanticaKGTool, +) + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) + +_SCRIPT = r""" +import sys +from types import SimpleNamespace + +class _BlockLangchain: + def find_spec(self, fullname, path=None, target=None): + if fullname == "langchain_core" or fullname.startswith("langchain_core."): + raise ImportError("langchain_core blocked for degradation test") + return None + +sys.meta_path.insert(0, _BlockLangchain()) +for name in list(sys.modules): + if name == "langchain_core" or name.startswith("langchain_core."): + del sys.modules[name] + +from integrations.langchain.retriever import LANGCHAIN_AVAILABLE as RET_AVAIL +from integrations.langchain.vectorstore import ( + LANGCHAIN_AVAILABLE as VS_AVAIL, + SemanticaVectorStore, +) +from integrations.langchain.tools import ( + LANGCHAIN_AVAILABLE as TOOL_AVAIL, + SemanticaKGTool, + SemanticaDecisionTool, +) +from integrations.langchain.retriever import SemanticaRetriever, _get_document + +assert RET_AVAIL is False and VS_AVAIL is False and TOOL_AVAIL is False + +retriever = SemanticaRetriever(graph=SimpleNamespace(), hops=2) +assert retriever.hops == 2 + +store = SemanticaVectorStore(hybrid=SimpleNamespace(), tags=["x"]) +assert store.hybrid is not None + +graph = SimpleNamespace(query=lambda q, limit=10: [{"q": q, "limit": limit}]) +assert SemanticaKGTool(graph).build() is None +assert SemanticaDecisionTool(graph).build() is None + +try: + _get_document(page_content="x") + raise SystemExit("expected RuntimeError from _get_document") +except RuntimeError as exc: + assert "langchain-core" in str(exc) + +print("DEGRADATION_OK") +""" + + +def test_importable_and_functional_without_langchain(): + result = subprocess.run( + [sys.executable, "-c", _SCRIPT], + cwd=REPO_ROOT, + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, ( + f"subprocess failed:\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" + ) + assert "DEGRADATION_OK" in result.stdout + + +@pytest.mark.skipif(LANGCHAIN_AVAILABLE, reason="langchain-core is installed") +def test_tools_build_returns_none_without_langchain(): + graph = SimpleNamespace() + assert SemanticaKGTool(graph).build() is None + assert SemanticaDecisionTool(graph).build() is None diff --git a/tests/integrations/langchain/test_langchain_integration.py b/tests/integrations/langchain/test_langchain_integration.py new file mode 100644 index 00000000..88812e06 --- /dev/null +++ b/tests/integrations/langchain/test_langchain_integration.py @@ -0,0 +1,231 @@ +""" +Tests for integrations/langchain. + +Adapter behavior is always exercised (hit parsing, seed/fallback, tool JSON). +LangChain-present paths use pytest.importorskip; degradation without +langchain-core is covered in test_degradation.py via a subprocess so it still +runs when langchain-core is installed in this env. +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from integrations.langchain import ( + LANGCHAIN_AVAILABLE, + SemanticaDecisionTool, + SemanticaKGTool, + SemanticaRetriever, + SemanticaVectorStore, +) +from integrations.langchain.retriever import _hit_content, _hit_id, _hit_type +from integrations.langchain.tools import QueryDecisionsInput, QueryGraphInput + +# HybridSearch.search() returns {id, score, distance, metadata} โ€” content lives +# inside metadata, and id is a vector id, not a graph node id. +_HYBRID_HIT = { + "id": "vec_0", + "score": 0.91, + "distance": 0.09, + "metadata": { + "node_id": "alice", + "content": "Alice is a developer", + "node_type": "person", + "source": "graph", + }, +} + + +def test_exports_exist(): + assert callable(SemanticaRetriever) + assert callable(SemanticaVectorStore) + assert callable(SemanticaKGTool) + assert callable(SemanticaDecisionTool) + + +def test_version(): + from integrations.langchain import __version__ + + assert __version__ == "0.1.0" + + +# --------------------------------------------------------------------------- +# Hit parsing (the Qodo high-severity finding) +# --------------------------------------------------------------------------- +def test_hit_id_prefers_metadata_node_id_over_vector_id(): + assert _hit_id(_HYBRID_HIT) == "alice" + assert _hit_id({"node_id": "n1"}) == "n1" + assert _hit_id({"id": "n2"}) == "n2" + + +def test_hit_id_unwraps_context_graph_query_shape(): + hit = { + "node": { + "id": "alice", + "type": "person", + "properties": {"content": "Alice"}, + }, + "score": 1.0, + "content": "Alice is a developer", + } + assert _hit_id(hit) == "alice" + assert _hit_content(hit) == "Alice is a developer" + assert _hit_type(hit) == "person" + + +def test_hit_content_and_type_read_nested_metadata(): + assert _hit_content(_HYBRID_HIT) == "Alice is a developer" + assert _hit_type(_HYBRID_HIT) == "person" + assert _hit_content({"id": "x"}) == "" + + +# --------------------------------------------------------------------------- +# Retriever +# --------------------------------------------------------------------------- +def test_empty_seed_returns_empty(): + graph = MagicMock() + graph.query.return_value = [] + retriever = SemanticaRetriever(graph=graph, top_k=5) + assert retriever._seed_results("query") == [] + assert retriever.hops == 2 + + +def test_seed_uses_hybrid_when_provided(): + graph = MagicMock() + hybrid = MagicMock() + hybrid.search.return_value = [_HYBRID_HIT] + retriever = SemanticaRetriever(graph=graph, hybrid=hybrid) + results = retriever._seed_results("query") + assert len(results) == 1 + hybrid.search.assert_called_once_with("query", k=10) + + +def test_graph_fallback_when_hybrid_fails(): + graph = MagicMock() + graph.query.return_value = [{"node_id": "n1", "content": "c1"}] + hybrid = MagicMock() + hybrid.search.side_effect = RuntimeError("down") + retriever = SemanticaRetriever(graph=graph, hybrid=hybrid) + results = retriever._seed_results("query") + assert len(results) == 1 + graph.query.assert_called_once() + + +def test_retriever_reads_hybrid_metadata_and_expands_by_node_id(): + pytest.importorskip("langchain_core") + graph = MagicMock() + graph.get_neighbors.return_value = [ + { + "id": "bob", + "type": "person", + "content": "Bob reports to Alice", + "weight": 0.8, + } + ] + hybrid = MagicMock() + hybrid.search.return_value = [_HYBRID_HIT] + retriever = SemanticaRetriever(graph=graph, hybrid=hybrid) + docs = retriever._get_relevant_documents("Alice") + assert docs[0].page_content == "Alice is a developer" + assert docs[0].metadata["node_id"] == "alice" + assert docs[0].metadata["source"] == "graph" + graph.get_neighbors.assert_called_once_with("alice", hops=2) + assert [d.metadata["node_id"] for d in docs] == ["alice", "bob"] + + +# --------------------------------------------------------------------------- +# VectorStore +# --------------------------------------------------------------------------- +def test_add_texts_delegates_to_vector_store(): + vs = MagicMock() + vs.add_documents.return_value = ["id1"] + store = SemanticaVectorStore(hybrid=MagicMock(), vector_store=vs) + assert store.add_texts(["hello"]) == ["id1"] + vs.add_documents.assert_called_once() + + +def test_add_texts_raises_without_vector_store(): + store = SemanticaVectorStore(hybrid=SimpleNamespace(vector_store=None)) + with pytest.raises(ValueError): + store.add_texts(["hello"]) + + +def test_from_texts_requires_hybrid_kwarg(): + with pytest.raises(ValueError): + SemanticaVectorStore.from_texts(["hello"], embedding=None) + + +def test_vectorstore_reads_hybrid_metadata(): + pytest.importorskip("langchain_core") + hybrid = MagicMock() + hybrid.search.return_value = [_HYBRID_HIT] + store = SemanticaVectorStore(hybrid=hybrid) + docs = store.similarity_search("Alice", k=1) + assert docs[0].page_content == "Alice is a developer" + assert docs[0].metadata["node_id"] == "alice" + assert docs[0].metadata["source"] == "graph" + pairs = store.similarity_search_with_score("Alice", k=1) + assert pairs[0][0].page_content == "Alice is a developer" + assert pairs[0][1] == pytest.approx(0.91) + + +# --------------------------------------------------------------------------- +# Tools โ€” JSON payload + BaseTool contract +# --------------------------------------------------------------------------- +def test_kg_tool_returns_full_valid_json(): + graph = MagicMock() + graph.query.return_value = [{"content": "x" * 5000, "id": i} for i in range(3)] + raw = SemanticaKGTool(graph)._run("q", limit=3) + parsed = json.loads(raw) + assert len(parsed) == 3 + assert len(parsed[0]["content"]) == 5000 + graph.query.assert_called_once_with("q", limit=3) + + +def test_tool_errors_are_json(): + graph = MagicMock() + graph.query.side_effect = RuntimeError("boom") + assert json.loads(SemanticaKGTool(graph)._run("q")) == {"error": "boom"} + assert json.loads(SemanticaDecisionTool(graph)._run("q")) == {"error": "boom"} + + +def test_decision_tool_empty_category_uses_insights(): + graph = MagicMock() + graph.get_decision_insights.return_value = {"n": 0} + assert json.loads(SemanticaDecisionTool(graph)._run("")) == {"n": 0} + + +def test_tools_are_base_tools_with_args_schema(): + pytest.importorskip("langchain_core") + from langchain_core.tools import BaseTool + + graph = MagicMock() + graph.query.return_value = [{"hit": True}] + kg = SemanticaKGTool(graph) + dec = SemanticaDecisionTool(graph) + assert isinstance(kg, BaseTool) + assert isinstance(dec, BaseTool) + assert kg.args_schema is QueryGraphInput + assert dec.args_schema is QueryDecisionsInput + assert kg.build() is kg + parsed = json.loads(kg.invoke({"query": "Alice", "limit": 5})) + assert parsed == [{"hit": True}] + + +@pytest.mark.skipif(not LANGCHAIN_AVAILABLE, reason="langchain-core not installed") +def test_kg_tool_invoke_with_context_graph(): + pytest.importorskip("langchain_core") + try: + from semantica.context import ContextGraph + except ImportError: + pytest.skip("ContextGraph import requires optional core deps") + + graph = ContextGraph() + graph.add_node(node_id="alice", node_type="person", content="Alice is a developer") + result = SemanticaKGTool(graph).invoke({"query": "Alice", "limit": 5}) + assert "Alice" in result + json.loads(result) diff --git a/tests/integrations/openclaw/__init__.py b/tests/integrations/openclaw/__init__.py new file mode 100644 index 00000000..6def8d9a --- /dev/null +++ b/tests/integrations/openclaw/__init__.py @@ -0,0 +1 @@ +# tests/integrations/openclaw package diff --git a/tests/integrations/openclaw/test_mcp_tool_ssrf.py b/tests/integrations/openclaw/test_mcp_tool_ssrf.py new file mode 100644 index 00000000..4c75c772 --- /dev/null +++ b/tests/integrations/openclaw/test_mcp_tool_ssrf.py @@ -0,0 +1,290 @@ +"""SSRF hardening tests for OpenClawKGTool. + +OpenClawKGTool is designed to speak to a locally-running Semantica REST server +(default: http://localhost:8000). The fix validates base_url at construction +time so that obviously wrong schemes (file://, ftp://, gopher://, etc.) and +malformed URLs are rejected immediately, while localhost and other private +addresses remain valid because allow_private_ips=True is the correct posture +for this tool's intended use case. + +These are construction-time tests; per-request SSRF guarding is not the +contract of this tool (its threat model is operator-configured base_url, not +untrusted per-call URLs). +""" + +from __future__ import annotations + +import pytest + +from integrations.openclaw.mcp_tool import OpenClawKGTool +from semantica.utils.exceptions import ValidationError + + +class TestOpenClawKGToolBaseUrlValidation: + """base_url is validated at __init__ time.""" + + # ------------------------------------------------------------------ + # Valid base_urls โ€” all must construct without raising + # ------------------------------------------------------------------ + + @pytest.mark.parametrize("url", [ + "http://localhost:8000", + "http://localhost", + "http://127.0.0.1:8000", + "http://127.0.0.1", + "https://localhost:8443", + "http://0.0.0.0:8000", + "http://192.168.1.10:8000", # LAN Semantica server + "http://10.0.0.5:8000", # corporate intranet deployment + "https://semantica.internal/api", + "https://semantica.example.com", + ]) + def test_valid_base_url_accepted(self, url): + """All reasonable operator-configured base_urls must be accepted.""" + tool = OpenClawKGTool(base_url=url) + assert tool.base_url == url.rstrip("/") + + # ------------------------------------------------------------------ + # Invalid schemes โ€” must raise at construction + # ------------------------------------------------------------------ + + @pytest.mark.parametrize("url", [ + "file:///etc/passwd", + "file://localhost/etc/shadow", + "ftp://example.com/", + "gopher://example.com/1", + "dict://example.com/", + "sftp://example.com/", + "ldap://example.com/", + "javascript:alert(1)", + ]) + def test_invalid_scheme_rejected(self, url): + """Non-HTTP(S) schemes must be rejected at construction time.""" + with pytest.raises((ValidationError, ValueError)): + OpenClawKGTool(base_url=url) + + # ------------------------------------------------------------------ + # Malformed URLs + # ------------------------------------------------------------------ + + def test_empty_string_rejected(self): + with pytest.raises((ValidationError, ValueError)): + OpenClawKGTool(base_url="") + + def test_no_scheme_rejected(self): + """A bare hostname without a scheme must be rejected.""" + with pytest.raises((ValidationError, ValueError)): + OpenClawKGTool(base_url="localhost:8000") + + def test_whitespace_only_rejected(self): + with pytest.raises((ValidationError, ValueError)): + OpenClawKGTool(base_url=" ") + + # ------------------------------------------------------------------ + # Default is the documented localhost value + # ------------------------------------------------------------------ + + def test_default_base_url_is_localhost(self): + """The default must remain http://localhost:8000 for backward compat.""" + tool = OpenClawKGTool() + assert tool.base_url == "http://localhost:8000" + + def test_trailing_slash_stripped(self): + """base_url trailing slash must be stripped so paths concatenate cleanly.""" + tool = OpenClawKGTool(base_url="http://localhost:8000/") + assert tool.base_url == "http://localhost:8000" + + def test_multiple_trailing_slashes_stripped(self): + tool = OpenClawKGTool(base_url="http://localhost:8000///") + assert tool.base_url == "http://localhost:8000" + + def test_leading_and_trailing_whitespace_stripped(self): + """Whitespace around a valid URL must be stripped before storage so + _post/_get don't build requests with space-padded URLs like + ' http://localhost:8000 /extract'.""" + tool = OpenClawKGTool(base_url=" http://localhost:8000 ") + assert tool.base_url == "http://localhost:8000" + + def test_whitespace_plus_trailing_slash_both_stripped(self): + tool = OpenClawKGTool(base_url=" http://localhost:8000/ ") + assert tool.base_url == "http://localhost:8000" + + +class TestOpenClawKGToolFallbackValidation: + """When semantica.ingest.ssrf is unavailable (ImportError path), the fallback + must perform the same structural checks as validate_url_for_request: + non-empty string, http/https scheme, netloc present, hostname present. + + The fallback is exercised by temporarily hiding semantica.ingest.ssrf + from sys.modules so the import inside __init__ raises ImportError. + """ + + @staticmethod + def _hide_ssrf(monkeypatch): + """Return a context in which semantica.ingest.ssrf appears unimportable.""" + import sys + monkeypatch.setitem(sys.modules, "semantica.ingest.ssrf", None) + + # ------------------------------------------------------------------ + # Valid URLs must still be accepted in the fallback path + # ------------------------------------------------------------------ + + @pytest.mark.parametrize("url", [ + "http://localhost:8000", + "http://127.0.0.1:8000", + "https://semantica.example.com", + ]) + def test_fallback_valid_url_accepted(self, url, monkeypatch): + self._hide_ssrf(monkeypatch) + tool = OpenClawKGTool(base_url=url) + assert tool.base_url == url.rstrip("/") + + # ------------------------------------------------------------------ + # Malformed URLs that the fallback previously let through + # ------------------------------------------------------------------ + + @pytest.mark.parametrize("url", [ + "http://", # scheme only, no netloc or hostname + "https://", # same + "http:///path", # empty hostname (netloc is present but hostname is None) + ]) + def test_fallback_no_netloc_rejected(self, url, monkeypatch): + """URLs with a valid scheme but missing netloc/hostname must be rejected + in the fallback path, matching validate_url_for_request's behaviour.""" + self._hide_ssrf(monkeypatch) + with pytest.raises((ValidationError, ValueError)): + OpenClawKGTool(base_url=url) + + def test_fallback_empty_string_rejected(self, monkeypatch): + self._hide_ssrf(monkeypatch) + with pytest.raises((ValidationError, ValueError)): + OpenClawKGTool(base_url="") + + def test_fallback_whitespace_only_rejected(self, monkeypatch): + self._hide_ssrf(monkeypatch) + with pytest.raises((ValidationError, ValueError)): + OpenClawKGTool(base_url=" ") + + def test_fallback_invalid_scheme_rejected(self, monkeypatch): + self._hide_ssrf(monkeypatch) + with pytest.raises((ValidationError, ValueError)): + OpenClawKGTool(base_url="file:///etc/passwd") + + def test_fallback_no_scheme_rejected(self, monkeypatch): + self._hide_ssrf(monkeypatch) + with pytest.raises((ValidationError, ValueError)): + OpenClawKGTool(base_url="localhost:8000") + + def test_fallback_whitespace_padded_valid_url_stored_clean(self, monkeypatch): + """Whitespace around a valid URL must be stripped before storage in the + fallback path too โ€” same guarantee as the normal path.""" + self._hide_ssrf(monkeypatch) + tool = OpenClawKGTool(base_url=" http://localhost:8000 ") + assert tool.base_url == "http://localhost:8000" + + +class TestOpenClawKGToolEndpointConstruction: + """Verify that per-method URLs are assembled from base_url + hardcoded paths. + + The endpoint strings are always literals defined in the class body โ€” + they are not caller-supplied โ€” so these tests confirm the URL assembly + logic is correct rather than testing SSRF guards on the endpoints. + + All HTTP calls are mocked so no real network connection is made. + """ + + def _mock_session(self, status: int = 200, body: bytes = b"{}") -> "MagicMock": + """Return a mock session whose post/get return a minimal JSON response.""" + from unittest.mock import MagicMock + mock_resp = MagicMock() + mock_resp.status_code = status + mock_resp.raise_for_status = MagicMock() + mock_resp.json.return_value = {} + session = MagicMock() + session.post.return_value = mock_resp + session.get.return_value = mock_resp + return session + + def test_post_url_constructed_from_base_url(self): + """_post must call session.post with the exact URL base_url+endpoint, + the supplied payload as json=, and the tool timeout. No real connection.""" + from unittest.mock import patch + + tool = OpenClawKGTool(base_url="http://localhost:8000") + mock_session = self._mock_session() + + with patch.object(tool, "_get_session", return_value=mock_session): + tool._post("/extract", {"text": "hello"}) + + mock_session.post.assert_called_once_with( + "http://localhost:8000/extract", + json={"text": "hello"}, + timeout=30, + ) + + def test_post_url_with_custom_base_url(self): + """base_url is reflected correctly in the outbound URL for _post.""" + from unittest.mock import patch + + tool = OpenClawKGTool(base_url="http://192.168.1.10:9000") + mock_session = self._mock_session() + + with patch.object(tool, "_get_session", return_value=mock_session): + tool._post("/decisions", {"decision": "deploy"}) + + mock_session.post.assert_called_once_with( + "http://192.168.1.10:9000/decisions", + json={"decision": "deploy"}, + timeout=30, + ) + + def test_get_url_constructed_from_base_url(self): + """_get must call session.get with the exact URL base_url+endpoint, + params={} when none are supplied, and the tool timeout.""" + from unittest.mock import patch + + tool = OpenClawKGTool(base_url="http://localhost:8000") + mock_session = self._mock_session() + + with patch.object(tool, "_get_session", return_value=mock_session): + tool._get("/analytics") + + mock_session.get.assert_called_once_with( + "http://localhost:8000/analytics", + params={}, + timeout=30, + ) + + def test_get_url_with_params(self): + """_get must forward supplied params to session.get.""" + from unittest.mock import patch + + tool = OpenClawKGTool(base_url="http://localhost:8000") + mock_session = self._mock_session() + + with patch.object(tool, "_get_session", return_value=mock_session): + tool._get("/decisions/search", {"q": "deploy", "limit": 5}) + + mock_session.get.assert_called_once_with( + "http://localhost:8000/decisions/search", + params={"q": "deploy", "limit": 5}, + timeout=30, + ) + + def test_custom_timeout_forwarded(self): + """A non-default timeout must reach session.post and session.get.""" + from unittest.mock import patch + + tool = OpenClawKGTool(base_url="http://localhost:8000", timeout=60) + mock_session = self._mock_session() + + with patch.object(tool, "_get_session", return_value=mock_session): + tool._post("/extract", {"text": "x"}) + tool._get("/analytics") + + assert mock_session.post.call_args.kwargs["timeout"] == 60 + assert mock_session.get.call_args.kwargs["timeout"] == 60 + + def test_repr_includes_base_url(self): + tool = OpenClawKGTool(base_url="http://localhost:9000") + assert "http://localhost:9000" in repr(tool) diff --git a/tests/kg/test_kg.py b/tests/kg/test_kg.py index a983694d..4d02b1e4 100644 --- a/tests/kg/test_kg.py +++ b/tests/kg/test_kg.py @@ -407,6 +407,69 @@ class TestTemporalGraphQuery(unittest.TestCase): self.assertEqual(result["num_relationships"], 1) + def test_analyze_evolution_stability_is_mean_duration_seconds(self): + day = 86400.0 + graph = { + "relationships": [ + { + "source": "1", + "target": "2", + "type": "a", + "valid_from": "2024-01-01", + "valid_until": "2024-01-02", # 1 day + }, + { + "source": "2", + "target": "3", + "type": "b", + "valid_from": "2024-01-01", + "valid_until": "2024-01-04", # 3 days + }, + ] + } + + result = self.query_engine.analyze_evolution(graph, metrics=["stability"]) + + # Mean of 1-day and 3-day durations == 2 days in seconds. + self.assertAlmostEqual(result["stability"], 2 * day) + + def test_analyze_evolution_stability_skips_unbounded_intervals(self): + graph = { + "relationships": [ + { + "source": "1", + "target": "2", + "type": "bounded", + "valid_from": "2024-01-01", + "valid_until": "2024-01-02", + }, + { + "source": "2", + "target": "3", + "type": "open", + "valid_from": "2024-01-01", + "valid_until": TemporalBound.OPEN, + }, + { + "source": "3", + "target": "4", + "type": "no-start", + "valid_until": "2024-06-01", + }, + ] + } + + result = self.query_engine.analyze_evolution(graph, metrics=["stability"]) + + # Only the fully bounded relationship contributes (1 day). + self.assertAlmostEqual(result["stability"], 86400.0) + + def test_analyze_evolution_stability_empty_is_zero(self): + result = self.query_engine.analyze_evolution( + {"relationships": []}, metrics=["stability"] + ) + self.assertEqual(result["stability"], 0) + def test_query_at_time_legacy_transaction_axis_uses_valid_from_when_recorded_missing(self): graph = { "relationships": [ diff --git a/tests/normalize/test_number_normalizer.py b/tests/normalize/test_number_normalizer.py index a7cf359c..2e06260c 100644 --- a/tests/normalize/test_number_normalizer.py +++ b/tests/normalize/test_number_normalizer.py @@ -1,4 +1,6 @@ import unittest + +from semantica.utils.exceptions import ValidationError from semantica.normalize.number_normalizer import ( NumberNormalizer, UnitConverter, @@ -32,6 +34,19 @@ class TestUnitConverter(unittest.TestCase): # 1 kg = 1000 g self.assertEqual(self.converter.convert_units(1, "kg", "g"), 1000.0) + def test_convert_accepts_aliases_for_category_validation(self): + # Aliases are part of the documented API, not just parsing syntax. + self.assertEqual(self.converter.convert_units(1, "feet", "m"), 0.3048) + self.assertEqual(self.converter.convert_units(1, "gal", "liter"), 3.78541) + + def test_convert_rejects_mismatched_categories_even_for_aliases(self): + # Both units normalize to canonical names first, so the category + # check sees real categories and rejects cross-category conversions. + with self.assertRaises(ValidationError): + self.converter.convert_units(1, "kg", "ft") + with self.assertRaises(ValidationError): + self.converter.convert_units(1, "gal", "lb") + def test_normalize_unit(self): self.assertEqual(self.converter.normalize_unit("km"), "kilometer") self.assertEqual(self.converter.normalize_unit("kgs"), "kilogram") diff --git a/tests/ontology/test_ontology_advanced.py b/tests/ontology/test_ontology_advanced.py index 98d5dc6e..cf77dbe0 100644 --- a/tests/ontology/test_ontology_advanced.py +++ b/tests/ontology/test_ontology_advanced.py @@ -340,6 +340,27 @@ class TestSHACLHierarchicalAndValidation(unittest.TestCase): ttl = gen.serialize(graph, format="turtle") self.assertIn("myorg.com", ttl) + # 24a โ€” an RDF namespace ending in `#` must not gain a trailing `/`, + # or every generated URI lands in a different namespace and SHACL + # validation silently targets nothing. + def test_hash_namespace_base_uri_is_not_mangled(self): + gen = self._make_gen(base_uri="http://example.org/manufacturing#") + self.assertEqual(gen.base_uri, "http://example.org/manufacturing#") + self.assertEqual(gen.shapes_uri, "http://example.org/manufacturing#shapes") + + graph = gen.generate(self._HIER_ONTOLOGY) + ttl = gen.serialize(graph, format="turtle") + self.assertNotIn("#/", ttl) + self.assertIn("manufacturing#", ttl) + + # 24b โ€” a `#`-terminated base is preserved verbatim, while slash runs + # are collapsed: Qodo review caught that `endswith(("/","#"))` left + # `.../ns////` intact, leaking a different namespace into emitted IRIs. + def test_slash_run_normalization_regression(self): + gen = self._make_gen(base_uri="http://example.org/ns////") + self.assertEqual(gen.base_uri, "http://example.org/ns/") + self.assertEqual(gen.shapes_uri, "http://example.org/ns/shapes") + # 25 def test_severity_warning(self): gen = self._make_gen(severity="Warning") @@ -525,6 +546,65 @@ class TestSHACLHierarchicalAndValidation(unittest.TestCase): self.assertEqual(mc[0].max_count, 2) # 33 + def test_public_run_shacl_validation_api(self): + """The public API validates data and retains the legacy alias.""" + try: + import pyshacl # noqa: F401 + import rdflib # noqa: F401 + except ImportError: + self.skipTest("pyshacl/rdflib not installed") + from semantica.ontology import run_shacl_validation + from semantica.ontology.ontology_validator import _run_pyshacl + + data = "@prefix ex: . ex:alice a ex:Person ." + shacl = """ + @prefix ex: . + @prefix sh: . + ex:PersonShape a sh:NodeShape ; sh:targetClass ex:Person ; + sh:property [ sh:path ex:name ; sh:minCount 1 ] . + """ + public_report = run_shacl_validation(data, shacl) + legacy_report = _run_pyshacl(data, shacl) + self.assertFalse(public_report.conforms) + self.assertEqual(public_report.violation_count, 1) + self.assertEqual(legacy_report.conforms, public_report.conforms) + self.assertEqual(legacy_report.violation_count, public_report.violation_count) + self.assertEqual( + [ + (v.focus_node, v.result_path, v.constraint, v.severity, v.message) + for v in legacy_report.violations + ], + [ + (v.focus_node, v.result_path, v.constraint, v.severity, v.message) + for v in public_report.violations + ], + ) + + # 34 + def test_public_run_shacl_validation_conforming_graph(self): + """The public API reports a valid graph without violations.""" + try: + import pyshacl # noqa: F401 + import rdflib # noqa: F401 + except ImportError: + self.skipTest("pyshacl/rdflib not installed") + from semantica.ontology import run_shacl_validation + + data = """ + @prefix ex: . + ex:alice a ex:Person ; ex:name "Alice" . + """ + shacl = """ + @prefix ex: . + @prefix sh: . + ex:PersonShape a sh:NodeShape ; sh:targetClass ex:Person ; + sh:property [ sh:path ex:name ; sh:minCount 1 ] . + """ + + report = run_shacl_validation(data, shacl) + + self.assertTrue(report.conforms) + self.assertEqual(report.violation_count, 0) def test_shacl_violation_to_dict(self): from semantica.ontology.ontology_validator import SHACLViolation v = SHACLViolation( diff --git a/tests/ontology/test_ontology_normalized_properties.py b/tests/ontology/test_ontology_normalized_properties.py new file mode 100644 index 00000000..5cef53d3 --- /dev/null +++ b/tests/ontology/test_ontology_normalized_properties.py @@ -0,0 +1,41 @@ +from semantica.ontology.class_inferrer import ClassInferrer +from semantica.ontology.ontology_generator import OntologyGenerator +from semantica.ontology.property_generator import PropertyGenerator + + +def _entities(): + return [ + { + "id": "e1", + "type": "software engineer", + "name": "Alice", + "email": "alice@example.org", + }, + { + "id": "e2", + "type": "software engineer", + "name": "Bob", + "email": "bob@example.org", + }, + ] + + +def test_property_generator_matches_normalized_class_names(): + entities = _entities() + classes = ClassInferrer().infer_classes(entities) + + properties = PropertyGenerator().infer_properties(entities, [], classes) + + email = next(prop for prop in properties if prop["name"] == "email") + assert email["domain"] == ["SoftwareEngineer"] + assert email["range"] == "xsd:string" + + +def test_ontology_pipeline_emits_data_properties_for_normalized_types(): + ontology = OntologyGenerator().generate_ontology( + {"entities": _entities(), "relationships": []} + ) + + email = next(prop for prop in ontology["properties"] if prop["name"] == "email") + assert email["domain"] == ["SoftwareEngineer"] + assert email["range"] == "xsd:string" diff --git a/tests/ontology/test_ontology_relationship_endpoints.py b/tests/ontology/test_ontology_relationship_endpoints.py new file mode 100644 index 00000000..a1251daf --- /dev/null +++ b/tests/ontology/test_ontology_relationship_endpoints.py @@ -0,0 +1,98 @@ +from semantica.ontology.ontology_generator import OntologyGenerator + + +def _object_property(ontology, name): + return next(prop for prop in ontology["properties"] if prop["name"] == name) + + +def test_id_based_relationship_endpoints_infer_domain_and_range(): + entities = [ + {"id": "p1", "type": "Person", "name": "Alice"}, + {"id": "p2", "type": "Person", "name": "Bob"}, + {"id": "o1", "type": "Organization", "name": "Acme"}, + {"id": "o2", "type": "Organization", "name": "Beta"}, + ] + relationships = [ + {"source_id": "p1", "target_id": "o1", "type": "works_for"}, + {"source_id": "p2", "target_id": "o2", "type": "works_for"}, + ] + + ontology = OntologyGenerator().generate_ontology( + {"entities": entities, "relationships": relationships} + ) + + works_for = _object_property(ontology, "worksFor") + assert works_for["domain"] == ["Person"] + assert works_for["range"] == ["Organization"] + + +def test_source_and_target_aliases_resolve_without_matching_missing_fields(): + entities = [ + {"id": "p1", "type": "Person", "name": "Alice"}, + {"id": "p2", "type": "Person", "name": "Bob"}, + {"id": "o1", "type": "Organization", "name": "Acme"}, + {"id": "o2", "type": "Organization", "name": "Beta"}, + ] + relationships = [ + {"source": "p1", "target": "o1", "type": "works_for"}, + {"source": "p2", "target": "o2", "type": "works_for"}, + ] + + ontology = OntologyGenerator().generate_ontology( + {"entities": entities, "relationships": relationships} + ) + + works_for = _object_property(ontology, "worksFor") + assert works_for["domain"] == ["Person"] + assert works_for["range"] == ["Organization"] + + +def test_explicit_relationship_endpoint_types_are_preserved(): + entities = [ + {"id": "p1", "type": "Person", "name": "Alice"}, + {"id": "o1", "type": "Organization", "name": "Acme"}, + ] + relationships = [ + { + "source_id": "p1", + "target_id": "o1", + "type": "works_for", + "source_type": "Employee", + "target_type": "Company", + } + ] + + ontology = OntologyGenerator(min_occurrences=1).generate_ontology( + {"entities": entities, "relationships": relationships} + ) + + works_for = _object_property(ontology, "worksFor") + assert works_for["domain"] == ["Employee"] + assert works_for["range"] == ["Company"] + + +def test_nested_endpoint_alias_skips_empty_id_and_uses_name(): + entities = [ + {"id": "p1", "type": "Person", "name": "Alice"}, + {"id": "o1", "type": "Organization", "name": "Acme"}, + ] + relationships = [ + { + "source": {"id": "", "name": "Alice"}, + "target": {"id": "", "name": "Acme"}, + "type": "works_for", + }, + { + "source": {"id": "", "name": "Alice"}, + "target": {"id": "", "name": "Acme"}, + "type": "works_for", + }, + ] + + ontology = OntologyGenerator().generate_ontology( + {"entities": entities, "relationships": relationships} + ) + + works_for = _object_property(ontology, "worksFor") + assert works_for["domain"] == ["Person"] + assert works_for["range"] == ["Organization"] diff --git a/tests/pipeline/test_pipeline.py b/tests/pipeline/test_pipeline.py index 3dc7f530..e681b3c7 100644 --- a/tests/pipeline/test_pipeline.py +++ b/tests/pipeline/test_pipeline.py @@ -66,6 +66,93 @@ class TestPipelineModule(unittest.TestCase): self.assertEqual(result.output, 12) # (5 + 1) * 2 = 12 self.assertEqual(pipeline.steps[0].status, StepStatus.COMPLETED) + def test_registered_step_handler_executes(self): + """A handler registered by step type should execute.""" + def increment(data): + return data + 1 + + builder = PipelineBuilder() + builder.register_step_handler("math", increment) + builder.add_step("increment", "math") + + result = ExecutionEngine().execute_pipeline( + builder.build("registered"), data=1 + ) + + self.assertTrue(result.success) + self.assertEqual(result.output, 2) + + def test_explicit_handler_does_not_receive_control_fields(self): + """Builder-only fields should not be passed to strict handlers.""" + def source(data): + return data + + def increment(data, amount): + return data + amount + + builder = PipelineBuilder() + builder.add_step("source", "source", handler=source) + step = builder.add_step( + "increment", + "math", + handler=increment, + dependencies=["source"], + amount=2, + ) + + result = ExecutionEngine().execute_pipeline( + builder.build("strict"), data=1 + ) + + self.assertTrue(result.success) + self.assertEqual(result.output, 3) + self.assertEqual(step.config, {"amount": 2}) + + def test_explicit_handler_overrides_registered_handler(self): + """An explicit step handler should take precedence over the registry.""" + builder = PipelineBuilder() + builder.register_step_handler("math", lambda data: data + 100) + builder.add_step("increment", "math", handler=lambda data: data + 1) + + result = ExecutionEngine().execute_pipeline( + builder.build("override"), data=1 + ) + + self.assertTrue(result.success) + self.assertEqual(result.output, 2) + + def test_step_without_handler_passes_input_through(self): + """A step with no explicit or registered handler should be a no-op.""" + builder = PipelineBuilder() + builder.add_step("passthrough", "unregistered") + + result = ExecutionEngine().execute_pipeline( + builder.build("handlerless"), data={"value": 1} + ) + + self.assertTrue(result.success) + self.assertEqual(result.output, {"value": 1}) + + def test_falsy_explicit_handler_is_invoked(self): + """A handler whose __bool__ is False must still be dispatched.""" + + class FalseyHandler: + def __bool__(self): + return False + + def __call__(self, data): + return {"explicit": data} + + builder = PipelineBuilder() + builder.add_step("step1", "mytype", handler=FalseyHandler()) + + result = ExecutionEngine().execute_pipeline( + builder.build("falsy"), data=1 + ) + + self.assertTrue(result.success) + self.assertEqual(result.output, {"explicit": 1}) + def test_execution_engine_failure(self): """Test pipeline failure handling.""" def failing_handler(data, **kwargs): @@ -112,8 +199,8 @@ class TestPipelineModule(unittest.TestCase): _ = semantica.pipeline - from semantica.pipeline import PipelineBuilder, PipelineValidator from semantica.deduplication import DuplicateDetector + from semantica.pipeline import PipelineBuilder, PipelineValidator builder = PipelineBuilder() builder.add_step("step1", "dummy") diff --git a/tests/pipeline/test_pipeline_serializer.py b/tests/pipeline/test_pipeline_serializer.py new file mode 100644 index 00000000..fe616ddd --- /dev/null +++ b/tests/pipeline/test_pipeline_serializer.py @@ -0,0 +1,103 @@ +import copy +import json + +import pytest + +from semantica.pipeline.pipeline_builder import PipelineBuilder, PipelineSerializer + + +@pytest.mark.parametrize("serialization_format", ["dict", "json"]) +def test_roundtrip_preserves_dependencies_and_delta_metadata(serialization_format): + builder = PipelineBuilder() + builder.add_step("extract", "source") + builder.add_step( + "index", + "sink", + delta_mode=True, + base_version_id="v1", + target_version_id="v2", + ) + builder.connect_steps("extract", "index") + pipeline = builder.build("incremental-index") + + serializer = PipelineSerializer() + serialized = serializer.serialize_pipeline(pipeline, format=serialization_format) + restored = serializer.deserialize_pipeline(serialized) + + index_step = next(step for step in restored.steps if step.name == "index") + assert index_step.dependencies == ["extract"] + assert index_step.delta_mode is True + assert index_step.base_version_id == "v1" + assert index_step.target_version_id == "v2" + + +@pytest.mark.parametrize("serialization_format", ["dict", "json"]) +def test_serialization_omits_runtime_handlers(serialization_format): + def handler(data, **config): + return data + + builder = PipelineBuilder() + builder.add_step("extract", "source", handler=handler, batch_size=10) + pipeline = builder.build("handler-pipeline") + + serializer = PipelineSerializer() + serialized = serializer.serialize_pipeline(pipeline, format=serialization_format) + serialized_data = ( + json.loads(serialized) if isinstance(serialized, str) else serialized + ) + + assert serialized_data["steps"][0]["config"] == {"batch_size": 10} + + restored = serializer.deserialize_pipeline(serialized) + assert restored.steps[0].handler is None + assert restored.steps[0].config == {"batch_size": 10} + + +def test_deserialization_ignores_legacy_stringified_handler(): + serialized = json.dumps( + { + "name": "legacy-handler-pipeline", + "steps": [ + { + "name": "extract", + "type": "source", + "config": { + "handler": "", + "batch_size": 10, + }, + "dependencies": [], + } + ], + } + ) + + restored = PipelineSerializer().deserialize_pipeline(serialized) + + assert restored.steps[0].handler is None + assert restored.steps[0].config == {"batch_size": 10} + + +def test_deserialization_does_not_mutate_caller_owned_dict(): + payload = { + "name": "legacy-handler-pipeline", + "steps": [ + { + "name": "extract", + "type": "source", + "config": { + "handler": "", + "batch_size": 10, + }, + "dependencies": [], + } + ], + } + snapshot = copy.deepcopy(payload) + + restored = PipelineSerializer().deserialize_pipeline(payload) + + assert payload == snapshot + assert "handler" in payload["steps"][0]["config"] + assert payload is not snapshot + assert restored.steps[0].handler is None + assert restored.steps[0].config == {"batch_size": 10} diff --git a/tests/reasoning/test_rule_actions.py b/tests/reasoning/test_rule_actions.py new file mode 100644 index 00000000..b4e11846 --- /dev/null +++ b/tests/reasoning/test_rule_actions.py @@ -0,0 +1,531 @@ +"""Tests for rule-driven actions (production-rule behaviour) on the Reasoner. + +Covers the L1 Action layer (Assert/Retract/Call/Emit), provenance logging of +fired actions (L2), and backward compatibility with the legacy Rule.handler +callback. +""" + +import unittest +from collections import UserDict + +from semantica.reasoning import ( + AssertAction, + CallAction, + EmitEventAction, + Fact, + Match, + Reasoner, + ReteEngine, + RetractAction, +) + + +class TestRuleActions(unittest.TestCase): + def setUp(self): + self.reasoner = Reasoner() + + def _add_person_parent_facts(self): + self.reasoner.add_fact("Person(John)") + self.reasoner.add_fact("Parent(John, Jane)") + + def test_assert_action_fires_and_substitutes_bindings(self): + rule = self.reasoner.add_rule( + "IF Person(?x) AND Parent(?x, ?y) THEN Child(?y, ?x)" + ) + rule.actions = [AssertAction("Adult(?x)")] + self._add_person_parent_facts() + + self.reasoner.forward_chain() + + # Action-asserted fact uses the match bindings (?x -> John). + self.assertIn("Adult(John)", self.reasoner.facts) + + def test_retract_action_removes_fact(self): + rule = self.reasoner.add_rule( + "IF Person(?x) AND Parent(?x, ?y) THEN Child(?y, ?x)" + ) + rule.actions = [RetractAction("Person(?x)")] + self._add_person_parent_facts() + + self.reasoner.forward_chain() + + self.assertNotIn("Person(John)", self.reasoner.facts) + + def test_call_action_invoked_with_bindings(self): + seen = {} + + def record(bindings, reasoner): + seen.update(bindings) + + rule = self.reasoner.add_rule( + "IF Person(?x) AND Parent(?x, ?y) THEN Child(?y, ?x)" + ) + rule.actions = [CallAction(record, name="record")] + self._add_person_parent_facts() + + self.reasoner.forward_chain() + + self.assertEqual(seen.get("x"), "John") + self.assertEqual(seen.get("y"), "Jane") + + def test_emit_event_action_delivers_to_sink(self): + events = [] + self.reasoner.on_event(lambda name, payload: events.append((name, payload))) + + rule = self.reasoner.add_rule( + "IF Person(?x) AND Parent(?x, ?y) THEN Child(?y, ?x)" + ) + rule.actions = [EmitEventAction("child_derived:?y")] + self._add_person_parent_facts() + + self.reasoner.forward_chain() + + self.assertEqual(len(events), 1) + name, payload = events[0] + self.assertEqual(name, "child_derived:Jane") + self.assertEqual(payload["bindings"]["x"], "John") + + def test_assert_action_write_back_to_knowledge_graph(self): + class FakeKG: + def __init__(self): + self.added = [] + + def add_fact(self, fact): + self.added.append(fact) + + kg = FakeKG() + reasoner = Reasoner(knowledge_graph=kg) + rule = reasoner.add_rule( + "IF Person(?x) AND Parent(?x, ?y) THEN Child(?y, ?x)" + ) + rule.actions = [AssertAction("Adult(?x)", write_back=True)] + reasoner.add_fact("Person(John)") + reasoner.add_fact("Parent(John, Jane)") + + reasoner.forward_chain() + + self.assertIn("Adult(John)", kg.added) + + def test_provenance_logs_fired_actions(self): + reasoner = Reasoner(provenance=True) + rule = reasoner.add_rule( + "IF Person(?x) AND Parent(?x, ?y) THEN Child(?y, ?x)" + ) + rule.actions = [AssertAction("Adult(?x)")] + reasoner.add_fact("Person(John)") + reasoner.add_fact("Parent(John, Jane)") + + reasoner.forward_chain() + + self.assertEqual(len(reasoner.action_log), 1) + entry = reasoner.action_log[0] + self.assertEqual(entry["action"], "AssertAction") + self.assertEqual(entry["rule_id"], rule.rule_id) + self.assertEqual(entry["bindings"]["x"], "John") + self.assertIn("Adult(John)", entry["description"]) + + def test_repeated_forward_chain_fires_same_activation_once(self): + calls = [] + + rule = self.reasoner.add_rule("IF Person(?x) THEN Adult(?x)") + rule.actions = [ + CallAction(lambda bindings, reasoner: calls.append(dict(bindings))) + ] + self.reasoner.add_fact("Person(John)") + + self.reasoner.forward_chain() + self.reasoner.forward_chain() + + self.assertEqual(calls, [{"x": "John"}]) + + def test_repeated_forward_chain_records_provenance_once(self): + reasoner = Reasoner(provenance=True) + rule = reasoner.add_rule("IF Person(?x) THEN Adult(?x)") + rule.actions = [AssertAction("Verified(?x)")] + reasoner.add_fact("Person(John)") + + reasoner.forward_chain() + reasoner.forward_chain() + + self.assertEqual(len(reasoner.action_log), 1) + + def test_new_binding_creates_a_new_activation(self): + calls = [] + rule = self.reasoner.add_rule("IF Person(?x) THEN Adult(?x)") + rule.actions = [ + CallAction(lambda bindings, reasoner: calls.append(bindings["x"])) + ] + self.reasoner.add_fact("Person(John)") + + self.reasoner.forward_chain() + self.reasoner.add_fact("Person(Jane)") + self.reasoner.forward_chain() + + self.assertCountEqual(calls, ["John", "Jane"]) + + def test_reset_action_history_allows_deliberate_replay(self): + calls = [] + rule = self.reasoner.add_rule("IF Person(?x) THEN Adult(?x)") + rule.actions = [CallAction(lambda bindings, reasoner: calls.append("called"))] + self.reasoner.add_fact("Person(John)") + + self.reasoner.forward_chain() + self.reasoner.reset_action_history() + self.reasoner.forward_chain() + + self.assertEqual(calls, ["called", "called"]) + + def test_clear_resets_action_history(self): + calls = [] + rule = self.reasoner.add_rule("IF Person(?x) THEN Adult(?x)") + rule.actions = [CallAction(lambda bindings, reasoner: calls.append("called"))] + self.reasoner.add_fact("Person(John)") + self.reasoner.forward_chain() + + self.reasoner.clear() + rule = self.reasoner.add_rule("IF Person(?x) THEN Adult(?x)") + rule.actions = [CallAction(lambda bindings, reasoner: calls.append("called"))] + self.reasoner.add_fact("Person(John)") + self.reasoner.forward_chain() + + self.assertEqual(calls, ["called", "called"]) + + def test_failed_action_is_not_retried_without_explicit_reset(self): + attempts = [] + + def fail(bindings, reasoner): + attempts.append(bindings["x"]) + raise RuntimeError("boom") + + rule = self.reasoner.add_rule("IF Person(?x) THEN Adult(?x)") + rule.actions = [CallAction(fail)] + self.reasoner.add_fact("Person(John)") + + self.reasoner.forward_chain() + self.reasoner.forward_chain() + self.assertEqual(attempts, ["John"]) + + self.reasoner.reset_action_history() + self.reasoner.forward_chain() + self.assertEqual(attempts, ["John", "John"]) + + def test_replacing_actions_in_place_requires_explicit_history_reset(self): + calls = [] + rule = self.reasoner.add_rule("IF Person(?x) THEN Adult(?x)") + rule.actions = [CallAction(lambda bindings, reasoner: calls.append("first"))] + self.reasoner.add_fact("Person(John)") + self.reasoner.forward_chain() + + rule.actions = [CallAction(lambda bindings, reasoner: calls.append("second"))] + self.reasoner.forward_chain() + self.assertEqual(calls, ["first"]) + + self.reasoner.reset_action_history() + self.reasoner.forward_chain() + self.assertEqual(calls, ["first", "second"]) + + def test_activation_is_recorded_before_reentrant_action_execution(self): + calls = [] + + def reenter(bindings, reasoner): + calls.append(bindings["x"]) + if len(calls) == 1: + reasoner.forward_chain() + + rule = self.reasoner.add_rule("IF Person(?x) THEN Adult(?x)") + rule.actions = [CallAction(reenter)] + self.reasoner.add_fact("Person(John)") + + self.reasoner.forward_chain() + + self.assertEqual(calls, ["John"]) + + def test_no_provenance_log_when_disabled(self): + rule = self.reasoner.add_rule( + "IF Person(?x) AND Parent(?x, ?y) THEN Child(?y, ?x)" + ) + rule.actions = [AssertAction("Adult(?x)")] + self._add_person_parent_facts() + + self.reasoner.forward_chain() + + self.assertEqual(self.reasoner.action_log, []) + + def test_legacy_handler_still_invoked(self): + calls = [] + + rule = self.reasoner.add_rule( + "IF Person(?x) AND Parent(?x, ?y) THEN Child(?y, ?x)" + ) + rule.handler = lambda bindings, reasoner: calls.append(bindings) + self._add_person_parent_facts() + + self.reasoner.forward_chain() + + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0]["x"], "John") + + def test_action_error_does_not_break_chain(self): + def boom(bindings, reasoner): + raise RuntimeError("boom") + + rule = self.reasoner.add_rule( + "IF Person(?x) AND Parent(?x, ?y) THEN Child(?y, ?x)" + ) + rule.actions = [CallAction(boom, name="boom"), AssertAction("Adult(?x)")] + self._add_person_parent_facts() + + # A failing action is logged but must not abort the pass; the later + # action still runs and the conclusion is still derived. + self.reasoner.forward_chain() + + self.assertIn("Adult(John)", self.reasoner.facts) + self.assertIn("Child(Jane, John)", self.reasoner.facts) + + +class TestRuleActionRegressions(unittest.TestCase): + """Regression coverage for the qodo-flagged bugs on PR #1096.""" + + def test_variable_substitution_no_prefix_collision(self): + # bug7: naive str.replace of "?x" would also corrupt "?xy". A + # token-aware substitution must bind ?x and ?xy independently. + reasoner = Reasoner() + rule = reasoner.add_rule("IF Pair(?x, ?xy) THEN Linked(?x, ?xy)") + rule.actions = [AssertAction("Tag(?x, ?xy)")] + reasoner.add_fact("Pair(John, Johny)") + + reasoner.forward_chain() + + self.assertIn("Tag(John, Johny)", reasoner.facts) + + def test_assert_write_back_to_canonical_knowledge_graph(self): + # bug1: a KG exposing only entities/relationships (no add_fact) must + # still receive the asserted fact via canonical translation. + from semantica.kg.knowledge_graph import KnowledgeGraph + + kg = KnowledgeGraph() + reasoner = Reasoner(knowledge_graph=kg) + rule = reasoner.add_rule("IF Person(?x) THEN Adult(?x)") + rule.actions = [AssertAction("Adult(?x)", write_back=True)] + reasoner.add_fact("Person(John)") + + reasoner.forward_chain() + + # A single-argument fact lands as an entity node. + self.assertTrue(any("John" in str(e) for e in kg.entities)) + + def test_write_back_unsupported_target_raises(self): + # bug1: an unsupported write-back target must fail loudly, not silently. + from semantica.reasoning.reasoner import _write_fact_to_graph + + with self.assertRaises(ValueError): + _write_fact_to_graph(object(), "Adult(John)") + + def test_provenance_entry_has_timestamp(self): + # bug3: action_log entries must be structured with a timestamp. + reasoner = Reasoner(provenance=True) + rule = reasoner.add_rule("IF Person(?x) THEN Adult(?x)") + rule.actions = [AssertAction("Adult(?x)")] + reasoner.add_fact("Person(John)") + + reasoner.forward_chain() + + entry = reasoner.action_log[0] + self.assertIn("timestamp", entry) + self.assertTrue(entry["timestamp"]) + + def test_action_fires_even_when_conclusion_already_known(self): + # bug4: previously an activation whose conclusion was already known + # skipped firing its actions. Now it must still fire exactly once. + reasoner = Reasoner() + rule = reasoner.add_rule("IF Person(?x) THEN Adult(?x)") + rule.actions = [AssertAction("Verified(?x)")] + reasoner.add_fact("Person(John)") + # Conclusion already present before the pass runs. + reasoner.add_fact("Adult(John)") + + reasoner.forward_chain() + + self.assertIn("Verified(John)", reasoner.facts) + + def test_retract_self_conclusion_terminates(self): + # bug5: a RetractAction removing its own premise previously re-fired + # every pass up to max_iterations. It must fire once and terminate. + reasoner = Reasoner() + rule = reasoner.add_rule("IF Person(?x) THEN Adult(?x)") + rule.actions = [RetractAction("Person(?x)")] + reasoner.add_fact("Person(John)") + + # Should return promptly without exhausting iterations. + reasoner.forward_chain() + + self.assertNotIn("Person(John)", reasoner.facts) + + def test_infer_with_results_preserves_confidence(self): + # bug9: confidence must survive to the InferenceResult objects. + reasoner = Reasoner() + rule = reasoner.add_rule("IF Person(?x) THEN Adult(?x)") + rule.confidence = 0.8 + + results = reasoner.infer_with_results(["Person(John)"]) + + self.assertTrue(results) + self.assertTrue(all(0.0 <= r.confidence <= 1.0 for r in results)) + self.assertAlmostEqual( + min(r.confidence for r in results), 0.8, places=6 + ) + + +class TestReteActionExecution(unittest.TestCase): + def setUp(self): + self.calls = [] + self.reasoner = Reasoner() + self.rule = self.reasoner.add_rule("IF Person(?x) THEN Adult(?x)") + self.rule.actions = [ + CallAction( + lambda bindings, reasoner: self.calls.append(dict(bindings)) + ) + ] + self.match = Match( + rule=self.rule, + facts=[Fact("person-1", "Person", ["John"])], + bindings={"x": "John"}, + ) + self.engine = ReteEngine(reasoner=self.reasoner) + + def test_rete_repeated_execute_matches_fires_activation_once(self): + first_results = self.engine.execute_matches([self.match]) + second_results = self.engine.execute_matches([self.match]) + + self.assertEqual(first_results, ["Adult(?x)"]) + self.assertEqual(second_results, ["Adult(?x)"]) + self.assertEqual(self.calls, [{"x": "John"}]) + + def test_rete_duplicate_match_preserves_results_but_fires_once(self): + results = self.engine.execute_matches([self.match, self.match]) + + self.assertEqual(results, ["Adult(?x)", "Adult(?x)"]) + self.assertEqual(self.calls, [{"x": "John"}]) + + def test_rete_distinct_fact_ids_create_distinct_activations(self): + other_match = Match( + rule=self.rule, + facts=[Fact("person-2", "Person", ["John"])], + bindings={"x": "John"}, + ) + + self.engine.execute_matches([self.match]) + self.engine.execute_matches([other_match]) + + self.assertEqual(self.calls, [{"x": "John"}, {"x": "John"}]) + + def test_rete_equivalent_nested_bindings_share_an_activation(self): + first_match = Match( + rule=self.rule, + facts=self.match.facts, + bindings={"x": {"a": 1, "b": 2}}, + ) + reordered_match = Match( + rule=self.rule, + facts=self.match.facts, + bindings={"x": {"b": 2, "a": 1}}, + ) + + self.engine.execute_matches([first_match]) + self.engine.execute_matches([reordered_match]) + + self.assertEqual(len(self.calls), 1) + + def test_rete_structured_fact_identity_avoids_separator_collisions(self): + first_match = Match( + rule=self.rule, + facts=[Fact("a", "b:C", [])], + bindings={"x": "John"}, + ) + colliding_text_match = Match( + rule=self.rule, + facts=[Fact("a:b", "C", [])], + bindings={"x": "John"}, + ) + + self.engine.execute_matches([first_match]) + self.engine.execute_matches([colliding_text_match]) + + self.assertEqual(len(self.calls), 2) + + def test_rete_cyclic_binding_preserves_results_and_deduplicates_actions(self): + cyclic_value = [] + cyclic_value.append(cyclic_value) + cyclic_match = Match( + rule=self.rule, + facts=self.match.facts, + bindings={"x": cyclic_value}, + ) + + first_results = self.engine.execute_matches([cyclic_match]) + second_results = self.engine.execute_matches([cyclic_match]) + + self.assertEqual(first_results, ["Adult(?x)"]) + self.assertEqual(second_results, ["Adult(?x)"]) + self.assertEqual(len(self.calls), 1) + + def test_rete_equivalent_mapping_implementations_share_an_activation(self): + first_match = Match( + rule=self.rule, + facts=self.match.facts, + bindings={"x": UserDict({"a": 1, "b": 2})}, + ) + reordered_match = Match( + rule=self.rule, + facts=self.match.facts, + bindings={"x": UserDict({"b": 2, "a": 1})}, + ) + + self.engine.execute_matches([first_match]) + self.engine.execute_matches([reordered_match]) + + self.assertEqual(len(self.calls), 1) + + def test_rete_key_error_does_not_suppress_conclusion(self): + class UnrepresentableValue: + def __repr__(self): + raise RuntimeError("cannot represent") + + invalid_match = Match( + rule=self.rule, + facts=self.match.facts, + bindings={"x": UnrepresentableValue()}, + ) + + results = self.engine.execute_matches([invalid_match]) + + self.assertEqual(results, ["Adult(?x)"]) + self.assertEqual(self.calls, []) + + def test_rete_reset_action_history_allows_deliberate_replay(self): + self.engine.execute_matches([self.match]) + + self.engine.reset_action_history() + self.engine.execute_matches([self.match]) + + self.assertEqual(self.calls, [{"x": "John"}, {"x": "John"}]) + + def test_rete_reset_allows_action_replay(self): + self.engine.execute_matches([self.match]) + + self.engine.reset() + self.engine.execute_matches([self.match]) + + self.assertEqual(self.calls, [{"x": "John"}, {"x": "John"}]) + + def test_rete_build_network_allows_action_replay(self): + self.engine.execute_matches([self.match]) + + self.engine.build_network([self.rule]) + self.engine.execute_matches([self.match]) + + self.assertEqual(self.calls, [{"x": "John"}, {"x": "John"}]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/reasoning/test_specialized_reasoners.py b/tests/reasoning/test_specialized_reasoners.py index 3a519171..51dbdc9a 100644 --- a/tests/reasoning/test_specialized_reasoners.py +++ b/tests/reasoning/test_specialized_reasoners.py @@ -30,6 +30,29 @@ class TestSpecializedReasoners(unittest.TestCase): binding_types = [b.get("x_type") for b in inferred.bindings] self.assertIn("Human", binding_types) + def test_execute_query_raises_not_implemented(self): + """Empty results must not pass as a valid answer (issue #1083). + + Both branches returned ``SPARQLQueryResult(bindings=[], variables=[])`` + -- with or without a triplet store -- so callers that trust an empty + result as "no matches" silently drew wrong conclusions. Until a real + execution path lands, refusing loudly is safer. + """ + reasoner = SPARQLReasoner() + with self.assertRaises(NotImplementedError): + reasoner.execute_query("SELECT ?s ?p ?o WHERE { ?s ?p ?o }") + + def test_execute_query_with_triplet_store_raises_not_implemented(self): + reasoner = SPARQLReasoner(triplet_store=object()) + with self.assertRaises(NotImplementedError): + reasoner.execute_query("SELECT ?s ?p ?o WHERE { ?s ?p ?o }") + + def test_execute_query_error_explains_why_the_query_is_refused(self): + reasoner = SPARQLReasoner() + with self.assertRaises(NotImplementedError) as ctx: + reasoner.execute_query("SELECT ?s WHERE { ?s ?p ?o }") + self.assertIn("not implemented", str(ctx.exception)) + def test_abductive_reasoner_generate_hypotheses(self): reasoner = AbductiveReasoner() reasoner.reasoner.add_rule("IF Disease(Flu) THEN Symptom(Fever)") diff --git a/tests/reproduce_issue_176.py b/tests/reproduce_issue_176.py deleted file mode 100644 index a24ba7c8..00000000 --- a/tests/reproduce_issue_176.py +++ /dev/null @@ -1,100 +0,0 @@ - -import unittest -from unittest.mock import MagicMock, patch -from semantica.semantic_extract.methods import extract_relations_llm, extract_entities_llm, extract_triplets_llm -from semantica.semantic_extract.ner_extractor import Entity - -class TestMaxTokensPropagation(unittest.TestCase): - @patch("semantica.semantic_extract.methods.create_provider") - def test_max_tokens_propagation_relations(self, mock_create_provider): - """Test that max_tokens is passed to generate_typed in extract_relations_llm.""" - # Setup mock - mock_llm = MagicMock() - mock_create_provider.return_value = mock_llm - mock_llm.is_available.return_value = True - - # Setup return value to avoid pydantic validation errors - mock_response = MagicMock() - mock_response.relations = [] - mock_llm.generate_typed.return_value = mock_response - - # Create dummy entities - entities = [Entity(text="Foo", label="ORG", start_char=0, end_char=3)] - - # Call the function with max_tokens - extract_relations_llm( - text="some text", - entities=entities, - provider="openai", - model="gpt-4", - max_tokens=128000 - ) - - # Check if generate_typed was called with max_tokens - args, kwargs = mock_llm.generate_typed.call_args - - print(f"Relations Call kwargs: {kwargs}") - - self.assertIn("max_tokens", kwargs) - self.assertEqual(kwargs["max_tokens"], 128000) - - @patch("semantica.semantic_extract.methods.create_provider") - def test_max_tokens_propagation_entities(self, mock_create_provider): - """Test that max_tokens is passed to generate_typed in extract_entities_llm.""" - # Setup mock - mock_llm = MagicMock() - mock_create_provider.return_value = mock_llm - mock_llm.is_available.return_value = True - - # Setup return value to avoid pydantic validation errors - mock_response = MagicMock() - mock_response.entities = [] - mock_llm.generate_typed.return_value = mock_response - - # Call the function with max_tokens - extract_entities_llm( - text="some text", - provider="openai", - model="gpt-4", - max_tokens=128000 - ) - - # Check if generate_typed was called with max_tokens - args, kwargs = mock_llm.generate_typed.call_args - - print(f"Entities Call kwargs: {kwargs}") - - self.assertIn("max_tokens", kwargs) - self.assertEqual(kwargs["max_tokens"], 128000) - - @patch("semantica.semantic_extract.methods.create_provider") - def test_max_tokens_propagation_triplets(self, mock_create_provider): - """Test that max_tokens is passed to generate_typed in extract_triplets_llm.""" - # Setup mock - mock_llm = MagicMock() - mock_create_provider.return_value = mock_llm - mock_llm.is_available.return_value = True - - # Setup return value to avoid pydantic validation errors - mock_response = MagicMock() - mock_response.triplets = [] - mock_llm.generate_typed.return_value = mock_response - - # Call the function with max_tokens - extract_triplets_llm( - text="some text", - provider="openai", - model="gpt-4", - max_tokens=128000 - ) - - # Check if generate_typed was called with max_tokens - args, kwargs = mock_llm.generate_typed.call_args - - print(f"Triplets Call kwargs: {kwargs}") - - self.assertIn("max_tokens", kwargs) - self.assertEqual(kwargs["max_tokens"], 128000) - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 44e6d2a9..da5e1d9c 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -1851,3 +1851,522 @@ class TestExitCodes: assert "Traceback" not in result.output, ( f"Traceback found for {argv}: {result.output}" ) + + +class TestDoctorEmbeddings: + """#994: doctor must surface non-functional embedding backends instead of + reporting all green. Default = import-level check; --deep-embeddings (or + SEMANTICA_DOCTOR_DEEP_EMBEDDINGS=1) instantiates via TextEmbedder.""" + + def _doctor_checks(self, runner, *extra): + result = runner.invoke(cli_module.main, ["doctor", "--json", *extra]) + _ok(result) + import json as _json + return {c["check"]: c for c in _json.loads(result.output)} + + def _with_fake_st(self, monkeypatch, **embedder_attrs): + fake_st = _fake_module( + __version__="9.9.9", + SentenceTransformer=object, + ) + monkeypatch.setitem(__import__("sys").modules, "sentence_transformers", fake_st) + + def test_doctor_reports_embedding_checks(self, runner): + checks = self._doctor_checks(runner) + assert "Embeddings (sentence-transformers)" in checks + assert "Embeddings (fastembed)" in checks + + def test_import_failure_is_fail_status_with_hint(self, runner, monkeypatch): + # Force the 'import sentence_transformers' inside _embedding_backend to + # raise ImportError regardless of whether the package is installed on + # this machine. Setting a module entry to None is the standard Python + # mechanism: any subsequent 'import ' raises + # "import of halted; None in sys.modules". + monkeypatch.setitem( + __import__("sys").modules, "sentence_transformers", None + ) + checks = self._doctor_checks(runner) + st = checks["Embeddings (sentence-transformers)"] + assert st["status"] == "fail" + assert st["hint"] == "pip install sentence-transformers" + + def test_deep_probe_detects_fallback_active(self, runner, monkeypatch): + self._with_fake_st(monkeypatch) + fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None) + + fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder) + monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod) + + checks = self._doctor_checks(runner, "--deep-embeddings") + st = checks["Embeddings (sentence-transformers)"] + assert st["status"] == "fail" + assert "hash fallback" in st["note"] + + def test_deep_probe_ok_when_model_loads(self, runner, monkeypatch): + self._with_fake_st(monkeypatch) + import numpy as np + fake_embedder = types.SimpleNamespace( + model=object(), + fastembed_model=None, + embed_text=lambda text: np.zeros(384, dtype=np.float32), + ) + fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder) + monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod) + + checks = self._doctor_checks(runner, "--deep-embeddings") + st = checks["Embeddings (sentence-transformers)"] + assert st["status"] == "ok" + assert "384-dim" in st["note"] + + def test_env_var_enables_deep_mode(self, runner, monkeypatch): + monkeypatch.setenv("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", "1") + self._with_fake_st(monkeypatch) + fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None) + fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder) + monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod) + + checks = self._doctor_checks(runner) + st = checks["Embeddings (sentence-transformers)"] + assert st["status"] == "fail" + assert "hash fallback" in st["note"] + + +class TestDoctorEmbeddingHintsAndEnv: + """Review follow-ups: deep failures must not carry the pip-install hint, + and the env toggle tolerates case/whitespace variants.""" + + def _doctor_checks(self, runner, *extra): + result = runner.invoke(cli_module.main, ["doctor", "--json", *extra]) + _ok(result) + import json as _json + return {c["check"]: c for c in _json.loads(result.output)} + + def _with_fake_st(self, monkeypatch): + fake_st = _fake_module( + __version__="9.9.9", + SentenceTransformer=object, + ) + monkeypatch.setitem(__import__("sys").modules, "sentence_transformers", fake_st) + + def test_deep_failure_hint_is_not_pip_install(self, runner, monkeypatch): + self._with_fake_st(monkeypatch) + fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None) + fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder) + monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod) + + checks = self._doctor_checks(runner, "--deep-embeddings") + st = checks["Embeddings (sentence-transformers)"] + assert st["status"] == "fail" + assert "pip install" not in (st["hint"] or ""), ( + "a deep probe failure means the package imported fine โ€” pointing " + "users at pip sends them to reinstall for a runtime/model problem" + ) + assert "runtime/model-load" in st["hint"] + + def test_env_var_tolerates_case_and_whitespace(self, runner, monkeypatch): + monkeypatch.setenv("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", " TRUE ") + self._with_fake_st(monkeypatch) + fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None) + fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder) + monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod) + + checks = self._doctor_checks(runner) + st = checks["Embeddings (sentence-transformers)"] + assert st["status"] == "fail" + assert "hash fallback" in st["note"], "padded/caps env value must enable deep mode" + + +class TestEmbedGenerateOutput: + """#994: `embed generate --output` must write files `embed index` can read.""" + + def _patch_generate(self, monkeypatch, retval): + import numpy as np + fake_emb = _fake_module(generate_embeddings=lambda *a, **k: np.asarray(retval)) + monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb) + + def test_writes_valid_parquet(self, runner, monkeypatch, tmp_path): + pytest.importorskip("pyarrow", reason="parquet writer regression needs pyarrow") + import numpy as np + import pandas as pd + self._patch_generate(monkeypatch, [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]) + out = tmp_path / "embeddings.parquet" + result = runner.invoke(cli_module.main, ["embed", "generate", "in.json", "--output", str(out)]) + _ok(result) + df = pd.read_parquet(out) + assert "embedding" in df.columns + assert len(df) == 2 + # Use allclose: the writer may store float32 or float64 depending on + # the model backend; exact == fails for float32-precision values. + assert np.allclose(df["embedding"].iloc[0], [0.1, 0.2, 0.3], atol=1e-6) + + def test_writes_1d_result_as_single_row_parquet(self, runner, monkeypatch, tmp_path): + pytest.importorskip("pyarrow", reason="parquet writer regression needs pyarrow") + import numpy as np + import pandas as pd + self._patch_generate(monkeypatch, [0.1, 0.2, 0.3]) + out = tmp_path / "embeddings.parquet" + result = runner.invoke(cli_module.main, ["embed", "generate", "in.json", "--output", str(out)]) + _ok(result) + df = pd.read_parquet(out) + assert len(df) == 1 + assert np.allclose(df["embedding"].iloc[0], [0.1, 0.2, 0.3], atol=1e-6) + + def test_writes_json_records_not_repr_strings(self, runner, monkeypatch, tmp_path): + import json as _json + import numpy as np + import pandas as pd + self._patch_generate(monkeypatch, [[0.1, 0.2], [0.3, 0.4]]) + out = tmp_path / "embeddings.json" + result = runner.invoke(cli_module.main, ["embed", "generate", "in.json", "--output", str(out)]) + _ok(result) + records = _json.loads(out.read_text(encoding="utf-8")) + assert records == [{"embedding": [0.1, 0.2]}, {"embedding": [0.3, 0.4]}] + # Verify embed index can read the file back (round-trip contract). + df = pd.read_json(out, orient="records") + vector_col = next( + (c for c in df.columns if isinstance(df[c].iloc[0], (list, np.ndarray))), + None, + ) + assert vector_col == "embedding", ( + f"embed index would not find a vector column; got columns {list(df.columns)}" + ) + + def test_rejects_unsupported_output_format(self, runner, monkeypatch, tmp_path): + self._patch_generate(monkeypatch, [[0.1, 0.2]]) + out = tmp_path / "embeddings.txt" + result = runner.invoke(cli_module.main, ["embed", "generate", "in.json", "--output", str(out)]) + assert result.exit_code != 0 + assert "Unsupported output format" in result.output + assert not out.exists() + + +class TestWriteResultOutput: + """Unit-level regression tests for _write_result_output(). + + Covers every branch: JSON, JSONL, CSV, unsupported extension, no-extension, + dict+JSONL, empty list, NumPy scalar/array values, and round-trip readback. + """ + + # โ”€โ”€ helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def _write(self, tmp_path, filename, result): + """Call _write_result_output and return the output Path.""" + from semantica.cli import _write_result_output + out = tmp_path / filename + _write_result_output(out, result) + return out + + # โ”€โ”€ JSON โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def test_json_dict_produces_valid_json(self, tmp_path): + import json + out = self._write(tmp_path, "r.json", {"pairs": 3, "score": 0.9}) + data = json.loads(out.read_text(encoding="utf-8")) + assert data == {"pairs": 3, "score": 0.9} + + def test_json_list_produces_valid_json(self, tmp_path): + import json + out = self._write(tmp_path, "r.json", [{"a": 1}, {"a": 2}]) + data = json.loads(out.read_text(encoding="utf-8")) + assert data == [{"a": 1}, {"a": 2}] + + def test_json_numpy_scalar_serialises_as_number_not_repr(self, tmp_path): + """np.float32 values must round-trip as JSON numbers, not repr strings.""" + import json + import numpy as np + out = self._write(tmp_path, "r.json", {"score": np.float32(0.95)}) + data = json.loads(out.read_text(encoding="utf-8")) + assert isinstance(data["score"], float), ( + f"expected float, got {type(data['score'])}: {data['score']!r}" + ) + assert abs(data["score"] - 0.95) < 1e-4 + + def test_json_numpy_array_serialises_as_list_not_repr(self, tmp_path): + """np.ndarray values must round-trip as JSON arrays, not '[0.1 0.2]' repr.""" + import json + import numpy as np + out = self._write(tmp_path, "r.json", {"vec": np.array([0.1, 0.2, 0.3])}) + data = json.loads(out.read_text(encoding="utf-8")) + assert isinstance(data["vec"], list), ( + f"expected list, got {type(data['vec'])}: {data['vec']!r}" + ) + assert len(data["vec"]) == 3 + + # โ”€โ”€ JSONL โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def test_jsonl_list_writes_one_object_per_line(self, tmp_path): + """Each item in a list result must occupy exactly one JSONL line.""" + import json + records = [{"id": "a", "score": 0.9}, {"id": "b", "score": 0.7}] + out = self._write(tmp_path, "r.jsonl", records) + lines = [l for l in out.read_text(encoding="utf-8").splitlines() if l.strip()] + assert len(lines) == 2 + assert json.loads(lines[0]) == {"id": "a", "score": 0.9} + assert json.loads(lines[1]) == {"id": "b", "score": 0.7} + + def test_jsonl_dict_writes_exactly_one_line(self, tmp_path): + """A dict result (e.g. ontology_align) must write one JSON object on one line, + not a pretty-printed multi-line block that pd.read_json(lines=True) cannot parse.""" + import json + import pandas as pd + result = {"total_entities": 10, "duplicate_pairs": 3} + out = self._write(tmp_path, "r.jsonl", result) + raw = out.read_text(encoding="utf-8") + lines = [l for l in raw.splitlines() if l.strip()] + # Exactly one line + assert len(lines) == 1, ( + f"Expected 1 JSONL line for dict result, got {len(lines)}:\n{raw!r}" + ) + # That line parses as valid JSON + parsed = json.loads(lines[0]) + assert parsed == result + # pd.read_json(lines=True) can read it back + df = pd.read_json(out, lines=True) + assert list(df.columns) == ["total_entities", "duplicate_pairs"] + + def test_jsonl_numpy_values_are_not_repr_strings(self, tmp_path): + """NumPy values inside JSONL lines must be proper JSON, not repr().""" + import json + import numpy as np + records = [{"score": np.float32(0.8), "tag": "x"}] + out = self._write(tmp_path, "r.jsonl", records) + line = out.read_text(encoding="utf-8").strip() + parsed = json.loads(line) + assert isinstance(parsed["score"], float) + + # โ”€โ”€ CSV โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def test_csv_list_of_dicts_produces_readable_csv(self, tmp_path): + import pandas as pd + rows = [{"entity_1": "Alice", "entity_2": "Bob", "similarity": 0.87}, + {"entity_1": "Carol", "entity_2": "Dave", "similarity": 0.72}] + out = self._write(tmp_path, "r.csv", rows) + df = pd.read_csv(out) + assert list(df.columns) == ["entity_1", "entity_2", "similarity"] + assert len(df) == 2 + assert abs(df["similarity"].iloc[0] - 0.87) < 1e-6 + + def test_csv_numpy_scalar_becomes_number_not_repr(self, tmp_path): + """np.float32 in a result row must not become a repr string in the CSV.""" + import numpy as np + import pandas as pd + rows = [{"label": "x", "score": np.float32(0.95)}] + out = self._write(tmp_path, "r.csv", rows) + df = pd.read_csv(out) + # The cell must be a numeric type, not a string like 'np.float32(0.95)' + assert df["score"].dtype.kind in ("f", "i"), ( + f"Expected numeric dtype, got {df['score'].dtype}: {df['score'].iloc[0]!r}" + ) + + def test_csv_empty_list_raises_clickexception(self, tmp_path): + """An empty result list must raise rather than create a headerless newline.""" + import click + from semantica.cli import _write_result_output + out = tmp_path / "empty.csv" + with pytest.raises(click.ClickException, match="No results to write"): + _write_result_output(out, []) + assert not out.exists() + + def test_csv_single_dict_written_as_one_row(self, tmp_path): + import pandas as pd + out = self._write(tmp_path, "r.csv", {"total": 5, "merged": 2}) + df = pd.read_csv(out) + assert len(df) == 1 + assert df["total"].iloc[0] == 5 + + # โ”€โ”€ unsupported / no-extension โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def test_unsupported_extension_raises_clickexception(self, tmp_path): + import click + from semantica.cli import _write_result_output + out = tmp_path / "report.txt" + with pytest.raises(click.ClickException, match="Unsupported output format"): + _write_result_output(out, {"k": "v"}) + assert not out.exists() + + def test_no_extension_raises_clickexception(self, tmp_path): + """No-extension paths must be rejected โ€” not silently renamed to .json โ€” + so the path reported to the user always matches the file created.""" + import click + from semantica.cli import _write_result_output + out = tmp_path / "report" + with pytest.raises(click.ClickException, match="Unsupported output format"): + _write_result_output(out, {"k": "v"}) + assert not out.exists() + assert not (tmp_path / "report.json").exists() + + def test_txt_extension_raises_clickexception(self, tmp_path): + """.txt is not a documented format and must be rejected, consistent with + _write_embeddings_output which also rejects it.""" + import click + from semantica.cli import _write_result_output + out = tmp_path / "r.txt" + with pytest.raises(click.ClickException, match="Unsupported output format"): + _write_result_output(out, {"k": "v"}) + assert not out.exists() + + def test_uppercase_extension_accepted(self, tmp_path): + """Extension matching must be case-insensitive (.CSV == .csv).""" + import pandas as pd + out = self._write(tmp_path, "r.CSV", [{"a": 1}]) + df = pd.read_csv(out) + assert len(df) == 1 + + +class TestDeduplicateOutput: + """CLI-level regression tests for deduplicate --output integration. + + Uses the same monkeypatching pattern as TestDeduplicate.test_detect_runtime_path: + patch _get_store and get_nodes at the graph_store.methods level, then patch + the deduplication module so no real model or DB is needed. + """ + + _ENTITIES = [ + {"id": "e1", "name": "Alice", "type": "Person"}, + {"id": "e2", "name": "Alice", "type": "Person"}, + ] + _DETECT_RESULT = [ + {"entity_1": "e1", "entity_2": "e2", "similarity": 0.9} + ] + + def _patch_dedup(self, monkeypatch): + """Wire graph store + deduplication mocks for the detect action.""" + entities = self._ENTITIES + detect_result = self._DETECT_RESULT + + class FakeStore: + def get_nodes(self, labels=None, properties=None, limit=100, **opts): + return entities + + monkeypatch.setattr( + "semantica.graph_store.methods._get_store", lambda: FakeStore() + ) + monkeypatch.setattr( + "semantica.graph_store.methods.get_nodes", lambda **kw: entities + ) + monkeypatch.setattr( + "semantica.deduplication.methods.detect_duplicates", + lambda *a, **k: detect_result, + raising=False, + ) + # The CLI imports from .deduplication directly; patch that too. + import types + fake_dedup = _fake_module(detect_duplicates=lambda *a, **k: detect_result) + fake_merger_inst = types.SimpleNamespace( + merge_duplicates=lambda *a, **k: detect_result + ) + fake_dedup.entity_merger = types.SimpleNamespace( + EntityMerger=lambda: fake_merger_inst + ) + monkeypatch.setitem( + __import__("sys").modules, "semantica.deduplication", fake_dedup + ) + monkeypatch.setitem( + __import__("sys").modules, + "semantica.deduplication.entity_merger", + fake_dedup.entity_merger, + ) + + def test_deduplicate_output_json_is_valid(self, runner, monkeypatch, tmp_path): + """deduplicate --output report.json must produce parseable JSON, not a repr.""" + import json + self._patch_dedup(monkeypatch) + out = tmp_path / "report.json" + result = runner.invoke( + cli_module.main, + ["deduplicate", "--action", "detect", "--output", str(out)], + ) + _ok(result) + assert out.exists(), f"output file not created; output: {result.output!r}" + data = json.loads(out.read_text(encoding="utf-8")) + assert isinstance(data, (list, dict)) + + def test_deduplicate_output_csv_is_readable(self, runner, monkeypatch, tmp_path): + """deduplicate --output report.csv (documented format) must produce valid CSV.""" + import pandas as pd + self._patch_dedup(monkeypatch) + out = tmp_path / "report.csv" + result = runner.invoke( + cli_module.main, + ["deduplicate", "--action", "detect", "--output", str(out)], + ) + _ok(result) + assert out.exists(), f"CSV file not created; output: {result.output!r}" + df = pd.read_csv(out) + assert len(df) >= 1 + + +class TestOntologyAlignOutput: + """CLI-level regression tests for ontology align --output integration. + + Uses runner.isolated_filesystem() so Click's exists=True source/target + validation passes, then patches semantica.ontology at the sys.modules level + before the import inside _action() fires โ€” same pattern as + TestOntology.test_align_import_error_is_clean. + """ + + _ALIGN_RESULT = { + "alignments": [{"source": "A", "target": "B", "score": 0.8}], + "total": 1, + } + + def _patch_align(self, monkeypatch, align_result=None): + result = align_result if align_result is not None else self._ALIGN_RESULT + import types + fake_gen = types.SimpleNamespace(align=lambda *a, **k: result) + fake_ontology = _fake_module( + OntologyGenerator=lambda **k: fake_gen, + ) + monkeypatch.setitem( + __import__("sys").modules, "semantica.ontology", fake_ontology + ) + + def test_ontology_align_output_json_is_valid(self, runner, monkeypatch, tmp_path): + """ontology align --output alignments.json must produce parseable JSON.""" + import json + self._patch_align(monkeypatch) + out = tmp_path / "alignments.json" + with runner.isolated_filesystem(): + open("s.ttl", "w").close() + open("t.ttl", "w").close() + result = runner.invoke( + cli_module.main, + ["ontology", "align", + "--source", "s.ttl", "--target", "t.ttl", + "--output", str(out)], + ) + _ok(result) + assert out.exists(), f"output file not created; output: {result.output!r}" + data = json.loads(out.read_text(encoding="utf-8")) + assert isinstance(data, dict) + assert "alignments" in data + + def test_ontology_align_output_jsonl_is_readable_by_pandas( + self, runner, monkeypatch, tmp_path + ): + """ontology align --output alignments.jsonl must produce valid JSONL: + exactly one JSON object per line, readable by pd.read_json(lines=True). + Regression for F2: dict result must NOT be pretty-printed across multiple + lines into a .jsonl file.""" + import pandas as pd + self._patch_align(monkeypatch) + out = tmp_path / "alignments.jsonl" + with runner.isolated_filesystem(): + open("s.ttl", "w").close() + open("t.ttl", "w").close() + result = runner.invoke( + cli_module.main, + ["ontology", "align", + "--source", "s.ttl", "--target", "t.ttl", + "--output", str(out)], + ) + _ok(result) + assert out.exists(), f"JSONL file not created; output: {result.output!r}" + raw = out.read_text(encoding="utf-8") + lines = [ln for ln in raw.splitlines() if ln.strip()] + assert len(lines) == 1, ( + f"Expected exactly 1 JSONL line for a dict result, got {len(lines)}:\n{raw!r}" + ) + # pd.read_json(lines=True) must succeed โ€” this is what the F2 bug broke. + df = pd.read_json(out, lines=True) + assert "alignments" in df.columns diff --git a/tests/test_embedding_providers.py b/tests/test_embedding_providers.py index e41de0db..6fcc8735 100644 --- a/tests/test_embedding_providers.py +++ b/tests/test_embedding_providers.py @@ -85,3 +85,134 @@ if __name__ == '__main__': runner = unittest.TextTestRunner(stream=f, verbosity=2) unittest.main(testRunner=runner, exit=False) + +class TestMethodDispatchRecursion(unittest.TestCase): + """#994: built-in aliases are registered in the method registry onto the + wrapper functions themselves, so dispatching through the registry called a + wrapper back into itself with the same default method โ€” a recursion storm + that surfaced as `maximum recursion depth exceeded` during model loading.""" + + def test_generate_embeddings_default_does_not_self_recurse(self): + from semantica.embeddings.methods import generate_embeddings + emb = generate_embeddings("recursion probe") + self.assertIsNotNone(emb) + + def test_embed_text_default_does_not_self_recurse(self): + # Use the deterministic hash fallback to avoid model download; + # "fallback" is registered as embed_text itself, so the identity + # guard is the thing being tested โ€” no sentence-transformers needed. + from semantica.embeddings.methods import embed_text + emb = embed_text("recursion probe", method="fallback") + self.assertIsNotNone(emb) + + def test_custom_registered_method_still_wins(self): + from semantica.embeddings.methods import method_registry + calls = [] + + def spy(data, *a, **k): + calls.append(data) + return {"custom": True} + + method_registry.register("generation", "my_custom_gen", spy) + try: + from semantica.embeddings.methods import generate_embeddings + out = generate_embeddings("payload", method="my_custom_gen") + self.assertEqual(out, {"custom": True}) + self.assertEqual(calls, ["payload"]) + finally: + method_registry.unregister("generation", "my_custom_gen") + + def test_provenance_wrapper_missing_generator_raises_attribute_error(self): + # Partially-initialised wrappers (failed __init__, pickle/copy probes) + # must raise AttributeError, not RecursionError via __getattr__. + from semantica.embeddings.embeddings_provenance import ( + EmbeddingGeneratorWithProvenance, + ) + bare = EmbeddingGeneratorWithProvenance.__new__( + EmbeddingGeneratorWithProvenance + ) + with self.assertRaises(AttributeError): + getattr(bare, "model") + + def test_calculate_similarity_cosine_does_not_self_recurse(self): + """calculate_similarity is registered under "cosine"/"euclidean" โ€” the + identity guard must prevent infinite recursion when those aliases fire.""" + import numpy as np + from semantica.embeddings.methods import calculate_similarity + e1 = np.array([1.0, 0.0, 0.0]) + e2 = np.array([0.0, 1.0, 0.0]) + result = calculate_similarity(e1, e2, method="cosine") + self.assertIsNotNone(result) + + def test_pool_embeddings_mean_does_not_self_recurse(self): + """pool_embeddings is registered under all pooling aliases โ€” the identity + guard must prevent infinite recursion for every built-in pooling method.""" + import numpy as np + from semantica.embeddings.methods import pool_embeddings + embs = np.array([[1.0, 2.0], [3.0, 4.0]]) + result = pool_embeddings(embs, method="mean") + self.assertIsNotNone(result) + + +class TestDeduplicationDispatchRecursion(unittest.TestCase): + """Indirect recursion in deduplication/methods.py: the private wrapper + functions (_multi_factor_similarity, _pairwise_detection, _graph_based_clustering) + are registered as handlers under their respective default method names and + call back into the public dispatch functions with the same method, creating + an indirect infinite recursion loop without an identity guard.""" + + def test_calculate_similarity_multi_factor_does_not_recurse(self): + """_multi_factor_similarity is registered under 'similarity/multi_factor' + and calls calculate_similarity(method='multi_factor'), which without a + guard would re-enter _multi_factor_similarity infinitely.""" + from semantica.deduplication.methods import calculate_similarity + e1 = {"name": "Apple Inc.", "type": "Company"} + e2 = {"name": "Apple", "type": "Company"} + result = calculate_similarity(e1, e2, method="multi_factor") + self.assertIsNotNone(result) + + def test_detect_duplicates_pairwise_does_not_recurse(self): + """_pairwise_detection is registered under 'detection/pairwise' and + calls detect_duplicates(method='pairwise') โ€” indirect loop without guard.""" + from semantica.deduplication.methods import detect_duplicates + entities = [ + {"id": "1", "name": "Alice"}, + {"id": "2", "name": "Bob"}, + ] + result = detect_duplicates(entities, method="pairwise") + self.assertIsNotNone(result) + + def test_build_clusters_graph_based_does_not_recurse(self): + """_graph_based_clustering is registered under 'clustering/graph_based' + and calls build_clusters(method='graph_based') โ€” indirect loop without guard.""" + from semantica.deduplication.methods import build_clusters + entities = [ + {"id": "1", "name": "Alice"}, + {"id": "2", "name": "Bob"}, + ] + result = build_clusters(entities, method="graph_based") + self.assertIsNotNone(result) + + def test_custom_deduplication_method_still_wins(self): + """A genuinely user-registered custom method must still take precedence + over the built-in implementation after the guard is added.""" + from semantica.deduplication.methods import ( + calculate_similarity, + ) + from semantica.deduplication.registry import method_registry + calls = [] + + def spy(e1, e2, **kw): + calls.append((e1, e2)) + from semantica.deduplication.similarity_calculator import SimilarityResult + return SimilarityResult(score=0.99, method="spy") + + method_registry.register("similarity", "spy_method", spy) + try: + e1 = {"name": "Alice"} + e2 = {"name": "Alice"} + result = calculate_similarity(e1, e2, method="spy_method") + self.assertEqual(result.score, 0.99) + self.assertEqual(len(calls), 1) + finally: + method_registry.unregister("similarity", "spy_method") diff --git a/tests/test_mcp_package_export_graph.py b/tests/test_mcp_package_export_graph.py new file mode 100644 index 00000000..b8d63228 --- /dev/null +++ b/tests/test_mcp_package_export_graph.py @@ -0,0 +1,232 @@ +"""Regression tests for the standalone mcp/ package export_graph tool. + +The mcp/ server (python -m mcp / python -m mcp.server) had two failures on +every RDF export format: + + 1. AttributeError: 'ContextGraph' object has no attribute 'get' + handle_export_graph() in mcp/tools/export.py called + RDFExporter().export_to_rdf(graph, ...) passing the raw ContextGraph + object instead of the canonical kg dict expected by the exporter. + + 2. stdout progress corruption + RDFExporter.__init__ instantiated the Semantica progress-tracker + singleton, which wrote a progress bar to sys.stdout before the + AttributeError was raised. stdout is the MCP stdio JSON-RPC transport, + so this interleaved non-JSON bytes corrupted framing for every client. + +Fixes applied: + - mcp/tools/export.py: convert with graph.to_kg_dict() before export_to_rdf() + - mcp/__init__.py: os.environ["SEMANTICA_DISABLE_PROGRESS"] = "1" at + package initialisation, before any tool handler can instantiate + RDFExporter and therefore before the tracker singleton is created. +""" + +from __future__ import annotations + +import io +import os +import sys +import subprocess +import unittest + +import semantica.utils.progress_tracker as _progress_module + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_graph(): + """Return a ContextGraph with two entities and one relationship.""" + from semantica.context.context_graph import ContextGraph + g = ContextGraph() + g.add_node("n1", node_type="entity") + g.add_node("n2", node_type="entity") + g.add_edge("n1", "n2", "related_to") + return g + + +def _reset_progress_singleton(): + """Destroy any cached progress-tracker singleton so the next call + to get_progress_tracker() reads the current environment variable.""" + _progress_module.ProgressTracker._instance = None + _progress_module._global_tracker = None + + +# --------------------------------------------------------------------------- +# RDF export correctness +# --------------------------------------------------------------------------- + +class TestMCPPackageExportGraphRDF(unittest.TestCase): + """handle_export_graph() must return a non-empty RDF string for every + supported RDF format, not an error dict.""" + + def setUp(self): + # Inject a known graph into the mcp/ session so handlers don't try to + # build a full ContextGraph (which requires heavy ML dependencies). + import mcp.session as _session + self._orig_graph = _session._graph + _session._graph = _make_graph() + + def tearDown(self): + import mcp.session as _session + _session._graph = self._orig_graph + + def test_turtle_returns_non_empty_string(self): + from mcp.tools.export import handle_export_graph + result = handle_export_graph({"format": "turtle"}) + self.assertNotIn("error", result, result) + self.assertIsInstance(result["data"], str) + self.assertGreater(len(result["data"]), 0) + # Turtle output must carry prefix declarations + self.assertIn("@prefix", result["data"]) + + def test_ttl_alias_returns_non_empty_string(self): + from mcp.tools.export import handle_export_graph + result = handle_export_graph({"format": "ttl"}) + self.assertNotIn("error", result, result) + self.assertIsInstance(result["data"], str) + self.assertGreater(len(result["data"]), 0) + + def test_nt_returns_non_empty_string(self): + from mcp.tools.export import handle_export_graph + result = handle_export_graph({"format": "nt"}) + self.assertNotIn("error", result, result) + self.assertIsInstance(result["data"], str) + self.assertGreater(len(result["data"]), 0) + + def test_xml_returns_non_empty_string(self): + from mcp.tools.export import handle_export_graph + result = handle_export_graph({"format": "xml"}) + self.assertNotIn("error", result, result) + self.assertIsInstance(result["data"], str) + self.assertGreater(len(result["data"]), 0) + + def test_jsonld_returns_non_empty_string(self): + from mcp.tools.export import handle_export_graph + result = handle_export_graph({"format": "json-ld"}) + self.assertNotIn("error", result, result) + self.assertIsInstance(result["data"], str) + self.assertGreater(len(result["data"]), 0) + + def test_all_rdf_formats_succeed(self): + from mcp.tools.export import handle_export_graph + for fmt in ("turtle", "ttl", "nt", "xml", "json-ld"): + with self.subTest(fmt=fmt): + result = handle_export_graph({"format": fmt}) + self.assertNotIn("error", result, f"format={fmt}: {result}") + self.assertIsInstance(result["data"], str) + self.assertGreater(len(result["data"]), 0) + + def test_rdf_branch_does_not_raise_context_graph_attribute_error(self): + """The pre-fix code passed ContextGraph directly to export_to_rdf(), + causing AttributeError: 'ContextGraph' object has no attribute 'get'. + Verify that error does not appear in the result.""" + from mcp.tools.export import handle_export_graph + result = handle_export_graph({"format": "turtle"}) + if "error" in result: + self.assertNotIn("'ContextGraph' object has no attribute 'get'", + result["error"]) + + +# --------------------------------------------------------------------------- +# stdout protection โ€” subprocess-based to avoid process-state cross-contamination +# --------------------------------------------------------------------------- + +class TestMCPPackageStdoutProtection(unittest.TestCase): + """The standalone mcp/ server must not write any progress bytes to stdout. + stdout is the MCP JSON-RPC transport channel. + + These tests use a subprocess to get a clean process state where + SEMANTICA_DISABLE_PROGRESS has not yet been set, so we can verify that + importing mcp and running an export produces no progress bytes on stdout. + """ + + def _run_in_subprocess(self, code: str, timeout: int = 30) -> subprocess.CompletedProcess: + """Run a Python snippet in a clean subprocess with the repo on sys.path.""" + repo_root = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..") + ) + env = os.environ.copy() + env["PYTHONPATH"] = repo_root + # Start with a clean slate โ€” no pre-set disable flag + env.pop("SEMANTICA_DISABLE_PROGRESS", None) + return subprocess.run( + [sys.executable, "-c", code], + cwd=repo_root, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + check=False, + ) + + def test_importing_mcp_sets_disable_progress(self): + """Importing the mcp package must set SEMANTICA_DISABLE_PROGRESS=1 + before any tool handler runs.""" + code = ( + "import os; " + "import mcp; " # triggers mcp/__init__.py + "print(os.environ.get('SEMANTICA_DISABLE_PROGRESS', 'NOT SET'))" + ) + result = self._run_in_subprocess(code) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("1", result.stdout) + + def test_rdf_export_writes_no_progress_to_stdout(self): + """An RDF export via handle_export_graph() must not write any Semantica + progress bytes to stdout. The only stdout bytes should be the explicit + print() call at the end of the snippet.""" + code = """ +import os, sys +# Ensure clean state +os.environ.pop("SEMANTICA_DISABLE_PROGRESS", None) + +import mcp # sets SEMANTICA_DISABLE_PROGRESS=1 +import mcp.session as session +from semantica.context.context_graph import ContextGraph + +g = ContextGraph() +g.add_node("n1", node_type="entity") +g.add_node("n2", node_type="entity") +g.add_edge("n1", "n2", "related_to") +session._graph = g + +# Intercept stdout writes to detect any progress output +written = [] +_orig = sys.stdout.write +def _capture(s): + written.append(s) + return _orig(s) +sys.stdout.write = _capture + +from mcp.tools.export import handle_export_graph +result = handle_export_graph({"format": "turtle"}) + +sys.stdout.write = _orig + +# Only our explicit output below should be in written +# (the sentinel line is added after restoring stdout) +progress_writes = [s for s in written] +print("RESULT_OK:" + str("error" not in result)) +print("STDOUT_WRITES:" + str(len(progress_writes))) +""" + proc = self._run_in_subprocess(code) + self.assertEqual(proc.returncode, 0, proc.stderr) + # Extract the printed lines + lines = proc.stdout.strip().splitlines() + result_ok_line = next((l for l in lines if l.startswith("RESULT_OK:")), None) + writes_line = next((l for l in lines if l.startswith("STDOUT_WRITES:")), None) + self.assertIsNotNone(result_ok_line, f"stdout: {proc.stdout!r}") + self.assertIsNotNone(writes_line, f"stdout: {proc.stdout!r}") + self.assertEqual(result_ok_line, "RESULT_OK:True", + f"export returned error; stdout={proc.stdout!r}, stderr={proc.stderr!r}") + n_writes = int(writes_line.split(":")[1]) + self.assertEqual(n_writes, 0, + f"Expected 0 progress writes to stdout, got {n_writes}; " + f"stdout={proc.stdout!r}") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mcp_server_export_graph.py b/tests/test_mcp_server_export_graph.py new file mode 100644 index 00000000..09179fd8 --- /dev/null +++ b/tests/test_mcp_server_export_graph.py @@ -0,0 +1,90 @@ +"""Regression tests for the MCP export_graph tool (issue: all branches broken). + +The MCP server's export_graph tool failed on every format in 0.6.5/0.6.6: + - json: JSONExporter().export(graph) called without the required file_path + argument -> TypeError, surfaced as {"error": ...} + - RDF: RDFExporter().export_to_rdf(graph, ...) received the ContextGraph + object instead of the canonical kg dict -> AttributeError + - all: the RDF path printed a rich progress bar to stdout, corrupting the + stdio JSON-RPC framing and hanging the client (observed: 300s + timeout over MCP, <1s directly). + +The fix: convert the graph with ContextGraph.to_kg_dict() before handing it to +the exporters, serialize json to a string, and force SEMANTICA_DISABLE_PROGRESS +for the server process (stdout is the protocol channel, not a console). +""" + +import json +import os +import unittest + +from semantica import mcp_server +from semantica.context import ContextGraph + + +def _graph_with_content() -> ContextGraph: + graph = ContextGraph(advanced_analytics=True) + graph.add_node("n1", node_type="entity", properties={"text": "hello"}) + graph.add_node("n2", node_type="entity", properties={"text": "world"}) + graph.add_edge("n1", "n2", "related_to") + return graph + + +class TestExportGraphTool(unittest.TestCase): + + def setUp(self): + self._old_graph = mcp_server._graph + mcp_server._graph = _graph_with_content() + + def tearDown(self): + mcp_server._graph = self._old_graph + + def test_json_branch_returns_string_data_not_error(self): + result = mcp_server._tool_export_graph({"format": "json"}) + self.assertNotIn("error", result) + self.assertEqual(result["format"], "json") + payload = json.loads(result["data"]) + self.assertEqual(len(payload["entities"]), 2) + self.assertEqual(len(payload["relationships"]), 1) + + def test_jsonld_branch_returns_string_data_not_error(self): + result = mcp_server._tool_export_graph({"format": "json-ld"}) + self.assertNotIn("error", result) + self.assertEqual(result["format"], "json-ld") + self.assertIsInstance(result["data"], str) + self.assertGreater(len(result["data"]), 0) + + def test_turtle_branch_returns_string_data_not_error(self): + result = mcp_server._tool_export_graph({"format": "turtle"}) + self.assertNotIn("error", result) + self.assertIsInstance(result["data"], str) + self.assertIn("@prefix", result["data"]) + + def test_all_rdf_formats_succeed(self): + for fmt in ("turtle", "ttl", "nt", "xml", "json-ld"): + with self.subTest(fmt=fmt): + result = mcp_server._tool_export_graph({"format": fmt}) + self.assertNotIn("error", result, fmt) + self.assertIsInstance(result["data"], str) + + def test_progress_is_disabled_for_the_server_process(self): + self.assertEqual(os.environ.get("SEMANTICA_DISABLE_PROGRESS"), "1") + + def test_unsupported_format_returns_error_not_mislabeled_json(self): + """A format outside the declared enum (typo, unsupported value, or a + client that skips schema validation) must error, not silently return + JSON data mislabeled with the requested format string.""" + result = mcp_server._tool_export_graph({"format": "yaml"}) + self.assertIn("error", result) + self.assertIn("yaml", result["error"]) + + def test_export_graph_schema_enum_matches_handled_formats(self): + """The tool's declared inputSchema enum must not drift from the set + of formats the handler actually accepts.""" + tool = next(t for t in mcp_server.TOOLS if t["name"] == "export_graph") + schema_enum = set(tool["inputSchema"]["properties"]["format"]["enum"]) + self.assertEqual(schema_enum, set(mcp_server._EXPORT_GRAPH_FORMATS)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_progress_tracker_regressions.py b/tests/test_progress_tracker_regressions.py index ac4c09da..b42a885b 100644 --- a/tests/test_progress_tracker_regressions.py +++ b/tests/test_progress_tracker_regressions.py @@ -12,6 +12,7 @@ import semantica.utils.progress_tracker as progress_module @pytest.fixture(autouse=True) def reset_progress_singletons(monkeypatch): monkeypatch.delenv("SEMANTICA_DISABLE_PROGRESS", raising=False) + monkeypatch.delenv("SEMANTICA_FORCE_PROGRESS", raising=False) progress_module.ProgressTracker._instance = None progress_module._global_tracker = None yield @@ -19,6 +20,40 @@ def reset_progress_singletons(monkeypatch): progress_module._global_tracker = None +class _FakeStdout: + """Minimal stdout stand-in with controllable TTY reporting.""" + + encoding = "utf-8" + + def __init__(self, tty): + self._tty = tty + self.written = [] + + def isatty(self): + return self._tty + + def write(self, text): + self.written.append(text) + return len(text) + + def flush(self): + pass + + +def _use_stdout(monkeypatch, tty): + """Point sys.stdout at a fake with the given TTY behaviour, outside Jupyter.""" + stream = _FakeStdout(tty=tty) + monkeypatch.setattr(sys, "stdout", stream) + monkeypatch.setattr( + progress_module.ProgressTracker, "_detect_jupyter", lambda *_: False + ) + return stream + + +def _displays_of(tracker, display_cls): + return [d for d in tracker.displays if isinstance(d, display_cls)] + + def _install_tracker_as_singleton(tracker: progress_module.ProgressTracker) -> None: progress_module.ProgressTracker._instance = tracker progress_module._global_tracker = tracker @@ -100,6 +135,67 @@ def test_disable_progress_env_prevents_reenable(monkeypatch): assert tracker.start_tracking(module="core", submodule="test") == "" +def test_console_display_omitted_when_stdout_is_not_a_tty(monkeypatch): + _use_stdout(monkeypatch, tty=False) + + tracker = progress_module.ProgressTracker(use_emoji=False, update_interval=0) + + assert _displays_of(tracker, progress_module.ConsoleProgressDisplay) == [] + + +def test_console_display_present_when_stdout_is_a_tty(monkeypatch): + _use_stdout(monkeypatch, tty=True) + + tracker = progress_module.ProgressTracker(use_emoji=False, update_interval=0) + + assert _displays_of(tracker, progress_module.ConsoleProgressDisplay) + + +def test_file_display_survives_non_tty_stdout(monkeypatch): + _use_stdout(monkeypatch, tty=False) + + tracker = progress_module.ProgressTracker(use_emoji=False, update_interval=0) + + assert _displays_of(tracker, progress_module.FileProgressDisplay) + + +def test_force_progress_env_restores_console_display_on_non_tty(monkeypatch): + monkeypatch.setenv("SEMANTICA_FORCE_PROGRESS", "1") + _use_stdout(monkeypatch, tty=False) + + tracker = progress_module.ProgressTracker(use_emoji=False, update_interval=0) + + assert _displays_of(tracker, progress_module.ConsoleProgressDisplay) + + +def test_disable_progress_env_beats_force_progress_env(monkeypatch): + monkeypatch.setenv("SEMANTICA_DISABLE_PROGRESS", "1") + monkeypatch.setenv("SEMANTICA_FORCE_PROGRESS", "1") + stream = _use_stdout(monkeypatch, tty=False) + + tracker = progress_module.ProgressTracker(use_emoji=False, update_interval=0) + _install_tracker_as_singleton(tracker) + + assert tracker.enabled is False + assert tracker.start_tracking(module="core", submodule="test") == "" + assert stream.written == [] + + +def test_non_tty_stdout_stays_silent_after_module_reenables_tracker(monkeypatch): + stream = _use_stdout(monkeypatch, tty=False) + tracker = progress_module.ProgressTracker(use_emoji=False, update_interval=0) + _install_tracker_as_singleton(tracker) + + # Mirrors the ~20 modules that do `self.progress_tracker.enabled = True`. + tracker.enabled = True + tracking_id = tracker.start_tracking( + module="core", submodule="Semantica", message="Building" + ) + tracker.update_progress(tracking_id, processed=1, total=1, message="Processing") + + assert stream.written == [] + + def test_build_knowledge_base_subprocess_does_not_deadlock(): root = Path(__file__).resolve().parents[1] runtime_dir = root / "test_data" / "runtime" / f"build-regression-{os.getpid()}" diff --git a/tests/test_reproduce_issue_176.py b/tests/test_reproduce_issue_176.py new file mode 100644 index 00000000..79b24076 --- /dev/null +++ b/tests/test_reproduce_issue_176.py @@ -0,0 +1,334 @@ + +import unittest +from unittest.mock import MagicMock, patch +from semantica.semantic_extract.methods import extract_relations_llm, extract_entities_llm, extract_triplets_llm +from semantica.semantic_extract.ner_extractor import Entity + +class TestMaxTokensPropagation(unittest.TestCase): + @patch("semantica.semantic_extract.methods.create_provider") + def test_max_tokens_propagation_relations(self, mock_create_provider): + """Test that max_tokens is passed to generate_typed in extract_relations_llm.""" + # Setup mock + mock_llm = MagicMock() + mock_create_provider.return_value = mock_llm + mock_llm.is_available.return_value = True + + # Setup return value to avoid pydantic validation errors + mock_response = MagicMock() + mock_response.relations = [] + mock_llm.generate_typed.return_value = mock_response + + # Create dummy entities + entities = [Entity(text="Foo", label="ORG", start_char=0, end_char=3)] + + # Call the function with max_tokens + extract_relations_llm( + text="some text", + entities=entities, + provider="openai", + model="gpt-4", + max_tokens=128000 + ) + + # Check if generate_typed was called with max_tokens + args, kwargs = mock_llm.generate_typed.call_args + + print(f"Relations Call kwargs: {kwargs}") + + self.assertIn("max_tokens", kwargs) + self.assertEqual(kwargs["max_tokens"], 128000) + + @patch("semantica.semantic_extract.methods.create_provider") + def test_max_tokens_propagation_entities(self, mock_create_provider): + """Test that max_tokens is passed to generate_typed in extract_entities_llm.""" + # Setup mock + mock_llm = MagicMock() + mock_create_provider.return_value = mock_llm + mock_llm.is_available.return_value = True + + # Setup return value to avoid pydantic validation errors + mock_response = MagicMock() + mock_response.entities = [] + mock_llm.generate_typed.return_value = mock_response + + # Call the function with max_tokens + extract_entities_llm( + text="some text", + provider="openai", + model="gpt-4", + max_tokens=128000 + ) + + # Check if generate_typed was called with max_tokens + args, kwargs = mock_llm.generate_typed.call_args + + print(f"Entities Call kwargs: {kwargs}") + + self.assertIn("max_tokens", kwargs) + self.assertEqual(kwargs["max_tokens"], 128000) + + @patch("semantica.semantic_extract.methods.create_provider") + def test_max_tokens_propagation_triplets(self, mock_create_provider): + """Test that max_tokens is passed to generate_typed in extract_triplets_llm.""" + # Setup mock + mock_llm = MagicMock() + mock_create_provider.return_value = mock_llm + mock_llm.is_available.return_value = True + + # Setup return value to avoid pydantic validation errors + mock_response = MagicMock() + mock_response.triplets = [] + mock_llm.generate_typed.return_value = mock_response + + # Call the function with max_tokens + extract_triplets_llm( + text="some text", + provider="openai", + model="gpt-4", + max_tokens=128000 + ) + + # Check if generate_typed was called with max_tokens + args, kwargs = mock_llm.generate_typed.call_args + + print(f"Triplets Call kwargs: {kwargs}") + + self.assertIn("max_tokens", kwargs) + self.assertEqual(kwargs["max_tokens"], 128000) + + +class TestCacheKeyIncludesGenerationParams(unittest.TestCase): + """Regression tests for the cache-key bug: two calls with identical extraction + inputs but different generation settings must NOT share a cached result. + + Before the fix, extract_relations_llm (and entities/triplets) built + cache_params without generation kwargs, so max_tokens=4096 and + max_tokens=128000 hashed to the same key. The second call would return the + first cached result without ever running generate_typed again. + """ + + def _make_mock_llm(self, relations=None, entities=None, triplets=None): + mock_llm = MagicMock() + mock_llm.is_available.return_value = True + resp = MagicMock() + resp.relations = relations if relations is not None else [] + resp.entities = entities if entities is not None else [] + resp.triplets = triplets if triplets is not None else [] + mock_llm.generate_typed.return_value = resp + return mock_llm + + @patch("semantica.semantic_extract.methods.create_provider") + def test_relations_different_max_tokens_bypass_cache(self, mock_create_provider): + """Two relation extraction calls with the same text/entities but different + max_tokens must each call generate_typed (2 calls total), not reuse the + first cached result.""" + from semantica.semantic_extract.methods import _result_cache + _result_cache.clear("relations") + + mock_llm = self._make_mock_llm() + mock_create_provider.return_value = mock_llm + + entities = [Entity(text="Foo", label="ORG", start_char=0, end_char=3)] + + extract_relations_llm( + text="some text", entities=entities, + provider="openai", model="gpt-4", max_tokens=4096 + ) + extract_relations_llm( + text="some text", entities=entities, + provider="openai", model="gpt-4", max_tokens=128000 + ) + + # generate_typed must have been called twice โ€” once per unique key + self.assertEqual( + mock_llm.generate_typed.call_count, 2, + "Different max_tokens values must produce different cache keys; " + "second call must not reuse the first cached result." + ) + + @patch("semantica.semantic_extract.methods.create_provider") + def test_relations_same_max_tokens_uses_cache(self, mock_create_provider): + """Two identical calls must reuse the cache (generate_typed called once).""" + from semantica.semantic_extract.methods import _result_cache + _result_cache.clear("relations") + + mock_llm = self._make_mock_llm() + mock_create_provider.return_value = mock_llm + + entities = [Entity(text="Foo", label="ORG", start_char=0, end_char=3)] + + extract_relations_llm( + text="some text", entities=entities, + provider="openai", model="gpt-4", max_tokens=4096 + ) + extract_relations_llm( + text="some text", entities=entities, + provider="openai", model="gpt-4", max_tokens=4096 + ) + + self.assertEqual( + mock_llm.generate_typed.call_count, 1, + "Identical calls must reuse the cache." + ) + + @patch("semantica.semantic_extract.methods.create_provider") + def test_relations_different_temperature_bypass_cache(self, mock_create_provider): + """Different temperature values must also produce different cache keys.""" + from semantica.semantic_extract.methods import _result_cache + _result_cache.clear("relations") + + mock_llm = self._make_mock_llm() + mock_create_provider.return_value = mock_llm + + entities = [Entity(text="Bar", label="PERSON", start_char=0, end_char=3)] + + extract_relations_llm( + text="other text", entities=entities, + provider="openai", model="gpt-4", temperature=0.0 + ) + extract_relations_llm( + text="other text", entities=entities, + provider="openai", model="gpt-4", temperature=1.0 + ) + + self.assertEqual(mock_llm.generate_typed.call_count, 2) + + @patch("semantica.semantic_extract.methods.create_provider") + def test_entities_different_max_tokens_bypass_cache(self, mock_create_provider): + """extract_entities_llm: different max_tokens must bypass cache.""" + from semantica.semantic_extract.methods import _result_cache + _result_cache.clear("entities") + + mock_llm = self._make_mock_llm() + mock_create_provider.return_value = mock_llm + + extract_entities_llm( + text="some entity text", provider="openai", model="gpt-4", + max_tokens=4096 + ) + extract_entities_llm( + text="some entity text", provider="openai", model="gpt-4", + max_tokens=128000 + ) + + self.assertEqual(mock_llm.generate_typed.call_count, 2) + + @patch("semantica.semantic_extract.methods.create_provider") + def test_triplets_different_max_tokens_bypass_cache(self, mock_create_provider): + """extract_triplets_llm: different max_tokens must bypass cache.""" + from semantica.semantic_extract.methods import _result_cache + _result_cache.clear("triplets") + + mock_llm = self._make_mock_llm() + mock_create_provider.return_value = mock_llm + + extract_triplets_llm( + text="some triplet text", provider="openai", model="gpt-4", + max_tokens=4096 + ) + extract_triplets_llm( + text="some triplet text", provider="openai", model="gpt-4", + max_tokens=128000 + ) + + self.assertEqual(mock_llm.generate_typed.call_count, 2) + + +class TestCacheKeyIncludesProviderSpecificGenerationParams(unittest.TestCase): + """Regression tests for provider-specific generation params that aren't part + of the common OpenAI-shaped kwargs (max_tokens, temperature, etc.) but still + change provider output and must therefore also change the cache key. + + See providers.py: AnthropicProvider.generate/generate_structured read + 'system' and 'stop_sequences' via a manual pass-through loop (not + _add_if_set); GeminiProvider.generate reads 'candidate_count' and + 'stop_sequences'; OllamaProvider._build_options reads 'repeat_penalty' and + 'num_ctx'/'context_window'. + """ + + def _make_mock_llm(self): + mock_llm = MagicMock() + mock_llm.is_available.return_value = True + resp = MagicMock() + resp.relations = [] + mock_llm.generate_typed.return_value = resp + return mock_llm + + @patch("semantica.semantic_extract.methods.create_provider") + def test_relations_different_system_prompt_bypass_cache(self, mock_create_provider): + """Anthropic 'system' prompt changes output; must not share a cache entry.""" + from semantica.semantic_extract.methods import _result_cache + _result_cache.clear("relations") + + mock_llm = self._make_mock_llm() + mock_create_provider.return_value = mock_llm + + entities = [Entity(text="Foo", label="ORG", start_char=0, end_char=3)] + + extract_relations_llm( + text="some text", entities=entities, + provider="anthropic", model="claude-3-sonnet-20240229", + system="Extract only ORG relations." + ) + extract_relations_llm( + text="some text", entities=entities, + provider="anthropic", model="claude-3-sonnet-20240229", + system="Extract only PERSON relations." + ) + + self.assertEqual( + mock_llm.generate_typed.call_count, 2, + "Different 'system' prompts must produce different cache keys." + ) + + @patch("semantica.semantic_extract.methods.create_provider") + def test_relations_different_stop_sequences_bypass_cache(self, mock_create_provider): + """Anthropic/Gemini 'stop_sequences' must also be part of the cache key.""" + from semantica.semantic_extract.methods import _result_cache + _result_cache.clear("relations") + + mock_llm = self._make_mock_llm() + mock_create_provider.return_value = mock_llm + + entities = [Entity(text="Foo", label="ORG", start_char=0, end_char=3)] + + extract_relations_llm( + text="some text", entities=entities, + provider="anthropic", model="claude-3-sonnet-20240229", + stop_sequences=["\n\n"] + ) + extract_relations_llm( + text="some text", entities=entities, + provider="anthropic", model="claude-3-sonnet-20240229", + stop_sequences=["STOP"] + ) + + self.assertEqual(mock_llm.generate_typed.call_count, 2) + + @patch("semantica.semantic_extract.methods.create_provider") + def test_relations_different_repeat_penalty_bypass_cache(self, mock_create_provider): + """Ollama 'repeat_penalty' must also be part of the cache key.""" + from semantica.semantic_extract.methods import _result_cache + _result_cache.clear("relations") + + mock_llm = self._make_mock_llm() + mock_create_provider.return_value = mock_llm + + entities = [Entity(text="Foo", label="ORG", start_char=0, end_char=3)] + + extract_relations_llm( + text="some text", entities=entities, + provider="ollama", model="llama2", + repeat_penalty=1.0 + ) + extract_relations_llm( + text="some text", entities=entities, + provider="ollama", model="llama2", + repeat_penalty=1.5 + ) + + self.assertEqual(mock_llm.generate_typed.call_count, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_seed_manager.py b/tests/test_seed_manager.py index 95d490a1..1f07ed0b 100644 --- a/tests/test_seed_manager.py +++ b/tests/test_seed_manager.py @@ -5,6 +5,9 @@ import json import csv from pathlib import Path from unittest.mock import MagicMock, patch + +import requests + from semantica.seed.seed_manager import SeedDataManager, SeedDataSource, SeedData from semantica.utils.exceptions import ProcessingError @@ -263,6 +266,61 @@ def test_load_from_api_does_not_mutate_empty_headers_dict(mock_guard, seed_manag guard_headers = call_kwargs.get("headers", {}) assert guard_headers.get("Authorization") == "Bearer key" + +# requests.exceptions.RequestException subclasses OSError, so network failures raised +# by request_with_ssrf_guard used to be reported as "requests library not available" +# by the obsolete ImportError / OSError handler. They must surface the real cause. +@pytest.mark.parametrize( + "error", + [ + requests.exceptions.ConnectionError("connection refused"), + requests.exceptions.Timeout("timed out"), + requests.exceptions.HTTPError("500 Server Error"), + ], +) +@patch("semantica.seed.seed_manager.request_with_ssrf_guard") +def test_load_from_api_request_failure_reports_real_cause(mock_guard, error, seed_manager): + mock_guard.side_effect = error + + with pytest.raises(ProcessingError) as excinfo: + seed_manager.load_from_api(api_url="http://api.example.com", endpoint="users") + + message = str(excinfo.value) + assert "Failed to load from API" in message + assert str(error) in message + assert "requests library not available" not in message + assert excinfo.value.__cause__ is error + + +@patch("semantica.seed.seed_manager.request_with_ssrf_guard") +def test_load_from_api_http_status_error_reports_real_cause(mock_guard, seed_manager): + http_error = requests.exceptions.HTTPError("404 Client Error: Not Found") + mock_response = MagicMock() + mock_response.raise_for_status.side_effect = http_error + mock_guard.return_value = mock_response + + with pytest.raises(ProcessingError) as excinfo: + seed_manager.load_from_api(api_url="http://api.example.com", endpoint="users") + + message = str(excinfo.value) + assert "404 Client Error: Not Found" in message + assert "requests library not available" not in message + mock_response.json.assert_not_called() + + +@patch("semantica.seed.seed_manager.request_with_ssrf_guard") +def test_load_from_api_invalid_json_reports_real_cause(mock_guard, seed_manager): + mock_response = MagicMock() + mock_response.json.side_effect = ValueError("Expecting value: line 1 column 1") + mock_guard.return_value = mock_response + + with pytest.raises(ProcessingError) as excinfo: + seed_manager.load_from_api(api_url="http://api.example.com") + + message = str(excinfo.value) + assert "Failed to load from API" in message + assert "Expecting value" in message + def test_load_source(seed_manager, temp_data_dir): json_file = temp_data_dir / "source.json" with open(json_file, "w") as f: diff --git a/tests/triplet_store/test_oxigraph_store.py b/tests/triplet_store/test_oxigraph_store.py index cbd3c979..9415795d 100644 --- a/tests/triplet_store/test_oxigraph_store.py +++ b/tests/triplet_store/test_oxigraph_store.py @@ -159,3 +159,89 @@ def test_missing_optional_dependency_has_install_hint(): ): with pytest.raises(ImportError, match="tripletstore-oxigraph"): _store() + + +def test_on_disk_add_triplets_calls_flush(tmp_path): + """add_triplets on a disk-backed store must flush once after the batch. + + The pyoxigraph background-thread flush "might lag a little bit"; an + explicit flush after the batch closes that race without fsyncing on + every individual write. This test verifies the contract directly + without relying on CPython destructor timing. + """ + store = OxigraphStore(path=tmp_path / "oxigraph") + with patch.object(store, "flush") as mock_flush: + store.add_triplets([ + Triplet(EX + "alice", EX + "knows", EX + "bob"), + Triplet(EX + "bob", EX + "knows", EX + "carol"), + ]) + mock_flush.assert_called_once() + + +def test_on_disk_add_triplet_does_not_flush(tmp_path): + """add_triplet (single write) must NOT flush on every call. + + Individual writes are committed to the store in memory; the caller is + responsible for calling flush() when a hard durability boundary is + needed. Flushing on every add_triplet() call would fsync on every + write, causing a severe throughput regression for workloads that write + triplets one at a time. + """ + store = OxigraphStore(path=tmp_path / "oxigraph") + with patch.object(store, "flush") as mock_flush: + store.add_triplet(Triplet(EX + "alice", EX + "knows", EX + "bob")) + mock_flush.assert_not_called() + + +def test_in_memory_add_triplets_does_not_flush(tmp_path): + """In-memory stores must not call flush() โ€” there is nothing to flush.""" + store = OxigraphStore() # no path โ†’ in-memory + with patch.object(store, "flush") as mock_flush: + store.add_triplet(Triplet(EX + "alice", EX + "knows", EX + "bob")) + store.add_triplets([Triplet(EX + "bob", EX + "knows", EX + "carol")]) + mock_flush.assert_not_called() + + +def test_on_disk_add_triplets_is_durable_on_reopen(tmp_path): + """End-to-end durability: a batch written via add_triplets and closed + cleanly survives a reopen. + + This is an integration test for the full add_triplets โ†’ flush โ†’ close โ†’ + reopen lifecycle. The durability contract here is provided by the + explicit ``store.flush()`` call before deletion; the internal flush + inside add_triplets reduces (but does not eliminate) the crash-window + race. The authoritative unit test for the internal flush behaviour is + ``test_on_disk_add_triplets_calls_flush``. + """ + path = tmp_path / "oxigraph" + store = OxigraphStore(path=path) + store.add_triplets([ + Triplet(EX + "alice", EX + "knows", EX + "bob"), + Triplet(EX + "bob", EX + "knows", EX + "carol"), + ]) + store.flush() # belt-and-suspenders: ensures close is clean + del store + gc.collect() + + reopened = OxigraphStore(path=path) + assert len(reopened.get_triplets()) == 2 + + +def test_storage_path_is_accepted_as_alias_for_path(tmp_path): + """Regression: ``storage_path=...`` used to be silently swallowed by + ``**config`` (the __init__ parameter is named ``path``), so the store + silently degraded to in-memory with no warning. It must now be accepted + as an alias consistent with other Semantica stores (e.g. ProvenanceManager).""" + storage_path = tmp_path / "oxigraph" + + store = OxigraphStore(storage_path=str(storage_path)) + + assert store.path == str(storage_path) + # and it must actually persist (proves the alias wired through to the + # on-disk path, not just set the attribute) + store.add_triplet(Triplet(EX + "alice", EX + "knows", EX + "bob")) + del store + gc.collect() + + reopened = OxigraphStore(storage_path=str(storage_path)) + assert len(reopened.get_triplets()) == 1 diff --git a/tests/triplet_store/test_rdf4j_store.py b/tests/triplet_store/test_rdf4j_store.py index 4ef4f16b..03a9b90b 100644 --- a/tests/triplet_store/test_rdf4j_store.py +++ b/tests/triplet_store/test_rdf4j_store.py @@ -22,6 +22,71 @@ def _make_connected_store(): CONSTRUCT_QUERY = "CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }" +class TestRDF4JStoreInitialization(unittest.TestCase): + + def test_explicit_repository_id_selects_repository(self): + response = MagicMock(status_code=200) + + with patch( + "semantica.triplet_store.rdf4j_store.requests.get", + return_value=response, + ) as mock_get: + store = RDF4JStore( + endpoint="http://localhost:8080/rdf4j-server/", + repository_id="semantica", + ) + + self.assertEqual(store.repository_id, "semantica") + mock_get.assert_called_once_with( + "http://localhost:8080/rdf4j-server/repositories/semantica", + timeout=30, + auth=None, + ) + + def test_repository_id_is_encoded_as_a_single_url_path_segment(self): + response = MagicMock(status_code=200) + + with patch( + "semantica.triplet_store.rdf4j_store.requests.get", + return_value=response, + ) as mock_get: + store = RDF4JStore( + endpoint="http://localhost:8080/rdf4j-server", + repository_id="team/repo ?#", + ) + + self.assertEqual(store.repository_id, "team/repo ?#") + mock_get.assert_called_once_with( + "http://localhost:8080/rdf4j-server/repositories/team%2Frepo%20%3F%23", + timeout=30, + auth=None, + ) + self.assertEqual( + store._get_sparql_endpoint(), + "http://localhost:8080/rdf4j-server/repositories/team%2Frepo%20%3F%23", + ) + self.assertEqual( + store._get_update_endpoint(), + "http://localhost:8080/rdf4j-server/repositories/" + "team%2Frepo%20%3F%23/statements", + ) + + transaction_response = MagicMock() + transaction_response.headers = {"Location": "/transactions/tx-1"} + with patch( + "semantica.triplet_store.rdf4j_store.requests.post", + return_value=transaction_response, + ) as mock_post: + self.assertEqual(store.begin_transaction(), "tx-1") + + mock_post.assert_called_once_with( + "http://localhost:8080/rdf4j-server/repositories/" + "team%2Frepo%20%3F%23/transactions", + timeout=30, + auth=None, + ) + + class TestRDF4JStoreIsConstructQuery(unittest.TestCase): def test_detects_uppercase(self): self.assertTrue(_make_connected_store()._is_construct_query( diff --git a/tests/utils/test_normalize_graph_payload.py b/tests/utils/test_normalize_graph_payload.py index 02b53219..6f64aa22 100644 --- a/tests/utils/test_normalize_graph_payload.py +++ b/tests/utils/test_normalize_graph_payload.py @@ -474,6 +474,77 @@ class TestIsRecordBoundary(unittest.TestCase): ) +class TestKeyDisplayBounds(unittest.TestCase): + """Exception messages must not scale with caller-controlled keys (#1001). + + The validation boundary interpolates supplied keys straight into error + messages, so an extremely large key produced an equally large exception + string -- and, through the export wrappers that log the full exception, + an equally large log entry. The displayed key is truncated to a bounded + length while the supplied payload itself is never modified. + """ + + def test_unrecognized_key_display_is_bounded(self): + with self.assertRaises(ValidationError) as ctx: + normalize_graph_payload({"x" * 1_000_000: [ENTITY]}) + + message = str(ctx.exception) + self.assertLess(len(message), 300) + self.assertIn("x" * 64 + "โ€ฆ", message) + # Truncating the supplied key must not cost the actionable part. + self.assertIn("no recognized key", message) + self.assertIn("entities", message) + + def test_dropped_record_key_display_is_bounded(self): + with self.assertRaises(ValidationError) as ctx: + normalize_graph_payload({"entities": [], "y" * 1_000_000: [ENTITY]}) + + message = str(ctx.exception) + self.assertLess(len(message), 300) + self.assertIn("y" * 64 + "โ€ฆ", message) + self.assertIn("holds records", message) + + def test_short_keys_are_displayed_in_full(self): + with self.assertRaises(ValidationError) as ctx: + normalize_graph_payload({"short_key": [ENTITY]}) + + self.assertIn("'short_key'", str(ctx.exception)) + + def test_bounded_display_does_not_mutate_the_payload(self): + big_key = "z" * 1_000_000 + payload = {big_key: [ENTITY]} + + with self.assertRaises(ValidationError): + normalize_graph_payload(payload) + + self.assertEqual(list(payload), [big_key]) + self.assertEqual(payload[big_key], [ENTITY]) + + def test_many_unrecognized_keys_are_summarized(self): + """Per-key truncation does not bound the number of keys shown. + + A payload carrying many short unrecognized keys would still size the + message (and the log entry that records it), so the count of + displayed keys is bounded too. + """ + payload = {f"key_{i}": [ENTITY] for i in range(100)} + with self.assertRaises(ValidationError) as ctx: + normalize_graph_payload(payload) + + message = str(ctx.exception) + self.assertLess(len(message), 500) + self.assertIn("and 92 more", message) + + def test_many_dropped_record_keys_are_summarized(self): + payload = {"entities": [], **{f"data_{i}": [ENTITY] for i in range(100)}} + with self.assertRaises(ValidationError) as ctx: + normalize_graph_payload(payload) + + message = str(ctx.exception) + self.assertLess(len(message), 500) + self.assertIn("and 92 more", message) + + @dataclass class _DataclassNode: id: str diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 5bbe3be3..054c010c 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -31,21 +31,48 @@ class TestHelpers(unittest.TestCase): dict2 = {"b": {"d": 3}, "e": 4} merged = helpers.merge_dicts(dict1, dict2, deep=True) self.assertEqual(merged, {"a": 1, "b": {"c": 2, "d": 3}, "e": 4}) + def test_flatten_dict(self): - data = {"a": {"b": 1, "c": 2}} + """Basic nested flattening with multiple sibling keys.""" + data = {"a": {"b": 1, "c": 2}, "d": 3} result = helpers.flatten_dict(data) - self.assertEqual(result, {"a.b": 1, "a.c": 2}) + self.assertEqual(result, {"a.b": 1, "a.c": 2, "d": 3}) + + def test_flatten_dict_deeply_nested(self): + """Deeply nested structure is fully flattened.""" + self.assertEqual( + helpers.flatten_dict({"a": {"b": {"c": 1}}}), + {"a.b.c": 1}, + ) + + def test_flatten_dict_custom_separator(self): + """Custom separator is used in generated keys.""" + self.assertEqual( + helpers.flatten_dict({"a": {"b": 1}}, sep="__"), + {"a__b": 1}, + ) + + def test_flatten_dict_empty(self): + """Empty input returns empty output.""" + self.assertEqual(helpers.flatten_dict({}), {}) def test_flatten_dict_key_collision(self): - data = { - "a.b": 1, - "a": { - "b": 2 - } - } + """#1010 regression: a top-level key containing the separator must not + silently overwrite a value produced from a nested dict when both resolve + to the same flattened key. Before the fix, {'a.b': 1, 'a': {'b': 2}} + silently dropped one value; now it raises ValueError.""" + with self.assertRaises(ValueError) as ctx: + helpers.flatten_dict({"a.b": 1, "a": {"b": 2}}) + self.assertIn("Key collision", str(ctx.exception)) + self.assertIn("a.b", str(ctx.exception)) - with self.assertRaises(ValueError): - helpers.flatten_dict(data) + def test_flatten_dict_no_false_positive(self): + """Similar-looking keys that produce distinct flattened keys must not + trigger the collision guard.""" + self.assertEqual( + helpers.flatten_dict({"a.b": 1, "a": {"c": 2}}), + {"a.b": 1, "a.c": 2}, + ) def test_safe_import_returns_module_and_flag(self): module, available = helpers.safe_import("json") diff --git a/tests/vector_store/test_vector_store.py b/tests/vector_store/test_vector_store.py index 219f88c2..80ff3e59 100644 --- a/tests/vector_store/test_vector_store.py +++ b/tests/vector_store/test_vector_store.py @@ -243,5 +243,19 @@ class TestVectorStore(unittest.TestCase): shutil.rmtree(tmpdir, ignore_errors=True) +class TestCreateIndexFunction(unittest.TestCase): + """create_index() forwards vector_store_config's defaults into VectorIndexer, + which already receives backend/dimension as explicit args. Regression for the + 'got multiple values for keyword argument dimension' crash on the default + (unmocked) config, hit by e.g. `semantica embed index`.""" + + def test_create_index_with_default_config(self): + from semantica.vector_store.methods import create_index + + vectors = [np.array([0.1, 0.2, 0.3]), np.array([0.4, 0.5, 0.6])] + index = create_index(vectors, ids=["a", "b"]) + self.assertIsNotNone(index) + + if __name__ == '__main__': unittest.main()