mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-30 04:40:16 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0384a8de30 | ||
|
|
f3d5932c24 | ||
|
|
a5aac7e22a | ||
|
|
47446ebdde | ||
|
|
c8a591e89e | ||
|
|
ab86127e4e | ||
|
|
30592b1285 | ||
|
|
aceb69a5bc | ||
|
|
e13c953bd8 | ||
|
|
85d6ccd0a5 | ||
|
|
19ff5bf200 | ||
|
|
4bf525d409 | ||
|
|
8858beb6d9 | ||
|
|
d3183d0ab3 | ||
|
|
da642f12fa | ||
|
|
5376f046ca | ||
|
|
56d9e9a857 | ||
|
|
dfd668c206 | ||
|
|
e74d0a274d | ||
|
|
b570794515 | ||
|
|
a94cec3b36 | ||
|
|
f9b1295d14 |
@@ -33,15 +33,29 @@ jobs:
|
||||
- name: Install Explorer frontend dependencies
|
||||
working-directory: explorer
|
||||
run: npm ci
|
||||
- name: Install Playwright Chromium
|
||||
working-directory: explorer
|
||||
run: npx playwright install --with-deps chromium
|
||||
- name: Test Explorer frontend
|
||||
working-directory: explorer
|
||||
run: |
|
||||
npm run test:graph-store
|
||||
npm run test:graph-workspace
|
||||
npm run test:plugin-registry
|
||||
npm run test:deterministic-e2e
|
||||
- name: Build Explorer frontend
|
||||
working-directory: explorer
|
||||
run: npm run build
|
||||
- name: Install Explorer backend test dependencies
|
||||
run: |
|
||||
# Run the deterministic backend path before the all-extras CI
|
||||
# environment is installed. The Explorer extra supplies the
|
||||
# production API dependencies without importing optional vector
|
||||
# providers such as Pinecone during test collection.
|
||||
pip install -e ".[explorer]" pytest==9.1.1
|
||||
- name: Test deterministic Explorer backend path
|
||||
run: |
|
||||
pytest -q tests/explorer/test_explorer_deterministic_rendering_e2e.py
|
||||
- name: Install pinned Python dependencies
|
||||
run: |
|
||||
pip install -r requirements-ci.txt
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
# meaningful state carried over from a failed attempt.
|
||||
- name: Initialize CodeQL (attempt 1)
|
||||
id: codeql-init-1
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 2)
|
||||
id: codeql-init-2
|
||||
if: steps.codeql-init-1.outcome == 'failure'
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -52,17 +52,17 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 3)
|
||||
id: codeql-init-3
|
||||
if: steps.codeql-init-2.outcome == 'failure'
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
with:
|
||||
languages: python
|
||||
queries: security-and-quality
|
||||
config-file: .github/codeql/codeql-config.yml
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/autobuild@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
with:
|
||||
category: "/language:python"
|
||||
upload: false
|
||||
@@ -72,7 +72,7 @@ jobs:
|
||||
# Uploads results only when Default Setup is not active.
|
||||
# If Default Setup is still enabled, this step skips gracefully
|
||||
# instead of failing the workflow with HTTP 409.
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.codeql.outputs.sarif-output }}
|
||||
category: "/language:python"
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
# avoiding the guardian.cmd/checkov exit-code bug in the MSDO wrapper.
|
||||
tools: eslint,templateanalyzer,terrascan
|
||||
- name: Upload results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.msdo.outputs.sarifFile }}
|
||||
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
}
|
||||
|
||||
- name: Upload Checkov results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: reports/checkov.sarif
|
||||
|
||||
@@ -213,6 +213,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
|
||||
- **RETE engine matched every fact against every rule — `AlphaNode._matches()` and `BetaNode._can_join()` were placeholder stubs that always returned `True`** (closes #300)
|
||||
- `semantica/reasoning/rete_engine.py` shipped a Rete network whose per-condition alpha test and cross-condition beta join were both `return True` stubs, so `match_patterns()` fired every rule for every fact regardless of predicate, arity, or shared-variable consistency
|
||||
- New module-level `unify_condition()` reuses the regex-based approach from `Reasoner._match_pattern()`: a condition pattern like `Person(?x)` / `Parent(?x, ?y)` is compiled against a fact's `predicate(arg, ...)` string, `?var` becomes a named capture group, and a variable seen twice within one condition (e.g. `Loves(?x, ?x)`) becomes a backreference, so it only unifies when both positions hold the same value. Returns the bindings dict or `None`
|
||||
- Reworked propagation to carry partial-match **tokens** instead of bare facts: a new `Token` dataclass bundles the accumulated `facts` with the consistent `bindings`. `AlphaNode` emits a single-fact token per match; `BetaNode.join()` merges a left token with a right token, concatenating their facts in condition order and returning the merged token only when shared variables agree (conflicting values → `None`, no join). Terminal activations carry the full fact list and accumulated bindings through to the emitted match
|
||||
- This fixes a P1 chained-join defect: rules with three or more conditions (e.g. `Person(?x)`, `Parent(?x, ?y)`, `Located(?y, ?z)`) previously lost bindings and accumulated wrong facts at the third join, and a conflicting third condition could spuriously fire. Beta nodes now keep both `left_tokens` and `right_tokens` memories and join each new token against every token on the opposite side, so deep chains stay binding-consistent and third-level conflicts are correctly suppressed
|
||||
- Fixed an adjacent network-topology bug surfaced by the above: newly created beta nodes were never appended to their input nodes' `children`, so tokens could not propagate; propagation was reworked to support chained joins and to thread bindings end-to-end
|
||||
- Reconciled with the rule-actions/provenance layer (#1096) merged after this fix was opened: `execute_matches()` still dedupes and fires `Rule.actions`/legacy `handler` through a bound `Reasoner` via `_make_activation_key`, now sourced from the Token model's own `bindings` instead of the interim `_bindings_for_rule()` regex re-extraction, which is removed as redundant
|
||||
- New `tests/reasoning/test_rete_engine.py`: `unify_condition` unit cases (single/multi variable, literal args, predicate mismatch, repeated-variable equality), alpha match/reject, beta consistent-join vs conflict-reject, end-to-end rules (single-condition fires only the matching fact; multi-condition join fires only on consistent bindings), and a `TestThreeConditionChain` suite (valid three-condition match, third-level conflict suppression, insertion-order independence, `Match.facts` complete and in condition order, multiple left tokens joining one right fact, parity against `Reasoner._match_rule()`, and `reset()` clearing all token memory)
|
||||
|
||||
- **KG provenance tests asserted on generated ID strings instead of stored records, and `kg_provenance.py` was missed by the `utcnow` sweep** (closes #946) by @pravit-amp
|
||||
- The KG workflow and integration suites checked that a tracker call returned an ID matching a prefix (`assert cent_id.startswith("centrality_")`) without ever reading the record back, so an ID generator that returned a well-formed string and wrote nothing would have passed. Worse, some of those calls named tracker methods that do not exist anywhere in `semantica/` (`track_layer_analysis`, `track_centrality_score`), so the assertions were satisfied with no real interaction behind them
|
||||
- Those tests now read provenance back through `get_provenance()` and assert on algorithm metadata, and call the methods that actually persist records. Verified by mutation rather than by a green run alone: neutering the manager's storage write (`self.storage.store(...)` → no-op) fails 10 tests
|
||||
|
||||
+2
-1
@@ -106,7 +106,8 @@
|
||||
"integrations/langchain",
|
||||
"integrations/docling",
|
||||
"integrations/snowflake",
|
||||
"integrations/databricks"
|
||||
"integrations/databricks",
|
||||
"integrations/salesforce"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
---
|
||||
title: "Salesforce Integration"
|
||||
description: "Ingest CRM records from Salesforce sObjects and SOQL queries into Semantica's KG pipeline."
|
||||
icon: "cloud"
|
||||
---
|
||||
|
||||
> Extract Accounts, Contacts, Opportunities, and custom objects from Salesforce into Semantica with username/password/security-token, JWT bearer, or session-based authentication.
|
||||
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
# Install with Salesforce support
|
||||
pip install "semantica[db-salesforce]"
|
||||
|
||||
# Or install the connector separately
|
||||
pip install simple-salesforce>=1.12.0
|
||||
```
|
||||
|
||||
|
||||
## Basic Usage
|
||||
|
||||
```python
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
import os
|
||||
|
||||
ingestor = SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
domain=os.getenv("SALESFORCE_DOMAIN", "login"), # "test" for sandbox
|
||||
)
|
||||
|
||||
data = ingestor.ingest_sobject("Account", fields=["Id", "Name", "Industry"], limit=1000)
|
||||
print(f"Retrieved {data.row_count} of {data.total_size} matching records")
|
||||
print(f"Columns: {data.columns}")
|
||||
```
|
||||
|
||||
<Tip>
|
||||
Use environment variables (or a `.env` file with `python-dotenv`) to keep credentials out of source code. `SalesforceIngestor()` with no arguments reads from `SALESFORCE_*` environment variables automatically.
|
||||
</Tip>
|
||||
|
||||
|
||||
## Authentication Methods
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Username / Password / Security Token">
|
||||
```python
|
||||
import os
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
|
||||
ingestor = SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
domain="login", # production; use "test" for sandbox
|
||||
)
|
||||
```
|
||||
Set the required environment variables before running:
|
||||
```bash
|
||||
export SALESFORCE_USERNAME="your-username@example.com"
|
||||
export SALESFORCE_PASSWORD="your-password"
|
||||
export SALESFORCE_SECURITY_TOKEN="your-security-token"
|
||||
```
|
||||
The standard server-side flow. The security token is appended to the
|
||||
password during Salesforce SOAP login. Generate or reset it under
|
||||
**Settings → My Personal Information → Reset My Security Token**.
|
||||
</Tab>
|
||||
<Tab title="JWT Bearer (Recommended for CI/CD)">
|
||||
```python
|
||||
import os
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
|
||||
ingestor = SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
consumer_key=os.getenv("SALESFORCE_CONSUMER_KEY"),
|
||||
privatekey_file=os.getenv("SALESFORCE_PRIVATE_KEY_FILE"),
|
||||
domain="login", # or "test" for sandbox
|
||||
)
|
||||
```
|
||||
```bash
|
||||
export SALESFORCE_USERNAME="your-username@example.com"
|
||||
export SALESFORCE_CONSUMER_KEY="your-connected-app-consumer-key"
|
||||
export SALESFORCE_PRIVATE_KEY_FILE="/path/to/server.key"
|
||||
```
|
||||
The JWT bearer flow authenticates with a signed token — no password
|
||||
is transmitted. Ideal for server-to-server integrations and CI/CD
|
||||
pipelines. Requires a Salesforce connected app configured with
|
||||
**Use digital signatures** and the pre-authorised user listed under
|
||||
**Manage → Profiles / Permission Sets**.
|
||||
|
||||
If you prefer to pass the key material as a string instead of a file
|
||||
path, use `SALESFORCE_PRIVATE_KEY` (the PEM contents) in place of
|
||||
`SALESFORCE_PRIVATE_KEY_FILE`.
|
||||
</Tab>
|
||||
<Tab title="Session ID + Instance URL">
|
||||
```python
|
||||
ingestor = SalesforceIngestor(
|
||||
session_id=os.getenv("SALESFORCE_SESSION_ID"),
|
||||
instance_url=os.getenv("SALESFORCE_INSTANCE_URL"),
|
||||
)
|
||||
```
|
||||
Use this when your environment already manages the OAuth token
|
||||
lifecycle (e.g. a connected app obtaining tokens via the web-server
|
||||
or device flow). Pass the access token as `session_id` and the full
|
||||
instance URL (e.g. `https://myorg.my.salesforce.com`) as
|
||||
`instance_url`.
|
||||
</Tab>
|
||||
<Tab title="Sandbox">
|
||||
```python
|
||||
import os
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
|
||||
ingestor = SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
domain="test", # routes to test.salesforce.com
|
||||
)
|
||||
```
|
||||
```bash
|
||||
export SALESFORCE_USERNAME="your-sandbox-username@example.com.sandbox"
|
||||
export SALESFORCE_PASSWORD="your-password"
|
||||
export SALESFORCE_SECURITY_TOKEN="your-security-token"
|
||||
export SALESFORCE_DOMAIN="test"
|
||||
```
|
||||
Replace `domain="login"` with `domain="test"` (or set
|
||||
`SALESFORCE_DOMAIN=test` in your environment) to connect to a
|
||||
developer or full sandbox.
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
### Environment variables
|
||||
|
||||
All constructor parameters have environment-variable fallbacks:
|
||||
|
||||
| Variable | Parameter | Default |
|
||||
|---|---|---|
|
||||
| `SALESFORCE_USERNAME` | `username` | — |
|
||||
| `SALESFORCE_PASSWORD` | `password` | — |
|
||||
| `SALESFORCE_SECURITY_TOKEN` | `security_token` | — |
|
||||
| `SALESFORCE_DOMAIN` | `domain` | `"login"` |
|
||||
| `SALESFORCE_INSTANCE_URL` | `instance_url` | — |
|
||||
| `SALESFORCE_SESSION_ID` | `session_id` | — |
|
||||
| `SALESFORCE_CONSUMER_KEY` | `consumer_key` | — |
|
||||
| `SALESFORCE_PRIVATE_KEY_FILE` | `privatekey_file` | — |
|
||||
| `SALESFORCE_PRIVATE_KEY` | `privatekey` | — |
|
||||
| `SALESFORCE_API_VERSION` | `api_version` | library default (`59.0`) |
|
||||
|
||||
|
||||
## Object Ingestion
|
||||
|
||||
### Ingest a standard object
|
||||
|
||||
```python
|
||||
data = ingestor.ingest_sobject(
|
||||
"Account",
|
||||
fields=["Id", "Name", "Industry", "AnnualRevenue", "BillingCity"],
|
||||
where="Type = 'Customer' AND AnnualRevenue > 1000000",
|
||||
order_by="Name ASC",
|
||||
limit=5000,
|
||||
)
|
||||
print(f"Retrieved {data.row_count} of {data.total_size} matching records")
|
||||
```
|
||||
|
||||
<Note>
|
||||
`data.row_count` is the number of records in `data.data` (i.e. what was actually returned after any `limit`). `data.total_size` is Salesforce's `totalSize` — the number of records matching the query *before* the limit. Compare them to know whether you got all results.
|
||||
</Note>
|
||||
|
||||
### Ingest a custom object
|
||||
|
||||
Custom objects end with `__c` in their API name:
|
||||
|
||||
```python
|
||||
data = ingestor.ingest_sobject(
|
||||
"My_Custom_Object__c",
|
||||
fields=["Id", "Name", "Custom_Field__c"],
|
||||
)
|
||||
```
|
||||
|
||||
Relationship traversal fields (`Owner.Name`) are also supported:
|
||||
|
||||
```python
|
||||
data = ingestor.ingest_sobject(
|
||||
"Contact",
|
||||
fields=["Id", "Name", "Email", "Account.Name", "Owner.Name"],
|
||||
limit=10000,
|
||||
)
|
||||
```
|
||||
|
||||
### Let Semantica choose the fields
|
||||
|
||||
When `fields` is omitted, all selectable fields are fetched via `describe()`
|
||||
(one extra API call). Compound address and geolocation fields (`type=address`,
|
||||
`type=location`) are automatically excluded — select their components
|
||||
(`BillingStreet`, `BillingCity`, `Location__Latitude__s`, …) individually if
|
||||
you need them.
|
||||
|
||||
```python
|
||||
data = ingestor.ingest_sobject("Opportunity")
|
||||
```
|
||||
|
||||
|
||||
## Raw SOQL Ingestion
|
||||
|
||||
Pass any valid SOQL query verbatim — pagination is handled automatically:
|
||||
|
||||
```python
|
||||
data = ingestor.ingest_query("""
|
||||
SELECT Id, Name, StageName, Amount, CloseDate,
|
||||
Account.Name, Owner.Name
|
||||
FROM Opportunity
|
||||
WHERE IsClosed = false
|
||||
ORDER BY CloseDate ASC
|
||||
""")
|
||||
print(f"Open opportunities: {data.row_count}")
|
||||
```
|
||||
|
||||
The query is passed to the Salesforce REST API unchanged. The caller is
|
||||
responsible for SOQL correctness and safety.
|
||||
|
||||
<Warning>
|
||||
`ingest_query` does not validate or sanitise the SOQL string. Use
|
||||
`ingest_sobject` (which validates sObject names, field names, and WHERE/ORDER
|
||||
BY fragments) when building queries from application-controlled inputs.
|
||||
</Warning>
|
||||
|
||||
|
||||
## Document Export
|
||||
|
||||
Convert ingested records to the Semantica document format for use with
|
||||
`GraphBuilder`:
|
||||
|
||||
```python
|
||||
documents = ingestor.export_as_documents(
|
||||
data,
|
||||
id_field="Id", # default; Salesforce 18-char record Id
|
||||
text_fields=["Name", "Description"], # omit to join all string fields
|
||||
)
|
||||
|
||||
print(f"Created {len(documents)} documents")
|
||||
# Each document:
|
||||
# {
|
||||
# "id": "001xx000003GYk2AAG",
|
||||
# "text": "Acme Corp Enterprise software company",
|
||||
# "metadata": {
|
||||
# "source": "salesforce",
|
||||
# "sobject": "Account",
|
||||
# "instance_url": "https://myorg.my.salesforce.com",
|
||||
# "row_data": { ... full cleaned record ... }
|
||||
# }
|
||||
# }
|
||||
```
|
||||
|
||||
Feed the documents directly into `GraphBuilder`:
|
||||
|
||||
```python
|
||||
from semantica.kg import GraphBuilder
|
||||
|
||||
builder = GraphBuilder()
|
||||
kg = builder.build(documents)
|
||||
```
|
||||
|
||||
|
||||
## Object and Schema Discovery
|
||||
|
||||
```python
|
||||
# List all accessible sObjects
|
||||
sobject_names = ingestor.list_sobjects()
|
||||
print(sobject_names[:10]) # ["Account", "Case", "Contact", ...]
|
||||
|
||||
# Inspect fields for a specific sObject
|
||||
schema = ingestor.get_sobject_schema("Account")
|
||||
for field in schema["fields"]:
|
||||
print(f"{field['name']}: {field['type']} (nillable={field['nillable']})")
|
||||
```
|
||||
|
||||
|
||||
## Context Manager
|
||||
|
||||
Prefer the context manager for long-running jobs — it opens one connection on
|
||||
entry and closes it on exit, so every ingestion call inside the `with` block
|
||||
reuses the same authenticated session:
|
||||
|
||||
```python
|
||||
with SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
) as sf:
|
||||
accounts = sf.ingest_sobject("Account", limit=10000)
|
||||
contacts = sf.ingest_sobject("Contact", limit=10000)
|
||||
sobjects = sf.list_sobjects()
|
||||
```
|
||||
|
||||
|
||||
## Convenience Function
|
||||
|
||||
Use `ingest_salesforce()` for one-liner ingestion:
|
||||
|
||||
```python
|
||||
from semantica.ingest import ingest_salesforce
|
||||
|
||||
# Fetch records
|
||||
data = ingest_salesforce(
|
||||
method="sobject",
|
||||
sobject_name="Account",
|
||||
fields=["Id", "Name", "Industry"],
|
||||
limit=500,
|
||||
)
|
||||
|
||||
# Execute raw SOQL (credentials from environment variables)
|
||||
data = ingest_salesforce(
|
||||
method="query",
|
||||
soql="SELECT Id, Name FROM Contact WHERE IsActive = true",
|
||||
)
|
||||
|
||||
# Ingest + export to documents in one step
|
||||
docs = ingest_salesforce(
|
||||
method="documents",
|
||||
sobject_name="Account",
|
||||
text_fields=["Name", "Description"],
|
||||
limit=1000,
|
||||
)
|
||||
|
||||
# List accessible sObjects
|
||||
sobject_names = ingest_salesforce(method="list_sobjects")
|
||||
```
|
||||
|
||||
Or use the unified `ingest()` dispatcher:
|
||||
|
||||
```python
|
||||
from semantica.ingest import ingest
|
||||
|
||||
result = ingest(
|
||||
None,
|
||||
source_type="salesforce",
|
||||
method="sobject",
|
||||
sobject_name="Account",
|
||||
fields=["Id", "Name"],
|
||||
limit=500,
|
||||
)
|
||||
data = result["data"] # SalesforceData
|
||||
```
|
||||
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
```python
|
||||
import os
|
||||
from semantica.ingest import SalesforceConnector
|
||||
|
||||
connector = SalesforceConnector(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
)
|
||||
if not connector.test_connection():
|
||||
print("Connection failed: check username, password, security token, and domain")
|
||||
```
|
||||
|
||||
Common causes of authentication failures:
|
||||
|
||||
- **Wrong domain**: production orgs use `domain="login"`; sandboxes use `domain="test"`.
|
||||
- **Stale security token**: reset it under **Settings → Reset My Security Token**. The new token is emailed to you.
|
||||
- **IP restriction**: your org's trusted IP ranges may block the originating IP. Check **Setup → Network Access**.
|
||||
- **API access disabled**: ensure the connected profile has the **API Enabled** permission.
|
||||
|
||||
|
||||
## See Also
|
||||
|
||||
- [Ingest Module](../reference/ingest) — Full `SalesforceIngestor` API and all other ingestors.
|
||||
- [Snowflake Integration](snowflake) — Relational warehouse connector with a similar design.
|
||||
- [Databricks Integration](databricks) — Lakehouse connector.
|
||||
- [Installation](../installation) — All optional dependency extras.
|
||||
- [Knowledge Graph](../reference/kg) — Build a KG from ingested Salesforce data.
|
||||
@@ -0,0 +1,195 @@
|
||||
"""
|
||||
Deterministic Explorer Rendering E2E Example.
|
||||
|
||||
Demonstrates building, serializing, and reloading a deterministic 4-node,
|
||||
3-edge knowledge graph baseline for visual inspection in Semantica Explorer (#1037).
|
||||
|
||||
Graph topology:
|
||||
Alice (Person, #63E6FF) --WORKS_AT--> Acme (Organization, #A78BFA)
|
||||
Bob (Person, #63E6FF) --KNOWS--> Alice (Person, #63E6FF)
|
||||
Acme (Organization, #A78BFA) --LOCATED_IN--> New York (Location, #34D399)
|
||||
|
||||
Clean Checkout Prerequisites:
|
||||
1. Python backend dependencies:
|
||||
pip install -e ".[explorer]"
|
||||
2. Frontend workspace dependencies:
|
||||
cd explorer && npm install && cd ..
|
||||
|
||||
Usage:
|
||||
# 1. Generate the deterministic graph baseline:
|
||||
python examples/explorer_deterministic_rendering_example.py
|
||||
|
||||
# 2. Launch Explorer with local dev authentication (Option A - Dev mode):
|
||||
# Terminal 1 (Backend API):
|
||||
SEMANTICA_ALLOW_ANONYMOUS=true python -m semantica.explorer --graph explorer_e2e_test_graph.json --port 8000 --no-browser
|
||||
# Terminal 2 (Frontend UI):
|
||||
cd explorer && npm run dev
|
||||
# Open http://localhost:5173
|
||||
|
||||
# 2. Launch Explorer (Option B - Standalone CLI server):
|
||||
SEMANTICA_ALLOW_ANONYMOUS=true python -m semantica.explorer --graph explorer_e2e_test_graph.json --port 8000
|
||||
# Open http://localhost:8000
|
||||
|
||||
# Secure authentication alternative:
|
||||
export SEMANTICA_API_KEY="your-secret-api-key"
|
||||
python -m semantica.explorer --graph explorer_e2e_test_graph.json --port 8000
|
||||
# Send HTTP header: X-API-Key: your-secret-api-key
|
||||
|
||||
Verification Checklist:
|
||||
- Exactly 4 nodes visible on canvas:
|
||||
* Alice (Person, #63E6FF)
|
||||
* Bob (Person, #63E6FF)
|
||||
* Acme (Organization, #A78BFA)
|
||||
* New York (Location, #34D399)
|
||||
- Exactly 3 directed edges with canonical relationship labels:
|
||||
* Alice -> Acme (WORKS_AT)
|
||||
* Bob -> Alice (KNOWS)
|
||||
* Acme -> New York (LOCATED_IN)
|
||||
- Zoom behavior:
|
||||
* Zoom in to Inspection tier (ratio <= 0.5): directional arrows and node labels scale clearly.
|
||||
* Zoom out to Overview tier (ratio > 1.2): layout remains stable and non-colliding.
|
||||
- Hover & Selection interactions:
|
||||
* Hover over 'Alice': node halo triggers; incident edges (WORKS_AT, KNOWS) highlight in local context.
|
||||
* Click an edge: Inspector panel confirms edgeType ('WORKS_AT', 'KNOWS', or 'LOCATED_IN').
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.session import GraphSession
|
||||
|
||||
|
||||
def build_deterministic_graph() -> ContextGraph:
|
||||
"""Build the exact 4-node, 3-edge graph specified in #1037."""
|
||||
graph = ContextGraph(advanced_analytics=False)
|
||||
|
||||
# 1. Add exactly 4 nodes
|
||||
graph.add_node(
|
||||
"alice",
|
||||
node_type="Person",
|
||||
content="Alice",
|
||||
color="#63E6FF",
|
||||
)
|
||||
graph.add_node(
|
||||
"bob",
|
||||
node_type="Person",
|
||||
content="Bob",
|
||||
color="#63E6FF",
|
||||
)
|
||||
graph.add_node(
|
||||
"acme",
|
||||
node_type="Organization",
|
||||
content="Acme",
|
||||
color="#A78BFA",
|
||||
)
|
||||
graph.add_node(
|
||||
"new_york",
|
||||
node_type="Location",
|
||||
content="New York",
|
||||
color="#34D399",
|
||||
)
|
||||
|
||||
# 2. Add exactly 3 directed edges
|
||||
graph.add_edge("alice", "acme", edge_type="WORKS_AT", weight=1.0)
|
||||
graph.add_edge("bob", "alice", edge_type="KNOWS", weight=1.0)
|
||||
graph.add_edge("acme", "new_york", edge_type="LOCATED_IN", weight=1.0)
|
||||
|
||||
return graph
|
||||
|
||||
|
||||
def main() -> None:
|
||||
print("=" * 75)
|
||||
print("Semantica Explorer Deterministic Graph Generator (#1037)")
|
||||
print("=" * 75)
|
||||
|
||||
print("1. Building deterministic ContextGraph...")
|
||||
graph = build_deterministic_graph()
|
||||
print(
|
||||
f" ✓ Graph built with {len(graph.nodes)} nodes "
|
||||
f"and {len(graph.edges)} edges."
|
||||
)
|
||||
|
||||
output_path = Path("explorer_e2e_test_graph.json").resolve()
|
||||
print(f"2. Persisting graph to '{output_path.name}'...")
|
||||
graph.save_to_file(str(output_path))
|
||||
print(f" ✓ Graph saved to {output_path}")
|
||||
|
||||
# Verify JSON format
|
||||
with open(output_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
assert len(data.get("nodes", [])) == 4
|
||||
assert len(data.get("edges", [])) == 3
|
||||
|
||||
print("3. Verifying reload via GraphSession.from_file()...")
|
||||
session = GraphSession.from_file(str(output_path))
|
||||
stats = session.get_stats()
|
||||
nodes, total_nodes = session.get_nodes()
|
||||
edges, total_edges = session.get_edges()
|
||||
|
||||
assert stats["node_count"] == 4
|
||||
assert stats["edge_count"] == 3
|
||||
assert total_nodes == 4
|
||||
assert total_edges == 3
|
||||
|
||||
print(
|
||||
f" ✓ Graph reloaded successfully without mutation "
|
||||
f"(nodes: {total_nodes}, edges: {total_edges}).\n"
|
||||
)
|
||||
|
||||
print("=" * 75)
|
||||
print("Clean Checkout Prerequisites:")
|
||||
print("=" * 75)
|
||||
print(" pip install -e '.[explorer]'")
|
||||
print(" cd explorer && npm install && cd ..\n")
|
||||
|
||||
print("=" * 75)
|
||||
print("Reproduction instructions to view in Semantica Explorer:")
|
||||
print("=" * 75)
|
||||
print("Option A (Frontend dev server + API backend — recommended for development):")
|
||||
print(
|
||||
f" 1. Backend: SEMANTICA_ALLOW_ANONYMOUS=true python -m semantica.explorer "
|
||||
f"--graph {output_path} --port 8000 --no-browser"
|
||||
)
|
||||
print(" 2. Frontend: cd explorer && npm run dev")
|
||||
print(" 3. Open http://localhost:5173 to inspect the graph canvas.\n")
|
||||
|
||||
print("Option B (Standalone Explorer CLI server):")
|
||||
print(
|
||||
f" SEMANTICA_ALLOW_ANONYMOUS=true python -m semantica.explorer "
|
||||
f"--graph {output_path} --port 8000"
|
||||
)
|
||||
print(" Open http://localhost:8000\n")
|
||||
|
||||
print("Secure Authentication Alternative:")
|
||||
print(" export SEMANTICA_API_KEY='your-secret-api-key'")
|
||||
print(
|
||||
f" python -m semantica.explorer --graph {output_path} --port 8000"
|
||||
)
|
||||
print(" Send header: 'X-API-Key: your-secret-api-key'\n")
|
||||
|
||||
print("=" * 75)
|
||||
print("Verification Checklist:")
|
||||
print("=" * 75)
|
||||
print(" 1. Nodes (4 total):")
|
||||
print(" - Alice (Person, #63E6FF)")
|
||||
print(" - Bob (Person, #63E6FF)")
|
||||
print(" - Acme (Organization, #A78BFA)")
|
||||
print(" - New York (Location, #34D399)")
|
||||
print(" 2. Directed Edges & Canonical Labels (3 total):")
|
||||
print(" - Alice -> Acme [WORKS_AT]")
|
||||
print(" - Bob -> Alice [KNOWS]")
|
||||
print(" - Acme -> New York [LOCATED_IN]")
|
||||
print(" 3. Zoom Interactions:")
|
||||
print(" - Inspection tier (zoom in): directional arrows & labels remain legible.")
|
||||
print(" - Overview tier (zoom out): nodes and edges maintain layout integrity.")
|
||||
print(" 4. Hover & Selection Interactions:")
|
||||
print(" - Hover Alice: node halo triggers and incident edges (WORKS_AT, KNOWS) highlight.")
|
||||
print(" - Click edge: Inspector panel displays edgeType label ('WORKS_AT', 'KNOWS', 'LOCATED_IN').")
|
||||
print("=" * 75)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -9,7 +9,8 @@
|
||||
"lint": "eslint .",
|
||||
"preview": "vite preview",
|
||||
"test:graph-store": "node --test tests/graphStore.multi-edge.test.mjs",
|
||||
"test:graph-workspace": "node --import tsx --test tests/markdownContentViewer.test.ts tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts",
|
||||
"test:graph-workspace": "node --import tsx --test tests/markdownContentViewer.test.ts tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts tests/deterministicExplorerRendering.test.ts",
|
||||
"test:deterministic-e2e": "node --import tsx --test tests/deterministicExplorerRendering.e2e.ts",
|
||||
"test:plugin-registry": "node --import tsx --test tests/pluginRegistry.temporal.test.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
|
||||
@@ -41,6 +41,7 @@ import {
|
||||
} from "./plugins";
|
||||
import { explorationEffectsShouldLoad, neighborhoodPanelShouldLoad, temporalOverlayShouldLoad } from "./pluginRegistryPredicates";
|
||||
import { shouldFetchTemporalBounds, shouldFetchTemporalSnapshot } from "./temporalLifecyclePredicates";
|
||||
import { createTemporalSnapshotGuards, type TemporalSnapshotResponse } from "./temporalSnapshotGuards";
|
||||
import type { LinkPrediction, PathResponse } from "./GraphInspectorPanel";
|
||||
import type { GraphSceneHandle, GraphSceneRuntime } from "./scene";
|
||||
import type {
|
||||
@@ -1479,6 +1480,23 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
summary?.edgeCount,
|
||||
]);
|
||||
|
||||
// Guards the snapshot lifecycle: at most one in-flight request per scrubber
|
||||
// position (identical-`at` polls are deduplicated, breaking the idle/play
|
||||
// polling loop), applied snapshots are cached and re-applied on revisit, and
|
||||
// a response applies only while the scrubber is still on its position
|
||||
// (out-of-order responses cannot clobber the active-node count).
|
||||
const temporalSnapshotGuardsRef = useRef<ReturnType<typeof createTemporalSnapshotGuards> | null>(null);
|
||||
if (temporalSnapshotGuardsRef.current === null) {
|
||||
temporalSnapshotGuardsRef.current = createTemporalSnapshotGuards();
|
||||
}
|
||||
const temporalSnapshotGuards = temporalSnapshotGuardsRef.current;
|
||||
|
||||
// A new graph summary means the graph data was replaced (reload/retry);
|
||||
// snapshots cached against the previous graph are stale, so reset all state.
|
||||
useEffect(() => {
|
||||
temporalSnapshotGuards.reset();
|
||||
}, [summary]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!canFetchTemporalSnapshot) {
|
||||
return;
|
||||
@@ -1488,37 +1506,67 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return;
|
||||
}
|
||||
|
||||
const atMs = debouncedTime.getTime();
|
||||
const { seq, cached } = temporalSnapshotGuards.begin(atMs);
|
||||
if (seq === null) {
|
||||
// An identical request is already in flight: one request per position.
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
const applyData = (data: TemporalSnapshotResponse) => {
|
||||
const nextActiveIds = new Set(data.active_node_ids);
|
||||
requestAnimationFrame(() => {
|
||||
if (cancelled) return;
|
||||
if (!temporalSnapshotGuards.shouldApply(atMs, seq)) {
|
||||
// The scrubber moved on (or this request was superseded): release the
|
||||
// position so a return to it refetches instead of stalling.
|
||||
temporalSnapshotGuards.finish(atMs, seq);
|
||||
return;
|
||||
}
|
||||
const previous = prevActiveIdsRef.current;
|
||||
previous.forEach((id) => {
|
||||
if (!nextActiveIds.has(id) && graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", true);
|
||||
}
|
||||
});
|
||||
nextActiveIds.forEach((id) => {
|
||||
if (graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", false);
|
||||
}
|
||||
});
|
||||
prevActiveIdsRef.current = nextActiveIds;
|
||||
setActiveNodeCount(data.active_node_count);
|
||||
setGraphVersion((current) => current + 1);
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
temporalSnapshotGuards.apply(atMs, seq, data);
|
||||
});
|
||||
};
|
||||
|
||||
if (cached) {
|
||||
// Returning to a position whose snapshot was already applied: re-apply
|
||||
// the cached result without a network request.
|
||||
applyData(cached);
|
||||
return;
|
||||
}
|
||||
|
||||
const applySnapshot = async () => {
|
||||
try {
|
||||
const at = debouncedTime.toISOString();
|
||||
const response = await fetch(`/api/temporal/snapshot?at=${encodeURIComponent(at)}`);
|
||||
if (!response.ok || cancelled) return;
|
||||
|
||||
const data: { active_node_ids: string[]; active_node_count: number } = await response.json();
|
||||
if (!response.ok) {
|
||||
// A failed request must be retryable if the scrubber returns.
|
||||
if (!cancelled) temporalSnapshotGuards.finish(atMs, seq);
|
||||
return;
|
||||
}
|
||||
if (cancelled) return;
|
||||
|
||||
const nextActiveIds = new Set(data.active_node_ids);
|
||||
requestAnimationFrame(() => {
|
||||
if (cancelled) return;
|
||||
const previous = prevActiveIdsRef.current;
|
||||
previous.forEach((id) => {
|
||||
if (!nextActiveIds.has(id) && graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", true);
|
||||
}
|
||||
});
|
||||
nextActiveIds.forEach((id) => {
|
||||
if (graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", false);
|
||||
}
|
||||
});
|
||||
prevActiveIdsRef.current = nextActiveIds;
|
||||
setActiveNodeCount(data.active_node_count);
|
||||
setGraphVersion((current) => current + 1);
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
});
|
||||
const data: TemporalSnapshotResponse = await response.json();
|
||||
if (cancelled) return;
|
||||
applyData(data);
|
||||
} catch (fetchError) {
|
||||
temporalSnapshotGuards.finish(atMs, seq);
|
||||
if (!cancelled) {
|
||||
console.error("[Temporal] Snapshot fetch failed", fetchError);
|
||||
}
|
||||
@@ -1528,6 +1576,8 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
applySnapshot();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
// A cancelled request must be retryable when its position is revisited.
|
||||
temporalSnapshotGuards.finish(atMs, seq);
|
||||
};
|
||||
}, [
|
||||
canFetchTemporalSnapshot,
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* Guards for the temporal snapshot fetch/apply lifecycle.
|
||||
*
|
||||
* The snapshot effect previously fetched /api/temporal/snapshot with no
|
||||
* idempotency or ordering protection. Upstream churn (timeline recreation
|
||||
* while bounds settle, play ticks resetting the playhead, drag events) could
|
||||
* re-request the same `at` repeatedly, and responses could arrive after the
|
||||
* scrubber had moved on.
|
||||
*
|
||||
* The guards enforce:
|
||||
* - at most one in-flight request per scrubber position (identical `at`
|
||||
* values are deduplicated while a request is pending, breaking the
|
||||
* idle/play polling loop);
|
||||
* - successful snapshots are cached per position and re-applied when the
|
||||
* scrubber returns (play wrap-around, back-scrubbing) without a refetch;
|
||||
* - a response is applied only while the scrubber is still on its position,
|
||||
* so out-of-order responses cannot clobber a newer position's count;
|
||||
* - failed, cancelled, or superseded requests release their position so it
|
||||
* can be fetched again on the next visit;
|
||||
* - `reset()` drops all state when the underlying graph data is replaced
|
||||
* (reload/retry), because cached snapshots describe the previous graph.
|
||||
*
|
||||
* `createTemporalSnapshotGuards()` is stateful by design.
|
||||
*/
|
||||
|
||||
export interface TemporalSnapshotResponse {
|
||||
active_node_ids: string[];
|
||||
active_node_count: number;
|
||||
}
|
||||
|
||||
export interface TemporalSnapshotRequest {
|
||||
/** null when the request was deduplicated because one is already in flight. */
|
||||
seq: number | null;
|
||||
/** The snapshot previously applied for this position, when revisiting it. */
|
||||
cached: TemporalSnapshotResponse | null;
|
||||
}
|
||||
|
||||
export interface TemporalSnapshotGuards {
|
||||
/** Begin (or dedupe) a request for `atMs`; marks it as the current position. */
|
||||
begin(atMs: number): TemporalSnapshotRequest;
|
||||
/** True when the response for `atMs`/`seq` may be applied (scrubber still on `atMs`). */
|
||||
shouldApply(atMs: number, seq: number): boolean;
|
||||
/** Record a successful application and cache its snapshot for revisits. */
|
||||
apply(atMs: number, seq: number, data: TemporalSnapshotResponse): void;
|
||||
/** Release a position whose request failed, was cancelled, or was superseded. */
|
||||
finish(atMs: number, seq: number): void;
|
||||
/** Drop all state; call when the underlying graph data is replaced (reload). */
|
||||
reset(): void;
|
||||
}
|
||||
|
||||
interface SnapshotEntry {
|
||||
seq: number;
|
||||
/** null while the request is in flight (or before the first success). */
|
||||
data: TemporalSnapshotResponse | null;
|
||||
}
|
||||
|
||||
/** Upper bound on cached positions so long scrubbing sessions stay bounded. */
|
||||
const MAX_CACHED_POSITIONS = 256;
|
||||
|
||||
export function createTemporalSnapshotGuards(): TemporalSnapshotGuards {
|
||||
const entries = new Map<number, SnapshotEntry>();
|
||||
let latestRequestSeq = 0;
|
||||
let currentAtMs: number | null = null;
|
||||
|
||||
const evictOldest = () => {
|
||||
while (entries.size > MAX_CACHED_POSITIONS) {
|
||||
const oldestAtMs = entries.keys().next().value;
|
||||
if (oldestAtMs === undefined) return;
|
||||
entries.delete(oldestAtMs);
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
begin(atMs) {
|
||||
const existing = entries.get(atMs);
|
||||
if (existing && existing.data === null) {
|
||||
// Identical request already in flight: dedupe, but the scrubber is here now.
|
||||
currentAtMs = atMs;
|
||||
return { seq: null, cached: null };
|
||||
}
|
||||
latestRequestSeq += 1;
|
||||
const seq = latestRequestSeq;
|
||||
entries.set(atMs, { seq, data: existing?.data ?? null });
|
||||
currentAtMs = atMs;
|
||||
evictOldest();
|
||||
return { seq, cached: existing?.data ?? null };
|
||||
},
|
||||
|
||||
shouldApply(atMs, seq) {
|
||||
return atMs === currentAtMs && entries.get(atMs)?.seq === seq;
|
||||
},
|
||||
|
||||
apply(atMs, seq, data) {
|
||||
const entry = entries.get(atMs);
|
||||
if (entry && entry.seq === seq) {
|
||||
entry.data = data;
|
||||
}
|
||||
},
|
||||
|
||||
finish(atMs, seq) {
|
||||
const entry = entries.get(atMs);
|
||||
if (entry && entry.seq === seq && entry.data === null) {
|
||||
entries.delete(atMs);
|
||||
}
|
||||
},
|
||||
|
||||
reset() {
|
||||
entries.clear();
|
||||
latestRequestSeq = 0;
|
||||
currentAtMs = null;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import test from "node:test";
|
||||
import { chromium, type Page } from "playwright";
|
||||
|
||||
const PORT = 4173;
|
||||
const BASE_URL = `http://127.0.0.1:${PORT}`;
|
||||
|
||||
const nodes = [
|
||||
{ id: "alice", type: "Person", content: "Alice", properties: {} },
|
||||
{ id: "bob", type: "Person", content: "Bob", properties: {} },
|
||||
{ id: "acme", type: "Organization", content: "Acme", properties: {} },
|
||||
{ id: "new_york", type: "Location", content: "New York", properties: {} },
|
||||
];
|
||||
|
||||
const edges = [
|
||||
{ id: "edge_alice_acme", familyId: "edge_alice_acme", source: "alice", target: "acme", type: "WORKS_AT", weight: 1, properties: {} },
|
||||
{ id: "edge_bob_alice", familyId: "edge_bob_alice", source: "bob", target: "alice", type: "KNOWS", weight: 1, properties: {} },
|
||||
{ id: "edge_acme_new_york", familyId: "edge_acme_new_york", source: "acme", target: "new_york", type: "LOCATED_IN", weight: 1, properties: {} },
|
||||
];
|
||||
|
||||
let server: ChildProcess | undefined;
|
||||
|
||||
async function startVite(): Promise<void> {
|
||||
server = spawn("npm", ["run", "dev", "--", "--host", "127.0.0.1", "--port", String(PORT)], {
|
||||
cwd: process.cwd(),
|
||||
stdio: "ignore",
|
||||
});
|
||||
|
||||
for (let attempt = 0; attempt < 50; attempt += 1) {
|
||||
try {
|
||||
const response = await fetch(BASE_URL);
|
||||
if (response.ok) return;
|
||||
} catch {
|
||||
// Vite is still starting.
|
||||
}
|
||||
await delay(100);
|
||||
}
|
||||
throw new Error("Vite did not become ready");
|
||||
}
|
||||
|
||||
async function installApiFixture(page: Page): Promise<void> {
|
||||
await page.route("**/api/graph/**", async (route) => {
|
||||
const pathname = new URL(route.request().url()).pathname;
|
||||
if (pathname === "/api/graph/stats") {
|
||||
await route.fulfill({ json: { node_count: 4, edge_count: 3 } });
|
||||
} else if (pathname === "/api/graph/nodes") {
|
||||
await route.fulfill({ json: { nodes, total: nodes.length, skip: 0, limit: 1000, next_cursor: null } });
|
||||
} else if (pathname === "/api/graph/edges") {
|
||||
await route.fulfill({ json: { edges, total: edges.length, skip: 0, limit: 1000, next_cursor: null } });
|
||||
} else {
|
||||
await route.continue();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test("real Explorer loading path hydrates and renders API edge labels", async (t) => {
|
||||
await startVite();
|
||||
t.after(async () => {
|
||||
server?.kill();
|
||||
});
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
executablePath: process.env.CHROMIUM_PATH || (existsSync("/usr/bin/chromium") ? "/usr/bin/chromium" : undefined),
|
||||
});
|
||||
t.after(() => browser.close());
|
||||
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
|
||||
await page.addInitScript(() => {
|
||||
const captured = (window as Window & { __capturedCanvasText?: string[] }).__capturedCanvasText = [];
|
||||
const originalFillText = CanvasRenderingContext2D.prototype.fillText;
|
||||
CanvasRenderingContext2D.prototype.fillText = function (text: string, ...args: [number, number, number?, number?]) {
|
||||
captured.push(String(text));
|
||||
return originalFillText.call(this, text, ...args);
|
||||
};
|
||||
});
|
||||
await installApiFixture(page);
|
||||
await page.goto(BASE_URL);
|
||||
await page.getByRole("button", { name: /Open Semantica Explorer/ }).click();
|
||||
|
||||
await page.locator("canvas").nth(0).waitFor({ state: "attached" });
|
||||
await page.waitForFunction(() => document.querySelectorAll("canvas").length >= 2);
|
||||
await page.waitForFunction(() => {
|
||||
const labels = (window as Window & { __capturedCanvasText?: string[] }).__capturedCanvasText ?? [];
|
||||
return ["WORKS_AT", "KNOWS", "LOCATED_IN"].every((label) => labels.includes(label));
|
||||
}, undefined, { timeout: 10_000 });
|
||||
|
||||
const capturedLabels = await page.evaluate(() => (window as Window & { __capturedCanvasText?: string[] }).__capturedCanvasText ?? []);
|
||||
for (const label of ["WORKS_AT", "KNOWS", "LOCATED_IN"]) {
|
||||
assert.ok(capturedLabels.includes(label), `Expected rendered edge label ${label}`);
|
||||
}
|
||||
assert.ok(capturedLabels.includes("Alice"));
|
||||
|
||||
await page.getByRole("button", { name: "Zoom In" }).click();
|
||||
await page.waitForTimeout(250);
|
||||
const labelsAfterZoom = await page.evaluate(() => (window as Window & { __capturedCanvasText?: string[] }).__capturedCanvasText ?? []);
|
||||
for (const label of ["WORKS_AT", "KNOWS", "LOCATED_IN"]) {
|
||||
assert.ok(labelsAfterZoom.includes(label), `Expected edge label ${label} after zoom`);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,508 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
batchMergeEdges,
|
||||
batchMergeNodes,
|
||||
clearGraph,
|
||||
graph,
|
||||
} from "../src/store/graphStore.ts";
|
||||
import {
|
||||
buildStructuralDistanceSnapshot,
|
||||
classifyFullGraphEdge,
|
||||
resolveDisplayGraph,
|
||||
resolveEdgeElementStyle,
|
||||
resolveEdgeVisualState,
|
||||
resolveNodeElementStyle,
|
||||
resolveNodeVisualState,
|
||||
shouldForceNodeLabel,
|
||||
} from "../src/workspaces/GraphWorkspace/graphSceneState.ts";
|
||||
import { GRAPH_THEME, type GraphZoomTier } from "../src/workspaces/GraphWorkspace/graphTheme.ts";
|
||||
|
||||
test.beforeEach(() => {
|
||||
clearGraph();
|
||||
});
|
||||
|
||||
test.after(() => {
|
||||
clearGraph();
|
||||
});
|
||||
|
||||
/**
|
||||
* Loads the canonical 4-node, 3-edge deterministic test graph (Semantica #1037).
|
||||
*
|
||||
* Graph structure:
|
||||
* Alice (Person) --WORKS_AT--> Acme (Organization)
|
||||
* Bob (Person) --KNOWS--> Alice (Person)
|
||||
* Acme (Organization) --LOCATED_IN--> New York (Location)
|
||||
*/
|
||||
function loadDeterministicTestGraph() {
|
||||
batchMergeNodes([
|
||||
{
|
||||
id: "alice",
|
||||
attributes: {
|
||||
label: "Alice",
|
||||
content: "Alice",
|
||||
x: 0,
|
||||
y: 0,
|
||||
size: 8,
|
||||
color: "#63E6FF",
|
||||
baseColor: "#63E6FF",
|
||||
nodeType: "Person",
|
||||
semanticGroup: "Person",
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "bob",
|
||||
attributes: {
|
||||
label: "Bob",
|
||||
content: "Bob",
|
||||
x: -50,
|
||||
y: 0,
|
||||
size: 8,
|
||||
color: "#63E6FF",
|
||||
baseColor: "#63E6FF",
|
||||
nodeType: "Person",
|
||||
semanticGroup: "Person",
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "acme",
|
||||
attributes: {
|
||||
label: "Acme",
|
||||
content: "Acme",
|
||||
x: 50,
|
||||
y: 0,
|
||||
size: 8,
|
||||
color: "#A78BFA",
|
||||
baseColor: "#A78BFA",
|
||||
nodeType: "Organization",
|
||||
semanticGroup: "Organization",
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "new_york",
|
||||
attributes: {
|
||||
label: "New York",
|
||||
content: "New York",
|
||||
x: 100,
|
||||
y: 0,
|
||||
size: 8,
|
||||
color: "#34D399",
|
||||
baseColor: "#34D399",
|
||||
nodeType: "Location",
|
||||
semanticGroup: "Location",
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
batchMergeEdges([
|
||||
{
|
||||
id: "edge_alice_acme",
|
||||
source: "alice",
|
||||
target: "acme",
|
||||
attributes: {
|
||||
edgeId: "edge_alice_acme",
|
||||
edgeType: "WORKS_AT",
|
||||
weight: 1.0,
|
||||
visualPriority: 0.8,
|
||||
baseSize: 0.8,
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "edge_bob_alice",
|
||||
source: "bob",
|
||||
target: "alice",
|
||||
attributes: {
|
||||
edgeId: "edge_bob_alice",
|
||||
edgeType: "KNOWS",
|
||||
weight: 1.0,
|
||||
visualPriority: 0.8,
|
||||
baseSize: 0.8,
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "edge_acme_new_york",
|
||||
source: "acme",
|
||||
target: "new_york",
|
||||
attributes: {
|
||||
edgeId: "edge_acme_new_york",
|
||||
edgeType: "LOCATED_IN",
|
||||
weight: 1.0,
|
||||
visualPriority: 0.8,
|
||||
baseSize: 0.8,
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
test("deterministic graph contains exactly 4 nodes and 3 edges in store", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
assert.equal(graph.order, 4, "Expected exactly 4 nodes");
|
||||
assert.equal(graph.size, 3, "Expected exactly 3 edges");
|
||||
|
||||
// Verify node identities and labels
|
||||
const alice = graph.getNodeAttributes("alice");
|
||||
const bob = graph.getNodeAttributes("bob");
|
||||
const acme = graph.getNodeAttributes("acme");
|
||||
const newYork = graph.getNodeAttributes("new_york");
|
||||
|
||||
assert.equal(alice.label, "Alice");
|
||||
assert.equal(alice.nodeType, "Person");
|
||||
assert.equal(alice.color, "#63E6FF");
|
||||
|
||||
assert.equal(bob.label, "Bob");
|
||||
assert.equal(bob.nodeType, "Person");
|
||||
assert.equal(bob.color, "#63E6FF");
|
||||
|
||||
assert.equal(acme.label, "Acme");
|
||||
assert.equal(acme.nodeType, "Organization");
|
||||
assert.equal(acme.color, "#A78BFA");
|
||||
|
||||
assert.equal(newYork.label, "New York");
|
||||
assert.equal(newYork.nodeType, "Location");
|
||||
assert.equal(newYork.color, "#34D399");
|
||||
|
||||
// Verify edge connectivity and canonical edgeType labels
|
||||
const edgeAliceAcme = graph.getEdgeAttributes("edge_alice_acme");
|
||||
const edgeBobAlice = graph.getEdgeAttributes("edge_bob_alice");
|
||||
const edgeAcmeNewYork = graph.getEdgeAttributes("edge_acme_new_york");
|
||||
|
||||
assert.equal(edgeAliceAcme.edgeType, "WORKS_AT");
|
||||
assert.equal(graph.source("edge_alice_acme"), "alice");
|
||||
assert.equal(graph.target("edge_alice_acme"), "acme");
|
||||
|
||||
assert.equal(edgeBobAlice.edgeType, "KNOWS");
|
||||
assert.equal(graph.source("edge_bob_alice"), "bob");
|
||||
assert.equal(graph.target("edge_bob_alice"), "alice");
|
||||
|
||||
assert.equal(edgeAcmeNewYork.edgeType, "LOCATED_IN");
|
||||
assert.equal(graph.source("edge_acme_new_york"), "acme");
|
||||
assert.equal(graph.target("edge_acme_new_york"), "new_york");
|
||||
});
|
||||
|
||||
test("display graph resolution preserves all 4 nodes and 3 edges in full view", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: false });
|
||||
|
||||
assert.equal(displayGraph.order, 4);
|
||||
assert.equal(displayGraph.size, 3);
|
||||
assert.ok(displayGraph.hasNode("alice"));
|
||||
assert.ok(displayGraph.hasNode("bob"));
|
||||
assert.ok(displayGraph.hasNode("acme"));
|
||||
assert.ok(displayGraph.hasNode("new_york"));
|
||||
assert.ok(displayGraph.hasEdge("edge_alice_acme"));
|
||||
assert.ok(displayGraph.hasEdge("edge_bob_alice"));
|
||||
assert.ok(displayGraph.hasEdge("edge_acme_new_york"));
|
||||
});
|
||||
|
||||
test("structural distance calculation resolves correct hop counts across the 3-edge chain", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
// From Bob: Bob (0) -> Alice (1) -> Acme (2) -> New York (3)
|
||||
const distances = buildStructuralDistanceSnapshot(graph, "bob", 3);
|
||||
|
||||
assert.equal(distances.bob, 0);
|
||||
assert.equal(distances.alice, 1);
|
||||
assert.equal(distances.acme, 2);
|
||||
assert.equal(distances.new_york, 3);
|
||||
});
|
||||
|
||||
test("edge rendering and canonical edge labels remain legible across zoom tiers and inspection modes", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
const canonicalEdges = [
|
||||
{ id: "edge_alice_acme", source: "alice", target: "acme", label: "WORKS_AT" },
|
||||
{ id: "edge_bob_alice", source: "bob", target: "alice", label: "KNOWS" },
|
||||
{ id: "edge_acme_new_york", source: "acme", target: "new_york", label: "LOCATED_IN" },
|
||||
];
|
||||
|
||||
// 1. Edge attributes preserve canonical edgeType labels in graph store:
|
||||
for (const item of canonicalEdges) {
|
||||
const attrs = graph.getEdgeAttributes(item.id);
|
||||
assert.equal(attrs.edgeType, item.label, `Edge ${item.id} must have edgeType ${item.label}`);
|
||||
assert.equal(graph.source(item.id), item.source);
|
||||
assert.equal(graph.target(item.id), item.target);
|
||||
}
|
||||
|
||||
// 2. In active context / neighbor state across all zoom tiers (overview, structure, inspection):
|
||||
const allTiers: GraphZoomTier[] = ["overview", "structure", "inspection"];
|
||||
for (const tier of allTiers) {
|
||||
for (const item of canonicalEdges) {
|
||||
const attrs = graph.getEdgeAttributes(item.id);
|
||||
const contextStyle = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
tier,
|
||||
"neighbor",
|
||||
attrs,
|
||||
item.source,
|
||||
item.target,
|
||||
"full",
|
||||
item.id,
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
contextStyle.hidden,
|
||||
false,
|
||||
`Edge ${item.id} (${item.label}) in context state 'neighbor' must be visible in zoom tier '${tier}'`,
|
||||
);
|
||||
assert.ok(
|
||||
contextStyle.size !== undefined && contextStyle.size > 0,
|
||||
`Edge ${item.id} (${item.label}) must have positive render size in zoom tier '${tier}'`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. In selected state in inspection zoom tier (close examination of edge details and label):
|
||||
for (const item of canonicalEdges) {
|
||||
const attrs = graph.getEdgeAttributes(item.id);
|
||||
const selectedStyle = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"inspection",
|
||||
"selected",
|
||||
attrs,
|
||||
item.source,
|
||||
item.target,
|
||||
"full",
|
||||
item.id,
|
||||
"selected",
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
selectedStyle.hidden,
|
||||
false,
|
||||
`Selected edge ${item.id} (${item.label}) must be visible in inspection zoom tier`,
|
||||
);
|
||||
assert.ok(
|
||||
selectedStyle.size !== undefined && selectedStyle.size > 0,
|
||||
`Selected edge ${item.id} (${item.label}) must have positive render size`,
|
||||
);
|
||||
}
|
||||
|
||||
// 4. Verify inspection zoom tier camera and arrow rendering settings
|
||||
assert.equal(GRAPH_THEME.zoomTiers.inspection.showContextualArrows, true);
|
||||
assert.equal(GRAPH_THEME.zoomTiers.inspection.showCurves, true);
|
||||
});
|
||||
|
||||
test("node hover interaction preserves edge visibility and highlights canonical incident edge types", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
// Scenario 1: Hover Alice
|
||||
// Incident edges: Alice -> Acme (WORKS_AT) and Bob -> Alice (KNOWS)
|
||||
const aliceAttrs = graph.getNodeAttributes("alice");
|
||||
const aliceVisual = resolveNodeVisualState("alice", "structure", "alice", "", "", new Set(), new Set(), new Set());
|
||||
assert.equal(aliceVisual, "hovered");
|
||||
|
||||
const aliceStyle = resolveNodeElementStyle(GRAPH_THEME, "structure", "hovered", aliceAttrs, "Alice");
|
||||
assert.equal(aliceStyle.forceLabel, true, "Hovered Alice must force-render label");
|
||||
assert.equal(aliceStyle.label, "Alice");
|
||||
assert.equal(aliceStyle.showHalo, true, "Hovered Alice must show interactive halo");
|
||||
|
||||
const aliceIncidentEdges = new Set(["edge_alice_acme", "edge_bob_alice"]);
|
||||
|
||||
// Edge Alice -> Acme (WORKS_AT) under Alice hover
|
||||
const aliceAcmeAttrs = graph.getEdgeAttributes("edge_alice_acme");
|
||||
assert.equal(aliceAcmeAttrs.edgeType, "WORKS_AT");
|
||||
const aliceAcmeState = resolveEdgeVisualState(
|
||||
"edge_alice_acme",
|
||||
"alice",
|
||||
"acme",
|
||||
"structure",
|
||||
"alice",
|
||||
"",
|
||||
"",
|
||||
new Set(),
|
||||
new Set(),
|
||||
aliceIncidentEdges,
|
||||
);
|
||||
assert.equal(aliceAcmeState, "hovered");
|
||||
const aliceAcmeStyle = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"structure",
|
||||
"hovered",
|
||||
aliceAcmeAttrs,
|
||||
"alice",
|
||||
"acme",
|
||||
"full",
|
||||
"edge_alice_acme",
|
||||
);
|
||||
assert.equal(aliceAcmeStyle.hidden, false, "Incident edge WORKS_AT must remain visible on hover");
|
||||
assert.ok(aliceAcmeStyle.size !== undefined && aliceAcmeStyle.size > 0);
|
||||
|
||||
// Edge Bob -> Alice (KNOWS) under Alice hover
|
||||
const bobAliceAttrs = graph.getEdgeAttributes("edge_bob_alice");
|
||||
assert.equal(bobAliceAttrs.edgeType, "KNOWS");
|
||||
const bobAliceState = resolveEdgeVisualState(
|
||||
"edge_bob_alice",
|
||||
"bob",
|
||||
"alice",
|
||||
"structure",
|
||||
"alice",
|
||||
"",
|
||||
"",
|
||||
new Set(),
|
||||
new Set(),
|
||||
aliceIncidentEdges,
|
||||
);
|
||||
assert.equal(bobAliceState, "hovered");
|
||||
const bobAliceStyle = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"structure",
|
||||
"hovered",
|
||||
bobAliceAttrs,
|
||||
"bob",
|
||||
"alice",
|
||||
"full",
|
||||
"edge_bob_alice",
|
||||
);
|
||||
assert.equal(bobAliceStyle.hidden, false, "Incident edge KNOWS must remain visible on hover");
|
||||
|
||||
// Non-incident edge Acme -> New York (LOCATED_IN) under Alice hover
|
||||
const acmeNyAttrs = graph.getEdgeAttributes("edge_acme_new_york");
|
||||
assert.equal(acmeNyAttrs.edgeType, "LOCATED_IN");
|
||||
const acmeNyState = resolveEdgeVisualState(
|
||||
"edge_acme_new_york",
|
||||
"acme",
|
||||
"new_york",
|
||||
"structure",
|
||||
"alice",
|
||||
"",
|
||||
"",
|
||||
new Set(),
|
||||
new Set(),
|
||||
aliceIncidentEdges,
|
||||
);
|
||||
assert.equal(acmeNyState, "muted");
|
||||
|
||||
// Scenario 2: Hover Acme
|
||||
// Incident edges: Alice -> Acme (WORKS_AT) and Acme -> New York (LOCATED_IN)
|
||||
const acmeAttrs = graph.getNodeAttributes("acme");
|
||||
const acmeStyle = resolveNodeElementStyle(GRAPH_THEME, "structure", "hovered", acmeAttrs, "Acme");
|
||||
assert.equal(acmeStyle.forceLabel, true);
|
||||
assert.equal(acmeStyle.label, "Acme");
|
||||
|
||||
const acmeIncidentEdges = new Set(["edge_alice_acme", "edge_acme_new_york"]);
|
||||
const acmeNyHoverState = resolveEdgeVisualState(
|
||||
"edge_acme_new_york",
|
||||
"acme",
|
||||
"new_york",
|
||||
"structure",
|
||||
"acme",
|
||||
"",
|
||||
"",
|
||||
new Set(),
|
||||
new Set(),
|
||||
acmeIncidentEdges,
|
||||
);
|
||||
assert.equal(acmeNyHoverState, "hovered");
|
||||
const acmeNyHoverStyle = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"structure",
|
||||
"hovered",
|
||||
acmeNyAttrs,
|
||||
"acme",
|
||||
"new_york",
|
||||
"full",
|
||||
"edge_acme_new_york",
|
||||
);
|
||||
assert.equal(acmeNyHoverStyle.hidden, false, "Incident edge LOCATED_IN must remain visible on hover");
|
||||
|
||||
// Scenario 3: Hover Bob
|
||||
// Incident edge: Bob -> Alice (KNOWS)
|
||||
const bobAttrs = graph.getNodeAttributes("bob");
|
||||
const bobStyle = resolveNodeElementStyle(GRAPH_THEME, "structure", "hovered", bobAttrs, "Bob");
|
||||
assert.equal(bobStyle.forceLabel, true);
|
||||
assert.equal(bobStyle.label, "Bob");
|
||||
|
||||
const bobIncidentEdges = new Set(["edge_bob_alice"]);
|
||||
const bobAliceHoverState = resolveEdgeVisualState(
|
||||
"edge_bob_alice",
|
||||
"bob",
|
||||
"alice",
|
||||
"structure",
|
||||
"bob",
|
||||
"",
|
||||
"",
|
||||
new Set(),
|
||||
new Set(),
|
||||
bobIncidentEdges,
|
||||
);
|
||||
assert.equal(bobAliceHoverState, "hovered");
|
||||
});
|
||||
|
||||
test("edge selection maintains canonical edge type labels and active visual state", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
const edgeCases = [
|
||||
{ id: "edge_alice_acme", source: "alice", target: "acme", label: "WORKS_AT" },
|
||||
{ id: "edge_bob_alice", source: "bob", target: "alice", label: "KNOWS" },
|
||||
{ id: "edge_acme_new_york", source: "acme", target: "new_york", label: "LOCATED_IN" },
|
||||
];
|
||||
|
||||
for (const { id, source, target, label } of edgeCases) {
|
||||
const attrs = graph.getEdgeAttributes(id);
|
||||
assert.equal(attrs.edgeType, label);
|
||||
|
||||
const visualState = resolveEdgeVisualState(
|
||||
id,
|
||||
source,
|
||||
target,
|
||||
"inspection",
|
||||
null,
|
||||
"",
|
||||
id, // selected edge
|
||||
new Set(),
|
||||
new Set(),
|
||||
);
|
||||
assert.equal(visualState, "selected", `Selected edge ${id} must resolve to 'selected' state`);
|
||||
|
||||
const style = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"inspection",
|
||||
"selected",
|
||||
attrs,
|
||||
source,
|
||||
target,
|
||||
"full",
|
||||
id,
|
||||
"selected",
|
||||
);
|
||||
|
||||
assert.equal(style.hidden, false, `Selected edge ${id} (${label}) must not be hidden`);
|
||||
assert.ok(
|
||||
style.size !== undefined && style.size > 0,
|
||||
`Selected edge ${id} (${label}) must have positive render size`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("node labels remain forced visible during hover, selection, and inspection zoom tier", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
const nodes = ["alice", "bob", "acme", "new_york"];
|
||||
|
||||
for (const nid of nodes) {
|
||||
const attrs = graph.getNodeAttributes(nid);
|
||||
|
||||
// Hover state forces label visibility
|
||||
const hoverForcesLabel = shouldForceNodeLabel(GRAPH_THEME, "structure", "hovered", attrs, 0);
|
||||
assert.equal(hoverForcesLabel, true, `Node ${nid} label must force visible on hover`);
|
||||
|
||||
// Selected state forces label visibility
|
||||
const selectForcesLabel = shouldForceNodeLabel(GRAPH_THEME, "structure", "selected", attrs, 0);
|
||||
assert.equal(selectForcesLabel, true, `Node ${nid} label must force visible on selection`);
|
||||
|
||||
// Resolved style emits actual string label
|
||||
const style = resolveNodeElementStyle(GRAPH_THEME, "inspection", "hovered", attrs, attrs.label);
|
||||
assert.equal(style.forceLabel, true);
|
||||
assert.equal(style.label, attrs.label);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,150 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { createTemporalSnapshotGuards } from "../src/workspaces/GraphWorkspace/temporalSnapshotGuards.ts";
|
||||
|
||||
const POSITION_1 = new Date("2023-07-02T00:00:00Z").getTime();
|
||||
const POSITION_2 = new Date("2024-01-02T00:00:00Z").getTime();
|
||||
const POSITION_3 = new Date("2024-07-02T00:00:00Z").getTime();
|
||||
|
||||
const SNAPSHOT = { active_node_ids: ["n1", "n2"], active_node_count: 2 };
|
||||
|
||||
// ── begin: one request per scrubber position ─────────────────────────────────
|
||||
|
||||
test("begin: a new position returns a fresh request sequence", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
assert.deepEqual(guards.begin(POSITION_1), { seq: 1, cached: null });
|
||||
});
|
||||
|
||||
test("begin: an identical in-flight request is deduplicated (no duplicate fetch)", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
guards.begin(POSITION_1);
|
||||
assert.deepEqual(guards.begin(POSITION_1), { seq: null, cached: null });
|
||||
});
|
||||
|
||||
test("begin: distinct positions request independently", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
assert.equal(guards.begin(POSITION_1).seq, 1);
|
||||
assert.equal(guards.begin(POSITION_2).seq, 2);
|
||||
});
|
||||
|
||||
test("begin: revisiting an applied position returns its cached snapshot", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
const revisit = guards.begin(POSITION_1);
|
||||
assert.equal(revisit.seq, 2);
|
||||
assert.deepEqual(revisit.cached, SNAPSHOT);
|
||||
});
|
||||
|
||||
test("begin: a failed position (finished) can be requested again", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.finish(POSITION_1, seq);
|
||||
const retry = guards.begin(POSITION_1);
|
||||
assert.equal(retry.seq, 2);
|
||||
assert.equal(retry.cached, null);
|
||||
});
|
||||
|
||||
test("finish: does not clear a position whose snapshot was already applied", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.finish(POSITION_1, seq);
|
||||
assert.deepEqual(guards.begin(POSITION_1).cached, SNAPSHOT);
|
||||
});
|
||||
|
||||
test("finish: a stale sequence cannot release a newer request's position", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const first = guards.begin(POSITION_1);
|
||||
guards.finish(POSITION_1, first.seq);
|
||||
guards.begin(POSITION_1); // seq 2, in flight again
|
||||
guards.finish(POSITION_1, first.seq); // stale seq: must not release seq 2
|
||||
assert.deepEqual(guards.begin(POSITION_1), { seq: null, cached: null });
|
||||
});
|
||||
|
||||
// ── shouldApply: applied only while the scrubber is on that position ─────────
|
||||
|
||||
test("shouldApply: the current position's response is applied", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq), true);
|
||||
});
|
||||
|
||||
test("shouldApply: a response for a position the scrubber left is discarded", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq: seq1 } = guards.begin(POSITION_1);
|
||||
guards.begin(POSITION_2);
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq1), false);
|
||||
assert.equal(guards.shouldApply(POSITION_2, 2), true);
|
||||
});
|
||||
|
||||
test("shouldApply: a late response for the position the scrubber returned to is applied", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq: seq1 } = guards.begin(POSITION_1);
|
||||
const { seq: seq2 } = guards.begin(POSITION_2);
|
||||
guards.begin(POSITION_1); // back to 1: deduplicated, no new request
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq1), true);
|
||||
assert.equal(guards.shouldApply(POSITION_2, seq2), false);
|
||||
});
|
||||
|
||||
test("shouldApply: an unknown sequence is discarded", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
guards.begin(POSITION_1);
|
||||
assert.equal(guards.shouldApply(POSITION_1, 99), false);
|
||||
});
|
||||
|
||||
test("shouldApply: after a reset no pre-reset response applies", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.reset();
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq), false);
|
||||
});
|
||||
|
||||
// ── apply: caching for revisits ─────────────────────────────────────────────
|
||||
|
||||
test("apply: stores the snapshot so a revisit re-applies it without a request", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.begin(POSITION_2);
|
||||
assert.deepEqual(guards.begin(POSITION_1).cached, SNAPSHOT);
|
||||
});
|
||||
|
||||
test("apply: play wrap-around re-applies the wrapped-to position's snapshot", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.begin(POSITION_2);
|
||||
guards.begin(POSITION_3);
|
||||
const wrap = guards.begin(POSITION_1);
|
||||
assert.deepEqual(wrap.cached, SNAPSHOT);
|
||||
assert.equal(guards.shouldApply(POSITION_1, wrap.seq), true);
|
||||
});
|
||||
|
||||
// ── reset: graph reload ─────────────────────────────────────────────────────
|
||||
|
||||
test("reset: clears requested and cached state so positions refetch", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.reset();
|
||||
const fresh = guards.begin(POSITION_1);
|
||||
assert.equal(fresh.seq, 1);
|
||||
assert.equal(fresh.cached, null);
|
||||
});
|
||||
|
||||
// ── cache bound ─────────────────────────────────────────────────────────────
|
||||
|
||||
test("cache: oldest positions are evicted when the cache is full", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const count = 300;
|
||||
for (let i = 0; i < count; i++) {
|
||||
const { seq } = guards.begin(POSITION_1 + i * 1000);
|
||||
guards.apply(POSITION_1 + i * 1000, seq, SNAPSHOT);
|
||||
}
|
||||
const oldest = guards.begin(POSITION_1);
|
||||
assert.equal(oldest.cached, null); // evicted: must refetch on revisit
|
||||
const newest = guards.begin(POSITION_1 + (count - 1) * 1000);
|
||||
assert.deepEqual(newest.cached, SNAPSHOT); // still cached
|
||||
});
|
||||
+2
-1
@@ -124,12 +124,13 @@ shacl = ["pyshacl>=0.25.0"]
|
||||
db-snowflake = ["snowflake-connector-python>=4.6.0", "cryptography>=49.0.0"]
|
||||
db-databricks = ["databricks-sdk>=0.60.0", "databricks-sql-connector>=4.0.0"]
|
||||
db-arrow = ["pyarrow>=24.0.0"]
|
||||
db-salesforce = ["simple-salesforce>=1.12.0"]
|
||||
ingest-parquet = ["pyarrow>=24.0.0"]
|
||||
ingest-arrow = ["pyarrow>=24.0.0"]
|
||||
ingest-sap = ["requests>=2.28.0"]
|
||||
|
||||
db-all = [
|
||||
"semantica[db-snowflake,db-databricks,db-arrow]"
|
||||
"semantica[db-snowflake,db-databricks,db-salesforce,db-arrow]"
|
||||
]
|
||||
|
||||
# ---- Embedding / Models ----
|
||||
|
||||
@@ -1286,13 +1286,19 @@ class AgentMemory:
|
||||
"""
|
||||
return self.retrieve(content, max_results=limit, **kwargs)
|
||||
|
||||
def find_by_entity(self, entity_id: str, limit: int = 10) -> List[Dict[str, Any]]:
|
||||
def find_by_entity(
|
||||
self, entity_id: str, limit: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Find by entity.
|
||||
|
||||
Args:
|
||||
entity_id: Entity ID to search for
|
||||
limit: Maximum results (default: 10)
|
||||
limit: Maximum results. None (the default) returns ALL matches.
|
||||
The previous default of 10 silently truncated results — an
|
||||
erasure workflow computing "what references this entity"
|
||||
from a truncated page would leave the remainder live
|
||||
(#1018). Callers that want pagination pass an explicit limit.
|
||||
|
||||
Returns:
|
||||
List of memory dicts containing the entity
|
||||
@@ -1308,9 +1314,9 @@ class AgentMemory:
|
||||
if mem_dict:
|
||||
results.append(mem_dict)
|
||||
break
|
||||
if len(results) >= limit:
|
||||
if limit is not None and len(results) >= limit:
|
||||
break
|
||||
return results[:limit]
|
||||
return results if limit is None else results[:limit]
|
||||
|
||||
def find_by_relationship(
|
||||
self, relationship_type: str, limit: int = 10
|
||||
|
||||
@@ -306,10 +306,10 @@ class JSONExporter:
|
||||
|
||||
self.logger.debug(f"Exporting {len(entities)} entity(ies) to JSON")
|
||||
|
||||
# Build JSON data with JSON-LD context
|
||||
# Build JSON data with JSON-LD context. No @vocab: it would expand
|
||||
# every bare key in the caller's entity dicts into ns# (#1146).
|
||||
json_data = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"entities": {"@id": "semantica:entities", "@container": "@list"},
|
||||
},
|
||||
@@ -339,7 +339,6 @@ class JSONExporter:
|
||||
"""
|
||||
json_data = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"relationships": {
|
||||
"@id": "semantica:relationships",
|
||||
@@ -434,11 +433,14 @@ class JSONExporter:
|
||||
Returns:
|
||||
Dictionary in JSON-LD format with @context, @graph/@value, and metadata
|
||||
"""
|
||||
# Initialize JSON-LD structure with context
|
||||
# Initialize JSON-LD structure with context. No @vocab: for a generic
|
||||
# payload it turned whatever bare keys the caller happened to use into
|
||||
# ns# terms (#1146). Undeclared terms now simply expand to nothing,
|
||||
# which is standard JSON-LD behaviour for a context that does not
|
||||
# know them; the raw payload is still in the document.
|
||||
jsonld = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": "https://semantica.dev/ns#",
|
||||
"semantica": SEMANTICA_NS,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -598,13 +600,21 @@ class JSONExporter:
|
||||
Returns:
|
||||
Dictionary in JSON-LD format with @context, @id, @type, and graph data
|
||||
"""
|
||||
# Initialize JSON-LD structure with RDF context
|
||||
# Initialize JSON-LD structure with RDF context. No @vocab: it applied
|
||||
# to every bare term in caller data, so an extracted type like "ORG"
|
||||
# became ns#ORG and a metadata key like "source" collided with the
|
||||
# real sem:source object property (#1146). Only explicit semantica:
|
||||
# terms resolve now, and the caller's metadata dict is typed @json so
|
||||
# it survives as one rdf:JSON literal instead of expanding its keys.
|
||||
jsonld = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": "https://semantica.dev/ns#",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
||||
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
||||
"semantica:metadata": {
|
||||
"@id": "semantica:metadata",
|
||||
"@type": "@json",
|
||||
},
|
||||
},
|
||||
# Minted from the graph's own content rather than the wall clock
|
||||
# (#1147): re-exporting an unchanged graph must produce the same
|
||||
@@ -664,14 +674,23 @@ class JSONExporter:
|
||||
entity_text = entity.get("text") or entity.get("label", "unknown")
|
||||
entity_id = entity.get("id") or mint_entity_iri(entity_text)
|
||||
|
||||
# The caller's type label is data, not a class we define: minting it
|
||||
# into @type expanded it through @vocab into ns#ORG and friends, terms
|
||||
# that look official but do not exist (#1146). The node is always a
|
||||
# semantica:Entity and the label travels as semantica:type, exactly
|
||||
# how _relationship_to_jsonld has always carried the relationship type.
|
||||
jsonld = {
|
||||
"@id": entity_id,
|
||||
"@type": entity.get("type") or "semantica:Entity",
|
||||
"@type": "semantica:Entity",
|
||||
"semantica:text": entity.get("text") or entity.get("label", ""),
|
||||
"semantica:confidence": entity.get("confidence", 1.0),
|
||||
}
|
||||
entity_type = entity.get("type")
|
||||
if entity_type:
|
||||
jsonld["semantica:type"] = entity_type
|
||||
|
||||
# Add metadata if present
|
||||
# Add metadata if present. The @json term definition on
|
||||
# semantica:metadata keeps the whole dict one rdf:JSON literal.
|
||||
if "metadata" in entity:
|
||||
jsonld["semantica:metadata"] = entity["metadata"]
|
||||
|
||||
|
||||
@@ -1226,11 +1226,14 @@ class RDFSerializer:
|
||||
metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None))
|
||||
graph_uri: Optional[str] = options.pop("graph_uri", None)
|
||||
|
||||
# Initialize JSON-LD structure with context
|
||||
# Initialize JSON-LD structure with context. No @vocab: it applied to
|
||||
# every bare term in caller data, so an extracted type like "ORG"
|
||||
# became ns#ORG and a metadata key like "source" collided with the
|
||||
# real sem:source object property (#1146). Only explicit semantica:
|
||||
# terms resolve now.
|
||||
jsonld = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": "https://semantica.dev/ns#",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
||||
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
||||
},
|
||||
@@ -1252,11 +1255,20 @@ class RDFSerializer:
|
||||
# and was dropped in full by a JSON-LD parser, silently.
|
||||
entity_id = entity.get("id") or mint_entity_iri(entity.get("text", ""))
|
||||
|
||||
# The caller's type label is data, not a class we define: minting
|
||||
# it into @type expanded it through @vocab into ns#ORG and
|
||||
# friends, terms that look official but do not exist (#1146).
|
||||
# The node is always a semantica:Entity and the label travels as
|
||||
# semantica:type, matching the relationship node below and
|
||||
# JSONExporter._entity_to_jsonld.
|
||||
node = {
|
||||
"@id": entity_id,
|
||||
"@type": entity.get("type", "semantica:Entity"),
|
||||
"@type": "semantica:Entity",
|
||||
"semantica:text": entity.get("text") or entity.get("label", ""),
|
||||
}
|
||||
entity_type = entity.get("type")
|
||||
if entity_type:
|
||||
node["semantica:type"] = entity_type
|
||||
confidence = normalize_confidence(entity.get("confidence", 1.0))
|
||||
if confidence is None:
|
||||
self.logger.warning(
|
||||
|
||||
@@ -130,7 +130,10 @@ Example Usage:
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from typing import Any, Dict, Tuple
|
||||
from typing import TYPE_CHECKING, Any, Dict, Tuple
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .salesforce_ingestor import SalesforceConnector, SalesforceData, SalesforceIngestor
|
||||
|
||||
from .config import IngestConfig, ingest_config
|
||||
from .file_ingestor import (
|
||||
@@ -152,6 +155,7 @@ from .methods import (
|
||||
ingest_parquet,
|
||||
ingest_public_api,
|
||||
ingest_repository,
|
||||
ingest_salesforce,
|
||||
ingest_stream,
|
||||
ingest_web,
|
||||
ingest_xml,
|
||||
@@ -235,6 +239,10 @@ _LAZY_EXPORTS: Dict[str, Tuple[str, str]] = {
|
||||
# XML ingestion
|
||||
"XMLIngestor": (".xml_ingestor", "XMLIngestor"),
|
||||
"XMLIngestionData": (".xml_ingestor", "XMLIngestionData"),
|
||||
# Salesforce ingestion
|
||||
"SalesforceIngestor": (".salesforce_ingestor", "SalesforceIngestor"),
|
||||
"SalesforceData": (".salesforce_ingestor", "SalesforceData"),
|
||||
"SalesforceConnector": (".salesforce_ingestor", "SalesforceConnector"),
|
||||
}
|
||||
|
||||
_OPTIONAL_DEPENDENCY_MESSAGES = {
|
||||
@@ -262,6 +270,11 @@ _OPTIONAL_DEPENDENCY_MESSAGES = {
|
||||
"Arrow ingestion requires optional dependency 'pyarrow'. "
|
||||
"Install it before importing ArrowIngestor or using ingest_arrow()."
|
||||
),
|
||||
".salesforce_ingestor": (
|
||||
"Salesforce ingestion requires optional dependency 'simple-salesforce'. "
|
||||
"Install it with: pip install \"semantica[db-salesforce]\" "
|
||||
"or: pip install simple-salesforce>=1.12.0"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -276,7 +289,7 @@ def __getattr__(name: str) -> Any:
|
||||
except ModuleNotFoundError as exc:
|
||||
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
|
||||
missing_name = getattr(exc, "name", None)
|
||||
if message and missing_name in {"git", "bs4", "pyarrow"}:
|
||||
if message and missing_name in {"git", "bs4", "pyarrow", "simple_salesforce"}:
|
||||
raise ImportError(message) from exc
|
||||
raise
|
||||
|
||||
@@ -366,6 +379,10 @@ __all__ = [
|
||||
# XML ingestion
|
||||
"XMLIngestor",
|
||||
"XMLIngestionData",
|
||||
# Salesforce ingestion
|
||||
"SalesforceIngestor",
|
||||
"SalesforceData",
|
||||
"SalesforceConnector",
|
||||
# Registry and Methods
|
||||
"MethodRegistry",
|
||||
"method_registry",
|
||||
@@ -377,6 +394,7 @@ __all__ = [
|
||||
"ingest_repository",
|
||||
"ingest_email",
|
||||
"ingest_database",
|
||||
"ingest_salesforce",
|
||||
"ingest_ontology",
|
||||
"ingest_arrow",
|
||||
"ingest_parquet",
|
||||
|
||||
@@ -186,8 +186,13 @@ class IngestConfig:
|
||||
self._method_configs[method] = config
|
||||
|
||||
def get_method_config(self, method: str) -> Dict:
|
||||
"""Get method-specific configuration."""
|
||||
return self._method_configs.get(method, {})
|
||||
"""Get method-specific configuration.
|
||||
|
||||
Returns a **copy** of the stored method configuration so callers can
|
||||
safely mutate it (e.g. to merge per-call options) without poisoning the
|
||||
global configuration for subsequent calls.
|
||||
"""
|
||||
return dict(self._method_configs.get(method, {}))
|
||||
|
||||
def get_all(self) -> Dict[str, Any]:
|
||||
"""Get all configuration."""
|
||||
|
||||
@@ -875,56 +875,95 @@ schema = connector.get_schema(engine)
|
||||
print(f" {table_name}: {[col['name'] for col in columns]}")
|
||||
```
|
||||
|
||||
## SAP OData Ingestion
|
||||
## Salesforce CRM Ingestion
|
||||
|
||||
`SAPIngestor` reads an Entity Set from a SAP OData service — S/4HANA Cloud,
|
||||
SuccessFactors, or an on-prem NetWeaver Gateway over its REST surface. It
|
||||
follows OData v2/v4 server-driven pagination and flattens each record into a
|
||||
document dict via `export_as_documents()`.
|
||||
Salesforce ingestion requires `simple-salesforce`:
|
||||
|
||||
Install with `pip install 'semantica[ingest-sap]'`.
|
||||
|
||||
### Connector Construction & Authentication
|
||||
|
||||
```python
|
||||
from semantica.ingest import SAPIngestor
|
||||
|
||||
# OAuth2 client-credentials (BTP / S/4HANA Cloud)
|
||||
ing = SAPIngestor(
|
||||
base_url="https://my-sap.example.com/sap/opu/odata/sap/API_BUSINESS_PARTNER",
|
||||
client_id="...", client_secret="...",
|
||||
token_url="https://my-sap.example.com/oauth/token",
|
||||
)
|
||||
# On-prem NetWeaver often uses Basic auth instead — swap the block above for:
|
||||
# ing = SAPIngestor(base_url="...", username="erp_user", password="...")
|
||||
```bash
|
||||
pip install "semantica[db-salesforce]"
|
||||
```
|
||||
|
||||
### Entity-Set Ingestion & Document Export
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
# 1. Discover entity sets + field types from $metadata
|
||||
sets = ing.discover_service()
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
import os
|
||||
|
||||
# 2. Page-walk an Entity Set (v2/v4 next links handled automatically)
|
||||
partners = ing.ingest_entity_set(
|
||||
entity_set="A_BusinessPartnerSet",
|
||||
select="BusinessPartner,BusinessPartnerFullName",
|
||||
top=1000,
|
||||
ingestor = SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
domain="login", # "test" for sandbox
|
||||
)
|
||||
|
||||
# 3. Flatten to document dicts that GraphBuilder can consume directly
|
||||
docs = ing.export_as_documents(partners)
|
||||
# Ingest Account records
|
||||
data = ingestor.ingest_sobject(
|
||||
"Account",
|
||||
fields=["Id", "Name", "Industry", "BillingCity"],
|
||||
where="Type = 'Customer'",
|
||||
limit=5000,
|
||||
)
|
||||
print(f"Retrieved {data.row_count} of {data.total_size} matching records")
|
||||
```
|
||||
|
||||
- Use `expand="to_Item"` on a sales-order header set to pull nested line items
|
||||
in one request — handy for modeling order → line-item → material relations.
|
||||
- Every outbound request, including the OAuth2 token exchange, is routed through
|
||||
the SSRF guard, and pagination never follows a next link that points to a
|
||||
different host than the service root.
|
||||
`SalesforceIngestor()` with no arguments reads from `SALESFORCE_USERNAME`, `SALESFORCE_PASSWORD`, `SALESFORCE_SECURITY_TOKEN`, and `SALESFORCE_DOMAIN` environment variables automatically.
|
||||
|
||||
### Custom Objects and Raw SOQL
|
||||
|
||||
```python
|
||||
# Custom object (API name ends in __c)
|
||||
data = ingestor.ingest_sobject("My_Custom_Object__c", fields=["Id", "Name", "Custom_Field__c"])
|
||||
|
||||
# Raw SOQL GÇö pagination is handled automatically
|
||||
data = ingestor.ingest_query("""
|
||||
SELECT Id, Name, StageName, Amount
|
||||
FROM Opportunity
|
||||
WHERE IsClosed = false
|
||||
ORDER BY CloseDate ASC
|
||||
""")
|
||||
print(f"Open opportunities: {data.row_count}")
|
||||
```
|
||||
|
||||
### Document Export
|
||||
|
||||
```python
|
||||
documents = ingestor.export_as_documents(
|
||||
data,
|
||||
id_field="Id", # Salesforce 18-char record Id
|
||||
text_fields=["Name", "Description"],
|
||||
)
|
||||
# Each document: {"id": "001...", "text": "...", "metadata": {"source": "salesforce", ...}}
|
||||
```
|
||||
|
||||
### Convenience Function
|
||||
|
||||
```python
|
||||
from semantica.ingest import ingest_salesforce
|
||||
|
||||
# Fetch records
|
||||
data = ingest_salesforce(
|
||||
method="sobject",
|
||||
sobject_name="Account",
|
||||
fields=["Id", "Name"],
|
||||
limit=500,
|
||||
)
|
||||
|
||||
# Ingest and export as documents in one call
|
||||
docs = ingest_salesforce(
|
||||
method="documents",
|
||||
sobject_name="Account",
|
||||
text_fields=["Name", "Description"],
|
||||
)
|
||||
|
||||
# Using the unified dispatcher
|
||||
from semantica.ingest import ingest
|
||||
result = ingest(None, source_type="salesforce", method="sobject",
|
||||
sobject_name="Account", fields=["Id", "Name"])
|
||||
data = result["data"]
|
||||
```
|
||||
|
||||
See [Salesforce Integration](https://docs.getsemantica.ai/integrations/salesforce) for full documentation including sandbox, schema discovery, pagination details, and troubleshooting.
|
||||
|
||||
> **Security Note:** Never hardcode credentials (`client_secret`, `password`);
|
||||
> pass them via environment variables (`SAP_CLIENT_SECRET`, `SAP_PASSWORD`) or a
|
||||
> secrets manager.
|
||||
|
||||
## MCP Server Ingestion
|
||||
|
||||
@@ -1664,3 +1703,54 @@ for source_type, source_list in sources.items():
|
||||
for batch in process_in_batches(large_dataset, batch_size=1000):
|
||||
result = ingest(batch)
|
||||
```
|
||||
|
||||
## SAP OData Ingestion
|
||||
|
||||
`SAPIngestor` reads an Entity Set from a SAP OData service — S/4HANA Cloud,
|
||||
SuccessFactors, or an on-prem NetWeaver Gateway over its REST surface. It
|
||||
follows OData v2/v4 server-driven pagination and flattens each record into a
|
||||
document dict via `export_as_documents()`.
|
||||
|
||||
Install with `pip install 'semantica[ingest-sap]'`.
|
||||
|
||||
### Connector Construction & Authentication
|
||||
|
||||
```python
|
||||
from semantica.ingest import SAPIngestor
|
||||
|
||||
# OAuth2 client-credentials (BTP / S/4HANA Cloud)
|
||||
ing = SAPIngestor(
|
||||
base_url="https://my-sap.example.com/sap/opu/odata/sap/API_BUSINESS_PARTNER",
|
||||
client_id="...", client_secret="...",
|
||||
token_url="https://my-sap.example.com/oauth/token",
|
||||
)
|
||||
# On-prem NetWeaver often uses Basic auth instead — swap the block above for:
|
||||
# ing = SAPIngestor(base_url="...", username="erp_user", password="...")
|
||||
```
|
||||
|
||||
### Entity-Set Ingestion & Document Export
|
||||
|
||||
```python
|
||||
# 1. Discover entity sets + field types from $metadata
|
||||
sets = ing.discover_service()
|
||||
|
||||
# 2. Page-walk an Entity Set (v2/v4 next links handled automatically)
|
||||
partners = ing.ingest_entity_set(
|
||||
entity_set="A_BusinessPartnerSet",
|
||||
select="BusinessPartner,BusinessPartnerFullName",
|
||||
top=1000,
|
||||
)
|
||||
|
||||
# 3. Flatten to document dicts that GraphBuilder can consume directly
|
||||
docs = ing.export_as_documents(partners)
|
||||
```
|
||||
|
||||
- Use `expand="to_Item"` on a sales-order header set to pull nested line items
|
||||
in one request — handy for modeling order → line-item → material relations.
|
||||
- Every outbound request, including the OAuth2 token exchange, is routed through
|
||||
the SSRF guard, and pagination never follows a next link that points to a
|
||||
different host than the service root.
|
||||
|
||||
> **Security Note:** Never hardcode credentials (`client_secret`, `password`);
|
||||
> pass them via environment variables (`SAP_CLIENT_SECRET`, `SAP_PASSWORD`) or a
|
||||
> secrets manager.
|
||||
|
||||
@@ -203,6 +203,7 @@ if TYPE_CHECKING:
|
||||
from .ontology_ingestor import OntologyData
|
||||
from .parquet_ingestor import ParquetData
|
||||
from .public_api_ingestor import PublicAPIDetection
|
||||
from .salesforce_ingestor import SalesforceData
|
||||
from .stream_ingestor import StreamProcessor
|
||||
from .web_ingestor import WebContent
|
||||
from .xml_ingestor import XMLIngestionData
|
||||
@@ -1126,6 +1127,213 @@ def ingest_database(
|
||||
raise
|
||||
|
||||
|
||||
def ingest_salesforce(
|
||||
source: Optional[Dict[str, Any]] = None,
|
||||
method: str = "sobject",
|
||||
**kwargs,
|
||||
) -> Union["SalesforceData", List[Dict[str, Any]], Dict[str, Any]]:
|
||||
"""Ingest data from Salesforce CRM (convenience function).
|
||||
|
||||
A user-friendly wrapper around :class:`~semantica.ingest.SalesforceIngestor`
|
||||
that connects, ingests, and returns data in a single call.
|
||||
|
||||
Args:
|
||||
source: Optional credential/configuration dictionary. Keys mirror the
|
||||
:class:`~semantica.ingest.SalesforceConnector` constructor:
|
||||
``username``, ``password``, ``security_token``, ``domain``
|
||||
(``"login"`` for production, ``"test"`` for sandbox),
|
||||
``instance_url``, ``session_id``, ``api_version``.
|
||||
When ``None``, credentials are read from environment variables
|
||||
(``SALESFORCE_USERNAME`` / ``SALESFORCE_PASSWORD`` /
|
||||
``SALESFORCE_SECURITY_TOKEN`` etc.).
|
||||
method: Ingestion method:
|
||||
|
||||
* ``"sobject"`` *(default)* — fetch records from a named sObject
|
||||
(requires ``sobject_name`` kwarg).
|
||||
* ``"query"`` — execute a raw SOQL query string (requires
|
||||
``soql`` kwarg).
|
||||
* ``"list_sobjects"`` — return a sorted list of accessible sObject
|
||||
API names.
|
||||
* ``"schema"`` — return field metadata for a named sObject
|
||||
(requires ``sobject_name`` kwarg).
|
||||
* ``"documents"`` — ingest an sObject and convert to the Semantica
|
||||
document format in one step (requires ``sobject_name`` kwarg;
|
||||
optional ``text_fields`` and ``id_field`` kwargs).
|
||||
|
||||
**kwargs: Additional options forwarded to the ingestor method.
|
||||
Common kwargs for ``"sobject"`` / ``"documents"``:
|
||||
|
||||
* ``sobject_name`` — Salesforce sObject API name (e.g.
|
||||
``"Account"``, ``"My_Custom__c"``).
|
||||
* ``fields`` — list of field API names to select. When omitted
|
||||
all selectable fields are fetched via ``describe()``.
|
||||
* ``where`` — SOQL ``WHERE`` clause fragment (trusted input only).
|
||||
* ``order_by`` — SOQL ``ORDER BY`` clause fragment.
|
||||
* ``limit`` — maximum number of records.
|
||||
|
||||
For ``"query"``:
|
||||
|
||||
* ``soql`` — full SOQL query string.
|
||||
|
||||
For ``"schema"``:
|
||||
|
||||
* ``sobject_name`` — sObject to describe.
|
||||
|
||||
Returns:
|
||||
* ``"sobject"`` / ``"query"`` → :class:`~semantica.ingest.SalesforceData`
|
||||
* ``"documents"`` → ``List[Dict[str, Any]]`` (Semantica document format)
|
||||
* ``"list_sobjects"`` → ``List[str]``
|
||||
* ``"schema"`` → ``Dict[str, Any]``
|
||||
|
||||
Raises:
|
||||
:class:`~semantica.utils.exceptions.ConfigurationError`: If
|
||||
``simple-salesforce`` is not installed.
|
||||
:class:`~semantica.utils.exceptions.ValidationError`: If credentials
|
||||
are incomplete or an sObject / field name is invalid.
|
||||
:class:`~semantica.utils.exceptions.ProcessingError`: If the
|
||||
Salesforce API call fails.
|
||||
|
||||
Examples::
|
||||
|
||||
>>> from semantica.ingest import ingest_salesforce
|
||||
|
||||
>>> # Fetch Account records (credentials from env vars)
|
||||
>>> data = ingest_salesforce(
|
||||
... method="sobject",
|
||||
... sobject_name="Account",
|
||||
... fields=["Id", "Name", "Industry"],
|
||||
... limit=500,
|
||||
... )
|
||||
|
||||
>>> # Execute a raw SOQL query (credentials from environment variables)
|
||||
>>> data = ingest_salesforce(
|
||||
... method="query",
|
||||
... soql="SELECT Id, Name FROM Contact WHERE IsActive = true",
|
||||
... )
|
||||
|
||||
>>> # Ingest and export as documents for GraphBuilder in one step
|
||||
>>> docs = ingest_salesforce(
|
||||
... method="documents",
|
||||
... sobject_name="Account",
|
||||
... text_fields=["Name", "Description"],
|
||||
... limit=1000,
|
||||
... )
|
||||
|
||||
>>> # List all accessible sObjects in the connected org
|
||||
>>> sobject_names = ingest_salesforce(method="list_sobjects")
|
||||
|
||||
>>> # Use sandbox org
|
||||
>>> data = ingest_salesforce(
|
||||
... method="sobject",
|
||||
... sobject_name="Account",
|
||||
... ) # set SALESFORCE_DOMAIN=test in environment for sandbox
|
||||
"""
|
||||
# Registry hook — allows callers to register a custom "salesforce" method
|
||||
custom_method = method_registry.get("salesforce", method)
|
||||
if custom_method and custom_method != ingest_salesforce:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source,
|
||||
fallback_on_custom_error=fallback, **kwargs,
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .salesforce_ingestor import SalesforceIngestor
|
||||
except ModuleNotFoundError as exc:
|
||||
if _is_missing_dependency(exc, "simple_salesforce"):
|
||||
raise _missing_optional_dependency(
|
||||
"Salesforce ingestion", "simple-salesforce"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
# Unpack credential dict (if given); everything else stays in kwargs.
|
||||
creds: Dict[str, Any] = {}
|
||||
if source is not None:
|
||||
if not isinstance(source, dict):
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() source must be a credential dict or None. "
|
||||
"Pass sobject_name / soql as keyword arguments."
|
||||
)
|
||||
creds = dict(source)
|
||||
|
||||
# Merge any ingest_config method config under "salesforce".
|
||||
# get_method_config() now returns a copy, so this dict is safe to mutate.
|
||||
# We build the final connector config in order of increasing priority:
|
||||
# 1. base method config (lowest — global defaults set by operator)
|
||||
# 2. per-call credential dict supplied via `source`
|
||||
# 3. per-call connector params supplied as kwargs
|
||||
# Credentials are extracted from kwargs and removed so they don't also
|
||||
# flow into the ingest method call (which doesn't understand them).
|
||||
_CONNECTOR_PARAMS = frozenset({
|
||||
"username", "password", "security_token", "domain",
|
||||
"instance_url", "session_id", "api_version",
|
||||
})
|
||||
connector_kwargs = {k: v for k, v in kwargs.items() if k in _CONNECTOR_PARAMS}
|
||||
for k in _CONNECTOR_PARAMS:
|
||||
kwargs.pop(k, None)
|
||||
|
||||
# Build a fresh per-call config dict — never mutate the global store.
|
||||
config: Dict[str, Any] = {
|
||||
**ingest_config.get_method_config("salesforce"), # base (already a copy)
|
||||
**creds, # source dict credentials
|
||||
**connector_kwargs, # kwarg credentials
|
||||
}
|
||||
|
||||
ingestor = SalesforceIngestor(**config)
|
||||
|
||||
if method == "sobject":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='sobject' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
return ingestor.ingest_sobject(sobject_name, **kwargs)
|
||||
|
||||
elif method == "query":
|
||||
soql = kwargs.pop("soql", None)
|
||||
if not soql:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='query' requires "
|
||||
"soql keyword argument."
|
||||
)
|
||||
return ingestor.ingest_query(soql, **kwargs)
|
||||
|
||||
elif method == "list_sobjects":
|
||||
return ingestor.list_sobjects()
|
||||
|
||||
elif method == "schema":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='schema' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
return ingestor.get_sobject_schema(sobject_name)
|
||||
|
||||
elif method == "documents":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='documents' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
id_field = kwargs.pop("id_field", "Id")
|
||||
text_fields = kwargs.pop("text_fields", None)
|
||||
data = ingestor.ingest_sobject(sobject_name, **kwargs)
|
||||
return ingestor.export_as_documents(data, id_field=id_field,
|
||||
text_fields=text_fields)
|
||||
|
||||
else:
|
||||
raise ProcessingError(
|
||||
f"Unknown ingest_salesforce method: {method!r}. "
|
||||
"Valid methods: 'sobject', 'query', 'list_sobjects', 'schema', "
|
||||
"'documents'."
|
||||
)
|
||||
|
||||
|
||||
def ingest_mcp(
|
||||
source: Union[str, Dict[str, Any]],
|
||||
method: str = "resources",
|
||||
@@ -1306,6 +1514,7 @@ def ingest(
|
||||
- "ontology": Ontology ingestion
|
||||
- "parquet": Apache Parquet file or directory ingestion
|
||||
- "xml": XML file or directory ingestion
|
||||
- "salesforce": Salesforce CRM ingestion (pass credentials via kwargs)
|
||||
method: Optional specific ingestion method
|
||||
**kwargs: Additional options passed to ingestor
|
||||
|
||||
@@ -1428,6 +1637,9 @@ def ingest(
|
||||
return {"ontology": ingest_ontology(sources, method=method or "file", **kwargs)}
|
||||
elif source_type == "mcp":
|
||||
return {"data": ingest_mcp(sources, method=method or "resources", **kwargs)}
|
||||
elif source_type == "salesforce":
|
||||
return {"data": ingest_salesforce(sources,
|
||||
method=method or "sobject", **kwargs)}
|
||||
else:
|
||||
raise ProcessingError(f"Unknown source type: {source_type}")
|
||||
|
||||
@@ -1540,3 +1752,9 @@ method_registry.register("ontology", "file", ingest_ontology)
|
||||
method_registry.register("ontology", "directory", ingest_ontology)
|
||||
method_registry.register("ingest", "default", ingest)
|
||||
method_registry.register("ingest", "unified", ingest)
|
||||
method_registry.register("salesforce", "default", ingest_salesforce)
|
||||
method_registry.register("salesforce", "sobject", ingest_salesforce)
|
||||
method_registry.register("salesforce", "query", ingest_salesforce)
|
||||
method_registry.register("salesforce", "list_sobjects", ingest_salesforce)
|
||||
method_registry.register("salesforce", "schema", ingest_salesforce)
|
||||
method_registry.register("salesforce", "documents", ingest_salesforce)
|
||||
|
||||
@@ -66,6 +66,7 @@ class MethodRegistry:
|
||||
"parquet": {},
|
||||
"arrow": {},
|
||||
"xml": {},
|
||||
"salesforce": {},
|
||||
"ingest": {},
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -117,6 +117,8 @@ class PropertyGenerator:
|
||||
data_properties = self._infer_data_properties(entities, classes, **options)
|
||||
properties.extend(data_properties)
|
||||
|
||||
properties = self._coalesce_normalized_properties(properties)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
@@ -193,6 +195,67 @@ class PropertyGenerator:
|
||||
|
||||
return properties
|
||||
|
||||
def _coalesce_normalized_properties(
|
||||
self, properties: List[Dict[str, Any]]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Merge same-kind properties that normalize to the same name."""
|
||||
property_kinds = defaultdict(set)
|
||||
for prop in properties:
|
||||
property_kinds[prop["name"]].add(prop.get("type"))
|
||||
|
||||
collisions = {
|
||||
name: sorted(kind for kind in kinds if kind is not None)
|
||||
for name, kinds in property_kinds.items()
|
||||
if len({kind for kind in kinds if kind is not None}) > 1
|
||||
}
|
||||
if collisions:
|
||||
raise ValidationError(
|
||||
"Normalized property names cannot be shared by object and "
|
||||
"data properties.",
|
||||
validation_context={"property_kind_collisions": collisions},
|
||||
)
|
||||
|
||||
merged: Dict[tuple, Dict[str, Any]] = {}
|
||||
result = []
|
||||
for prop in properties:
|
||||
key = (prop.get("type"), prop["name"])
|
||||
existing = merged.get(key)
|
||||
if existing is None:
|
||||
merged[key] = prop
|
||||
result.append(prop)
|
||||
continue
|
||||
|
||||
existing["domain"] = self._merge_property_values(
|
||||
existing.get("domain", []), prop.get("domain", [])
|
||||
)
|
||||
if prop.get("type") == "object":
|
||||
existing["range"] = self._merge_property_values(
|
||||
existing.get("range", []), prop.get("range", [])
|
||||
)
|
||||
existing_metadata = existing.setdefault("metadata", {})
|
||||
existing_metadata["occurrence_count"] = (
|
||||
existing_metadata.get("occurrence_count", 0)
|
||||
+ prop.get("metadata", {}).get("occurrence_count", 0)
|
||||
)
|
||||
elif existing.get("range") != prop.get("range"):
|
||||
existing["range"] = self._get_more_general_type(
|
||||
existing["range"], prop["range"]
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _merge_property_values(current: Any, incoming: Any) -> List[Any]:
|
||||
"""Merge scalar-or-list property values while preserving input order."""
|
||||
values = list(current) if isinstance(current, list) else [current]
|
||||
incoming_values = (
|
||||
incoming if isinstance(incoming, list) else [incoming]
|
||||
)
|
||||
for value in incoming_values:
|
||||
if value not in values:
|
||||
values.append(value)
|
||||
return [value for value in values if value is not None]
|
||||
|
||||
def _infer_data_properties(
|
||||
self, entities: List[Dict[str, Any]], classes: List[Dict[str, Any]], **options
|
||||
) -> List[Dict[str, Any]]:
|
||||
|
||||
@@ -70,7 +70,8 @@ sem:metadata a owl:AnnotationProperty ;
|
||||
rdfs:label "metadata" ;
|
||||
rdfs:comment """Free-form metadata carried through from extraction. An
|
||||
annotation property because its value is an arbitrary structure rather than a
|
||||
modelled one.""" ;
|
||||
modelled one; in the JSON-LD export the whole mapping is written as one
|
||||
rdf:JSON literal so caller keys never expand into this namespace (#1146).""" ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
# ── Relationship terms (JSON-LD export) ──────────────────────────────────────
|
||||
@@ -96,10 +97,10 @@ sem:target a owl:ObjectProperty ;
|
||||
|
||||
sem:type a owl:DatatypeProperty ;
|
||||
rdfs:label "type" ;
|
||||
rdfs:comment """The relationship type as a label, as emitted in the JSON-LD
|
||||
export. Distinct from rdf:type, which relates a node to a class rather than to
|
||||
a string.""" ;
|
||||
rdfs:domain sem:Relationship ;
|
||||
rdfs:comment """The entity or relationship type as a label, as emitted in
|
||||
the JSON-LD export. Distinct from rdf:type, which relates a node to a class
|
||||
rather than to a string. Emitted for both entities and relationships, so the
|
||||
domain is left open rather than tied to sem:Relationship.""" ;
|
||||
rdfs:range xsd:string ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
|
||||
@@ -41,69 +41,134 @@ from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .reasoner import Fact, Rule, _make_activation_key
|
||||
|
||||
logger = get_logger("rete_engine")
|
||||
|
||||
def _extract_bindings(condition: Any, fact: Fact) -> Dict[str, Any]:
|
||||
"""Extract ``?var`` bindings by matching a condition pattern against a fact.
|
||||
|
||||
``condition`` is the pattern stored on the alpha node (typically a string
|
||||
like ``"Person(?x)"``); ``fact`` is the working-memory :class:`Fact`. The
|
||||
fact's canonical string form (``Predicate(arg1, arg2, ...)``) is matched
|
||||
against the pattern using the same ``?\\w+`` placeholder convention as the
|
||||
Reasoner, so downstream actions receive real bindings (e.g. ``{"x": "John"}``)
|
||||
instead of the empty dict that previously left ``?x`` placeholders
|
||||
unsubstituted.
|
||||
def _build_condition_regex(
|
||||
pattern: str,
|
||||
initial_bindings: Optional[Dict[str, str]] = None,
|
||||
) -> str:
|
||||
"""Build an anchored regex string for a condition pattern.
|
||||
|
||||
Returns an empty dict when the condition is not a string pattern or does
|
||||
not match -- callers treat that as "no bindings extracted".
|
||||
Splits the pattern on ``?var`` placeholders, escaping the literal
|
||||
segments so surrounding parentheses/commas match literally. Variables
|
||||
become named groups (or backreferences when repeated); variables already
|
||||
present in ``initial_bindings`` are inlined as their literal value.
|
||||
|
||||
Args:
|
||||
pattern: The condition pattern string (e.g. ``"Person(?x)"``).
|
||||
initial_bindings: Bindings already established upstream. Variables
|
||||
already bound are matched as literals rather than captured.
|
||||
|
||||
Returns:
|
||||
An anchored regex string (``^...$``) suitable for ``re.compile`` /
|
||||
``re.match``.
|
||||
"""
|
||||
if not isinstance(condition, str):
|
||||
return {}
|
||||
|
||||
segments = re.split(r"(\?\w+)", condition)
|
||||
bindings = initial_bindings or {}
|
||||
segments = re.split(r"(\?\w+)", pattern)
|
||||
seen_vars: Set[str] = set()
|
||||
p_regex = ""
|
||||
for seg in segments:
|
||||
if seg.startswith("?"):
|
||||
var_name = seg[1:]
|
||||
if var_name in seen_vars:
|
||||
if var_name in bindings:
|
||||
# Already bound — require the exact literal value.
|
||||
p_regex += re.escape(bindings[var_name])
|
||||
elif var_name in seen_vars:
|
||||
# Same variable used twice — enforce a backreference.
|
||||
p_regex += f"(?P={var_name})"
|
||||
else:
|
||||
p_regex += f"(?P<{var_name}>.+?)"
|
||||
seen_vars.add(var_name)
|
||||
else:
|
||||
p_regex += re.escape(seg)
|
||||
p_regex = f"^{p_regex}$"
|
||||
return f"^{p_regex}$"
|
||||
|
||||
|
||||
def unify_condition(
|
||||
condition: Any,
|
||||
fact: Fact,
|
||||
initial_bindings: Optional[Dict[str, str]] = None,
|
||||
) -> Optional[Dict[str, str]]:
|
||||
"""Unify a condition pattern against a fact.
|
||||
|
||||
A condition is a pattern string such as ``"Person(?x)"`` or
|
||||
``"knows(?x, ?y)"`` where tokens beginning with ``?`` are variables.
|
||||
The fact is rendered via its ``__str__`` representation
|
||||
(``predicate(arg1, arg2)``) and matched against the pattern.
|
||||
|
||||
This mirrors ``Reasoner._match_pattern`` but is self-contained so the
|
||||
RETE engine does not need a live ``Reasoner`` instance.
|
||||
|
||||
Args:
|
||||
condition: The condition pattern (string). Non-string conditions
|
||||
are stringified before matching.
|
||||
fact: The fact to test.
|
||||
initial_bindings: Bindings already established upstream. Variables
|
||||
already bound must match the corresponding literal in the fact.
|
||||
|
||||
Returns:
|
||||
A dict of variable bindings if the fact unifies with the condition,
|
||||
otherwise ``None``.
|
||||
"""
|
||||
bindings = dict(initial_bindings or {})
|
||||
pattern = condition if isinstance(condition, str) else str(condition)
|
||||
fact_str = str(fact)
|
||||
|
||||
# Build the anchored regex once (variables already bound are inlined as
|
||||
# literals). See ``_build_condition_regex`` for the segment handling.
|
||||
p_regex = _build_condition_regex(pattern, bindings)
|
||||
|
||||
try:
|
||||
match = re.match(p_regex, str(fact))
|
||||
except re.error:
|
||||
return {}
|
||||
match = re.match(p_regex, fact_str)
|
||||
except re.error as e:
|
||||
logger.warning(
|
||||
"unify_condition failed to compile/match condition "
|
||||
"%r (regex: %r) against fact %r: %s",
|
||||
pattern,
|
||||
p_regex,
|
||||
fact_str,
|
||||
e,
|
||||
)
|
||||
return None
|
||||
except Exception as e: # noqa: BLE001 - mirror Reasoner._match_pattern
|
||||
logger.warning(
|
||||
"unify_condition unexpected error matching condition "
|
||||
"%r (regex: %r) against fact %r: %s",
|
||||
pattern,
|
||||
p_regex,
|
||||
fact_str,
|
||||
e,
|
||||
)
|
||||
return None
|
||||
if not match:
|
||||
return {}
|
||||
return {k: v for k, v in match.groupdict().items() if v is not None}
|
||||
return None
|
||||
|
||||
|
||||
def _bindings_for_rule(rule: Rule, facts: List[Fact]) -> Dict[str, Any]:
|
||||
"""Merge ``?var`` bindings from matching a rule's conditions against facts.
|
||||
|
||||
Each fact is matched against every condition of the rule; the first
|
||||
condition that yields bindings for a fact contributes them. Bindings from
|
||||
all facts are merged so multi-condition (joined) rules receive the full
|
||||
variable environment. Later conflicting values do not overwrite earlier
|
||||
ones, preserving the binding that a join already validated.
|
||||
"""
|
||||
bindings: Dict[str, Any] = {}
|
||||
for fact in facts:
|
||||
for condition in rule.conditions:
|
||||
extracted = _extract_bindings(condition, fact)
|
||||
if not extracted:
|
||||
continue
|
||||
for key, value in extracted.items():
|
||||
bindings.setdefault(key, value)
|
||||
break
|
||||
for var, value in match.groupdict().items():
|
||||
if var in bindings and bindings[var] != value:
|
||||
return None # Binding conflict.
|
||||
bindings[var] = value
|
||||
return bindings
|
||||
|
||||
|
||||
@dataclass
|
||||
class Token:
|
||||
"""A partial match flowing through the Rete network.
|
||||
|
||||
A token represents an ordered collection of concrete facts that have
|
||||
been unified so far, together with the consistent variable bindings
|
||||
accumulated across those facts.
|
||||
|
||||
Alpha nodes emit single-fact tokens. Beta nodes merge a left token and
|
||||
a right token into a new token whose ``facts`` are the concatenation of
|
||||
both sides (preserving condition order) and whose ``bindings`` are the
|
||||
consistent union of both sides.
|
||||
"""
|
||||
|
||||
facts: List[Fact] = field(default_factory=list)
|
||||
bindings: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Match:
|
||||
"""Pattern match."""
|
||||
@@ -128,19 +193,68 @@ class AlphaNode(ReteNode):
|
||||
def __init__(self, node_id: str, condition: Any):
|
||||
super().__init__(node_id)
|
||||
self.condition = condition
|
||||
self.matches: List[Fact] = []
|
||||
# Single-fact tokens produced by unifying each matched fact with
|
||||
# this node's condition.
|
||||
self.tokens: List[Token] = []
|
||||
# Pre-compile the condition regex once. Alpha nodes never have
|
||||
# initial bindings, so the pattern is stable for the node's lifetime
|
||||
# and every incoming fact reuses this compiled matcher instead of
|
||||
# rebuilding it (avoids repeated regex construction overhead).
|
||||
pattern = condition if isinstance(condition, str) else str(condition)
|
||||
self._compiled: Optional[re.Pattern] = None
|
||||
try:
|
||||
self._compiled = re.compile(_build_condition_regex(pattern))
|
||||
except re.error as e:
|
||||
logger.warning(
|
||||
"AlphaNode %r failed to compile condition %r: %s; "
|
||||
"node will never match",
|
||||
node_id,
|
||||
pattern,
|
||||
e,
|
||||
)
|
||||
|
||||
def add_fact(self, fact: Fact) -> bool:
|
||||
"""Add fact if it matches condition."""
|
||||
if self._matches(fact):
|
||||
self.matches.append(fact)
|
||||
return True
|
||||
return False
|
||||
def add_fact(self, fact: Fact) -> Optional[Token]:
|
||||
"""Add fact if it matches the condition, returning its token.
|
||||
|
||||
def _matches(self, fact: Fact) -> bool:
|
||||
"""Check if fact matches condition."""
|
||||
# Simple matching - can be enhanced
|
||||
return True
|
||||
Returns the single-fact ``Token`` produced by unification when the
|
||||
fact matches, otherwise ``None``.
|
||||
"""
|
||||
bindings = self._matches(fact)
|
||||
if bindings is not None:
|
||||
token = Token(facts=[fact], bindings=dict(bindings))
|
||||
self.tokens.append(token)
|
||||
return token
|
||||
return None
|
||||
|
||||
def _matches(self, fact: Fact) -> Optional[Dict[str, str]]:
|
||||
"""Check if fact matches the alpha node condition.
|
||||
|
||||
Uses the pre-compiled regex built in ``__init__`` for performance,
|
||||
since RETE evaluates many facts against every alpha node.
|
||||
|
||||
Returns the variable bindings produced by unification if the fact
|
||||
matches, otherwise ``None``. An empty dict signals a match with no
|
||||
variables (still distinct from ``None``).
|
||||
"""
|
||||
if self._compiled is None:
|
||||
# Compilation failed at build time; treat as non-matching.
|
||||
return None
|
||||
fact_str = str(fact)
|
||||
try:
|
||||
match = self._compiled.match(fact_str)
|
||||
except Exception as e: # noqa: BLE001 - mirror unify_condition
|
||||
logger.warning(
|
||||
"AlphaNode %r unexpected error matching condition "
|
||||
"%r against fact %r: %s",
|
||||
self.node_id,
|
||||
self.condition,
|
||||
fact_str,
|
||||
e,
|
||||
)
|
||||
return None
|
||||
if not match:
|
||||
return None
|
||||
return match.groupdict()
|
||||
|
||||
|
||||
class BetaNode(ReteNode):
|
||||
@@ -150,19 +264,28 @@ class BetaNode(ReteNode):
|
||||
super().__init__(node_id)
|
||||
self.left = left
|
||||
self.right = right
|
||||
self.matches: List[Tuple[Fact, Fact]] = []
|
||||
# Token memories for each side. Incoming tokens are stored here so
|
||||
# that later-arriving tokens on the opposite side can be joined
|
||||
# against every token already seen (chained joins).
|
||||
self.left_tokens: List[Token] = []
|
||||
self.right_tokens: List[Token] = []
|
||||
|
||||
def join(self, left_fact: Fact, right_fact: Fact) -> bool:
|
||||
"""Join facts from left and right nodes."""
|
||||
if self._can_join(left_fact, right_fact):
|
||||
self.matches.append((left_fact, right_fact))
|
||||
return True
|
||||
return False
|
||||
def join(self, left_token: Token, right_token: Token) -> Optional[Token]:
|
||||
"""Join a left token with a right token.
|
||||
|
||||
def _can_join(self, left_fact: Fact, right_fact: Fact) -> bool:
|
||||
"""Check if facts can be joined."""
|
||||
# Simple join logic - can be enhanced
|
||||
return True
|
||||
Returns a new merged ``Token`` (facts concatenated in condition
|
||||
order, bindings unified) when the two tokens are consistent,
|
||||
otherwise ``None`` on a binding conflict.
|
||||
"""
|
||||
merged = dict(left_token.bindings)
|
||||
for var, value in right_token.bindings.items():
|
||||
if var in merged and merged[var] != value:
|
||||
return None # Binding conflict — cannot join.
|
||||
merged[var] = value
|
||||
return Token(
|
||||
facts=list(left_token.facts) + list(right_token.facts),
|
||||
bindings=merged,
|
||||
)
|
||||
|
||||
|
||||
class TerminalNode(ReteNode):
|
||||
@@ -248,12 +371,16 @@ class ReteEngine:
|
||||
self._add_rule_to_network(rule)
|
||||
|
||||
self.logger.info(
|
||||
f"Built Rete network with {len(self.network)} nodes for {len(rules)} rules"
|
||||
f"Built Rete network with {len(self.network)} nodes "
|
||||
f"for {len(rules)} rules"
|
||||
)
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Built Rete network with {len(self.network)} nodes for {len(rules)} rules",
|
||||
message=(
|
||||
f"Built Rete network with {len(self.network)} nodes "
|
||||
f"for {len(rules)} rules"
|
||||
),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
@@ -281,6 +408,10 @@ class ReteEngine:
|
||||
self.node_counter += 1
|
||||
beta_node = BetaNode(node_id, current, alpha_nodes[i])
|
||||
self.network[node_id] = beta_node
|
||||
# Wire the beta node as a child of both its inputs so facts
|
||||
# propagating from either side reach the join.
|
||||
current.children.append(beta_node)
|
||||
alpha_nodes[i].children.append(beta_node)
|
||||
current = beta_node
|
||||
final_node = current
|
||||
else:
|
||||
@@ -311,40 +442,58 @@ class ReteEngine:
|
||||
# Find matching alpha nodes
|
||||
for node_id, node in self.network.items():
|
||||
if isinstance(node, AlphaNode):
|
||||
if node.add_fact(fact):
|
||||
# Propagate to children
|
||||
self._propagate_from_alpha(node, fact)
|
||||
token = node.add_fact(fact)
|
||||
if token is not None:
|
||||
# Propagate the single-fact token to children.
|
||||
self._propagate_token(node, token)
|
||||
|
||||
def _propagate_from_alpha(self, alpha_node: AlphaNode, fact: Fact) -> None:
|
||||
"""Propagate from alpha node to children."""
|
||||
for child in alpha_node.children:
|
||||
def _propagate_token(self, source: ReteNode, token: Token) -> None:
|
||||
"""Propagate ``token`` (arriving from ``source``) to its children.
|
||||
|
||||
A ``Token`` carries the ordered facts and consistent bindings of a
|
||||
partial match. Beta children attempt joins and, on success, emit a
|
||||
new merged token downstream; terminal children turn the token into a
|
||||
rule activation using the token's complete facts and bindings.
|
||||
"""
|
||||
for child in source.children:
|
||||
if isinstance(child, BetaNode):
|
||||
# Join with matches from left side
|
||||
for left_fact in alpha_node.matches:
|
||||
if child.join(left_fact, fact):
|
||||
# Propagate to children
|
||||
for grandchild in child.children:
|
||||
if isinstance(grandchild, TerminalNode):
|
||||
facts = [left_fact, fact]
|
||||
match = Match(
|
||||
rule=grandchild.rule,
|
||||
facts=facts,
|
||||
bindings=_bindings_for_rule(
|
||||
grandchild.rule, facts
|
||||
),
|
||||
confidence=1.0,
|
||||
)
|
||||
grandchild.activate(match)
|
||||
self._propagate_to_beta(child, source, token)
|
||||
elif isinstance(child, TerminalNode):
|
||||
# Direct activation
|
||||
match = Match(
|
||||
rule=child.rule,
|
||||
facts=[fact],
|
||||
bindings=_bindings_for_rule(child.rule, [fact]),
|
||||
facts=list(token.facts),
|
||||
bindings=dict(token.bindings),
|
||||
confidence=1.0,
|
||||
)
|
||||
child.activate(match)
|
||||
|
||||
def _propagate_to_beta(
|
||||
self,
|
||||
beta: "BetaNode",
|
||||
source: ReteNode,
|
||||
token: Token,
|
||||
) -> None:
|
||||
"""Attempt joins at ``beta`` for a token arriving from one side.
|
||||
|
||||
The incoming token is stored in the corresponding side's memory,
|
||||
then joined against every token already recorded on the opposite
|
||||
side. Each successful join produces a new merged token that is
|
||||
propagated further downstream, enabling correct chained joins across
|
||||
three or more conditions.
|
||||
"""
|
||||
if source is beta.left:
|
||||
beta.left_tokens.append(token)
|
||||
for right_token in list(beta.right_tokens):
|
||||
merged = beta.join(token, right_token)
|
||||
if merged is not None:
|
||||
self._propagate_token(beta, merged)
|
||||
elif source is beta.right:
|
||||
beta.right_tokens.append(token)
|
||||
for left_token in list(beta.left_tokens):
|
||||
merged = beta.join(left_token, token)
|
||||
if merged is not None:
|
||||
self._propagate_token(beta, merged)
|
||||
|
||||
def match_patterns(self, facts: Optional[List[Fact]] = None) -> List[Match]:
|
||||
"""
|
||||
Match patterns using Rete algorithm.
|
||||
@@ -468,8 +617,11 @@ class ReteEngine:
|
||||
self.facts.clear()
|
||||
self.reset_action_history()
|
||||
for node in self.network.values():
|
||||
if isinstance(node, AlphaNode) or isinstance(node, BetaNode):
|
||||
node.matches.clear()
|
||||
if isinstance(node, AlphaNode):
|
||||
node.tokens.clear()
|
||||
elif isinstance(node, BetaNode):
|
||||
node.left_tokens.clear()
|
||||
node.right_tokens.clear()
|
||||
elif isinstance(node, TerminalNode):
|
||||
node.activations.clear()
|
||||
|
||||
|
||||
@@ -64,6 +64,8 @@ class SlidingWindowChunker:
|
||||
raise ValidationError("overlap must be non-negative")
|
||||
if self.overlap >= self.chunk_size:
|
||||
raise ValidationError("overlap must be less than chunk_size")
|
||||
if self.stride <= 0:
|
||||
raise ValidationError("stride must be positive")
|
||||
|
||||
def chunk(self, text: str, **options) -> List[Chunk]:
|
||||
"""
|
||||
@@ -215,15 +217,20 @@ class SlidingWindowChunker:
|
||||
Returns:
|
||||
list: List of chunks
|
||||
"""
|
||||
if overlap_size is None:
|
||||
return self.chunk(text)
|
||||
if overlap_size < 0:
|
||||
raise ValidationError("overlap_size must be non-negative")
|
||||
if overlap_size >= self.chunk_size:
|
||||
raise ValidationError("overlap_size must be less than chunk_size")
|
||||
|
||||
original_overlap = self.overlap
|
||||
if overlap_size is not None:
|
||||
original_stride = self.stride
|
||||
|
||||
try:
|
||||
self.overlap = overlap_size
|
||||
self.stride = self.chunk_size - self.overlap
|
||||
|
||||
chunks = self.chunk(text)
|
||||
|
||||
# Restore original overlap
|
||||
self.overlap = original_overlap
|
||||
self.stride = self.chunk_size - self.overlap
|
||||
|
||||
return chunks
|
||||
return self.chunk(text)
|
||||
finally:
|
||||
self.overlap = original_overlap
|
||||
self.stride = original_stride
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""AgentMemory.find_by_entity returns all matches by default (#1018).
|
||||
|
||||
The previous default limit of 10 silently truncated results, making erasure
|
||||
workflows incomplete for entities with more than 10 memories: a caller
|
||||
computing "what references this entity" from a truncated page would leave
|
||||
the remainder live. The unbounded default is deliberate — an erasure check
|
||||
cannot paginate — while callers that want a page still pass an explicit
|
||||
limit. (Previously lived in tests/test_seed_manager.py; moved to the
|
||||
AgentMemory area per review.)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
|
||||
|
||||
from semantica.context.agent_memory import AgentMemory
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def memory_with_15():
|
||||
mem = AgentMemory()
|
||||
for i in range(15):
|
||||
mem.store(
|
||||
content=f"fact {i} about entity",
|
||||
entities=[{"id": "e1", "name": "Entity", "type": "thing"}],
|
||||
)
|
||||
return mem
|
||||
|
||||
|
||||
class TestFindByEntityLimit:
|
||||
def test_returns_all_matches_by_default(self, memory_with_15):
|
||||
results = memory_with_15.find_by_entity("e1")
|
||||
assert len(results) == 15, f"expected 15 (all), got {len(results)}"
|
||||
|
||||
def test_explicit_limit_still_works(self, memory_with_15):
|
||||
assert len(memory_with_15.find_by_entity("e1", limit=5)) == 5
|
||||
|
||||
def test_no_matches_returns_empty(self):
|
||||
assert AgentMemory().find_by_entity("nonexistent") == []
|
||||
@@ -24,14 +24,19 @@ import math
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from pydantic import ValidationError
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the imports below, which need that extra.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.app import create_app
|
||||
from semantica.explorer.routes.decisions import _node_to_decision
|
||||
from semantica.explorer.schemas import DecisionResponse
|
||||
from semantica.explorer.session import GraphSession
|
||||
from fastapi.testclient import TestClient # noqa: E402
|
||||
from pydantic import ValidationError # noqa: E402
|
||||
|
||||
from semantica.context.context_graph import ContextGraph # noqa: E402
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.routes.decisions import _node_to_decision # noqa: E402
|
||||
from semantica.explorer.schemas import DecisionResponse # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -9,16 +9,15 @@ import networkx as nx
|
||||
import pytest
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.app import create_app
|
||||
from semantica.explorer.session import GraphSession
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip(
|
||||
"starlette TestClient is required for explorer tests. Install semantica[explorer].",
|
||||
allow_module_level=True,
|
||||
)
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
|
||||
from starlette.testclient import TestClient # noqa: E402
|
||||
|
||||
|
||||
|
||||
@@ -1104,7 +1103,7 @@ class TestBidirectionalPathRoute:
|
||||
# _classify_distance unit tests — issue #472
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
from semantica.utils.helpers import classify_path_distance
|
||||
from semantica.utils.helpers import classify_path_distance # noqa: E402
|
||||
|
||||
|
||||
class _FakeSimilarity:
|
||||
|
||||
@@ -13,16 +13,15 @@ browsers can't set custom headers on a WebSocket handshake.
|
||||
import pytest
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.app import create_app
|
||||
from semantica.explorer.session import GraphSession
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip(
|
||||
"starlette TestClient is required for explorer tests. Install semantica[explorer].",
|
||||
allow_module_level=True,
|
||||
)
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
|
||||
from starlette.testclient import TestClient # noqa: E402
|
||||
|
||||
|
||||
def _build_sample_graph() -> ContextGraph:
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
"""
|
||||
End-to-end integration test for deterministic Explorer rendering (Issue #1037).
|
||||
|
||||
Validates:
|
||||
1. Building the canonical 4-node, 3-edge graph using ContextGraph.
|
||||
2. Serialization via save_to_file().
|
||||
3. Reloading via load_from_file() and GraphSession.from_file() without mutation.
|
||||
4. Explorer HTTP API serving exact nodes, edges, edge types, and connectivity.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the explorer imports below, which pull fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from semantica.context.context_graph import ContextGraph # noqa: E402
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip(
|
||||
"starlette TestClient required. Install semantica[explorer].",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
def _build_deterministic_graph() -> ContextGraph:
|
||||
"""Build the exact graph requested in Semantica #1037."""
|
||||
graph = ContextGraph(advanced_analytics=False)
|
||||
|
||||
graph.add_node("alice", "Person", content="Alice")
|
||||
graph.add_node("bob", "Person", content="Bob")
|
||||
graph.add_node("acme", "Organization", content="Acme")
|
||||
graph.add_node("new_york", "Location", content="New York")
|
||||
|
||||
graph.add_edge("alice", "acme", edge_type="WORKS_AT")
|
||||
graph.add_edge("bob", "alice", edge_type="KNOWS")
|
||||
graph.add_edge("acme", "new_york", edge_type="LOCATED_IN")
|
||||
|
||||
return graph
|
||||
|
||||
|
||||
class TestExplorerDeterministicRenderingE2E:
|
||||
"""E2E suite for deterministic graph rendering and session loading."""
|
||||
|
||||
def test_build_and_in_memory_structure(self):
|
||||
graph = _build_deterministic_graph()
|
||||
|
||||
assert len(graph.nodes) == 4
|
||||
assert len(graph.edges) == 3
|
||||
|
||||
node_map = {
|
||||
nid: (node.node_type, node.content) for nid, node in graph.nodes.items()
|
||||
}
|
||||
assert node_map["alice"] == ("Person", "Alice")
|
||||
assert node_map["bob"] == ("Person", "Bob")
|
||||
assert node_map["acme"] == ("Organization", "Acme")
|
||||
assert node_map["new_york"] == ("Location", "New York")
|
||||
|
||||
edge_tuples = {(e.source_id, e.target_id, e.edge_type) for e in graph.edges}
|
||||
assert edge_tuples == {
|
||||
("alice", "acme", "WORKS_AT"),
|
||||
("bob", "alice", "KNOWS"),
|
||||
("acme", "new_york", "LOCATED_IN"),
|
||||
}
|
||||
|
||||
def test_serialization_and_deserialization(self, tmp_path: Path):
|
||||
graph = _build_deterministic_graph()
|
||||
output_file = tmp_path / "explorer_deterministic_graph.json"
|
||||
|
||||
graph.save_to_file(str(output_file))
|
||||
assert output_file.exists()
|
||||
|
||||
# Validate JSON structure
|
||||
with open(output_file, "r", encoding="utf-8") as f:
|
||||
raw_data = json.load(f)
|
||||
|
||||
assert "nodes" in raw_data
|
||||
assert "edges" in raw_data
|
||||
assert len(raw_data["nodes"]) == 4
|
||||
assert len(raw_data["edges"]) == 3
|
||||
|
||||
# Reload with ContextGraph.load_from_file
|
||||
reloaded_graph = ContextGraph(advanced_analytics=False)
|
||||
reloaded_graph.load_from_file(str(output_file))
|
||||
|
||||
assert len(reloaded_graph.nodes) == 4
|
||||
assert len(reloaded_graph.edges) == 3
|
||||
reloaded_nodes = {
|
||||
nid: (node.node_type, node.content)
|
||||
for nid, node in reloaded_graph.nodes.items()
|
||||
}
|
||||
assert reloaded_nodes["alice"] == ("Person", "Alice")
|
||||
assert reloaded_nodes["bob"] == ("Person", "Bob")
|
||||
assert reloaded_nodes["acme"] == ("Organization", "Acme")
|
||||
assert reloaded_nodes["new_york"] == ("Location", "New York")
|
||||
|
||||
reloaded_edges = {
|
||||
(e.source_id, e.target_id, e.edge_type) for e in reloaded_graph.edges
|
||||
}
|
||||
assert reloaded_edges == {
|
||||
("alice", "acme", "WORKS_AT"),
|
||||
("bob", "alice", "KNOWS"),
|
||||
("acme", "new_york", "LOCATED_IN"),
|
||||
}
|
||||
|
||||
def test_session_loading_and_stats(self, tmp_path: Path):
|
||||
graph = _build_deterministic_graph()
|
||||
output_file = tmp_path / "explorer_deterministic_graph.json"
|
||||
graph.save_to_file(str(output_file))
|
||||
|
||||
session = GraphSession.from_file(str(output_file))
|
||||
stats = session.get_stats()
|
||||
|
||||
assert stats["node_count"] == 4
|
||||
assert stats["edge_count"] == 3
|
||||
|
||||
def test_explorer_api_endpoints_with_deterministic_graph(self, tmp_path: Path):
|
||||
graph = _build_deterministic_graph()
|
||||
output_file = tmp_path / "explorer_deterministic_graph.json"
|
||||
graph.save_to_file(str(output_file))
|
||||
|
||||
session = GraphSession.from_file(str(output_file))
|
||||
app = create_app(session=session)
|
||||
|
||||
with TestClient(app) as client:
|
||||
# 1. Health & Info
|
||||
health = client.get("/api/health")
|
||||
assert health.status_code == 200
|
||||
assert health.json() == {"status": "ok"}
|
||||
|
||||
info = client.get("/api/info")
|
||||
assert info.status_code == 200
|
||||
assert info.json()["status"] == "active"
|
||||
|
||||
# 2. Stats
|
||||
stats = client.get("/api/graph/stats")
|
||||
assert stats.status_code == 200
|
||||
stats_data = stats.json()
|
||||
assert stats_data["node_count"] == 4
|
||||
assert stats_data["edge_count"] == 3
|
||||
|
||||
# 3. Nodes endpoint
|
||||
nodes_res = client.get("/api/graph/nodes")
|
||||
assert nodes_res.status_code == 200
|
||||
nodes_data = nodes_res.json()
|
||||
assert nodes_data["total"] == 4
|
||||
assert len(nodes_data["nodes"]) == 4
|
||||
|
||||
returned_nodes = {
|
||||
n["id"]: (n["type"], n["content"]) for n in nodes_data["nodes"]
|
||||
}
|
||||
assert returned_nodes["alice"] == ("Person", "Alice")
|
||||
assert returned_nodes["bob"] == ("Person", "Bob")
|
||||
assert returned_nodes["acme"] == ("Organization", "Acme")
|
||||
assert returned_nodes["new_york"] == ("Location", "New York")
|
||||
|
||||
# 4. Individual node lookups
|
||||
for node_id in ["alice", "bob", "acme", "new_york"]:
|
||||
node_res = client.get(f"/api/graph/node/{node_id}")
|
||||
assert node_res.status_code == 200
|
||||
assert node_res.json()["id"] == node_id
|
||||
|
||||
# 5. Edges endpoint
|
||||
edges_res = client.get("/api/graph/edges")
|
||||
assert edges_res.status_code == 200
|
||||
edges_data = edges_res.json()
|
||||
assert edges_data["total"] == 3
|
||||
assert len(edges_data["edges"]) == 3
|
||||
|
||||
returned_edges = {
|
||||
(e["source"], e["target"], e["type"]) for e in edges_data["edges"]
|
||||
}
|
||||
assert returned_edges == {
|
||||
("alice", "acme", "WORKS_AT"),
|
||||
("bob", "alice", "KNOWS"),
|
||||
("acme", "new_york", "LOCATED_IN"),
|
||||
}
|
||||
|
||||
def test_deterministic_graph_auth_enforcement(self, tmp_path: Path, monkeypatch):
|
||||
graph = _build_deterministic_graph()
|
||||
output_file = tmp_path / "explorer_deterministic_graph.json"
|
||||
graph.save_to_file(str(output_file))
|
||||
|
||||
session = GraphSession.from_file(str(output_file))
|
||||
app = create_app(session=session)
|
||||
|
||||
with TestClient(app) as client:
|
||||
# 1. Unconfigured auth (no SEMANTICA_ALLOW_ANONYMOUS, no SEMANTICA_API_KEY) -> 503
|
||||
monkeypatch.delenv("SEMANTICA_ALLOW_ANONYMOUS", raising=False)
|
||||
monkeypatch.delenv("SEMANTICA_API_KEY", raising=False)
|
||||
unconfigured_res = client.get("/api/graph/stats")
|
||||
assert unconfigured_res.status_code == 503
|
||||
|
||||
# 2. Configured SEMANTICA_API_KEY without header -> 401
|
||||
test_key = "secret-test-key-1037"
|
||||
monkeypatch.setenv("SEMANTICA_API_KEY", test_key)
|
||||
unauthorized_res = client.get("/api/graph/nodes")
|
||||
assert unauthorized_res.status_code == 401
|
||||
|
||||
# 3. Configured SEMANTICA_API_KEY with valid X-API-Key header -> 200
|
||||
authorized_res = client.get(
|
||||
"/api/graph/nodes",
|
||||
headers={"X-API-Key": test_key},
|
||||
)
|
||||
assert authorized_res.status_code == 200
|
||||
assert authorized_res.json()["total"] == 4
|
||||
|
||||
# 4. Explicit local development opt-in: SEMANTICA_ALLOW_ANONYMOUS=true -> 200 without header
|
||||
monkeypatch.setenv("SEMANTICA_ALLOW_ANONYMOUS", "true")
|
||||
monkeypatch.delenv("SEMANTICA_API_KEY", raising=False)
|
||||
anon_res = client.get("/api/graph/edges")
|
||||
assert anon_res.status_code == 200
|
||||
assert anon_res.json()["total"] == 3
|
||||
|
||||
@@ -27,7 +27,12 @@ import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.explorer.routes import ontology as ontology_mod
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from semantica.explorer.routes import ontology as ontology_mod # noqa: E402
|
||||
|
||||
|
||||
def _start_local_server():
|
||||
|
||||
@@ -21,7 +21,12 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.explorer.routes import ontology as ontology_mod
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from semantica.explorer.routes import ontology as ontology_mod # noqa: E402
|
||||
|
||||
|
||||
def _fake_getaddrinfo(host, *args, **kwargs):
|
||||
|
||||
@@ -6,17 +6,16 @@ from urllib.parse import quote
|
||||
import pytest
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.app import create_app
|
||||
from semantica.explorer.routes.ontology import OntologyEntry
|
||||
from semantica.explorer.session import GraphSession
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip(
|
||||
"starlette TestClient is required for explorer tests. Install semantica[explorer].",
|
||||
allow_module_level=True,
|
||||
)
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.routes.ontology import OntologyEntry # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
|
||||
from starlette.testclient import TestClient # noqa: E402
|
||||
|
||||
|
||||
def _build_ontology_graph() -> ContextGraph:
|
||||
|
||||
@@ -5,13 +5,18 @@ Tests for ProvenanceManager wiring into Explorer routes and application startup.
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from starlette.testclient import TestClient
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the starlette/explorer imports below, which need that extra.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.app import create_app
|
||||
from semantica.explorer.session import GraphSession
|
||||
from semantica.provenance import ProvenanceManager
|
||||
from semantica.provenance.storage import SQLiteStorage
|
||||
from starlette.testclient import TestClient # noqa: E402
|
||||
|
||||
from semantica.context.context_graph import ContextGraph # noqa: E402
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
from semantica.provenance import ProvenanceManager # noqa: E402
|
||||
from semantica.provenance.storage import SQLiteStorage # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -2,7 +2,14 @@
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from semantica.explorer.routes.provenance import (
|
||||
import pytest
|
||||
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from semantica.explorer.routes.provenance import ( # noqa: E402
|
||||
_add_chain_edges,
|
||||
_build_provenance,
|
||||
_render_markdown,
|
||||
|
||||
@@ -14,18 +14,17 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.app import create_app
|
||||
from semantica.explorer.session import GraphSession
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip(
|
||||
"starlette TestClient is required for explorer tests. Install semantica[explorer].",
|
||||
allow_module_level=True,
|
||||
)
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
|
||||
import semantica.explorer.routes.sparql as sparql_mod
|
||||
from starlette.testclient import TestClient # noqa: E402
|
||||
|
||||
import semantica.explorer.routes.sparql as sparql_mod # noqa: E402
|
||||
|
||||
|
||||
def _build_sample_graph() -> ContextGraph:
|
||||
|
||||
@@ -2,12 +2,19 @@
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
import pytest
|
||||
|
||||
from semantica.explorer.dependencies import get_session
|
||||
from semantica.explorer.routes.vocabulary import router
|
||||
from semantica.utils.skos import validate_skos_hierarchy
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from fastapi import FastAPI # noqa: E402
|
||||
from fastapi.testclient import TestClient # noqa: E402
|
||||
|
||||
from semantica.explorer.dependencies import get_session # noqa: E402
|
||||
from semantica.explorer.routes.vocabulary import router # noqa: E402
|
||||
from semantica.utils.skos import validate_skos_hierarchy # noqa: E402
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(router)
|
||||
|
||||
@@ -67,7 +67,7 @@ def test_merging_repeated_exports_yields_one_graph_node(tmp_path):
|
||||
assert len(kg_nodes) == 1
|
||||
|
||||
entity_nodes = set(
|
||||
merged.subjects(RDF.type, URIRef("https://semantica.dev/vocab/ORG"))
|
||||
merged.subjects(RDF.type, URIRef("https://semantica.dev/ns#Entity"))
|
||||
)
|
||||
assert len(entity_nodes) == 1
|
||||
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Caller data must never expand into the Semantica namespace (#1146).
|
||||
|
||||
``@vocab`` used to sit in every JSON-LD context pointing at ``ns#``, so every
|
||||
bare term in caller data expanded into it: an extracted type like ``"ORG"``
|
||||
became ``ns#ORG`` (a term the vocabulary does not define), and a metadata key
|
||||
like ``"source"`` collided with the real ``sem:source`` object property,
|
||||
attaching a plain string to a property whose range is a resource. The fix
|
||||
removes ``@vocab`` outright: only explicit ``semantica:``-prefixed terms
|
||||
resolve, caller type labels travel as ``semantica:type`` strings, and caller
|
||||
metadata survives as one ``rdf:JSON`` literal.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from rdflib import RDF, Graph, Literal, URIRef
|
||||
|
||||
from semantica.export.json_exporter import JSONExporter
|
||||
from semantica.export.rdf_exporter import RDFExporter, SEMANTICA_NS
|
||||
|
||||
NS = SEMANTICA_NS
|
||||
E1 = "https://example.org/e1"
|
||||
|
||||
KG = {
|
||||
"entities": [
|
||||
{
|
||||
"id": E1,
|
||||
"text": "Acme",
|
||||
"type": "ORG",
|
||||
"metadata": {"source": "crm_export_2024"},
|
||||
}
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"source_id": E1,
|
||||
"target_id": "https://example.org/e2",
|
||||
"type": "employs",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _jsonld_file(exporter, kind, tmp_path, name):
|
||||
path = tmp_path / name
|
||||
if kind == "knowledge_graph":
|
||||
exporter.export_knowledge_graph(KG, path, format="json-ld")
|
||||
elif kind == "entities":
|
||||
exporter.export_entities(KG["entities"], path, format="json-ld")
|
||||
elif kind == "relationships":
|
||||
exporter.export_relationships(KG["relationships"], path, format="json-ld")
|
||||
elif kind == "generic":
|
||||
exporter.export({"note": "plain payload, no @id"}, path, format="json-ld")
|
||||
else:
|
||||
raise AssertionError(kind)
|
||||
return json.loads(path.read_text())
|
||||
|
||||
|
||||
def test_no_jsonld_context_declares_a_vocab(tmp_path):
|
||||
exporter = JSONExporter()
|
||||
for kind in ("knowledge_graph", "entities", "relationships", "generic"):
|
||||
context = _jsonld_file(exporter, kind, tmp_path, f"{kind}.jsonld")[
|
||||
"@context"
|
||||
]
|
||||
assert "@vocab" not in context, f"{kind}: @vocab expands caller data"
|
||||
|
||||
context = json.loads(RDFExporter().export_to_rdf(KG, format="jsonld"))[
|
||||
"@context"
|
||||
]
|
||||
assert "@vocab" not in context
|
||||
|
||||
|
||||
def test_extracted_type_labels_stay_out_of_the_namespace(tmp_path):
|
||||
path = tmp_path / "kg.jsonld"
|
||||
JSONExporter().export_knowledge_graph(KG, path, format="json-ld")
|
||||
|
||||
graph = Graph()
|
||||
graph.parse(str(path), format="json-ld")
|
||||
|
||||
assert (None, RDF.type, URIRef(NS + "ORG")) not in graph, (
|
||||
"the caller's type label was minted as a class in ns#"
|
||||
)
|
||||
assert (URIRef(E1), RDF.type, URIRef(NS + "Entity")) in graph
|
||||
assert (URIRef(E1), URIRef(NS + "type"), Literal("ORG")) in graph, (
|
||||
"the label itself must survive, as a string"
|
||||
)
|
||||
|
||||
|
||||
def test_metadata_keys_stay_out_of_the_namespace(tmp_path):
|
||||
path = tmp_path / "kg.jsonld"
|
||||
JSONExporter().export_knowledge_graph(KG, path, format="json-ld")
|
||||
|
||||
graph = Graph()
|
||||
graph.parse(str(path), format="json-ld")
|
||||
|
||||
assert (None, URIRef(NS + "source"), Literal("crm_export_2024")) not in (
|
||||
graph
|
||||
), "caller metadata value attached to the real sem:source object property"
|
||||
for _, _, o in graph.triples((None, URIRef(NS + "source"), None)):
|
||||
assert not isinstance(o, Literal), (
|
||||
"sem:source has a resource range but received a plain literal"
|
||||
)
|
||||
|
||||
literals = [
|
||||
o
|
||||
for o in graph.objects(None, URIRef(NS + "metadata"))
|
||||
if isinstance(o, Literal)
|
||||
]
|
||||
assert literals, "the metadata dict was dropped instead of preserved"
|
||||
assert literals[0].datatype == RDF.JSON
|
||||
assert json.loads(str(literals[0])) == {"source": "crm_export_2024"}
|
||||
|
||||
|
||||
def test_rdf_exporter_jsonld_keeps_type_labels_out_of_the_namespace():
|
||||
graph = Graph()
|
||||
graph.parse(
|
||||
data=RDFExporter().export_to_rdf(KG, format="jsonld"), format="json-ld"
|
||||
)
|
||||
|
||||
assert (None, RDF.type, URIRef(NS + "ORG")) not in graph
|
||||
assert (URIRef(E1), RDF.type, URIRef(NS + "Entity")) in graph
|
||||
assert (URIRef(E1), URIRef(NS + "type"), Literal("ORG")) in graph
|
||||
@@ -0,0 +1,51 @@
|
||||
import pytest
|
||||
|
||||
from semantica.ontology.class_inferrer import ClassInferrer
|
||||
from semantica.ontology.property_generator import PropertyGenerator
|
||||
from semantica.utils.exceptions import ValidationError
|
||||
|
||||
|
||||
def test_same_kind_normalized_object_properties_are_merged():
|
||||
entities = [
|
||||
{"id": "p1", "type": "Person", "name": "Alice"},
|
||||
{"id": "o1", "type": "Organization", "name": "Acme"},
|
||||
]
|
||||
classes = ClassInferrer(min_occurrences=1).infer_classes(entities)
|
||||
relationships = [
|
||||
{
|
||||
"source_type": "Person",
|
||||
"target_type": "Organization",
|
||||
"type": "works_for",
|
||||
},
|
||||
{
|
||||
"source_type": "Person",
|
||||
"target_type": "Organization",
|
||||
"type": "worksFor",
|
||||
},
|
||||
]
|
||||
|
||||
properties = PropertyGenerator().infer_properties(
|
||||
entities, relationships, classes, min_occurrences=1
|
||||
)
|
||||
|
||||
works_for = [prop for prop in properties if prop["name"] == "worksFor"]
|
||||
assert len(works_for) == 1
|
||||
assert works_for[0]["domain"] == ["Person"]
|
||||
assert works_for[0]["range"] == ["Organization"]
|
||||
|
||||
|
||||
def test_normalized_name_cannot_be_both_object_and_data_property():
|
||||
entities = [
|
||||
{"id": "p1", "type": "Person", "value": "Alice"},
|
||||
{"id": "p2", "type": "Person", "value": "Bob"},
|
||||
]
|
||||
classes = ClassInferrer(min_occurrences=1).infer_classes(entities)
|
||||
relationships = [
|
||||
{"source_type": "Person", "target_type": "Person", "type": "value"},
|
||||
{"source_type": "Person", "target_type": "Person", "type": "value"},
|
||||
]
|
||||
|
||||
with pytest.raises(ValidationError, match="object and data"):
|
||||
PropertyGenerator().infer_properties(
|
||||
entities, relationships, classes, min_occurrences=1
|
||||
)
|
||||
@@ -0,0 +1,366 @@
|
||||
"""Tests for the RETE engine pattern matching (issue #300).
|
||||
|
||||
These tests verify that ``AlphaNode._matches`` and ``BetaNode._can_join`` no
|
||||
longer behave like the old always-``True`` stubs, and that the network as a
|
||||
whole only fires rules whose conditions actually unify with the facts.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
import re
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from semantica.reasoning import rete_engine
|
||||
from semantica.reasoning.reasoner import Fact, Rule
|
||||
from semantica.reasoning.rete_engine import (
|
||||
AlphaNode,
|
||||
BetaNode,
|
||||
ReteEngine,
|
||||
unify_condition,
|
||||
)
|
||||
|
||||
|
||||
class TestUnifyCondition(unittest.TestCase):
|
||||
def test_single_variable_binds(self):
|
||||
fact = Fact("f1", "Person", ["John"])
|
||||
bindings = unify_condition("Person(?x)", fact)
|
||||
self.assertEqual(bindings, {"x": "John"})
|
||||
|
||||
def test_predicate_mismatch_returns_none(self):
|
||||
fact = Fact("f1", "Company", ["Google"])
|
||||
self.assertIsNone(unify_condition("Person(?x)", fact))
|
||||
|
||||
def test_two_arguments_bind(self):
|
||||
fact = Fact("f2", "Parent", ["John", "Mary"])
|
||||
bindings = unify_condition("Parent(?x, ?y)", fact)
|
||||
self.assertEqual(bindings, {"x": "John", "y": "Mary"})
|
||||
|
||||
def test_literal_argument_must_match(self):
|
||||
fact = Fact("f3", "Parent", ["John", "Mary"])
|
||||
self.assertIsNone(unify_condition("Parent(Bob, ?y)", fact))
|
||||
self.assertEqual(unify_condition("Parent(John, ?y)", fact), {"y": "Mary"})
|
||||
|
||||
def test_repeated_variable_requires_equal_values(self):
|
||||
loves_self = Fact("f4", "Loves", ["John", "John"])
|
||||
loves_other = Fact("f5", "Loves", ["John", "Mary"])
|
||||
self.assertEqual(unify_condition("Loves(?x, ?x)", loves_self), {"x": "John"})
|
||||
self.assertIsNone(unify_condition("Loves(?x, ?x)", loves_other))
|
||||
|
||||
def test_regex_error_logs_warning_and_returns_none(self):
|
||||
"""A regex compilation error is logged with context and yields None."""
|
||||
fact = Fact("f6", "Person", ["John"])
|
||||
with mock.patch.object(
|
||||
rete_engine.re,
|
||||
"match",
|
||||
side_effect=re.error("bad pattern"),
|
||||
), self.assertLogs("semantica.rete_engine", level="WARNING") as captured:
|
||||
result = unify_condition("Person(?x)", fact)
|
||||
self.assertIsNone(result)
|
||||
joined = "\n".join(captured.output)
|
||||
self.assertIn("Person(?x)", joined)
|
||||
self.assertIn("Person(John)", joined)
|
||||
self.assertIn("bad pattern", joined)
|
||||
|
||||
def test_unexpected_error_logs_warning_and_returns_none(self):
|
||||
"""An unexpected error is also logged and swallowed as None."""
|
||||
fact = Fact("f7", "Person", ["John"])
|
||||
with mock.patch.object(
|
||||
rete_engine.re,
|
||||
"match",
|
||||
side_effect=RuntimeError("boom"),
|
||||
), self.assertLogs("semantica.rete_engine", level="WARNING") as captured:
|
||||
result = unify_condition("Person(?x)", fact)
|
||||
self.assertIsNone(result)
|
||||
self.assertIn("boom", "\n".join(captured.output))
|
||||
|
||||
|
||||
class TestAlphaNode(unittest.TestCase):
|
||||
def test_matches_stores_bindings(self):
|
||||
node = AlphaNode("a1", "Person(?x)")
|
||||
fact = Fact("f1", "Person", ["John"])
|
||||
token = node.add_fact(fact)
|
||||
self.assertIsNotNone(token)
|
||||
assert token is not None # narrow type for the checker
|
||||
self.assertEqual(token.facts, [fact])
|
||||
self.assertEqual(token.bindings, {"x": "John"})
|
||||
self.assertIn(token, node.tokens)
|
||||
|
||||
def test_non_matching_fact_rejected(self):
|
||||
node = AlphaNode("a1", "Person(?x)")
|
||||
fact = Fact("f1", "Company", ["Google"])
|
||||
self.assertIsNone(node.add_fact(fact))
|
||||
self.assertEqual(node.tokens, [])
|
||||
|
||||
def test_uses_precompiled_regex(self):
|
||||
"""AlphaNode compiles its condition once and reuses it per fact."""
|
||||
node = AlphaNode("a1", "Person(?x)")
|
||||
self.assertIsNotNone(node._compiled)
|
||||
# Matching goes through the compiled matcher, not unify_condition.
|
||||
with mock.patch.object(rete_engine, "unify_condition") as unify:
|
||||
fact = Fact("f1", "Person", ["John"])
|
||||
token = node.add_fact(fact)
|
||||
unify.assert_not_called()
|
||||
self.assertIsNotNone(token)
|
||||
assert token is not None
|
||||
self.assertEqual(token.bindings, {"x": "John"})
|
||||
|
||||
def test_bad_condition_never_matches_and_logs(self):
|
||||
"""A condition that fails to compile logs a warning and never fires."""
|
||||
with mock.patch.object(
|
||||
rete_engine,
|
||||
"_build_condition_regex",
|
||||
return_value="(unbalanced",
|
||||
), self.assertLogs("semantica.rete_engine", level="WARNING") as captured:
|
||||
node = AlphaNode("bad", "Person(?x)")
|
||||
self.assertIsNone(node._compiled)
|
||||
self.assertIn("failed to compile", "\n".join(captured.output))
|
||||
fact = Fact("f1", "Person", ["John"])
|
||||
self.assertIsNone(node.add_fact(fact))
|
||||
self.assertEqual(node.tokens, [])
|
||||
|
||||
|
||||
class TestBetaNode(unittest.TestCase):
|
||||
def test_join_consistent_bindings(self):
|
||||
left = AlphaNode("a1", "Parent(?x, ?y)")
|
||||
right = AlphaNode("a2", "Person(?x)")
|
||||
beta = BetaNode("b1", left, right)
|
||||
|
||||
parent = Fact("f1", "Parent", ["John", "Mary"])
|
||||
person = Fact("f2", "Person", ["John"])
|
||||
left_token = left.add_fact(parent)
|
||||
right_token = right.add_fact(person)
|
||||
assert left_token is not None and right_token is not None
|
||||
|
||||
merged = beta.join(left_token, right_token)
|
||||
self.assertIsNotNone(merged)
|
||||
assert merged is not None # narrow type for the checker
|
||||
self.assertEqual(merged.bindings, {"x": "John", "y": "Mary"})
|
||||
# Facts are concatenated left-then-right in condition order.
|
||||
self.assertEqual(merged.facts, [parent, person])
|
||||
|
||||
def test_join_conflicting_bindings_rejected(self):
|
||||
left = AlphaNode("a1", "Parent(?x, ?y)")
|
||||
right = AlphaNode("a2", "Person(?x)")
|
||||
beta = BetaNode("b1", left, right)
|
||||
|
||||
parent = Fact("f1", "Parent", ["John", "Mary"])
|
||||
# ?x conflicts: John vs Alice
|
||||
person = Fact("f2", "Person", ["Alice"])
|
||||
left_token = left.add_fact(parent)
|
||||
right_token = right.add_fact(person)
|
||||
assert left_token is not None and right_token is not None
|
||||
|
||||
self.assertIsNone(beta.join(left_token, right_token))
|
||||
|
||||
|
||||
class TestReteEngineEndToEnd(unittest.TestCase):
|
||||
def test_only_matching_rule_fires(self):
|
||||
engine = ReteEngine()
|
||||
rule = Rule(
|
||||
rule_id="r1",
|
||||
name="person rule",
|
||||
conditions=["Person(?x)"],
|
||||
conclusion="Mortal(?x)",
|
||||
)
|
||||
engine.build_network([rule])
|
||||
|
||||
engine.add_fact(Fact("f1", "Person", ["John"]))
|
||||
engine.add_fact(Fact("f2", "Company", ["Google"])) # should NOT fire
|
||||
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(len(matches), 1)
|
||||
self.assertEqual(matches[0].bindings, {"x": "John"})
|
||||
|
||||
def test_multi_condition_join(self):
|
||||
engine = ReteEngine()
|
||||
rule = Rule(
|
||||
rule_id="r1",
|
||||
name="child rule",
|
||||
conditions=["Person(?x)", "Parent(?x, ?y)"],
|
||||
conclusion="Child(?y, ?x)",
|
||||
)
|
||||
engine.build_network([rule])
|
||||
|
||||
engine.add_fact(Fact("f1", "Person", ["John"]))
|
||||
engine.add_fact(Fact("f2", "Parent", ["John", "Mary"]))
|
||||
# Unrelated parent whose ?x does not match any Person -> no activation.
|
||||
engine.add_fact(Fact("f3", "Parent", ["Bob", "Sue"]))
|
||||
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(len(matches), 1)
|
||||
self.assertEqual(matches[0].bindings, {"x": "John", "y": "Mary"})
|
||||
|
||||
def test_no_activation_when_join_inconsistent(self):
|
||||
engine = ReteEngine()
|
||||
rule = Rule(
|
||||
rule_id="r1",
|
||||
name="child rule",
|
||||
conditions=["Person(?x)", "Parent(?x, ?y)"],
|
||||
conclusion="Child(?y, ?x)",
|
||||
)
|
||||
engine.build_network([rule])
|
||||
|
||||
engine.add_fact(Fact("f1", "Person", ["John"]))
|
||||
engine.add_fact(Fact("f2", "Parent", ["Alice", "Mary"])) # ?x mismatch
|
||||
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(matches, [])
|
||||
|
||||
|
||||
class TestThreeConditionChain(unittest.TestCase):
|
||||
"""Chained beta joins across three or more conditions (issue #300).
|
||||
|
||||
These exercise the Token model: a token must accumulate the ordered
|
||||
facts and the consistent bindings of every condition, so that deep
|
||||
chains neither drop bindings nor duplicate facts, and a conflict on the
|
||||
third condition correctly suppresses activation.
|
||||
"""
|
||||
|
||||
def _three_condition_rule(self):
|
||||
return Rule(
|
||||
rule_id="r1",
|
||||
name="location chain",
|
||||
conditions=[
|
||||
"Person(?x)",
|
||||
"Parent(?x, ?y)",
|
||||
"Located(?y, ?z)",
|
||||
],
|
||||
conclusion="LivesNear(?x, ?z)",
|
||||
)
|
||||
|
||||
def test_three_condition_valid_match(self):
|
||||
engine = ReteEngine()
|
||||
engine.build_network([self._three_condition_rule()])
|
||||
|
||||
engine.add_fact(Fact("f1", "Person", ["John"]))
|
||||
engine.add_fact(Fact("f2", "Parent", ["John", "Mary"]))
|
||||
engine.add_fact(Fact("f3", "Located", ["Mary", "Paris"]))
|
||||
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(len(matches), 1)
|
||||
self.assertEqual(
|
||||
matches[0].bindings,
|
||||
{"x": "John", "y": "Mary", "z": "Paris"},
|
||||
)
|
||||
|
||||
def test_three_condition_third_level_conflict(self):
|
||||
engine = ReteEngine()
|
||||
engine.build_network([self._three_condition_rule()])
|
||||
|
||||
engine.add_fact(Fact("f1", "Person", ["John"]))
|
||||
engine.add_fact(Fact("f2", "Parent", ["John", "Mary"]))
|
||||
# ?y is bound to Mary, so a Located fact about Bob must not join.
|
||||
engine.add_fact(Fact("f3", "Located", ["Bob", "Paris"]))
|
||||
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(matches, [])
|
||||
|
||||
def test_fact_insertion_order_independent(self):
|
||||
# Whatever order facts arrive, the same single match must result.
|
||||
base_facts = [
|
||||
Fact("f1", "Person", ["John"]),
|
||||
Fact("f2", "Parent", ["John", "Mary"]),
|
||||
Fact("f3", "Located", ["Mary", "Paris"]),
|
||||
]
|
||||
expected = {"x": "John", "y": "Mary", "z": "Paris"}
|
||||
|
||||
for order in itertools.permutations(base_facts):
|
||||
engine = ReteEngine()
|
||||
engine.build_network([self._three_condition_rule()])
|
||||
for fact in order:
|
||||
engine.add_fact(fact)
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(len(matches), 1, f"order={order}")
|
||||
self.assertEqual(matches[0].bindings, expected)
|
||||
|
||||
def test_match_facts_complete_in_condition_order(self):
|
||||
engine = ReteEngine()
|
||||
engine.build_network([self._three_condition_rule()])
|
||||
|
||||
person = Fact("f1", "Person", ["John"])
|
||||
parent = Fact("f2", "Parent", ["John", "Mary"])
|
||||
located = Fact("f3", "Located", ["Mary", "Paris"])
|
||||
engine.add_fact(person)
|
||||
engine.add_fact(parent)
|
||||
engine.add_fact(located)
|
||||
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(len(matches), 1)
|
||||
# All three facts preserved, in condition order, no duplicates.
|
||||
self.assertEqual(matches[0].facts, [person, parent, located])
|
||||
|
||||
def test_multiple_left_tokens_join_one_right_fact(self):
|
||||
# Two Person/Parent chains sharing the same Located(?y, ?z) fact.
|
||||
engine = ReteEngine()
|
||||
engine.build_network([self._three_condition_rule()])
|
||||
|
||||
engine.add_fact(Fact("f1", "Person", ["John"]))
|
||||
engine.add_fact(Fact("f2", "Parent", ["John", "Mary"]))
|
||||
engine.add_fact(Fact("f3", "Person", ["Alice"]))
|
||||
engine.add_fact(Fact("f4", "Parent", ["Alice", "Mary"]))
|
||||
# One right fact should join with both accumulated left tokens.
|
||||
engine.add_fact(Fact("f5", "Located", ["Mary", "Paris"]))
|
||||
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(len(matches), 2)
|
||||
result = {m.bindings["x"]: m.bindings["z"] for m in matches}
|
||||
self.assertEqual(result, {"John": "Paris", "Alice": "Paris"})
|
||||
|
||||
def test_matches_reasoner_match_rule(self):
|
||||
from semantica.reasoning.reasoner import Reasoner
|
||||
|
||||
rule = self._three_condition_rule()
|
||||
facts = [
|
||||
Fact("f1", "Person", ["John"]),
|
||||
Fact("f2", "Parent", ["John", "Mary"]),
|
||||
Fact("f3", "Located", ["Mary", "Paris"]),
|
||||
]
|
||||
|
||||
# Reasoner works over stringified facts and returns
|
||||
# (conclusion, matched_facts, bindings) tuples from self.facts.
|
||||
reasoner = Reasoner()
|
||||
for fact in facts:
|
||||
reasoner.add_fact(str(fact))
|
||||
reasoner_matches = reasoner._match_rule(rule)
|
||||
|
||||
engine = ReteEngine()
|
||||
engine.build_network([rule])
|
||||
for fact in facts:
|
||||
engine.add_fact(fact)
|
||||
rete_matches = engine.match_patterns()
|
||||
|
||||
# Both engines must agree on the number of activations.
|
||||
self.assertEqual(len(rete_matches), len(reasoner_matches))
|
||||
self.assertEqual(len(rete_matches), 1)
|
||||
self.assertEqual(
|
||||
rete_matches[0].bindings,
|
||||
{"x": "John", "y": "Mary", "z": "Paris"},
|
||||
)
|
||||
# The RETE match must carry the instantiated conclusion facts too.
|
||||
conclusion, _, _ = reasoner_matches[0]
|
||||
self.assertEqual(conclusion, "LivesNear(John, Paris)")
|
||||
|
||||
def test_reset_clears_all_token_memory(self):
|
||||
engine = ReteEngine()
|
||||
engine.build_network([self._three_condition_rule()])
|
||||
|
||||
engine.add_fact(Fact("f1", "Person", ["John"]))
|
||||
engine.add_fact(Fact("f2", "Parent", ["John", "Mary"]))
|
||||
engine.add_fact(Fact("f3", "Located", ["Mary", "Paris"]))
|
||||
self.assertEqual(len(engine.match_patterns()), 1)
|
||||
|
||||
engine.reset()
|
||||
|
||||
# No stale facts, tokens or activations remain anywhere.
|
||||
self.assertEqual(engine.facts, [])
|
||||
for node in engine.network.values():
|
||||
if isinstance(node, AlphaNode):
|
||||
self.assertEqual(node.tokens, [])
|
||||
elif isinstance(node, BetaNode):
|
||||
self.assertEqual(node.left_tokens, [])
|
||||
self.assertEqual(node.right_tokens, [])
|
||||
self.assertEqual(engine.match_patterns(), [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -60,6 +60,11 @@ class TestSlidingWindowChunker:
|
||||
with pytest.raises(ValidationError):
|
||||
SlidingWindowChunker(chunk_size=100, overlap=100)
|
||||
|
||||
@pytest.mark.parametrize("stride", [0, -1])
|
||||
def test_init_rejects_non_positive_stride(self, stride):
|
||||
with pytest.raises(ValidationError, match="stride must be positive"):
|
||||
SlidingWindowChunker(chunk_size=100, stride=stride)
|
||||
|
||||
def test_empty_text_returns_empty(self):
|
||||
chunker = SlidingWindowChunker(chunk_size=50, overlap=10)
|
||||
assert chunker.chunk("") == []
|
||||
@@ -97,6 +102,49 @@ class TestSlidingWindowChunker:
|
||||
chunks = chunker.chunk_with_overlap(text, overlap_size=15)
|
||||
assert len(chunks) >= 2
|
||||
assert chunker.overlap == 0
|
||||
assert chunker.stride == 50
|
||||
|
||||
@pytest.mark.parametrize("overlap_size", [-1, 50, 51])
|
||||
def test_chunk_with_overlap_rejects_invalid_override(self, overlap_size):
|
||||
chunker = SlidingWindowChunker(chunk_size=50)
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
chunker.chunk_with_overlap(
|
||||
"non-empty input", overlap_size=overlap_size
|
||||
)
|
||||
|
||||
def test_chunk_with_overlap_accepts_largest_valid_override(self):
|
||||
chunker = SlidingWindowChunker(chunk_size=5)
|
||||
|
||||
chunks = chunker.chunk_with_overlap("abcdefghij", overlap_size=4)
|
||||
|
||||
assert [chunk.start_index for chunk in chunks] == list(range(10))
|
||||
|
||||
def test_chunk_with_overlap_restores_custom_stride(self):
|
||||
chunker = SlidingWindowChunker(chunk_size=10, overlap=2, stride=3)
|
||||
|
||||
chunker.chunk_with_overlap(
|
||||
"abcdefghijklmnopqrstuvwxyz", overlap_size=4
|
||||
)
|
||||
|
||||
assert chunker.overlap == 2
|
||||
assert chunker.stride == 3
|
||||
|
||||
def test_chunk_with_overlap_restores_state_when_chunk_raises(
|
||||
self, monkeypatch
|
||||
):
|
||||
chunker = SlidingWindowChunker(chunk_size=10, overlap=2, stride=3)
|
||||
|
||||
def raise_error(text):
|
||||
raise RuntimeError("chunk failed")
|
||||
|
||||
monkeypatch.setattr(chunker, "chunk", raise_error)
|
||||
|
||||
with pytest.raises(RuntimeError, match="chunk failed"):
|
||||
chunker.chunk_with_overlap("non-empty input", overlap_size=4)
|
||||
|
||||
assert chunker.overlap == 2
|
||||
assert chunker.stride == 3
|
||||
|
||||
def test_boundary_preservation_avoids_mid_word_when_possible(self):
|
||||
text = (
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -24,9 +24,20 @@ import pytest
|
||||
# rdf:/rdfs: namespaces) and neither the code nor this test caught it,
|
||||
# since both had the same bug. Importing the real function makes that class
|
||||
# of drift impossible.
|
||||
from semantica.explorer.routes.sparql import _is_read_only_query
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this import
|
||||
# fails on a plain dev install. Only the SPARQL class below needs it; the Cypher,
|
||||
# XXE, vector-serialization and SSRF classes in this module are independent, so
|
||||
# the skip is scoped to the one class rather than the whole file.
|
||||
try:
|
||||
from semantica.explorer.routes.sparql import _is_read_only_query
|
||||
except ImportError: # pragma: no cover - depends on the installed extras
|
||||
_is_read_only_query = None
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_is_read_only_query is None,
|
||||
reason="requires semantica[explorer] (fastapi)",
|
||||
)
|
||||
class TestSparqlReadOnlyValidation:
|
||||
"""Regression tests for SPARQL injection prevention."""
|
||||
|
||||
|
||||
@@ -443,4 +443,3 @@ def test_export_seed_data(seed_manager, temp_data_dir):
|
||||
rows = list(reader)
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["id"] == "1"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user