mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-30 04:40:16 +00:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
19ff5bf200 | ||
|
|
4bf525d409 | ||
|
|
8858beb6d9 | ||
|
|
d3183d0ab3 | ||
|
|
da642f12fa | ||
|
|
5376f046ca | ||
|
|
56d9e9a857 |
@@ -33,15 +33,29 @@ jobs:
|
||||
- name: Install Explorer frontend dependencies
|
||||
working-directory: explorer
|
||||
run: npm ci
|
||||
- name: Install Playwright Chromium
|
||||
working-directory: explorer
|
||||
run: npx playwright install --with-deps chromium
|
||||
- name: Test Explorer frontend
|
||||
working-directory: explorer
|
||||
run: |
|
||||
npm run test:graph-store
|
||||
npm run test:graph-workspace
|
||||
npm run test:plugin-registry
|
||||
npm run test:deterministic-e2e
|
||||
- name: Build Explorer frontend
|
||||
working-directory: explorer
|
||||
run: npm run build
|
||||
- name: Install Explorer backend test dependencies
|
||||
run: |
|
||||
# Run the deterministic backend path before the all-extras CI
|
||||
# environment is installed. The Explorer extra supplies the
|
||||
# production API dependencies without importing optional vector
|
||||
# providers such as Pinecone during test collection.
|
||||
pip install -e ".[explorer]" pytest==9.1.1
|
||||
- name: Test deterministic Explorer backend path
|
||||
run: |
|
||||
pytest -q tests/explorer/test_explorer_deterministic_rendering_e2e.py
|
||||
- name: Install pinned Python dependencies
|
||||
run: |
|
||||
pip install -r requirements-ci.txt
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
# meaningful state carried over from a failed attempt.
|
||||
- name: Initialize CodeQL (attempt 1)
|
||||
id: codeql-init-1
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 2)
|
||||
id: codeql-init-2
|
||||
if: steps.codeql-init-1.outcome == 'failure'
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -52,17 +52,17 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 3)
|
||||
id: codeql-init-3
|
||||
if: steps.codeql-init-2.outcome == 'failure'
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
with:
|
||||
languages: python
|
||||
queries: security-and-quality
|
||||
config-file: .github/codeql/codeql-config.yml
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/autobuild@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
with:
|
||||
category: "/language:python"
|
||||
upload: false
|
||||
@@ -72,7 +72,7 @@ jobs:
|
||||
# Uploads results only when Default Setup is not active.
|
||||
# If Default Setup is still enabled, this step skips gracefully
|
||||
# instead of failing the workflow with HTTP 409.
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.codeql.outputs.sarif-output }}
|
||||
category: "/language:python"
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
# avoiding the guardian.cmd/checkov exit-code bug in the MSDO wrapper.
|
||||
tools: eslint,templateanalyzer,terrascan
|
||||
- name: Upload results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.msdo.outputs.sarifFile }}
|
||||
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
}
|
||||
|
||||
- name: Upload Checkov results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: reports/checkov.sarif
|
||||
|
||||
+2
-1
@@ -106,7 +106,8 @@
|
||||
"integrations/langchain",
|
||||
"integrations/docling",
|
||||
"integrations/snowflake",
|
||||
"integrations/databricks"
|
||||
"integrations/databricks",
|
||||
"integrations/salesforce"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
---
|
||||
title: "Salesforce Integration"
|
||||
description: "Ingest CRM records from Salesforce sObjects and SOQL queries into Semantica's KG pipeline."
|
||||
icon: "cloud"
|
||||
---
|
||||
|
||||
> Extract Accounts, Contacts, Opportunities, and custom objects from Salesforce into Semantica with username/password/security-token, JWT bearer, or session-based authentication.
|
||||
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
# Install with Salesforce support
|
||||
pip install "semantica[db-salesforce]"
|
||||
|
||||
# Or install the connector separately
|
||||
pip install simple-salesforce>=1.12.0
|
||||
```
|
||||
|
||||
|
||||
## Basic Usage
|
||||
|
||||
```python
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
import os
|
||||
|
||||
ingestor = SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
domain=os.getenv("SALESFORCE_DOMAIN", "login"), # "test" for sandbox
|
||||
)
|
||||
|
||||
data = ingestor.ingest_sobject("Account", fields=["Id", "Name", "Industry"], limit=1000)
|
||||
print(f"Retrieved {data.row_count} of {data.total_size} matching records")
|
||||
print(f"Columns: {data.columns}")
|
||||
```
|
||||
|
||||
<Tip>
|
||||
Use environment variables (or a `.env` file with `python-dotenv`) to keep credentials out of source code. `SalesforceIngestor()` with no arguments reads from `SALESFORCE_*` environment variables automatically.
|
||||
</Tip>
|
||||
|
||||
|
||||
## Authentication Methods
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Username / Password / Security Token">
|
||||
```python
|
||||
import os
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
|
||||
ingestor = SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
domain="login", # production; use "test" for sandbox
|
||||
)
|
||||
```
|
||||
Set the required environment variables before running:
|
||||
```bash
|
||||
export SALESFORCE_USERNAME="your-username@example.com"
|
||||
export SALESFORCE_PASSWORD="your-password"
|
||||
export SALESFORCE_SECURITY_TOKEN="your-security-token"
|
||||
```
|
||||
The standard server-side flow. The security token is appended to the
|
||||
password during Salesforce SOAP login. Generate or reset it under
|
||||
**Settings → My Personal Information → Reset My Security Token**.
|
||||
</Tab>
|
||||
<Tab title="JWT Bearer (Recommended for CI/CD)">
|
||||
```python
|
||||
import os
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
|
||||
ingestor = SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
consumer_key=os.getenv("SALESFORCE_CONSUMER_KEY"),
|
||||
privatekey_file=os.getenv("SALESFORCE_PRIVATE_KEY_FILE"),
|
||||
domain="login", # or "test" for sandbox
|
||||
)
|
||||
```
|
||||
```bash
|
||||
export SALESFORCE_USERNAME="your-username@example.com"
|
||||
export SALESFORCE_CONSUMER_KEY="your-connected-app-consumer-key"
|
||||
export SALESFORCE_PRIVATE_KEY_FILE="/path/to/server.key"
|
||||
```
|
||||
The JWT bearer flow authenticates with a signed token — no password
|
||||
is transmitted. Ideal for server-to-server integrations and CI/CD
|
||||
pipelines. Requires a Salesforce connected app configured with
|
||||
**Use digital signatures** and the pre-authorised user listed under
|
||||
**Manage → Profiles / Permission Sets**.
|
||||
|
||||
If you prefer to pass the key material as a string instead of a file
|
||||
path, use `SALESFORCE_PRIVATE_KEY` (the PEM contents) in place of
|
||||
`SALESFORCE_PRIVATE_KEY_FILE`.
|
||||
</Tab>
|
||||
<Tab title="Session ID + Instance URL">
|
||||
```python
|
||||
ingestor = SalesforceIngestor(
|
||||
session_id=os.getenv("SALESFORCE_SESSION_ID"),
|
||||
instance_url=os.getenv("SALESFORCE_INSTANCE_URL"),
|
||||
)
|
||||
```
|
||||
Use this when your environment already manages the OAuth token
|
||||
lifecycle (e.g. a connected app obtaining tokens via the web-server
|
||||
or device flow). Pass the access token as `session_id` and the full
|
||||
instance URL (e.g. `https://myorg.my.salesforce.com`) as
|
||||
`instance_url`.
|
||||
</Tab>
|
||||
<Tab title="Sandbox">
|
||||
```python
|
||||
import os
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
|
||||
ingestor = SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
domain="test", # routes to test.salesforce.com
|
||||
)
|
||||
```
|
||||
```bash
|
||||
export SALESFORCE_USERNAME="your-sandbox-username@example.com.sandbox"
|
||||
export SALESFORCE_PASSWORD="your-password"
|
||||
export SALESFORCE_SECURITY_TOKEN="your-security-token"
|
||||
export SALESFORCE_DOMAIN="test"
|
||||
```
|
||||
Replace `domain="login"` with `domain="test"` (or set
|
||||
`SALESFORCE_DOMAIN=test` in your environment) to connect to a
|
||||
developer or full sandbox.
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
### Environment variables
|
||||
|
||||
All constructor parameters have environment-variable fallbacks:
|
||||
|
||||
| Variable | Parameter | Default |
|
||||
|---|---|---|
|
||||
| `SALESFORCE_USERNAME` | `username` | — |
|
||||
| `SALESFORCE_PASSWORD` | `password` | — |
|
||||
| `SALESFORCE_SECURITY_TOKEN` | `security_token` | — |
|
||||
| `SALESFORCE_DOMAIN` | `domain` | `"login"` |
|
||||
| `SALESFORCE_INSTANCE_URL` | `instance_url` | — |
|
||||
| `SALESFORCE_SESSION_ID` | `session_id` | — |
|
||||
| `SALESFORCE_CONSUMER_KEY` | `consumer_key` | — |
|
||||
| `SALESFORCE_PRIVATE_KEY_FILE` | `privatekey_file` | — |
|
||||
| `SALESFORCE_PRIVATE_KEY` | `privatekey` | — |
|
||||
| `SALESFORCE_API_VERSION` | `api_version` | library default (`59.0`) |
|
||||
|
||||
|
||||
## Object Ingestion
|
||||
|
||||
### Ingest a standard object
|
||||
|
||||
```python
|
||||
data = ingestor.ingest_sobject(
|
||||
"Account",
|
||||
fields=["Id", "Name", "Industry", "AnnualRevenue", "BillingCity"],
|
||||
where="Type = 'Customer' AND AnnualRevenue > 1000000",
|
||||
order_by="Name ASC",
|
||||
limit=5000,
|
||||
)
|
||||
print(f"Retrieved {data.row_count} of {data.total_size} matching records")
|
||||
```
|
||||
|
||||
<Note>
|
||||
`data.row_count` is the number of records in `data.data` (i.e. what was actually returned after any `limit`). `data.total_size` is Salesforce's `totalSize` — the number of records matching the query *before* the limit. Compare them to know whether you got all results.
|
||||
</Note>
|
||||
|
||||
### Ingest a custom object
|
||||
|
||||
Custom objects end with `__c` in their API name:
|
||||
|
||||
```python
|
||||
data = ingestor.ingest_sobject(
|
||||
"My_Custom_Object__c",
|
||||
fields=["Id", "Name", "Custom_Field__c"],
|
||||
)
|
||||
```
|
||||
|
||||
Relationship traversal fields (`Owner.Name`) are also supported:
|
||||
|
||||
```python
|
||||
data = ingestor.ingest_sobject(
|
||||
"Contact",
|
||||
fields=["Id", "Name", "Email", "Account.Name", "Owner.Name"],
|
||||
limit=10000,
|
||||
)
|
||||
```
|
||||
|
||||
### Let Semantica choose the fields
|
||||
|
||||
When `fields` is omitted, all selectable fields are fetched via `describe()`
|
||||
(one extra API call). Compound address and geolocation fields (`type=address`,
|
||||
`type=location`) are automatically excluded — select their components
|
||||
(`BillingStreet`, `BillingCity`, `Location__Latitude__s`, …) individually if
|
||||
you need them.
|
||||
|
||||
```python
|
||||
data = ingestor.ingest_sobject("Opportunity")
|
||||
```
|
||||
|
||||
|
||||
## Raw SOQL Ingestion
|
||||
|
||||
Pass any valid SOQL query verbatim — pagination is handled automatically:
|
||||
|
||||
```python
|
||||
data = ingestor.ingest_query("""
|
||||
SELECT Id, Name, StageName, Amount, CloseDate,
|
||||
Account.Name, Owner.Name
|
||||
FROM Opportunity
|
||||
WHERE IsClosed = false
|
||||
ORDER BY CloseDate ASC
|
||||
""")
|
||||
print(f"Open opportunities: {data.row_count}")
|
||||
```
|
||||
|
||||
The query is passed to the Salesforce REST API unchanged. The caller is
|
||||
responsible for SOQL correctness and safety.
|
||||
|
||||
<Warning>
|
||||
`ingest_query` does not validate or sanitise the SOQL string. Use
|
||||
`ingest_sobject` (which validates sObject names, field names, and WHERE/ORDER
|
||||
BY fragments) when building queries from application-controlled inputs.
|
||||
</Warning>
|
||||
|
||||
|
||||
## Document Export
|
||||
|
||||
Convert ingested records to the Semantica document format for use with
|
||||
`GraphBuilder`:
|
||||
|
||||
```python
|
||||
documents = ingestor.export_as_documents(
|
||||
data,
|
||||
id_field="Id", # default; Salesforce 18-char record Id
|
||||
text_fields=["Name", "Description"], # omit to join all string fields
|
||||
)
|
||||
|
||||
print(f"Created {len(documents)} documents")
|
||||
# Each document:
|
||||
# {
|
||||
# "id": "001xx000003GYk2AAG",
|
||||
# "text": "Acme Corp Enterprise software company",
|
||||
# "metadata": {
|
||||
# "source": "salesforce",
|
||||
# "sobject": "Account",
|
||||
# "instance_url": "https://myorg.my.salesforce.com",
|
||||
# "row_data": { ... full cleaned record ... }
|
||||
# }
|
||||
# }
|
||||
```
|
||||
|
||||
Feed the documents directly into `GraphBuilder`:
|
||||
|
||||
```python
|
||||
from semantica.kg import GraphBuilder
|
||||
|
||||
builder = GraphBuilder()
|
||||
kg = builder.build(documents)
|
||||
```
|
||||
|
||||
|
||||
## Object and Schema Discovery
|
||||
|
||||
```python
|
||||
# List all accessible sObjects
|
||||
sobject_names = ingestor.list_sobjects()
|
||||
print(sobject_names[:10]) # ["Account", "Case", "Contact", ...]
|
||||
|
||||
# Inspect fields for a specific sObject
|
||||
schema = ingestor.get_sobject_schema("Account")
|
||||
for field in schema["fields"]:
|
||||
print(f"{field['name']}: {field['type']} (nillable={field['nillable']})")
|
||||
```
|
||||
|
||||
|
||||
## Context Manager
|
||||
|
||||
Prefer the context manager for long-running jobs — it opens one connection on
|
||||
entry and closes it on exit, so every ingestion call inside the `with` block
|
||||
reuses the same authenticated session:
|
||||
|
||||
```python
|
||||
with SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
) as sf:
|
||||
accounts = sf.ingest_sobject("Account", limit=10000)
|
||||
contacts = sf.ingest_sobject("Contact", limit=10000)
|
||||
sobjects = sf.list_sobjects()
|
||||
```
|
||||
|
||||
|
||||
## Convenience Function
|
||||
|
||||
Use `ingest_salesforce()` for one-liner ingestion:
|
||||
|
||||
```python
|
||||
from semantica.ingest import ingest_salesforce
|
||||
|
||||
# Fetch records
|
||||
data = ingest_salesforce(
|
||||
method="sobject",
|
||||
sobject_name="Account",
|
||||
fields=["Id", "Name", "Industry"],
|
||||
limit=500,
|
||||
)
|
||||
|
||||
# Execute raw SOQL (credentials from environment variables)
|
||||
data = ingest_salesforce(
|
||||
method="query",
|
||||
soql="SELECT Id, Name FROM Contact WHERE IsActive = true",
|
||||
)
|
||||
|
||||
# Ingest + export to documents in one step
|
||||
docs = ingest_salesforce(
|
||||
method="documents",
|
||||
sobject_name="Account",
|
||||
text_fields=["Name", "Description"],
|
||||
limit=1000,
|
||||
)
|
||||
|
||||
# List accessible sObjects
|
||||
sobject_names = ingest_salesforce(method="list_sobjects")
|
||||
```
|
||||
|
||||
Or use the unified `ingest()` dispatcher:
|
||||
|
||||
```python
|
||||
from semantica.ingest import ingest
|
||||
|
||||
result = ingest(
|
||||
None,
|
||||
source_type="salesforce",
|
||||
method="sobject",
|
||||
sobject_name="Account",
|
||||
fields=["Id", "Name"],
|
||||
limit=500,
|
||||
)
|
||||
data = result["data"] # SalesforceData
|
||||
```
|
||||
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
```python
|
||||
import os
|
||||
from semantica.ingest import SalesforceConnector
|
||||
|
||||
connector = SalesforceConnector(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
)
|
||||
if not connector.test_connection():
|
||||
print("Connection failed: check username, password, security token, and domain")
|
||||
```
|
||||
|
||||
Common causes of authentication failures:
|
||||
|
||||
- **Wrong domain**: production orgs use `domain="login"`; sandboxes use `domain="test"`.
|
||||
- **Stale security token**: reset it under **Settings → Reset My Security Token**. The new token is emailed to you.
|
||||
- **IP restriction**: your org's trusted IP ranges may block the originating IP. Check **Setup → Network Access**.
|
||||
- **API access disabled**: ensure the connected profile has the **API Enabled** permission.
|
||||
|
||||
|
||||
## See Also
|
||||
|
||||
- [Ingest Module](../reference/ingest) — Full `SalesforceIngestor` API and all other ingestors.
|
||||
- [Snowflake Integration](snowflake) — Relational warehouse connector with a similar design.
|
||||
- [Databricks Integration](databricks) — Lakehouse connector.
|
||||
- [Installation](../installation) — All optional dependency extras.
|
||||
- [Knowledge Graph](../reference/kg) — Build a KG from ingested Salesforce data.
|
||||
@@ -0,0 +1,195 @@
|
||||
"""
|
||||
Deterministic Explorer Rendering E2E Example.
|
||||
|
||||
Demonstrates building, serializing, and reloading a deterministic 4-node,
|
||||
3-edge knowledge graph baseline for visual inspection in Semantica Explorer (#1037).
|
||||
|
||||
Graph topology:
|
||||
Alice (Person, #63E6FF) --WORKS_AT--> Acme (Organization, #A78BFA)
|
||||
Bob (Person, #63E6FF) --KNOWS--> Alice (Person, #63E6FF)
|
||||
Acme (Organization, #A78BFA) --LOCATED_IN--> New York (Location, #34D399)
|
||||
|
||||
Clean Checkout Prerequisites:
|
||||
1. Python backend dependencies:
|
||||
pip install -e ".[explorer]"
|
||||
2. Frontend workspace dependencies:
|
||||
cd explorer && npm install && cd ..
|
||||
|
||||
Usage:
|
||||
# 1. Generate the deterministic graph baseline:
|
||||
python examples/explorer_deterministic_rendering_example.py
|
||||
|
||||
# 2. Launch Explorer with local dev authentication (Option A - Dev mode):
|
||||
# Terminal 1 (Backend API):
|
||||
SEMANTICA_ALLOW_ANONYMOUS=true python -m semantica.explorer --graph explorer_e2e_test_graph.json --port 8000 --no-browser
|
||||
# Terminal 2 (Frontend UI):
|
||||
cd explorer && npm run dev
|
||||
# Open http://localhost:5173
|
||||
|
||||
# 2. Launch Explorer (Option B - Standalone CLI server):
|
||||
SEMANTICA_ALLOW_ANONYMOUS=true python -m semantica.explorer --graph explorer_e2e_test_graph.json --port 8000
|
||||
# Open http://localhost:8000
|
||||
|
||||
# Secure authentication alternative:
|
||||
export SEMANTICA_API_KEY="your-secret-api-key"
|
||||
python -m semantica.explorer --graph explorer_e2e_test_graph.json --port 8000
|
||||
# Send HTTP header: X-API-Key: your-secret-api-key
|
||||
|
||||
Verification Checklist:
|
||||
- Exactly 4 nodes visible on canvas:
|
||||
* Alice (Person, #63E6FF)
|
||||
* Bob (Person, #63E6FF)
|
||||
* Acme (Organization, #A78BFA)
|
||||
* New York (Location, #34D399)
|
||||
- Exactly 3 directed edges with canonical relationship labels:
|
||||
* Alice -> Acme (WORKS_AT)
|
||||
* Bob -> Alice (KNOWS)
|
||||
* Acme -> New York (LOCATED_IN)
|
||||
- Zoom behavior:
|
||||
* Zoom in to Inspection tier (ratio <= 0.5): directional arrows and node labels scale clearly.
|
||||
* Zoom out to Overview tier (ratio > 1.2): layout remains stable and non-colliding.
|
||||
- Hover & Selection interactions:
|
||||
* Hover over 'Alice': node halo triggers; incident edges (WORKS_AT, KNOWS) highlight in local context.
|
||||
* Click an edge: Inspector panel confirms edgeType ('WORKS_AT', 'KNOWS', or 'LOCATED_IN').
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.session import GraphSession
|
||||
|
||||
|
||||
def build_deterministic_graph() -> ContextGraph:
|
||||
"""Build the exact 4-node, 3-edge graph specified in #1037."""
|
||||
graph = ContextGraph(advanced_analytics=False)
|
||||
|
||||
# 1. Add exactly 4 nodes
|
||||
graph.add_node(
|
||||
"alice",
|
||||
node_type="Person",
|
||||
content="Alice",
|
||||
color="#63E6FF",
|
||||
)
|
||||
graph.add_node(
|
||||
"bob",
|
||||
node_type="Person",
|
||||
content="Bob",
|
||||
color="#63E6FF",
|
||||
)
|
||||
graph.add_node(
|
||||
"acme",
|
||||
node_type="Organization",
|
||||
content="Acme",
|
||||
color="#A78BFA",
|
||||
)
|
||||
graph.add_node(
|
||||
"new_york",
|
||||
node_type="Location",
|
||||
content="New York",
|
||||
color="#34D399",
|
||||
)
|
||||
|
||||
# 2. Add exactly 3 directed edges
|
||||
graph.add_edge("alice", "acme", edge_type="WORKS_AT", weight=1.0)
|
||||
graph.add_edge("bob", "alice", edge_type="KNOWS", weight=1.0)
|
||||
graph.add_edge("acme", "new_york", edge_type="LOCATED_IN", weight=1.0)
|
||||
|
||||
return graph
|
||||
|
||||
|
||||
def main() -> None:
|
||||
print("=" * 75)
|
||||
print("Semantica Explorer Deterministic Graph Generator (#1037)")
|
||||
print("=" * 75)
|
||||
|
||||
print("1. Building deterministic ContextGraph...")
|
||||
graph = build_deterministic_graph()
|
||||
print(
|
||||
f" ✓ Graph built with {len(graph.nodes)} nodes "
|
||||
f"and {len(graph.edges)} edges."
|
||||
)
|
||||
|
||||
output_path = Path("explorer_e2e_test_graph.json").resolve()
|
||||
print(f"2. Persisting graph to '{output_path.name}'...")
|
||||
graph.save_to_file(str(output_path))
|
||||
print(f" ✓ Graph saved to {output_path}")
|
||||
|
||||
# Verify JSON format
|
||||
with open(output_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
assert len(data.get("nodes", [])) == 4
|
||||
assert len(data.get("edges", [])) == 3
|
||||
|
||||
print("3. Verifying reload via GraphSession.from_file()...")
|
||||
session = GraphSession.from_file(str(output_path))
|
||||
stats = session.get_stats()
|
||||
nodes, total_nodes = session.get_nodes()
|
||||
edges, total_edges = session.get_edges()
|
||||
|
||||
assert stats["node_count"] == 4
|
||||
assert stats["edge_count"] == 3
|
||||
assert total_nodes == 4
|
||||
assert total_edges == 3
|
||||
|
||||
print(
|
||||
f" ✓ Graph reloaded successfully without mutation "
|
||||
f"(nodes: {total_nodes}, edges: {total_edges}).\n"
|
||||
)
|
||||
|
||||
print("=" * 75)
|
||||
print("Clean Checkout Prerequisites:")
|
||||
print("=" * 75)
|
||||
print(" pip install -e '.[explorer]'")
|
||||
print(" cd explorer && npm install && cd ..\n")
|
||||
|
||||
print("=" * 75)
|
||||
print("Reproduction instructions to view in Semantica Explorer:")
|
||||
print("=" * 75)
|
||||
print("Option A (Frontend dev server + API backend — recommended for development):")
|
||||
print(
|
||||
f" 1. Backend: SEMANTICA_ALLOW_ANONYMOUS=true python -m semantica.explorer "
|
||||
f"--graph {output_path} --port 8000 --no-browser"
|
||||
)
|
||||
print(" 2. Frontend: cd explorer && npm run dev")
|
||||
print(" 3. Open http://localhost:5173 to inspect the graph canvas.\n")
|
||||
|
||||
print("Option B (Standalone Explorer CLI server):")
|
||||
print(
|
||||
f" SEMANTICA_ALLOW_ANONYMOUS=true python -m semantica.explorer "
|
||||
f"--graph {output_path} --port 8000"
|
||||
)
|
||||
print(" Open http://localhost:8000\n")
|
||||
|
||||
print("Secure Authentication Alternative:")
|
||||
print(" export SEMANTICA_API_KEY='your-secret-api-key'")
|
||||
print(
|
||||
f" python -m semantica.explorer --graph {output_path} --port 8000"
|
||||
)
|
||||
print(" Send header: 'X-API-Key: your-secret-api-key'\n")
|
||||
|
||||
print("=" * 75)
|
||||
print("Verification Checklist:")
|
||||
print("=" * 75)
|
||||
print(" 1. Nodes (4 total):")
|
||||
print(" - Alice (Person, #63E6FF)")
|
||||
print(" - Bob (Person, #63E6FF)")
|
||||
print(" - Acme (Organization, #A78BFA)")
|
||||
print(" - New York (Location, #34D399)")
|
||||
print(" 2. Directed Edges & Canonical Labels (3 total):")
|
||||
print(" - Alice -> Acme [WORKS_AT]")
|
||||
print(" - Bob -> Alice [KNOWS]")
|
||||
print(" - Acme -> New York [LOCATED_IN]")
|
||||
print(" 3. Zoom Interactions:")
|
||||
print(" - Inspection tier (zoom in): directional arrows & labels remain legible.")
|
||||
print(" - Overview tier (zoom out): nodes and edges maintain layout integrity.")
|
||||
print(" 4. Hover & Selection Interactions:")
|
||||
print(" - Hover Alice: node halo triggers and incident edges (WORKS_AT, KNOWS) highlight.")
|
||||
print(" - Click edge: Inspector panel displays edgeType label ('WORKS_AT', 'KNOWS', 'LOCATED_IN').")
|
||||
print("=" * 75)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -9,7 +9,8 @@
|
||||
"lint": "eslint .",
|
||||
"preview": "vite preview",
|
||||
"test:graph-store": "node --test tests/graphStore.multi-edge.test.mjs",
|
||||
"test:graph-workspace": "node --import tsx --test tests/markdownContentViewer.test.ts tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts",
|
||||
"test:graph-workspace": "node --import tsx --test tests/markdownContentViewer.test.ts tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts tests/deterministicExplorerRendering.test.ts",
|
||||
"test:deterministic-e2e": "node --import tsx --test tests/deterministicExplorerRendering.e2e.ts",
|
||||
"test:plugin-registry": "node --import tsx --test tests/pluginRegistry.temporal.test.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
|
||||
@@ -41,6 +41,7 @@ import {
|
||||
} from "./plugins";
|
||||
import { explorationEffectsShouldLoad, neighborhoodPanelShouldLoad, temporalOverlayShouldLoad } from "./pluginRegistryPredicates";
|
||||
import { shouldFetchTemporalBounds, shouldFetchTemporalSnapshot } from "./temporalLifecyclePredicates";
|
||||
import { createTemporalSnapshotGuards, type TemporalSnapshotResponse } from "./temporalSnapshotGuards";
|
||||
import type { LinkPrediction, PathResponse } from "./GraphInspectorPanel";
|
||||
import type { GraphSceneHandle, GraphSceneRuntime } from "./scene";
|
||||
import type {
|
||||
@@ -1479,6 +1480,23 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
summary?.edgeCount,
|
||||
]);
|
||||
|
||||
// Guards the snapshot lifecycle: at most one in-flight request per scrubber
|
||||
// position (identical-`at` polls are deduplicated, breaking the idle/play
|
||||
// polling loop), applied snapshots are cached and re-applied on revisit, and
|
||||
// a response applies only while the scrubber is still on its position
|
||||
// (out-of-order responses cannot clobber the active-node count).
|
||||
const temporalSnapshotGuardsRef = useRef<ReturnType<typeof createTemporalSnapshotGuards> | null>(null);
|
||||
if (temporalSnapshotGuardsRef.current === null) {
|
||||
temporalSnapshotGuardsRef.current = createTemporalSnapshotGuards();
|
||||
}
|
||||
const temporalSnapshotGuards = temporalSnapshotGuardsRef.current;
|
||||
|
||||
// A new graph summary means the graph data was replaced (reload/retry);
|
||||
// snapshots cached against the previous graph are stale, so reset all state.
|
||||
useEffect(() => {
|
||||
temporalSnapshotGuards.reset();
|
||||
}, [summary]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!canFetchTemporalSnapshot) {
|
||||
return;
|
||||
@@ -1488,37 +1506,67 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return;
|
||||
}
|
||||
|
||||
const atMs = debouncedTime.getTime();
|
||||
const { seq, cached } = temporalSnapshotGuards.begin(atMs);
|
||||
if (seq === null) {
|
||||
// An identical request is already in flight: one request per position.
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
const applyData = (data: TemporalSnapshotResponse) => {
|
||||
const nextActiveIds = new Set(data.active_node_ids);
|
||||
requestAnimationFrame(() => {
|
||||
if (cancelled) return;
|
||||
if (!temporalSnapshotGuards.shouldApply(atMs, seq)) {
|
||||
// The scrubber moved on (or this request was superseded): release the
|
||||
// position so a return to it refetches instead of stalling.
|
||||
temporalSnapshotGuards.finish(atMs, seq);
|
||||
return;
|
||||
}
|
||||
const previous = prevActiveIdsRef.current;
|
||||
previous.forEach((id) => {
|
||||
if (!nextActiveIds.has(id) && graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", true);
|
||||
}
|
||||
});
|
||||
nextActiveIds.forEach((id) => {
|
||||
if (graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", false);
|
||||
}
|
||||
});
|
||||
prevActiveIdsRef.current = nextActiveIds;
|
||||
setActiveNodeCount(data.active_node_count);
|
||||
setGraphVersion((current) => current + 1);
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
temporalSnapshotGuards.apply(atMs, seq, data);
|
||||
});
|
||||
};
|
||||
|
||||
if (cached) {
|
||||
// Returning to a position whose snapshot was already applied: re-apply
|
||||
// the cached result without a network request.
|
||||
applyData(cached);
|
||||
return;
|
||||
}
|
||||
|
||||
const applySnapshot = async () => {
|
||||
try {
|
||||
const at = debouncedTime.toISOString();
|
||||
const response = await fetch(`/api/temporal/snapshot?at=${encodeURIComponent(at)}`);
|
||||
if (!response.ok || cancelled) return;
|
||||
|
||||
const data: { active_node_ids: string[]; active_node_count: number } = await response.json();
|
||||
if (!response.ok) {
|
||||
// A failed request must be retryable if the scrubber returns.
|
||||
if (!cancelled) temporalSnapshotGuards.finish(atMs, seq);
|
||||
return;
|
||||
}
|
||||
if (cancelled) return;
|
||||
|
||||
const nextActiveIds = new Set(data.active_node_ids);
|
||||
requestAnimationFrame(() => {
|
||||
if (cancelled) return;
|
||||
const previous = prevActiveIdsRef.current;
|
||||
previous.forEach((id) => {
|
||||
if (!nextActiveIds.has(id) && graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", true);
|
||||
}
|
||||
});
|
||||
nextActiveIds.forEach((id) => {
|
||||
if (graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", false);
|
||||
}
|
||||
});
|
||||
prevActiveIdsRef.current = nextActiveIds;
|
||||
setActiveNodeCount(data.active_node_count);
|
||||
setGraphVersion((current) => current + 1);
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
});
|
||||
const data: TemporalSnapshotResponse = await response.json();
|
||||
if (cancelled) return;
|
||||
applyData(data);
|
||||
} catch (fetchError) {
|
||||
temporalSnapshotGuards.finish(atMs, seq);
|
||||
if (!cancelled) {
|
||||
console.error("[Temporal] Snapshot fetch failed", fetchError);
|
||||
}
|
||||
@@ -1528,6 +1576,8 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
applySnapshot();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
// A cancelled request must be retryable when its position is revisited.
|
||||
temporalSnapshotGuards.finish(atMs, seq);
|
||||
};
|
||||
}, [
|
||||
canFetchTemporalSnapshot,
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* Guards for the temporal snapshot fetch/apply lifecycle.
|
||||
*
|
||||
* The snapshot effect previously fetched /api/temporal/snapshot with no
|
||||
* idempotency or ordering protection. Upstream churn (timeline recreation
|
||||
* while bounds settle, play ticks resetting the playhead, drag events) could
|
||||
* re-request the same `at` repeatedly, and responses could arrive after the
|
||||
* scrubber had moved on.
|
||||
*
|
||||
* The guards enforce:
|
||||
* - at most one in-flight request per scrubber position (identical `at`
|
||||
* values are deduplicated while a request is pending, breaking the
|
||||
* idle/play polling loop);
|
||||
* - successful snapshots are cached per position and re-applied when the
|
||||
* scrubber returns (play wrap-around, back-scrubbing) without a refetch;
|
||||
* - a response is applied only while the scrubber is still on its position,
|
||||
* so out-of-order responses cannot clobber a newer position's count;
|
||||
* - failed, cancelled, or superseded requests release their position so it
|
||||
* can be fetched again on the next visit;
|
||||
* - `reset()` drops all state when the underlying graph data is replaced
|
||||
* (reload/retry), because cached snapshots describe the previous graph.
|
||||
*
|
||||
* `createTemporalSnapshotGuards()` is stateful by design.
|
||||
*/
|
||||
|
||||
export interface TemporalSnapshotResponse {
|
||||
active_node_ids: string[];
|
||||
active_node_count: number;
|
||||
}
|
||||
|
||||
export interface TemporalSnapshotRequest {
|
||||
/** null when the request was deduplicated because one is already in flight. */
|
||||
seq: number | null;
|
||||
/** The snapshot previously applied for this position, when revisiting it. */
|
||||
cached: TemporalSnapshotResponse | null;
|
||||
}
|
||||
|
||||
export interface TemporalSnapshotGuards {
|
||||
/** Begin (or dedupe) a request for `atMs`; marks it as the current position. */
|
||||
begin(atMs: number): TemporalSnapshotRequest;
|
||||
/** True when the response for `atMs`/`seq` may be applied (scrubber still on `atMs`). */
|
||||
shouldApply(atMs: number, seq: number): boolean;
|
||||
/** Record a successful application and cache its snapshot for revisits. */
|
||||
apply(atMs: number, seq: number, data: TemporalSnapshotResponse): void;
|
||||
/** Release a position whose request failed, was cancelled, or was superseded. */
|
||||
finish(atMs: number, seq: number): void;
|
||||
/** Drop all state; call when the underlying graph data is replaced (reload). */
|
||||
reset(): void;
|
||||
}
|
||||
|
||||
interface SnapshotEntry {
|
||||
seq: number;
|
||||
/** null while the request is in flight (or before the first success). */
|
||||
data: TemporalSnapshotResponse | null;
|
||||
}
|
||||
|
||||
/** Upper bound on cached positions so long scrubbing sessions stay bounded. */
|
||||
const MAX_CACHED_POSITIONS = 256;
|
||||
|
||||
export function createTemporalSnapshotGuards(): TemporalSnapshotGuards {
|
||||
const entries = new Map<number, SnapshotEntry>();
|
||||
let latestRequestSeq = 0;
|
||||
let currentAtMs: number | null = null;
|
||||
|
||||
const evictOldest = () => {
|
||||
while (entries.size > MAX_CACHED_POSITIONS) {
|
||||
const oldestAtMs = entries.keys().next().value;
|
||||
if (oldestAtMs === undefined) return;
|
||||
entries.delete(oldestAtMs);
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
begin(atMs) {
|
||||
const existing = entries.get(atMs);
|
||||
if (existing && existing.data === null) {
|
||||
// Identical request already in flight: dedupe, but the scrubber is here now.
|
||||
currentAtMs = atMs;
|
||||
return { seq: null, cached: null };
|
||||
}
|
||||
latestRequestSeq += 1;
|
||||
const seq = latestRequestSeq;
|
||||
entries.set(atMs, { seq, data: existing?.data ?? null });
|
||||
currentAtMs = atMs;
|
||||
evictOldest();
|
||||
return { seq, cached: existing?.data ?? null };
|
||||
},
|
||||
|
||||
shouldApply(atMs, seq) {
|
||||
return atMs === currentAtMs && entries.get(atMs)?.seq === seq;
|
||||
},
|
||||
|
||||
apply(atMs, seq, data) {
|
||||
const entry = entries.get(atMs);
|
||||
if (entry && entry.seq === seq) {
|
||||
entry.data = data;
|
||||
}
|
||||
},
|
||||
|
||||
finish(atMs, seq) {
|
||||
const entry = entries.get(atMs);
|
||||
if (entry && entry.seq === seq && entry.data === null) {
|
||||
entries.delete(atMs);
|
||||
}
|
||||
},
|
||||
|
||||
reset() {
|
||||
entries.clear();
|
||||
latestRequestSeq = 0;
|
||||
currentAtMs = null;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import test from "node:test";
|
||||
import { chromium, type Page } from "playwright";
|
||||
|
||||
const PORT = 4173;
|
||||
const BASE_URL = `http://127.0.0.1:${PORT}`;
|
||||
|
||||
const nodes = [
|
||||
{ id: "alice", type: "Person", content: "Alice", properties: {} },
|
||||
{ id: "bob", type: "Person", content: "Bob", properties: {} },
|
||||
{ id: "acme", type: "Organization", content: "Acme", properties: {} },
|
||||
{ id: "new_york", type: "Location", content: "New York", properties: {} },
|
||||
];
|
||||
|
||||
const edges = [
|
||||
{ id: "edge_alice_acme", familyId: "edge_alice_acme", source: "alice", target: "acme", type: "WORKS_AT", weight: 1, properties: {} },
|
||||
{ id: "edge_bob_alice", familyId: "edge_bob_alice", source: "bob", target: "alice", type: "KNOWS", weight: 1, properties: {} },
|
||||
{ id: "edge_acme_new_york", familyId: "edge_acme_new_york", source: "acme", target: "new_york", type: "LOCATED_IN", weight: 1, properties: {} },
|
||||
];
|
||||
|
||||
let server: ChildProcess | undefined;
|
||||
|
||||
async function startVite(): Promise<void> {
|
||||
server = spawn("npm", ["run", "dev", "--", "--host", "127.0.0.1", "--port", String(PORT)], {
|
||||
cwd: process.cwd(),
|
||||
stdio: "ignore",
|
||||
});
|
||||
|
||||
for (let attempt = 0; attempt < 50; attempt += 1) {
|
||||
try {
|
||||
const response = await fetch(BASE_URL);
|
||||
if (response.ok) return;
|
||||
} catch {
|
||||
// Vite is still starting.
|
||||
}
|
||||
await delay(100);
|
||||
}
|
||||
throw new Error("Vite did not become ready");
|
||||
}
|
||||
|
||||
async function installApiFixture(page: Page): Promise<void> {
|
||||
await page.route("**/api/graph/**", async (route) => {
|
||||
const pathname = new URL(route.request().url()).pathname;
|
||||
if (pathname === "/api/graph/stats") {
|
||||
await route.fulfill({ json: { node_count: 4, edge_count: 3 } });
|
||||
} else if (pathname === "/api/graph/nodes") {
|
||||
await route.fulfill({ json: { nodes, total: nodes.length, skip: 0, limit: 1000, next_cursor: null } });
|
||||
} else if (pathname === "/api/graph/edges") {
|
||||
await route.fulfill({ json: { edges, total: edges.length, skip: 0, limit: 1000, next_cursor: null } });
|
||||
} else {
|
||||
await route.continue();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test("real Explorer loading path hydrates and renders API edge labels", async (t) => {
|
||||
await startVite();
|
||||
t.after(async () => {
|
||||
server?.kill();
|
||||
});
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
executablePath: process.env.CHROMIUM_PATH || (existsSync("/usr/bin/chromium") ? "/usr/bin/chromium" : undefined),
|
||||
});
|
||||
t.after(() => browser.close());
|
||||
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
|
||||
await page.addInitScript(() => {
|
||||
const captured = (window as Window & { __capturedCanvasText?: string[] }).__capturedCanvasText = [];
|
||||
const originalFillText = CanvasRenderingContext2D.prototype.fillText;
|
||||
CanvasRenderingContext2D.prototype.fillText = function (text: string, ...args: [number, number, number?, number?]) {
|
||||
captured.push(String(text));
|
||||
return originalFillText.call(this, text, ...args);
|
||||
};
|
||||
});
|
||||
await installApiFixture(page);
|
||||
await page.goto(BASE_URL);
|
||||
await page.getByRole("button", { name: /Open Semantica Explorer/ }).click();
|
||||
|
||||
await page.locator("canvas").nth(0).waitFor({ state: "attached" });
|
||||
await page.waitForFunction(() => document.querySelectorAll("canvas").length >= 2);
|
||||
await page.waitForFunction(() => {
|
||||
const labels = (window as Window & { __capturedCanvasText?: string[] }).__capturedCanvasText ?? [];
|
||||
return ["WORKS_AT", "KNOWS", "LOCATED_IN"].every((label) => labels.includes(label));
|
||||
}, undefined, { timeout: 10_000 });
|
||||
|
||||
const capturedLabels = await page.evaluate(() => (window as Window & { __capturedCanvasText?: string[] }).__capturedCanvasText ?? []);
|
||||
for (const label of ["WORKS_AT", "KNOWS", "LOCATED_IN"]) {
|
||||
assert.ok(capturedLabels.includes(label), `Expected rendered edge label ${label}`);
|
||||
}
|
||||
assert.ok(capturedLabels.includes("Alice"));
|
||||
|
||||
await page.getByRole("button", { name: "Zoom In" }).click();
|
||||
await page.waitForTimeout(250);
|
||||
const labelsAfterZoom = await page.evaluate(() => (window as Window & { __capturedCanvasText?: string[] }).__capturedCanvasText ?? []);
|
||||
for (const label of ["WORKS_AT", "KNOWS", "LOCATED_IN"]) {
|
||||
assert.ok(labelsAfterZoom.includes(label), `Expected edge label ${label} after zoom`);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,508 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
batchMergeEdges,
|
||||
batchMergeNodes,
|
||||
clearGraph,
|
||||
graph,
|
||||
} from "../src/store/graphStore.ts";
|
||||
import {
|
||||
buildStructuralDistanceSnapshot,
|
||||
classifyFullGraphEdge,
|
||||
resolveDisplayGraph,
|
||||
resolveEdgeElementStyle,
|
||||
resolveEdgeVisualState,
|
||||
resolveNodeElementStyle,
|
||||
resolveNodeVisualState,
|
||||
shouldForceNodeLabel,
|
||||
} from "../src/workspaces/GraphWorkspace/graphSceneState.ts";
|
||||
import { GRAPH_THEME, type GraphZoomTier } from "../src/workspaces/GraphWorkspace/graphTheme.ts";
|
||||
|
||||
test.beforeEach(() => {
|
||||
clearGraph();
|
||||
});
|
||||
|
||||
test.after(() => {
|
||||
clearGraph();
|
||||
});
|
||||
|
||||
/**
|
||||
* Loads the canonical 4-node, 3-edge deterministic test graph (Semantica #1037).
|
||||
*
|
||||
* Graph structure:
|
||||
* Alice (Person) --WORKS_AT--> Acme (Organization)
|
||||
* Bob (Person) --KNOWS--> Alice (Person)
|
||||
* Acme (Organization) --LOCATED_IN--> New York (Location)
|
||||
*/
|
||||
function loadDeterministicTestGraph() {
|
||||
batchMergeNodes([
|
||||
{
|
||||
id: "alice",
|
||||
attributes: {
|
||||
label: "Alice",
|
||||
content: "Alice",
|
||||
x: 0,
|
||||
y: 0,
|
||||
size: 8,
|
||||
color: "#63E6FF",
|
||||
baseColor: "#63E6FF",
|
||||
nodeType: "Person",
|
||||
semanticGroup: "Person",
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "bob",
|
||||
attributes: {
|
||||
label: "Bob",
|
||||
content: "Bob",
|
||||
x: -50,
|
||||
y: 0,
|
||||
size: 8,
|
||||
color: "#63E6FF",
|
||||
baseColor: "#63E6FF",
|
||||
nodeType: "Person",
|
||||
semanticGroup: "Person",
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "acme",
|
||||
attributes: {
|
||||
label: "Acme",
|
||||
content: "Acme",
|
||||
x: 50,
|
||||
y: 0,
|
||||
size: 8,
|
||||
color: "#A78BFA",
|
||||
baseColor: "#A78BFA",
|
||||
nodeType: "Organization",
|
||||
semanticGroup: "Organization",
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "new_york",
|
||||
attributes: {
|
||||
label: "New York",
|
||||
content: "New York",
|
||||
x: 100,
|
||||
y: 0,
|
||||
size: 8,
|
||||
color: "#34D399",
|
||||
baseColor: "#34D399",
|
||||
nodeType: "Location",
|
||||
semanticGroup: "Location",
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
batchMergeEdges([
|
||||
{
|
||||
id: "edge_alice_acme",
|
||||
source: "alice",
|
||||
target: "acme",
|
||||
attributes: {
|
||||
edgeId: "edge_alice_acme",
|
||||
edgeType: "WORKS_AT",
|
||||
weight: 1.0,
|
||||
visualPriority: 0.8,
|
||||
baseSize: 0.8,
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "edge_bob_alice",
|
||||
source: "bob",
|
||||
target: "alice",
|
||||
attributes: {
|
||||
edgeId: "edge_bob_alice",
|
||||
edgeType: "KNOWS",
|
||||
weight: 1.0,
|
||||
visualPriority: 0.8,
|
||||
baseSize: 0.8,
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "edge_acme_new_york",
|
||||
source: "acme",
|
||||
target: "new_york",
|
||||
attributes: {
|
||||
edgeId: "edge_acme_new_york",
|
||||
edgeType: "LOCATED_IN",
|
||||
weight: 1.0,
|
||||
visualPriority: 0.8,
|
||||
baseSize: 0.8,
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
test("deterministic graph contains exactly 4 nodes and 3 edges in store", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
assert.equal(graph.order, 4, "Expected exactly 4 nodes");
|
||||
assert.equal(graph.size, 3, "Expected exactly 3 edges");
|
||||
|
||||
// Verify node identities and labels
|
||||
const alice = graph.getNodeAttributes("alice");
|
||||
const bob = graph.getNodeAttributes("bob");
|
||||
const acme = graph.getNodeAttributes("acme");
|
||||
const newYork = graph.getNodeAttributes("new_york");
|
||||
|
||||
assert.equal(alice.label, "Alice");
|
||||
assert.equal(alice.nodeType, "Person");
|
||||
assert.equal(alice.color, "#63E6FF");
|
||||
|
||||
assert.equal(bob.label, "Bob");
|
||||
assert.equal(bob.nodeType, "Person");
|
||||
assert.equal(bob.color, "#63E6FF");
|
||||
|
||||
assert.equal(acme.label, "Acme");
|
||||
assert.equal(acme.nodeType, "Organization");
|
||||
assert.equal(acme.color, "#A78BFA");
|
||||
|
||||
assert.equal(newYork.label, "New York");
|
||||
assert.equal(newYork.nodeType, "Location");
|
||||
assert.equal(newYork.color, "#34D399");
|
||||
|
||||
// Verify edge connectivity and canonical edgeType labels
|
||||
const edgeAliceAcme = graph.getEdgeAttributes("edge_alice_acme");
|
||||
const edgeBobAlice = graph.getEdgeAttributes("edge_bob_alice");
|
||||
const edgeAcmeNewYork = graph.getEdgeAttributes("edge_acme_new_york");
|
||||
|
||||
assert.equal(edgeAliceAcme.edgeType, "WORKS_AT");
|
||||
assert.equal(graph.source("edge_alice_acme"), "alice");
|
||||
assert.equal(graph.target("edge_alice_acme"), "acme");
|
||||
|
||||
assert.equal(edgeBobAlice.edgeType, "KNOWS");
|
||||
assert.equal(graph.source("edge_bob_alice"), "bob");
|
||||
assert.equal(graph.target("edge_bob_alice"), "alice");
|
||||
|
||||
assert.equal(edgeAcmeNewYork.edgeType, "LOCATED_IN");
|
||||
assert.equal(graph.source("edge_acme_new_york"), "acme");
|
||||
assert.equal(graph.target("edge_acme_new_york"), "new_york");
|
||||
});
|
||||
|
||||
test("display graph resolution preserves all 4 nodes and 3 edges in full view", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: false });
|
||||
|
||||
assert.equal(displayGraph.order, 4);
|
||||
assert.equal(displayGraph.size, 3);
|
||||
assert.ok(displayGraph.hasNode("alice"));
|
||||
assert.ok(displayGraph.hasNode("bob"));
|
||||
assert.ok(displayGraph.hasNode("acme"));
|
||||
assert.ok(displayGraph.hasNode("new_york"));
|
||||
assert.ok(displayGraph.hasEdge("edge_alice_acme"));
|
||||
assert.ok(displayGraph.hasEdge("edge_bob_alice"));
|
||||
assert.ok(displayGraph.hasEdge("edge_acme_new_york"));
|
||||
});
|
||||
|
||||
test("structural distance calculation resolves correct hop counts across the 3-edge chain", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
// From Bob: Bob (0) -> Alice (1) -> Acme (2) -> New York (3)
|
||||
const distances = buildStructuralDistanceSnapshot(graph, "bob", 3);
|
||||
|
||||
assert.equal(distances.bob, 0);
|
||||
assert.equal(distances.alice, 1);
|
||||
assert.equal(distances.acme, 2);
|
||||
assert.equal(distances.new_york, 3);
|
||||
});
|
||||
|
||||
test("edge rendering and canonical edge labels remain legible across zoom tiers and inspection modes", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
const canonicalEdges = [
|
||||
{ id: "edge_alice_acme", source: "alice", target: "acme", label: "WORKS_AT" },
|
||||
{ id: "edge_bob_alice", source: "bob", target: "alice", label: "KNOWS" },
|
||||
{ id: "edge_acme_new_york", source: "acme", target: "new_york", label: "LOCATED_IN" },
|
||||
];
|
||||
|
||||
// 1. Edge attributes preserve canonical edgeType labels in graph store:
|
||||
for (const item of canonicalEdges) {
|
||||
const attrs = graph.getEdgeAttributes(item.id);
|
||||
assert.equal(attrs.edgeType, item.label, `Edge ${item.id} must have edgeType ${item.label}`);
|
||||
assert.equal(graph.source(item.id), item.source);
|
||||
assert.equal(graph.target(item.id), item.target);
|
||||
}
|
||||
|
||||
// 2. In active context / neighbor state across all zoom tiers (overview, structure, inspection):
|
||||
const allTiers: GraphZoomTier[] = ["overview", "structure", "inspection"];
|
||||
for (const tier of allTiers) {
|
||||
for (const item of canonicalEdges) {
|
||||
const attrs = graph.getEdgeAttributes(item.id);
|
||||
const contextStyle = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
tier,
|
||||
"neighbor",
|
||||
attrs,
|
||||
item.source,
|
||||
item.target,
|
||||
"full",
|
||||
item.id,
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
contextStyle.hidden,
|
||||
false,
|
||||
`Edge ${item.id} (${item.label}) in context state 'neighbor' must be visible in zoom tier '${tier}'`,
|
||||
);
|
||||
assert.ok(
|
||||
contextStyle.size !== undefined && contextStyle.size > 0,
|
||||
`Edge ${item.id} (${item.label}) must have positive render size in zoom tier '${tier}'`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. In selected state in inspection zoom tier (close examination of edge details and label):
|
||||
for (const item of canonicalEdges) {
|
||||
const attrs = graph.getEdgeAttributes(item.id);
|
||||
const selectedStyle = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"inspection",
|
||||
"selected",
|
||||
attrs,
|
||||
item.source,
|
||||
item.target,
|
||||
"full",
|
||||
item.id,
|
||||
"selected",
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
selectedStyle.hidden,
|
||||
false,
|
||||
`Selected edge ${item.id} (${item.label}) must be visible in inspection zoom tier`,
|
||||
);
|
||||
assert.ok(
|
||||
selectedStyle.size !== undefined && selectedStyle.size > 0,
|
||||
`Selected edge ${item.id} (${item.label}) must have positive render size`,
|
||||
);
|
||||
}
|
||||
|
||||
// 4. Verify inspection zoom tier camera and arrow rendering settings
|
||||
assert.equal(GRAPH_THEME.zoomTiers.inspection.showContextualArrows, true);
|
||||
assert.equal(GRAPH_THEME.zoomTiers.inspection.showCurves, true);
|
||||
});
|
||||
|
||||
test("node hover interaction preserves edge visibility and highlights canonical incident edge types", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
// Scenario 1: Hover Alice
|
||||
// Incident edges: Alice -> Acme (WORKS_AT) and Bob -> Alice (KNOWS)
|
||||
const aliceAttrs = graph.getNodeAttributes("alice");
|
||||
const aliceVisual = resolveNodeVisualState("alice", "structure", "alice", "", "", new Set(), new Set(), new Set());
|
||||
assert.equal(aliceVisual, "hovered");
|
||||
|
||||
const aliceStyle = resolveNodeElementStyle(GRAPH_THEME, "structure", "hovered", aliceAttrs, "Alice");
|
||||
assert.equal(aliceStyle.forceLabel, true, "Hovered Alice must force-render label");
|
||||
assert.equal(aliceStyle.label, "Alice");
|
||||
assert.equal(aliceStyle.showHalo, true, "Hovered Alice must show interactive halo");
|
||||
|
||||
const aliceIncidentEdges = new Set(["edge_alice_acme", "edge_bob_alice"]);
|
||||
|
||||
// Edge Alice -> Acme (WORKS_AT) under Alice hover
|
||||
const aliceAcmeAttrs = graph.getEdgeAttributes("edge_alice_acme");
|
||||
assert.equal(aliceAcmeAttrs.edgeType, "WORKS_AT");
|
||||
const aliceAcmeState = resolveEdgeVisualState(
|
||||
"edge_alice_acme",
|
||||
"alice",
|
||||
"acme",
|
||||
"structure",
|
||||
"alice",
|
||||
"",
|
||||
"",
|
||||
new Set(),
|
||||
new Set(),
|
||||
aliceIncidentEdges,
|
||||
);
|
||||
assert.equal(aliceAcmeState, "hovered");
|
||||
const aliceAcmeStyle = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"structure",
|
||||
"hovered",
|
||||
aliceAcmeAttrs,
|
||||
"alice",
|
||||
"acme",
|
||||
"full",
|
||||
"edge_alice_acme",
|
||||
);
|
||||
assert.equal(aliceAcmeStyle.hidden, false, "Incident edge WORKS_AT must remain visible on hover");
|
||||
assert.ok(aliceAcmeStyle.size !== undefined && aliceAcmeStyle.size > 0);
|
||||
|
||||
// Edge Bob -> Alice (KNOWS) under Alice hover
|
||||
const bobAliceAttrs = graph.getEdgeAttributes("edge_bob_alice");
|
||||
assert.equal(bobAliceAttrs.edgeType, "KNOWS");
|
||||
const bobAliceState = resolveEdgeVisualState(
|
||||
"edge_bob_alice",
|
||||
"bob",
|
||||
"alice",
|
||||
"structure",
|
||||
"alice",
|
||||
"",
|
||||
"",
|
||||
new Set(),
|
||||
new Set(),
|
||||
aliceIncidentEdges,
|
||||
);
|
||||
assert.equal(bobAliceState, "hovered");
|
||||
const bobAliceStyle = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"structure",
|
||||
"hovered",
|
||||
bobAliceAttrs,
|
||||
"bob",
|
||||
"alice",
|
||||
"full",
|
||||
"edge_bob_alice",
|
||||
);
|
||||
assert.equal(bobAliceStyle.hidden, false, "Incident edge KNOWS must remain visible on hover");
|
||||
|
||||
// Non-incident edge Acme -> New York (LOCATED_IN) under Alice hover
|
||||
const acmeNyAttrs = graph.getEdgeAttributes("edge_acme_new_york");
|
||||
assert.equal(acmeNyAttrs.edgeType, "LOCATED_IN");
|
||||
const acmeNyState = resolveEdgeVisualState(
|
||||
"edge_acme_new_york",
|
||||
"acme",
|
||||
"new_york",
|
||||
"structure",
|
||||
"alice",
|
||||
"",
|
||||
"",
|
||||
new Set(),
|
||||
new Set(),
|
||||
aliceIncidentEdges,
|
||||
);
|
||||
assert.equal(acmeNyState, "muted");
|
||||
|
||||
// Scenario 2: Hover Acme
|
||||
// Incident edges: Alice -> Acme (WORKS_AT) and Acme -> New York (LOCATED_IN)
|
||||
const acmeAttrs = graph.getNodeAttributes("acme");
|
||||
const acmeStyle = resolveNodeElementStyle(GRAPH_THEME, "structure", "hovered", acmeAttrs, "Acme");
|
||||
assert.equal(acmeStyle.forceLabel, true);
|
||||
assert.equal(acmeStyle.label, "Acme");
|
||||
|
||||
const acmeIncidentEdges = new Set(["edge_alice_acme", "edge_acme_new_york"]);
|
||||
const acmeNyHoverState = resolveEdgeVisualState(
|
||||
"edge_acme_new_york",
|
||||
"acme",
|
||||
"new_york",
|
||||
"structure",
|
||||
"acme",
|
||||
"",
|
||||
"",
|
||||
new Set(),
|
||||
new Set(),
|
||||
acmeIncidentEdges,
|
||||
);
|
||||
assert.equal(acmeNyHoverState, "hovered");
|
||||
const acmeNyHoverStyle = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"structure",
|
||||
"hovered",
|
||||
acmeNyAttrs,
|
||||
"acme",
|
||||
"new_york",
|
||||
"full",
|
||||
"edge_acme_new_york",
|
||||
);
|
||||
assert.equal(acmeNyHoverStyle.hidden, false, "Incident edge LOCATED_IN must remain visible on hover");
|
||||
|
||||
// Scenario 3: Hover Bob
|
||||
// Incident edge: Bob -> Alice (KNOWS)
|
||||
const bobAttrs = graph.getNodeAttributes("bob");
|
||||
const bobStyle = resolveNodeElementStyle(GRAPH_THEME, "structure", "hovered", bobAttrs, "Bob");
|
||||
assert.equal(bobStyle.forceLabel, true);
|
||||
assert.equal(bobStyle.label, "Bob");
|
||||
|
||||
const bobIncidentEdges = new Set(["edge_bob_alice"]);
|
||||
const bobAliceHoverState = resolveEdgeVisualState(
|
||||
"edge_bob_alice",
|
||||
"bob",
|
||||
"alice",
|
||||
"structure",
|
||||
"bob",
|
||||
"",
|
||||
"",
|
||||
new Set(),
|
||||
new Set(),
|
||||
bobIncidentEdges,
|
||||
);
|
||||
assert.equal(bobAliceHoverState, "hovered");
|
||||
});
|
||||
|
||||
test("edge selection maintains canonical edge type labels and active visual state", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
const edgeCases = [
|
||||
{ id: "edge_alice_acme", source: "alice", target: "acme", label: "WORKS_AT" },
|
||||
{ id: "edge_bob_alice", source: "bob", target: "alice", label: "KNOWS" },
|
||||
{ id: "edge_acme_new_york", source: "acme", target: "new_york", label: "LOCATED_IN" },
|
||||
];
|
||||
|
||||
for (const { id, source, target, label } of edgeCases) {
|
||||
const attrs = graph.getEdgeAttributes(id);
|
||||
assert.equal(attrs.edgeType, label);
|
||||
|
||||
const visualState = resolveEdgeVisualState(
|
||||
id,
|
||||
source,
|
||||
target,
|
||||
"inspection",
|
||||
null,
|
||||
"",
|
||||
id, // selected edge
|
||||
new Set(),
|
||||
new Set(),
|
||||
);
|
||||
assert.equal(visualState, "selected", `Selected edge ${id} must resolve to 'selected' state`);
|
||||
|
||||
const style = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"inspection",
|
||||
"selected",
|
||||
attrs,
|
||||
source,
|
||||
target,
|
||||
"full",
|
||||
id,
|
||||
"selected",
|
||||
);
|
||||
|
||||
assert.equal(style.hidden, false, `Selected edge ${id} (${label}) must not be hidden`);
|
||||
assert.ok(
|
||||
style.size !== undefined && style.size > 0,
|
||||
`Selected edge ${id} (${label}) must have positive render size`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("node labels remain forced visible during hover, selection, and inspection zoom tier", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
const nodes = ["alice", "bob", "acme", "new_york"];
|
||||
|
||||
for (const nid of nodes) {
|
||||
const attrs = graph.getNodeAttributes(nid);
|
||||
|
||||
// Hover state forces label visibility
|
||||
const hoverForcesLabel = shouldForceNodeLabel(GRAPH_THEME, "structure", "hovered", attrs, 0);
|
||||
assert.equal(hoverForcesLabel, true, `Node ${nid} label must force visible on hover`);
|
||||
|
||||
// Selected state forces label visibility
|
||||
const selectForcesLabel = shouldForceNodeLabel(GRAPH_THEME, "structure", "selected", attrs, 0);
|
||||
assert.equal(selectForcesLabel, true, `Node ${nid} label must force visible on selection`);
|
||||
|
||||
// Resolved style emits actual string label
|
||||
const style = resolveNodeElementStyle(GRAPH_THEME, "inspection", "hovered", attrs, attrs.label);
|
||||
assert.equal(style.forceLabel, true);
|
||||
assert.equal(style.label, attrs.label);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,150 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { createTemporalSnapshotGuards } from "../src/workspaces/GraphWorkspace/temporalSnapshotGuards.ts";
|
||||
|
||||
const POSITION_1 = new Date("2023-07-02T00:00:00Z").getTime();
|
||||
const POSITION_2 = new Date("2024-01-02T00:00:00Z").getTime();
|
||||
const POSITION_3 = new Date("2024-07-02T00:00:00Z").getTime();
|
||||
|
||||
const SNAPSHOT = { active_node_ids: ["n1", "n2"], active_node_count: 2 };
|
||||
|
||||
// ── begin: one request per scrubber position ─────────────────────────────────
|
||||
|
||||
test("begin: a new position returns a fresh request sequence", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
assert.deepEqual(guards.begin(POSITION_1), { seq: 1, cached: null });
|
||||
});
|
||||
|
||||
test("begin: an identical in-flight request is deduplicated (no duplicate fetch)", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
guards.begin(POSITION_1);
|
||||
assert.deepEqual(guards.begin(POSITION_1), { seq: null, cached: null });
|
||||
});
|
||||
|
||||
test("begin: distinct positions request independently", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
assert.equal(guards.begin(POSITION_1).seq, 1);
|
||||
assert.equal(guards.begin(POSITION_2).seq, 2);
|
||||
});
|
||||
|
||||
test("begin: revisiting an applied position returns its cached snapshot", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
const revisit = guards.begin(POSITION_1);
|
||||
assert.equal(revisit.seq, 2);
|
||||
assert.deepEqual(revisit.cached, SNAPSHOT);
|
||||
});
|
||||
|
||||
test("begin: a failed position (finished) can be requested again", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.finish(POSITION_1, seq);
|
||||
const retry = guards.begin(POSITION_1);
|
||||
assert.equal(retry.seq, 2);
|
||||
assert.equal(retry.cached, null);
|
||||
});
|
||||
|
||||
test("finish: does not clear a position whose snapshot was already applied", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.finish(POSITION_1, seq);
|
||||
assert.deepEqual(guards.begin(POSITION_1).cached, SNAPSHOT);
|
||||
});
|
||||
|
||||
test("finish: a stale sequence cannot release a newer request's position", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const first = guards.begin(POSITION_1);
|
||||
guards.finish(POSITION_1, first.seq);
|
||||
guards.begin(POSITION_1); // seq 2, in flight again
|
||||
guards.finish(POSITION_1, first.seq); // stale seq: must not release seq 2
|
||||
assert.deepEqual(guards.begin(POSITION_1), { seq: null, cached: null });
|
||||
});
|
||||
|
||||
// ── shouldApply: applied only while the scrubber is on that position ─────────
|
||||
|
||||
test("shouldApply: the current position's response is applied", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq), true);
|
||||
});
|
||||
|
||||
test("shouldApply: a response for a position the scrubber left is discarded", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq: seq1 } = guards.begin(POSITION_1);
|
||||
guards.begin(POSITION_2);
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq1), false);
|
||||
assert.equal(guards.shouldApply(POSITION_2, 2), true);
|
||||
});
|
||||
|
||||
test("shouldApply: a late response for the position the scrubber returned to is applied", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq: seq1 } = guards.begin(POSITION_1);
|
||||
const { seq: seq2 } = guards.begin(POSITION_2);
|
||||
guards.begin(POSITION_1); // back to 1: deduplicated, no new request
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq1), true);
|
||||
assert.equal(guards.shouldApply(POSITION_2, seq2), false);
|
||||
});
|
||||
|
||||
test("shouldApply: an unknown sequence is discarded", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
guards.begin(POSITION_1);
|
||||
assert.equal(guards.shouldApply(POSITION_1, 99), false);
|
||||
});
|
||||
|
||||
test("shouldApply: after a reset no pre-reset response applies", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.reset();
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq), false);
|
||||
});
|
||||
|
||||
// ── apply: caching for revisits ─────────────────────────────────────────────
|
||||
|
||||
test("apply: stores the snapshot so a revisit re-applies it without a request", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.begin(POSITION_2);
|
||||
assert.deepEqual(guards.begin(POSITION_1).cached, SNAPSHOT);
|
||||
});
|
||||
|
||||
test("apply: play wrap-around re-applies the wrapped-to position's snapshot", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.begin(POSITION_2);
|
||||
guards.begin(POSITION_3);
|
||||
const wrap = guards.begin(POSITION_1);
|
||||
assert.deepEqual(wrap.cached, SNAPSHOT);
|
||||
assert.equal(guards.shouldApply(POSITION_1, wrap.seq), true);
|
||||
});
|
||||
|
||||
// ── reset: graph reload ─────────────────────────────────────────────────────
|
||||
|
||||
test("reset: clears requested and cached state so positions refetch", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.reset();
|
||||
const fresh = guards.begin(POSITION_1);
|
||||
assert.equal(fresh.seq, 1);
|
||||
assert.equal(fresh.cached, null);
|
||||
});
|
||||
|
||||
// ── cache bound ─────────────────────────────────────────────────────────────
|
||||
|
||||
test("cache: oldest positions are evicted when the cache is full", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const count = 300;
|
||||
for (let i = 0; i < count; i++) {
|
||||
const { seq } = guards.begin(POSITION_1 + i * 1000);
|
||||
guards.apply(POSITION_1 + i * 1000, seq, SNAPSHOT);
|
||||
}
|
||||
const oldest = guards.begin(POSITION_1);
|
||||
assert.equal(oldest.cached, null); // evicted: must refetch on revisit
|
||||
const newest = guards.begin(POSITION_1 + (count - 1) * 1000);
|
||||
assert.deepEqual(newest.cached, SNAPSHOT); // still cached
|
||||
});
|
||||
+2
-1
@@ -124,12 +124,13 @@ shacl = ["pyshacl>=0.25.0"]
|
||||
db-snowflake = ["snowflake-connector-python>=4.6.0", "cryptography>=49.0.0"]
|
||||
db-databricks = ["databricks-sdk>=0.60.0", "databricks-sql-connector>=4.0.0"]
|
||||
db-arrow = ["pyarrow>=24.0.0"]
|
||||
db-salesforce = ["simple-salesforce>=1.12.0"]
|
||||
ingest-parquet = ["pyarrow>=24.0.0"]
|
||||
ingest-arrow = ["pyarrow>=24.0.0"]
|
||||
ingest-sap = ["requests>=2.28.0"]
|
||||
|
||||
db-all = [
|
||||
"semantica[db-snowflake,db-databricks,db-arrow]"
|
||||
"semantica[db-snowflake,db-databricks,db-salesforce,db-arrow]"
|
||||
]
|
||||
|
||||
# ---- Embedding / Models ----
|
||||
|
||||
@@ -306,10 +306,10 @@ class JSONExporter:
|
||||
|
||||
self.logger.debug(f"Exporting {len(entities)} entity(ies) to JSON")
|
||||
|
||||
# Build JSON data with JSON-LD context
|
||||
# Build JSON data with JSON-LD context. No @vocab: it would expand
|
||||
# every bare key in the caller's entity dicts into ns# (#1146).
|
||||
json_data = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"entities": {"@id": "semantica:entities", "@container": "@list"},
|
||||
},
|
||||
@@ -339,7 +339,6 @@ class JSONExporter:
|
||||
"""
|
||||
json_data = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"relationships": {
|
||||
"@id": "semantica:relationships",
|
||||
@@ -434,11 +433,14 @@ class JSONExporter:
|
||||
Returns:
|
||||
Dictionary in JSON-LD format with @context, @graph/@value, and metadata
|
||||
"""
|
||||
# Initialize JSON-LD structure with context
|
||||
# Initialize JSON-LD structure with context. No @vocab: for a generic
|
||||
# payload it turned whatever bare keys the caller happened to use into
|
||||
# ns# terms (#1146). Undeclared terms now simply expand to nothing,
|
||||
# which is standard JSON-LD behaviour for a context that does not
|
||||
# know them; the raw payload is still in the document.
|
||||
jsonld = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": "https://semantica.dev/ns#",
|
||||
"semantica": SEMANTICA_NS,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -598,13 +600,21 @@ class JSONExporter:
|
||||
Returns:
|
||||
Dictionary in JSON-LD format with @context, @id, @type, and graph data
|
||||
"""
|
||||
# Initialize JSON-LD structure with RDF context
|
||||
# Initialize JSON-LD structure with RDF context. No @vocab: it applied
|
||||
# to every bare term in caller data, so an extracted type like "ORG"
|
||||
# became ns#ORG and a metadata key like "source" collided with the
|
||||
# real sem:source object property (#1146). Only explicit semantica:
|
||||
# terms resolve now, and the caller's metadata dict is typed @json so
|
||||
# it survives as one rdf:JSON literal instead of expanding its keys.
|
||||
jsonld = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": "https://semantica.dev/ns#",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
||||
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
||||
"semantica:metadata": {
|
||||
"@id": "semantica:metadata",
|
||||
"@type": "@json",
|
||||
},
|
||||
},
|
||||
# Minted from the graph's own content rather than the wall clock
|
||||
# (#1147): re-exporting an unchanged graph must produce the same
|
||||
@@ -664,14 +674,23 @@ class JSONExporter:
|
||||
entity_text = entity.get("text") or entity.get("label", "unknown")
|
||||
entity_id = entity.get("id") or mint_entity_iri(entity_text)
|
||||
|
||||
# The caller's type label is data, not a class we define: minting it
|
||||
# into @type expanded it through @vocab into ns#ORG and friends, terms
|
||||
# that look official but do not exist (#1146). The node is always a
|
||||
# semantica:Entity and the label travels as semantica:type, exactly
|
||||
# how _relationship_to_jsonld has always carried the relationship type.
|
||||
jsonld = {
|
||||
"@id": entity_id,
|
||||
"@type": entity.get("type") or "semantica:Entity",
|
||||
"@type": "semantica:Entity",
|
||||
"semantica:text": entity.get("text") or entity.get("label", ""),
|
||||
"semantica:confidence": entity.get("confidence", 1.0),
|
||||
}
|
||||
entity_type = entity.get("type")
|
||||
if entity_type:
|
||||
jsonld["semantica:type"] = entity_type
|
||||
|
||||
# Add metadata if present
|
||||
# Add metadata if present. The @json term definition on
|
||||
# semantica:metadata keeps the whole dict one rdf:JSON literal.
|
||||
if "metadata" in entity:
|
||||
jsonld["semantica:metadata"] = entity["metadata"]
|
||||
|
||||
|
||||
@@ -1226,11 +1226,14 @@ class RDFSerializer:
|
||||
metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None))
|
||||
graph_uri: Optional[str] = options.pop("graph_uri", None)
|
||||
|
||||
# Initialize JSON-LD structure with context
|
||||
# Initialize JSON-LD structure with context. No @vocab: it applied to
|
||||
# every bare term in caller data, so an extracted type like "ORG"
|
||||
# became ns#ORG and a metadata key like "source" collided with the
|
||||
# real sem:source object property (#1146). Only explicit semantica:
|
||||
# terms resolve now.
|
||||
jsonld = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": "https://semantica.dev/ns#",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
||||
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
||||
},
|
||||
@@ -1252,11 +1255,20 @@ class RDFSerializer:
|
||||
# and was dropped in full by a JSON-LD parser, silently.
|
||||
entity_id = entity.get("id") or mint_entity_iri(entity.get("text", ""))
|
||||
|
||||
# The caller's type label is data, not a class we define: minting
|
||||
# it into @type expanded it through @vocab into ns#ORG and
|
||||
# friends, terms that look official but do not exist (#1146).
|
||||
# The node is always a semantica:Entity and the label travels as
|
||||
# semantica:type, matching the relationship node below and
|
||||
# JSONExporter._entity_to_jsonld.
|
||||
node = {
|
||||
"@id": entity_id,
|
||||
"@type": entity.get("type", "semantica:Entity"),
|
||||
"@type": "semantica:Entity",
|
||||
"semantica:text": entity.get("text") or entity.get("label", ""),
|
||||
}
|
||||
entity_type = entity.get("type")
|
||||
if entity_type:
|
||||
node["semantica:type"] = entity_type
|
||||
confidence = normalize_confidence(entity.get("confidence", 1.0))
|
||||
if confidence is None:
|
||||
self.logger.warning(
|
||||
|
||||
@@ -130,7 +130,10 @@ Example Usage:
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from typing import Any, Dict, Tuple
|
||||
from typing import TYPE_CHECKING, Any, Dict, Tuple
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .salesforce_ingestor import SalesforceConnector, SalesforceData, SalesforceIngestor
|
||||
|
||||
from .config import IngestConfig, ingest_config
|
||||
from .file_ingestor import (
|
||||
@@ -152,6 +155,7 @@ from .methods import (
|
||||
ingest_parquet,
|
||||
ingest_public_api,
|
||||
ingest_repository,
|
||||
ingest_salesforce,
|
||||
ingest_stream,
|
||||
ingest_web,
|
||||
ingest_xml,
|
||||
@@ -235,6 +239,10 @@ _LAZY_EXPORTS: Dict[str, Tuple[str, str]] = {
|
||||
# XML ingestion
|
||||
"XMLIngestor": (".xml_ingestor", "XMLIngestor"),
|
||||
"XMLIngestionData": (".xml_ingestor", "XMLIngestionData"),
|
||||
# Salesforce ingestion
|
||||
"SalesforceIngestor": (".salesforce_ingestor", "SalesforceIngestor"),
|
||||
"SalesforceData": (".salesforce_ingestor", "SalesforceData"),
|
||||
"SalesforceConnector": (".salesforce_ingestor", "SalesforceConnector"),
|
||||
}
|
||||
|
||||
_OPTIONAL_DEPENDENCY_MESSAGES = {
|
||||
@@ -262,6 +270,11 @@ _OPTIONAL_DEPENDENCY_MESSAGES = {
|
||||
"Arrow ingestion requires optional dependency 'pyarrow'. "
|
||||
"Install it before importing ArrowIngestor or using ingest_arrow()."
|
||||
),
|
||||
".salesforce_ingestor": (
|
||||
"Salesforce ingestion requires optional dependency 'simple-salesforce'. "
|
||||
"Install it with: pip install \"semantica[db-salesforce]\" "
|
||||
"or: pip install simple-salesforce>=1.12.0"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -276,7 +289,7 @@ def __getattr__(name: str) -> Any:
|
||||
except ModuleNotFoundError as exc:
|
||||
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
|
||||
missing_name = getattr(exc, "name", None)
|
||||
if message and missing_name in {"git", "bs4", "pyarrow"}:
|
||||
if message and missing_name in {"git", "bs4", "pyarrow", "simple_salesforce"}:
|
||||
raise ImportError(message) from exc
|
||||
raise
|
||||
|
||||
@@ -366,6 +379,10 @@ __all__ = [
|
||||
# XML ingestion
|
||||
"XMLIngestor",
|
||||
"XMLIngestionData",
|
||||
# Salesforce ingestion
|
||||
"SalesforceIngestor",
|
||||
"SalesforceData",
|
||||
"SalesforceConnector",
|
||||
# Registry and Methods
|
||||
"MethodRegistry",
|
||||
"method_registry",
|
||||
@@ -377,6 +394,7 @@ __all__ = [
|
||||
"ingest_repository",
|
||||
"ingest_email",
|
||||
"ingest_database",
|
||||
"ingest_salesforce",
|
||||
"ingest_ontology",
|
||||
"ingest_arrow",
|
||||
"ingest_parquet",
|
||||
|
||||
@@ -186,8 +186,13 @@ class IngestConfig:
|
||||
self._method_configs[method] = config
|
||||
|
||||
def get_method_config(self, method: str) -> Dict:
|
||||
"""Get method-specific configuration."""
|
||||
return self._method_configs.get(method, {})
|
||||
"""Get method-specific configuration.
|
||||
|
||||
Returns a **copy** of the stored method configuration so callers can
|
||||
safely mutate it (e.g. to merge per-call options) without poisoning the
|
||||
global configuration for subsequent calls.
|
||||
"""
|
||||
return dict(self._method_configs.get(method, {}))
|
||||
|
||||
def get_all(self) -> Dict[str, Any]:
|
||||
"""Get all configuration."""
|
||||
|
||||
@@ -875,56 +875,95 @@ schema = connector.get_schema(engine)
|
||||
print(f" {table_name}: {[col['name'] for col in columns]}")
|
||||
```
|
||||
|
||||
## SAP OData Ingestion
|
||||
## Salesforce CRM Ingestion
|
||||
|
||||
`SAPIngestor` reads an Entity Set from a SAP OData service — S/4HANA Cloud,
|
||||
SuccessFactors, or an on-prem NetWeaver Gateway over its REST surface. It
|
||||
follows OData v2/v4 server-driven pagination and flattens each record into a
|
||||
document dict via `export_as_documents()`.
|
||||
Salesforce ingestion requires `simple-salesforce`:
|
||||
|
||||
Install with `pip install 'semantica[ingest-sap]'`.
|
||||
|
||||
### Connector Construction & Authentication
|
||||
|
||||
```python
|
||||
from semantica.ingest import SAPIngestor
|
||||
|
||||
# OAuth2 client-credentials (BTP / S/4HANA Cloud)
|
||||
ing = SAPIngestor(
|
||||
base_url="https://my-sap.example.com/sap/opu/odata/sap/API_BUSINESS_PARTNER",
|
||||
client_id="...", client_secret="...",
|
||||
token_url="https://my-sap.example.com/oauth/token",
|
||||
)
|
||||
# On-prem NetWeaver often uses Basic auth instead — swap the block above for:
|
||||
# ing = SAPIngestor(base_url="...", username="erp_user", password="...")
|
||||
```bash
|
||||
pip install "semantica[db-salesforce]"
|
||||
```
|
||||
|
||||
### Entity-Set Ingestion & Document Export
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
# 1. Discover entity sets + field types from $metadata
|
||||
sets = ing.discover_service()
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
import os
|
||||
|
||||
# 2. Page-walk an Entity Set (v2/v4 next links handled automatically)
|
||||
partners = ing.ingest_entity_set(
|
||||
entity_set="A_BusinessPartnerSet",
|
||||
select="BusinessPartner,BusinessPartnerFullName",
|
||||
top=1000,
|
||||
ingestor = SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
domain="login", # "test" for sandbox
|
||||
)
|
||||
|
||||
# 3. Flatten to document dicts that GraphBuilder can consume directly
|
||||
docs = ing.export_as_documents(partners)
|
||||
# Ingest Account records
|
||||
data = ingestor.ingest_sobject(
|
||||
"Account",
|
||||
fields=["Id", "Name", "Industry", "BillingCity"],
|
||||
where="Type = 'Customer'",
|
||||
limit=5000,
|
||||
)
|
||||
print(f"Retrieved {data.row_count} of {data.total_size} matching records")
|
||||
```
|
||||
|
||||
- Use `expand="to_Item"` on a sales-order header set to pull nested line items
|
||||
in one request — handy for modeling order → line-item → material relations.
|
||||
- Every outbound request, including the OAuth2 token exchange, is routed through
|
||||
the SSRF guard, and pagination never follows a next link that points to a
|
||||
different host than the service root.
|
||||
`SalesforceIngestor()` with no arguments reads from `SALESFORCE_USERNAME`, `SALESFORCE_PASSWORD`, `SALESFORCE_SECURITY_TOKEN`, and `SALESFORCE_DOMAIN` environment variables automatically.
|
||||
|
||||
### Custom Objects and Raw SOQL
|
||||
|
||||
```python
|
||||
# Custom object (API name ends in __c)
|
||||
data = ingestor.ingest_sobject("My_Custom_Object__c", fields=["Id", "Name", "Custom_Field__c"])
|
||||
|
||||
# Raw SOQL GÇö pagination is handled automatically
|
||||
data = ingestor.ingest_query("""
|
||||
SELECT Id, Name, StageName, Amount
|
||||
FROM Opportunity
|
||||
WHERE IsClosed = false
|
||||
ORDER BY CloseDate ASC
|
||||
""")
|
||||
print(f"Open opportunities: {data.row_count}")
|
||||
```
|
||||
|
||||
### Document Export
|
||||
|
||||
```python
|
||||
documents = ingestor.export_as_documents(
|
||||
data,
|
||||
id_field="Id", # Salesforce 18-char record Id
|
||||
text_fields=["Name", "Description"],
|
||||
)
|
||||
# Each document: {"id": "001...", "text": "...", "metadata": {"source": "salesforce", ...}}
|
||||
```
|
||||
|
||||
### Convenience Function
|
||||
|
||||
```python
|
||||
from semantica.ingest import ingest_salesforce
|
||||
|
||||
# Fetch records
|
||||
data = ingest_salesforce(
|
||||
method="sobject",
|
||||
sobject_name="Account",
|
||||
fields=["Id", "Name"],
|
||||
limit=500,
|
||||
)
|
||||
|
||||
# Ingest and export as documents in one call
|
||||
docs = ingest_salesforce(
|
||||
method="documents",
|
||||
sobject_name="Account",
|
||||
text_fields=["Name", "Description"],
|
||||
)
|
||||
|
||||
# Using the unified dispatcher
|
||||
from semantica.ingest import ingest
|
||||
result = ingest(None, source_type="salesforce", method="sobject",
|
||||
sobject_name="Account", fields=["Id", "Name"])
|
||||
data = result["data"]
|
||||
```
|
||||
|
||||
See [Salesforce Integration](https://docs.getsemantica.ai/integrations/salesforce) for full documentation including sandbox, schema discovery, pagination details, and troubleshooting.
|
||||
|
||||
> **Security Note:** Never hardcode credentials (`client_secret`, `password`);
|
||||
> pass them via environment variables (`SAP_CLIENT_SECRET`, `SAP_PASSWORD`) or a
|
||||
> secrets manager.
|
||||
|
||||
## MCP Server Ingestion
|
||||
|
||||
@@ -1664,3 +1703,54 @@ for source_type, source_list in sources.items():
|
||||
for batch in process_in_batches(large_dataset, batch_size=1000):
|
||||
result = ingest(batch)
|
||||
```
|
||||
|
||||
## SAP OData Ingestion
|
||||
|
||||
`SAPIngestor` reads an Entity Set from a SAP OData service — S/4HANA Cloud,
|
||||
SuccessFactors, or an on-prem NetWeaver Gateway over its REST surface. It
|
||||
follows OData v2/v4 server-driven pagination and flattens each record into a
|
||||
document dict via `export_as_documents()`.
|
||||
|
||||
Install with `pip install 'semantica[ingest-sap]'`.
|
||||
|
||||
### Connector Construction & Authentication
|
||||
|
||||
```python
|
||||
from semantica.ingest import SAPIngestor
|
||||
|
||||
# OAuth2 client-credentials (BTP / S/4HANA Cloud)
|
||||
ing = SAPIngestor(
|
||||
base_url="https://my-sap.example.com/sap/opu/odata/sap/API_BUSINESS_PARTNER",
|
||||
client_id="...", client_secret="...",
|
||||
token_url="https://my-sap.example.com/oauth/token",
|
||||
)
|
||||
# On-prem NetWeaver often uses Basic auth instead — swap the block above for:
|
||||
# ing = SAPIngestor(base_url="...", username="erp_user", password="...")
|
||||
```
|
||||
|
||||
### Entity-Set Ingestion & Document Export
|
||||
|
||||
```python
|
||||
# 1. Discover entity sets + field types from $metadata
|
||||
sets = ing.discover_service()
|
||||
|
||||
# 2. Page-walk an Entity Set (v2/v4 next links handled automatically)
|
||||
partners = ing.ingest_entity_set(
|
||||
entity_set="A_BusinessPartnerSet",
|
||||
select="BusinessPartner,BusinessPartnerFullName",
|
||||
top=1000,
|
||||
)
|
||||
|
||||
# 3. Flatten to document dicts that GraphBuilder can consume directly
|
||||
docs = ing.export_as_documents(partners)
|
||||
```
|
||||
|
||||
- Use `expand="to_Item"` on a sales-order header set to pull nested line items
|
||||
in one request — handy for modeling order → line-item → material relations.
|
||||
- Every outbound request, including the OAuth2 token exchange, is routed through
|
||||
the SSRF guard, and pagination never follows a next link that points to a
|
||||
different host than the service root.
|
||||
|
||||
> **Security Note:** Never hardcode credentials (`client_secret`, `password`);
|
||||
> pass them via environment variables (`SAP_CLIENT_SECRET`, `SAP_PASSWORD`) or a
|
||||
> secrets manager.
|
||||
|
||||
@@ -203,6 +203,7 @@ if TYPE_CHECKING:
|
||||
from .ontology_ingestor import OntologyData
|
||||
from .parquet_ingestor import ParquetData
|
||||
from .public_api_ingestor import PublicAPIDetection
|
||||
from .salesforce_ingestor import SalesforceData
|
||||
from .stream_ingestor import StreamProcessor
|
||||
from .web_ingestor import WebContent
|
||||
from .xml_ingestor import XMLIngestionData
|
||||
@@ -1126,6 +1127,213 @@ def ingest_database(
|
||||
raise
|
||||
|
||||
|
||||
def ingest_salesforce(
|
||||
source: Optional[Dict[str, Any]] = None,
|
||||
method: str = "sobject",
|
||||
**kwargs,
|
||||
) -> Union["SalesforceData", List[Dict[str, Any]], Dict[str, Any]]:
|
||||
"""Ingest data from Salesforce CRM (convenience function).
|
||||
|
||||
A user-friendly wrapper around :class:`~semantica.ingest.SalesforceIngestor`
|
||||
that connects, ingests, and returns data in a single call.
|
||||
|
||||
Args:
|
||||
source: Optional credential/configuration dictionary. Keys mirror the
|
||||
:class:`~semantica.ingest.SalesforceConnector` constructor:
|
||||
``username``, ``password``, ``security_token``, ``domain``
|
||||
(``"login"`` for production, ``"test"`` for sandbox),
|
||||
``instance_url``, ``session_id``, ``api_version``.
|
||||
When ``None``, credentials are read from environment variables
|
||||
(``SALESFORCE_USERNAME`` / ``SALESFORCE_PASSWORD`` /
|
||||
``SALESFORCE_SECURITY_TOKEN`` etc.).
|
||||
method: Ingestion method:
|
||||
|
||||
* ``"sobject"`` *(default)* — fetch records from a named sObject
|
||||
(requires ``sobject_name`` kwarg).
|
||||
* ``"query"`` — execute a raw SOQL query string (requires
|
||||
``soql`` kwarg).
|
||||
* ``"list_sobjects"`` — return a sorted list of accessible sObject
|
||||
API names.
|
||||
* ``"schema"`` — return field metadata for a named sObject
|
||||
(requires ``sobject_name`` kwarg).
|
||||
* ``"documents"`` — ingest an sObject and convert to the Semantica
|
||||
document format in one step (requires ``sobject_name`` kwarg;
|
||||
optional ``text_fields`` and ``id_field`` kwargs).
|
||||
|
||||
**kwargs: Additional options forwarded to the ingestor method.
|
||||
Common kwargs for ``"sobject"`` / ``"documents"``:
|
||||
|
||||
* ``sobject_name`` — Salesforce sObject API name (e.g.
|
||||
``"Account"``, ``"My_Custom__c"``).
|
||||
* ``fields`` — list of field API names to select. When omitted
|
||||
all selectable fields are fetched via ``describe()``.
|
||||
* ``where`` — SOQL ``WHERE`` clause fragment (trusted input only).
|
||||
* ``order_by`` — SOQL ``ORDER BY`` clause fragment.
|
||||
* ``limit`` — maximum number of records.
|
||||
|
||||
For ``"query"``:
|
||||
|
||||
* ``soql`` — full SOQL query string.
|
||||
|
||||
For ``"schema"``:
|
||||
|
||||
* ``sobject_name`` — sObject to describe.
|
||||
|
||||
Returns:
|
||||
* ``"sobject"`` / ``"query"`` → :class:`~semantica.ingest.SalesforceData`
|
||||
* ``"documents"`` → ``List[Dict[str, Any]]`` (Semantica document format)
|
||||
* ``"list_sobjects"`` → ``List[str]``
|
||||
* ``"schema"`` → ``Dict[str, Any]``
|
||||
|
||||
Raises:
|
||||
:class:`~semantica.utils.exceptions.ConfigurationError`: If
|
||||
``simple-salesforce`` is not installed.
|
||||
:class:`~semantica.utils.exceptions.ValidationError`: If credentials
|
||||
are incomplete or an sObject / field name is invalid.
|
||||
:class:`~semantica.utils.exceptions.ProcessingError`: If the
|
||||
Salesforce API call fails.
|
||||
|
||||
Examples::
|
||||
|
||||
>>> from semantica.ingest import ingest_salesforce
|
||||
|
||||
>>> # Fetch Account records (credentials from env vars)
|
||||
>>> data = ingest_salesforce(
|
||||
... method="sobject",
|
||||
... sobject_name="Account",
|
||||
... fields=["Id", "Name", "Industry"],
|
||||
... limit=500,
|
||||
... )
|
||||
|
||||
>>> # Execute a raw SOQL query (credentials from environment variables)
|
||||
>>> data = ingest_salesforce(
|
||||
... method="query",
|
||||
... soql="SELECT Id, Name FROM Contact WHERE IsActive = true",
|
||||
... )
|
||||
|
||||
>>> # Ingest and export as documents for GraphBuilder in one step
|
||||
>>> docs = ingest_salesforce(
|
||||
... method="documents",
|
||||
... sobject_name="Account",
|
||||
... text_fields=["Name", "Description"],
|
||||
... limit=1000,
|
||||
... )
|
||||
|
||||
>>> # List all accessible sObjects in the connected org
|
||||
>>> sobject_names = ingest_salesforce(method="list_sobjects")
|
||||
|
||||
>>> # Use sandbox org
|
||||
>>> data = ingest_salesforce(
|
||||
... method="sobject",
|
||||
... sobject_name="Account",
|
||||
... ) # set SALESFORCE_DOMAIN=test in environment for sandbox
|
||||
"""
|
||||
# Registry hook — allows callers to register a custom "salesforce" method
|
||||
custom_method = method_registry.get("salesforce", method)
|
||||
if custom_method and custom_method != ingest_salesforce:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source,
|
||||
fallback_on_custom_error=fallback, **kwargs,
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .salesforce_ingestor import SalesforceIngestor
|
||||
except ModuleNotFoundError as exc:
|
||||
if _is_missing_dependency(exc, "simple_salesforce"):
|
||||
raise _missing_optional_dependency(
|
||||
"Salesforce ingestion", "simple-salesforce"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
# Unpack credential dict (if given); everything else stays in kwargs.
|
||||
creds: Dict[str, Any] = {}
|
||||
if source is not None:
|
||||
if not isinstance(source, dict):
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() source must be a credential dict or None. "
|
||||
"Pass sobject_name / soql as keyword arguments."
|
||||
)
|
||||
creds = dict(source)
|
||||
|
||||
# Merge any ingest_config method config under "salesforce".
|
||||
# get_method_config() now returns a copy, so this dict is safe to mutate.
|
||||
# We build the final connector config in order of increasing priority:
|
||||
# 1. base method config (lowest — global defaults set by operator)
|
||||
# 2. per-call credential dict supplied via `source`
|
||||
# 3. per-call connector params supplied as kwargs
|
||||
# Credentials are extracted from kwargs and removed so they don't also
|
||||
# flow into the ingest method call (which doesn't understand them).
|
||||
_CONNECTOR_PARAMS = frozenset({
|
||||
"username", "password", "security_token", "domain",
|
||||
"instance_url", "session_id", "api_version",
|
||||
})
|
||||
connector_kwargs = {k: v for k, v in kwargs.items() if k in _CONNECTOR_PARAMS}
|
||||
for k in _CONNECTOR_PARAMS:
|
||||
kwargs.pop(k, None)
|
||||
|
||||
# Build a fresh per-call config dict — never mutate the global store.
|
||||
config: Dict[str, Any] = {
|
||||
**ingest_config.get_method_config("salesforce"), # base (already a copy)
|
||||
**creds, # source dict credentials
|
||||
**connector_kwargs, # kwarg credentials
|
||||
}
|
||||
|
||||
ingestor = SalesforceIngestor(**config)
|
||||
|
||||
if method == "sobject":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='sobject' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
return ingestor.ingest_sobject(sobject_name, **kwargs)
|
||||
|
||||
elif method == "query":
|
||||
soql = kwargs.pop("soql", None)
|
||||
if not soql:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='query' requires "
|
||||
"soql keyword argument."
|
||||
)
|
||||
return ingestor.ingest_query(soql, **kwargs)
|
||||
|
||||
elif method == "list_sobjects":
|
||||
return ingestor.list_sobjects()
|
||||
|
||||
elif method == "schema":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='schema' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
return ingestor.get_sobject_schema(sobject_name)
|
||||
|
||||
elif method == "documents":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='documents' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
id_field = kwargs.pop("id_field", "Id")
|
||||
text_fields = kwargs.pop("text_fields", None)
|
||||
data = ingestor.ingest_sobject(sobject_name, **kwargs)
|
||||
return ingestor.export_as_documents(data, id_field=id_field,
|
||||
text_fields=text_fields)
|
||||
|
||||
else:
|
||||
raise ProcessingError(
|
||||
f"Unknown ingest_salesforce method: {method!r}. "
|
||||
"Valid methods: 'sobject', 'query', 'list_sobjects', 'schema', "
|
||||
"'documents'."
|
||||
)
|
||||
|
||||
|
||||
def ingest_mcp(
|
||||
source: Union[str, Dict[str, Any]],
|
||||
method: str = "resources",
|
||||
@@ -1306,6 +1514,7 @@ def ingest(
|
||||
- "ontology": Ontology ingestion
|
||||
- "parquet": Apache Parquet file or directory ingestion
|
||||
- "xml": XML file or directory ingestion
|
||||
- "salesforce": Salesforce CRM ingestion (pass credentials via kwargs)
|
||||
method: Optional specific ingestion method
|
||||
**kwargs: Additional options passed to ingestor
|
||||
|
||||
@@ -1428,6 +1637,9 @@ def ingest(
|
||||
return {"ontology": ingest_ontology(sources, method=method or "file", **kwargs)}
|
||||
elif source_type == "mcp":
|
||||
return {"data": ingest_mcp(sources, method=method or "resources", **kwargs)}
|
||||
elif source_type == "salesforce":
|
||||
return {"data": ingest_salesforce(sources,
|
||||
method=method or "sobject", **kwargs)}
|
||||
else:
|
||||
raise ProcessingError(f"Unknown source type: {source_type}")
|
||||
|
||||
@@ -1540,3 +1752,9 @@ method_registry.register("ontology", "file", ingest_ontology)
|
||||
method_registry.register("ontology", "directory", ingest_ontology)
|
||||
method_registry.register("ingest", "default", ingest)
|
||||
method_registry.register("ingest", "unified", ingest)
|
||||
method_registry.register("salesforce", "default", ingest_salesforce)
|
||||
method_registry.register("salesforce", "sobject", ingest_salesforce)
|
||||
method_registry.register("salesforce", "query", ingest_salesforce)
|
||||
method_registry.register("salesforce", "list_sobjects", ingest_salesforce)
|
||||
method_registry.register("salesforce", "schema", ingest_salesforce)
|
||||
method_registry.register("salesforce", "documents", ingest_salesforce)
|
||||
|
||||
@@ -66,6 +66,7 @@ class MethodRegistry:
|
||||
"parquet": {},
|
||||
"arrow": {},
|
||||
"xml": {},
|
||||
"salesforce": {},
|
||||
"ingest": {},
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -117,6 +117,8 @@ class PropertyGenerator:
|
||||
data_properties = self._infer_data_properties(entities, classes, **options)
|
||||
properties.extend(data_properties)
|
||||
|
||||
properties = self._coalesce_normalized_properties(properties)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
@@ -193,6 +195,67 @@ class PropertyGenerator:
|
||||
|
||||
return properties
|
||||
|
||||
def _coalesce_normalized_properties(
|
||||
self, properties: List[Dict[str, Any]]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Merge same-kind properties that normalize to the same name."""
|
||||
property_kinds = defaultdict(set)
|
||||
for prop in properties:
|
||||
property_kinds[prop["name"]].add(prop.get("type"))
|
||||
|
||||
collisions = {
|
||||
name: sorted(kind for kind in kinds if kind is not None)
|
||||
for name, kinds in property_kinds.items()
|
||||
if len({kind for kind in kinds if kind is not None}) > 1
|
||||
}
|
||||
if collisions:
|
||||
raise ValidationError(
|
||||
"Normalized property names cannot be shared by object and "
|
||||
"data properties.",
|
||||
validation_context={"property_kind_collisions": collisions},
|
||||
)
|
||||
|
||||
merged: Dict[tuple, Dict[str, Any]] = {}
|
||||
result = []
|
||||
for prop in properties:
|
||||
key = (prop.get("type"), prop["name"])
|
||||
existing = merged.get(key)
|
||||
if existing is None:
|
||||
merged[key] = prop
|
||||
result.append(prop)
|
||||
continue
|
||||
|
||||
existing["domain"] = self._merge_property_values(
|
||||
existing.get("domain", []), prop.get("domain", [])
|
||||
)
|
||||
if prop.get("type") == "object":
|
||||
existing["range"] = self._merge_property_values(
|
||||
existing.get("range", []), prop.get("range", [])
|
||||
)
|
||||
existing_metadata = existing.setdefault("metadata", {})
|
||||
existing_metadata["occurrence_count"] = (
|
||||
existing_metadata.get("occurrence_count", 0)
|
||||
+ prop.get("metadata", {}).get("occurrence_count", 0)
|
||||
)
|
||||
elif existing.get("range") != prop.get("range"):
|
||||
existing["range"] = self._get_more_general_type(
|
||||
existing["range"], prop["range"]
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _merge_property_values(current: Any, incoming: Any) -> List[Any]:
|
||||
"""Merge scalar-or-list property values while preserving input order."""
|
||||
values = list(current) if isinstance(current, list) else [current]
|
||||
incoming_values = (
|
||||
incoming if isinstance(incoming, list) else [incoming]
|
||||
)
|
||||
for value in incoming_values:
|
||||
if value not in values:
|
||||
values.append(value)
|
||||
return [value for value in values if value is not None]
|
||||
|
||||
def _infer_data_properties(
|
||||
self, entities: List[Dict[str, Any]], classes: List[Dict[str, Any]], **options
|
||||
) -> List[Dict[str, Any]]:
|
||||
|
||||
@@ -70,7 +70,8 @@ sem:metadata a owl:AnnotationProperty ;
|
||||
rdfs:label "metadata" ;
|
||||
rdfs:comment """Free-form metadata carried through from extraction. An
|
||||
annotation property because its value is an arbitrary structure rather than a
|
||||
modelled one.""" ;
|
||||
modelled one; in the JSON-LD export the whole mapping is written as one
|
||||
rdf:JSON literal so caller keys never expand into this namespace (#1146).""" ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
# ── Relationship terms (JSON-LD export) ──────────────────────────────────────
|
||||
@@ -96,10 +97,10 @@ sem:target a owl:ObjectProperty ;
|
||||
|
||||
sem:type a owl:DatatypeProperty ;
|
||||
rdfs:label "type" ;
|
||||
rdfs:comment """The relationship type as a label, as emitted in the JSON-LD
|
||||
export. Distinct from rdf:type, which relates a node to a class rather than to
|
||||
a string.""" ;
|
||||
rdfs:domain sem:Relationship ;
|
||||
rdfs:comment """The entity or relationship type as a label, as emitted in
|
||||
the JSON-LD export. Distinct from rdf:type, which relates a node to a class
|
||||
rather than to a string. Emitted for both entities and relationships, so the
|
||||
domain is left open rather than tied to sem:Relationship.""" ;
|
||||
rdfs:range xsd:string ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
"""
|
||||
End-to-end integration test for deterministic Explorer rendering (Issue #1037).
|
||||
|
||||
Validates:
|
||||
1. Building the canonical 4-node, 3-edge graph using ContextGraph.
|
||||
2. Serialization via save_to_file().
|
||||
3. Reloading via load_from_file() and GraphSession.from_file() without mutation.
|
||||
4. Explorer HTTP API serving exact nodes, edges, edge types, and connectivity.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.app import create_app
|
||||
from semantica.explorer.session import GraphSession
|
||||
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip(
|
||||
"starlette TestClient required. Install semantica[explorer].",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
def _build_deterministic_graph() -> ContextGraph:
|
||||
"""Build the exact graph requested in Semantica #1037."""
|
||||
graph = ContextGraph(advanced_analytics=False)
|
||||
|
||||
graph.add_node("alice", "Person", content="Alice")
|
||||
graph.add_node("bob", "Person", content="Bob")
|
||||
graph.add_node("acme", "Organization", content="Acme")
|
||||
graph.add_node("new_york", "Location", content="New York")
|
||||
|
||||
graph.add_edge("alice", "acme", edge_type="WORKS_AT")
|
||||
graph.add_edge("bob", "alice", edge_type="KNOWS")
|
||||
graph.add_edge("acme", "new_york", edge_type="LOCATED_IN")
|
||||
|
||||
return graph
|
||||
|
||||
|
||||
class TestExplorerDeterministicRenderingE2E:
|
||||
"""E2E suite for deterministic graph rendering and session loading."""
|
||||
|
||||
def test_build_and_in_memory_structure(self):
|
||||
graph = _build_deterministic_graph()
|
||||
|
||||
assert len(graph.nodes) == 4
|
||||
assert len(graph.edges) == 3
|
||||
|
||||
node_map = {
|
||||
nid: (node.node_type, node.content) for nid, node in graph.nodes.items()
|
||||
}
|
||||
assert node_map["alice"] == ("Person", "Alice")
|
||||
assert node_map["bob"] == ("Person", "Bob")
|
||||
assert node_map["acme"] == ("Organization", "Acme")
|
||||
assert node_map["new_york"] == ("Location", "New York")
|
||||
|
||||
edge_tuples = {(e.source_id, e.target_id, e.edge_type) for e in graph.edges}
|
||||
assert edge_tuples == {
|
||||
("alice", "acme", "WORKS_AT"),
|
||||
("bob", "alice", "KNOWS"),
|
||||
("acme", "new_york", "LOCATED_IN"),
|
||||
}
|
||||
|
||||
def test_serialization_and_deserialization(self, tmp_path: Path):
|
||||
graph = _build_deterministic_graph()
|
||||
output_file = tmp_path / "explorer_deterministic_graph.json"
|
||||
|
||||
graph.save_to_file(str(output_file))
|
||||
assert output_file.exists()
|
||||
|
||||
# Validate JSON structure
|
||||
with open(output_file, "r", encoding="utf-8") as f:
|
||||
raw_data = json.load(f)
|
||||
|
||||
assert "nodes" in raw_data
|
||||
assert "edges" in raw_data
|
||||
assert len(raw_data["nodes"]) == 4
|
||||
assert len(raw_data["edges"]) == 3
|
||||
|
||||
# Reload with ContextGraph.load_from_file
|
||||
reloaded_graph = ContextGraph(advanced_analytics=False)
|
||||
reloaded_graph.load_from_file(str(output_file))
|
||||
|
||||
assert len(reloaded_graph.nodes) == 4
|
||||
assert len(reloaded_graph.edges) == 3
|
||||
reloaded_nodes = {
|
||||
nid: (node.node_type, node.content)
|
||||
for nid, node in reloaded_graph.nodes.items()
|
||||
}
|
||||
assert reloaded_nodes["alice"] == ("Person", "Alice")
|
||||
assert reloaded_nodes["bob"] == ("Person", "Bob")
|
||||
assert reloaded_nodes["acme"] == ("Organization", "Acme")
|
||||
assert reloaded_nodes["new_york"] == ("Location", "New York")
|
||||
|
||||
reloaded_edges = {
|
||||
(e.source_id, e.target_id, e.edge_type) for e in reloaded_graph.edges
|
||||
}
|
||||
assert reloaded_edges == {
|
||||
("alice", "acme", "WORKS_AT"),
|
||||
("bob", "alice", "KNOWS"),
|
||||
("acme", "new_york", "LOCATED_IN"),
|
||||
}
|
||||
|
||||
def test_session_loading_and_stats(self, tmp_path: Path):
|
||||
graph = _build_deterministic_graph()
|
||||
output_file = tmp_path / "explorer_deterministic_graph.json"
|
||||
graph.save_to_file(str(output_file))
|
||||
|
||||
session = GraphSession.from_file(str(output_file))
|
||||
stats = session.get_stats()
|
||||
|
||||
assert stats["node_count"] == 4
|
||||
assert stats["edge_count"] == 3
|
||||
|
||||
def test_explorer_api_endpoints_with_deterministic_graph(self, tmp_path: Path):
|
||||
graph = _build_deterministic_graph()
|
||||
output_file = tmp_path / "explorer_deterministic_graph.json"
|
||||
graph.save_to_file(str(output_file))
|
||||
|
||||
session = GraphSession.from_file(str(output_file))
|
||||
app = create_app(session=session)
|
||||
|
||||
with TestClient(app) as client:
|
||||
# 1. Health & Info
|
||||
health = client.get("/api/health")
|
||||
assert health.status_code == 200
|
||||
assert health.json() == {"status": "ok"}
|
||||
|
||||
info = client.get("/api/info")
|
||||
assert info.status_code == 200
|
||||
assert info.json()["status"] == "active"
|
||||
|
||||
# 2. Stats
|
||||
stats = client.get("/api/graph/stats")
|
||||
assert stats.status_code == 200
|
||||
stats_data = stats.json()
|
||||
assert stats_data["node_count"] == 4
|
||||
assert stats_data["edge_count"] == 3
|
||||
|
||||
# 3. Nodes endpoint
|
||||
nodes_res = client.get("/api/graph/nodes")
|
||||
assert nodes_res.status_code == 200
|
||||
nodes_data = nodes_res.json()
|
||||
assert nodes_data["total"] == 4
|
||||
assert len(nodes_data["nodes"]) == 4
|
||||
|
||||
returned_nodes = {
|
||||
n["id"]: (n["type"], n["content"]) for n in nodes_data["nodes"]
|
||||
}
|
||||
assert returned_nodes["alice"] == ("Person", "Alice")
|
||||
assert returned_nodes["bob"] == ("Person", "Bob")
|
||||
assert returned_nodes["acme"] == ("Organization", "Acme")
|
||||
assert returned_nodes["new_york"] == ("Location", "New York")
|
||||
|
||||
# 4. Individual node lookups
|
||||
for node_id in ["alice", "bob", "acme", "new_york"]:
|
||||
node_res = client.get(f"/api/graph/node/{node_id}")
|
||||
assert node_res.status_code == 200
|
||||
assert node_res.json()["id"] == node_id
|
||||
|
||||
# 5. Edges endpoint
|
||||
edges_res = client.get("/api/graph/edges")
|
||||
assert edges_res.status_code == 200
|
||||
edges_data = edges_res.json()
|
||||
assert edges_data["total"] == 3
|
||||
assert len(edges_data["edges"]) == 3
|
||||
|
||||
returned_edges = {
|
||||
(e["source"], e["target"], e["type"]) for e in edges_data["edges"]
|
||||
}
|
||||
assert returned_edges == {
|
||||
("alice", "acme", "WORKS_AT"),
|
||||
("bob", "alice", "KNOWS"),
|
||||
("acme", "new_york", "LOCATED_IN"),
|
||||
}
|
||||
|
||||
def test_deterministic_graph_auth_enforcement(self, tmp_path: Path, monkeypatch):
|
||||
graph = _build_deterministic_graph()
|
||||
output_file = tmp_path / "explorer_deterministic_graph.json"
|
||||
graph.save_to_file(str(output_file))
|
||||
|
||||
session = GraphSession.from_file(str(output_file))
|
||||
app = create_app(session=session)
|
||||
|
||||
with TestClient(app) as client:
|
||||
# 1. Unconfigured auth (no SEMANTICA_ALLOW_ANONYMOUS, no SEMANTICA_API_KEY) -> 503
|
||||
monkeypatch.delenv("SEMANTICA_ALLOW_ANONYMOUS", raising=False)
|
||||
monkeypatch.delenv("SEMANTICA_API_KEY", raising=False)
|
||||
unconfigured_res = client.get("/api/graph/stats")
|
||||
assert unconfigured_res.status_code == 503
|
||||
|
||||
# 2. Configured SEMANTICA_API_KEY without header -> 401
|
||||
test_key = "secret-test-key-1037"
|
||||
monkeypatch.setenv("SEMANTICA_API_KEY", test_key)
|
||||
unauthorized_res = client.get("/api/graph/nodes")
|
||||
assert unauthorized_res.status_code == 401
|
||||
|
||||
# 3. Configured SEMANTICA_API_KEY with valid X-API-Key header -> 200
|
||||
authorized_res = client.get(
|
||||
"/api/graph/nodes",
|
||||
headers={"X-API-Key": test_key},
|
||||
)
|
||||
assert authorized_res.status_code == 200
|
||||
assert authorized_res.json()["total"] == 4
|
||||
|
||||
# 4. Explicit local development opt-in: SEMANTICA_ALLOW_ANONYMOUS=true -> 200 without header
|
||||
monkeypatch.setenv("SEMANTICA_ALLOW_ANONYMOUS", "true")
|
||||
monkeypatch.delenv("SEMANTICA_API_KEY", raising=False)
|
||||
anon_res = client.get("/api/graph/edges")
|
||||
assert anon_res.status_code == 200
|
||||
assert anon_res.json()["total"] == 3
|
||||
|
||||
@@ -67,7 +67,7 @@ def test_merging_repeated_exports_yields_one_graph_node(tmp_path):
|
||||
assert len(kg_nodes) == 1
|
||||
|
||||
entity_nodes = set(
|
||||
merged.subjects(RDF.type, URIRef("https://semantica.dev/vocab/ORG"))
|
||||
merged.subjects(RDF.type, URIRef("https://semantica.dev/ns#Entity"))
|
||||
)
|
||||
assert len(entity_nodes) == 1
|
||||
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Caller data must never expand into the Semantica namespace (#1146).
|
||||
|
||||
``@vocab`` used to sit in every JSON-LD context pointing at ``ns#``, so every
|
||||
bare term in caller data expanded into it: an extracted type like ``"ORG"``
|
||||
became ``ns#ORG`` (a term the vocabulary does not define), and a metadata key
|
||||
like ``"source"`` collided with the real ``sem:source`` object property,
|
||||
attaching a plain string to a property whose range is a resource. The fix
|
||||
removes ``@vocab`` outright: only explicit ``semantica:``-prefixed terms
|
||||
resolve, caller type labels travel as ``semantica:type`` strings, and caller
|
||||
metadata survives as one ``rdf:JSON`` literal.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from rdflib import RDF, Graph, Literal, URIRef
|
||||
|
||||
from semantica.export.json_exporter import JSONExporter
|
||||
from semantica.export.rdf_exporter import RDFExporter, SEMANTICA_NS
|
||||
|
||||
NS = SEMANTICA_NS
|
||||
E1 = "https://example.org/e1"
|
||||
|
||||
KG = {
|
||||
"entities": [
|
||||
{
|
||||
"id": E1,
|
||||
"text": "Acme",
|
||||
"type": "ORG",
|
||||
"metadata": {"source": "crm_export_2024"},
|
||||
}
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"source_id": E1,
|
||||
"target_id": "https://example.org/e2",
|
||||
"type": "employs",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _jsonld_file(exporter, kind, tmp_path, name):
|
||||
path = tmp_path / name
|
||||
if kind == "knowledge_graph":
|
||||
exporter.export_knowledge_graph(KG, path, format="json-ld")
|
||||
elif kind == "entities":
|
||||
exporter.export_entities(KG["entities"], path, format="json-ld")
|
||||
elif kind == "relationships":
|
||||
exporter.export_relationships(KG["relationships"], path, format="json-ld")
|
||||
elif kind == "generic":
|
||||
exporter.export({"note": "plain payload, no @id"}, path, format="json-ld")
|
||||
else:
|
||||
raise AssertionError(kind)
|
||||
return json.loads(path.read_text())
|
||||
|
||||
|
||||
def test_no_jsonld_context_declares_a_vocab(tmp_path):
|
||||
exporter = JSONExporter()
|
||||
for kind in ("knowledge_graph", "entities", "relationships", "generic"):
|
||||
context = _jsonld_file(exporter, kind, tmp_path, f"{kind}.jsonld")[
|
||||
"@context"
|
||||
]
|
||||
assert "@vocab" not in context, f"{kind}: @vocab expands caller data"
|
||||
|
||||
context = json.loads(RDFExporter().export_to_rdf(KG, format="jsonld"))[
|
||||
"@context"
|
||||
]
|
||||
assert "@vocab" not in context
|
||||
|
||||
|
||||
def test_extracted_type_labels_stay_out_of_the_namespace(tmp_path):
|
||||
path = tmp_path / "kg.jsonld"
|
||||
JSONExporter().export_knowledge_graph(KG, path, format="json-ld")
|
||||
|
||||
graph = Graph()
|
||||
graph.parse(str(path), format="json-ld")
|
||||
|
||||
assert (None, RDF.type, URIRef(NS + "ORG")) not in graph, (
|
||||
"the caller's type label was minted as a class in ns#"
|
||||
)
|
||||
assert (URIRef(E1), RDF.type, URIRef(NS + "Entity")) in graph
|
||||
assert (URIRef(E1), URIRef(NS + "type"), Literal("ORG")) in graph, (
|
||||
"the label itself must survive, as a string"
|
||||
)
|
||||
|
||||
|
||||
def test_metadata_keys_stay_out_of_the_namespace(tmp_path):
|
||||
path = tmp_path / "kg.jsonld"
|
||||
JSONExporter().export_knowledge_graph(KG, path, format="json-ld")
|
||||
|
||||
graph = Graph()
|
||||
graph.parse(str(path), format="json-ld")
|
||||
|
||||
assert (None, URIRef(NS + "source"), Literal("crm_export_2024")) not in (
|
||||
graph
|
||||
), "caller metadata value attached to the real sem:source object property"
|
||||
for _, _, o in graph.triples((None, URIRef(NS + "source"), None)):
|
||||
assert not isinstance(o, Literal), (
|
||||
"sem:source has a resource range but received a plain literal"
|
||||
)
|
||||
|
||||
literals = [
|
||||
o
|
||||
for o in graph.objects(None, URIRef(NS + "metadata"))
|
||||
if isinstance(o, Literal)
|
||||
]
|
||||
assert literals, "the metadata dict was dropped instead of preserved"
|
||||
assert literals[0].datatype == RDF.JSON
|
||||
assert json.loads(str(literals[0])) == {"source": "crm_export_2024"}
|
||||
|
||||
|
||||
def test_rdf_exporter_jsonld_keeps_type_labels_out_of_the_namespace():
|
||||
graph = Graph()
|
||||
graph.parse(
|
||||
data=RDFExporter().export_to_rdf(KG, format="jsonld"), format="json-ld"
|
||||
)
|
||||
|
||||
assert (None, RDF.type, URIRef(NS + "ORG")) not in graph
|
||||
assert (URIRef(E1), RDF.type, URIRef(NS + "Entity")) in graph
|
||||
assert (URIRef(E1), URIRef(NS + "type"), Literal("ORG")) in graph
|
||||
@@ -0,0 +1,51 @@
|
||||
import pytest
|
||||
|
||||
from semantica.ontology.class_inferrer import ClassInferrer
|
||||
from semantica.ontology.property_generator import PropertyGenerator
|
||||
from semantica.utils.exceptions import ValidationError
|
||||
|
||||
|
||||
def test_same_kind_normalized_object_properties_are_merged():
|
||||
entities = [
|
||||
{"id": "p1", "type": "Person", "name": "Alice"},
|
||||
{"id": "o1", "type": "Organization", "name": "Acme"},
|
||||
]
|
||||
classes = ClassInferrer(min_occurrences=1).infer_classes(entities)
|
||||
relationships = [
|
||||
{
|
||||
"source_type": "Person",
|
||||
"target_type": "Organization",
|
||||
"type": "works_for",
|
||||
},
|
||||
{
|
||||
"source_type": "Person",
|
||||
"target_type": "Organization",
|
||||
"type": "worksFor",
|
||||
},
|
||||
]
|
||||
|
||||
properties = PropertyGenerator().infer_properties(
|
||||
entities, relationships, classes, min_occurrences=1
|
||||
)
|
||||
|
||||
works_for = [prop for prop in properties if prop["name"] == "worksFor"]
|
||||
assert len(works_for) == 1
|
||||
assert works_for[0]["domain"] == ["Person"]
|
||||
assert works_for[0]["range"] == ["Organization"]
|
||||
|
||||
|
||||
def test_normalized_name_cannot_be_both_object_and_data_property():
|
||||
entities = [
|
||||
{"id": "p1", "type": "Person", "value": "Alice"},
|
||||
{"id": "p2", "type": "Person", "value": "Bob"},
|
||||
]
|
||||
classes = ClassInferrer(min_occurrences=1).infer_classes(entities)
|
||||
relationships = [
|
||||
{"source_type": "Person", "target_type": "Person", "type": "value"},
|
||||
{"source_type": "Person", "target_type": "Person", "type": "value"},
|
||||
]
|
||||
|
||||
with pytest.raises(ValidationError, match="object and data"):
|
||||
PropertyGenerator().infer_properties(
|
||||
entities, relationships, classes, min_occurrences=1
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user