diff --git a/docs/docs.json b/docs/docs.json
index d5713f4d..88000010 100644
--- a/docs/docs.json
+++ b/docs/docs.json
@@ -106,7 +106,8 @@
"integrations/langchain",
"integrations/docling",
"integrations/snowflake",
- "integrations/databricks"
+ "integrations/databricks",
+ "integrations/salesforce"
]
},
{
diff --git a/docs/integrations/salesforce.md b/docs/integrations/salesforce.md
new file mode 100644
index 00000000..e8b3e94a
--- /dev/null
+++ b/docs/integrations/salesforce.md
@@ -0,0 +1,350 @@
+---
+title: "Salesforce Integration"
+description: "Ingest CRM records from Salesforce sObjects and SOQL queries into Semantica's KG pipeline."
+icon: "cloud"
+---
+
+> Extract Accounts, Contacts, Opportunities, and custom objects from Salesforce into Semantica with username/password/security-token or session-based authentication.
+
+
+**JWT Bearer** and **Bulk API 2.0** authentication are not yet implemented. Use username/password/security-token for server-side integrations, or pass a pre-existing `session_id` + `instance_url` if your environment already manages OAuth tokens.
+
+
+
+## Installation
+
+```bash
+# Install with Salesforce support
+pip install "semantica[db-salesforce]"
+
+# Or install the connector separately
+pip install simple-salesforce>=1.12.0
+```
+
+
+## Basic Usage
+
+```python
+from semantica.ingest import SalesforceIngestor
+import os
+
+ingestor = SalesforceIngestor(
+ username=os.getenv("SALESFORCE_USERNAME"),
+ password=os.getenv("SALESFORCE_PASSWORD"),
+ security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
+ domain=os.getenv("SALESFORCE_DOMAIN", "login"), # "test" for sandbox
+)
+
+data = ingestor.ingest_sobject("Account", fields=["Id", "Name", "Industry"], limit=1000)
+print(f"Retrieved {data.row_count} of {data.total_size} matching records")
+print(f"Columns: {data.columns}")
+```
+
+
+Use environment variables (or a `.env` file with `python-dotenv`) to keep credentials out of source code. `SalesforceIngestor()` with no arguments reads from `SALESFORCE_*` environment variables automatically.
+
+
+
+## Authentication Methods
+
+
+
+ ```python
+ import os
+ from semantica.ingest import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(
+ username=os.getenv("SALESFORCE_USERNAME"),
+ password=os.getenv("SALESFORCE_PASSWORD"),
+ security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
+ domain="login", # production; use "test" for sandbox
+ )
+ ```
+ Set the required environment variables before running:
+ ```bash
+ export SALESFORCE_USERNAME="your-username@example.com"
+ export SALESFORCE_PASSWORD="your-password"
+ export SALESFORCE_SECURITY_TOKEN="your-security-token"
+ ```
+ The standard server-side flow. The security token is appended to the
+ password during Salesforce SOAP login. Generate or reset it under
+ **Settings → My Personal Information → Reset My Security Token**.
+
+
+ ```python
+ ingestor = SalesforceIngestor(
+ session_id=os.getenv("SALESFORCE_SESSION_ID"),
+ instance_url=os.getenv("SALESFORCE_INSTANCE_URL"),
+ )
+ ```
+ Use this when your environment already manages the OAuth token
+ lifecycle (e.g. a connected app obtaining tokens via the web-server
+ or device flow). Pass the access token as `session_id` and the full
+ instance URL (e.g. `https://myorg.my.salesforce.com`) as
+ `instance_url`.
+
+
+ ```python
+ import os
+ from semantica.ingest import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(
+ username=os.getenv("SALESFORCE_USERNAME"),
+ password=os.getenv("SALESFORCE_PASSWORD"),
+ security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
+ domain="test", # routes to test.salesforce.com
+ )
+ ```
+ ```bash
+ export SALESFORCE_USERNAME="your-sandbox-username@example.com.sandbox"
+ export SALESFORCE_PASSWORD="your-password"
+ export SALESFORCE_SECURITY_TOKEN="your-security-token"
+ export SALESFORCE_DOMAIN="test"
+ ```
+ Replace `domain="login"` with `domain="test"` (or set
+ `SALESFORCE_DOMAIN=test` in your environment) to connect to a
+ developer or full sandbox.
+
+
+
+### Environment variables
+
+All constructor parameters have environment-variable fallbacks:
+
+| Variable | Parameter | Default |
+|---|---|---|
+| `SALESFORCE_USERNAME` | `username` | — |
+| `SALESFORCE_PASSWORD` | `password` | — |
+| `SALESFORCE_SECURITY_TOKEN` | `security_token` | — |
+| `SALESFORCE_DOMAIN` | `domain` | `"login"` |
+| `SALESFORCE_INSTANCE_URL` | `instance_url` | — |
+| `SALESFORCE_SESSION_ID` | `session_id` | — |
+| `SALESFORCE_API_VERSION` | `api_version` | library default (`59.0`) |
+
+
+## Object Ingestion
+
+### Ingest a standard object
+
+```python
+data = ingestor.ingest_sobject(
+ "Account",
+ fields=["Id", "Name", "Industry", "AnnualRevenue", "BillingCity"],
+ where="Type = 'Customer' AND AnnualRevenue > 1000000",
+ order_by="Name ASC",
+ limit=5000,
+)
+print(f"Retrieved {data.row_count} of {data.total_size} matching records")
+```
+
+
+`data.row_count` is the number of records in `data.data` (i.e. what was actually returned after any `limit`). `data.total_size` is Salesforce's `totalSize` — the number of records matching the query *before* the limit. Compare them to know whether you got all results.
+
+
+### Ingest a custom object
+
+Custom objects end with `__c` in their API name:
+
+```python
+data = ingestor.ingest_sobject(
+ "My_Custom_Object__c",
+ fields=["Id", "Name", "Custom_Field__c"],
+)
+```
+
+Relationship traversal fields (`Owner.Name`) are also supported:
+
+```python
+data = ingestor.ingest_sobject(
+ "Contact",
+ fields=["Id", "Name", "Email", "Account.Name", "Owner.Name"],
+ limit=10000,
+)
+```
+
+### Let Semantica choose the fields
+
+When `fields` is omitted, all selectable fields are fetched via `describe()`
+(one extra API call). Compound address and geolocation fields (`type=address`,
+`type=location`) are automatically excluded — select their components
+(`BillingStreet`, `BillingCity`, `Location__Latitude__s`, …) individually if
+you need them.
+
+```python
+data = ingestor.ingest_sobject("Opportunity")
+```
+
+
+## Raw SOQL Ingestion
+
+Pass any valid SOQL query verbatim — pagination is handled automatically:
+
+```python
+data = ingestor.ingest_query("""
+ SELECT Id, Name, StageName, Amount, CloseDate,
+ Account.Name, Owner.Name
+ FROM Opportunity
+ WHERE IsClosed = false
+ ORDER BY CloseDate ASC
+""")
+print(f"Open opportunities: {data.row_count}")
+```
+
+The query is passed to the Salesforce REST API unchanged. The caller is
+responsible for SOQL correctness and safety.
+
+
+`ingest_query` does not validate or sanitise the SOQL string. Use
+`ingest_sobject` (which validates sObject names, field names, and WHERE/ORDER
+BY fragments) when building queries from application-controlled inputs.
+
+
+
+## Document Export
+
+Convert ingested records to the Semantica document format for use with
+`GraphBuilder`:
+
+```python
+documents = ingestor.export_as_documents(
+ data,
+ id_field="Id", # default; Salesforce 18-char record Id
+ text_fields=["Name", "Description"], # omit to join all string fields
+)
+
+print(f"Created {len(documents)} documents")
+# Each document:
+# {
+# "id": "001xx000003GYk2AAG",
+# "text": "Acme Corp Enterprise software company",
+# "metadata": {
+# "source": "salesforce",
+# "sobject": "Account",
+# "instance_url": "https://myorg.my.salesforce.com",
+# "row_data": { ... full cleaned record ... }
+# }
+# }
+```
+
+Feed the documents directly into `GraphBuilder`:
+
+```python
+from semantica.kg import GraphBuilder
+
+builder = GraphBuilder()
+kg = builder.build(documents)
+```
+
+
+## Object and Schema Discovery
+
+```python
+# List all accessible sObjects
+sobject_names = ingestor.list_sobjects()
+print(sobject_names[:10]) # ["Account", "Case", "Contact", ...]
+
+# Inspect fields for a specific sObject
+schema = ingestor.get_sobject_schema("Account")
+for field in schema["fields"]:
+ print(f"{field['name']}: {field['type']} (nillable={field['nillable']})")
+```
+
+
+## Context Manager
+
+Prefer the context manager for long-running jobs — it opens one connection on
+entry and closes it on exit, so every ingestion call inside the `with` block
+reuses the same authenticated session:
+
+```python
+with SalesforceIngestor(
+ username=os.getenv("SALESFORCE_USERNAME"),
+ password=os.getenv("SALESFORCE_PASSWORD"),
+ security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
+) as sf:
+ accounts = sf.ingest_sobject("Account", limit=10000)
+ contacts = sf.ingest_sobject("Contact", limit=10000)
+ sobjects = sf.list_sobjects()
+```
+
+
+## Convenience Function
+
+Use `ingest_salesforce()` for one-liner ingestion:
+
+```python
+from semantica.ingest import ingest_salesforce
+
+# Fetch records
+data = ingest_salesforce(
+ method="sobject",
+ sobject_name="Account",
+ fields=["Id", "Name", "Industry"],
+ limit=500,
+)
+
+# Execute raw SOQL (credentials from environment variables)
+data = ingest_salesforce(
+ method="query",
+ soql="SELECT Id, Name FROM Contact WHERE IsActive = true",
+)
+
+# Ingest + export to documents in one step
+docs = ingest_salesforce(
+ method="documents",
+ sobject_name="Account",
+ text_fields=["Name", "Description"],
+ limit=1000,
+)
+
+# List accessible sObjects
+sobject_names = ingest_salesforce(method="list_sobjects")
+```
+
+Or use the unified `ingest()` dispatcher:
+
+```python
+from semantica.ingest import ingest
+
+result = ingest(
+ None,
+ source_type="salesforce",
+ method="sobject",
+ sobject_name="Account",
+ fields=["Id", "Name"],
+ limit=500,
+)
+data = result["data"] # SalesforceData
+```
+
+
+## Troubleshooting
+
+```python
+import os
+from semantica.ingest import SalesforceConnector
+
+connector = SalesforceConnector(
+ username=os.getenv("SALESFORCE_USERNAME"),
+ password=os.getenv("SALESFORCE_PASSWORD"),
+ security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
+)
+if not connector.test_connection():
+ print("Connection failed: check username, password, security token, and domain")
+```
+
+Common causes of authentication failures:
+
+- **Wrong domain**: production orgs use `domain="login"`; sandboxes use `domain="test"`.
+- **Stale security token**: reset it under **Settings → Reset My Security Token**. The new token is emailed to you.
+- **IP restriction**: your org's trusted IP ranges may block the originating IP. Check **Setup → Network Access**.
+- **API access disabled**: ensure the connected profile has the **API Enabled** permission.
+
+
+## See Also
+
+- [Ingest Module](../reference/ingest) — Full `SalesforceIngestor` API and all other ingestors.
+- [Snowflake Integration](snowflake) — Relational warehouse connector with a similar design.
+- [Databricks Integration](databricks) — Lakehouse connector.
+- [Installation](../installation) — All optional dependency extras.
+- [Knowledge Graph](../reference/kg) — Build a KG from ingested Salesforce data.
diff --git a/pyproject.toml b/pyproject.toml
index 278b4c52..074d23a8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -124,11 +124,12 @@ shacl = ["pyshacl>=0.25.0"]
db-snowflake = ["snowflake-connector-python>=4.6.0", "cryptography>=49.0.0"]
db-databricks = ["databricks-sdk>=0.60.0", "databricks-sql-connector>=4.0.0"]
db-arrow = ["pyarrow>=24.0.0"]
+db-salesforce = ["simple-salesforce>=1.12.0"]
ingest-parquet = ["pyarrow>=24.0.0"]
ingest-arrow = ["pyarrow>=24.0.0"]
db-all = [
- "semantica[db-snowflake,db-databricks,db-arrow]"
+ "semantica[db-snowflake,db-databricks,db-salesforce,db-arrow]"
]
# ---- Embedding / Models ----
diff --git a/semantica/ingest/__init__.py b/semantica/ingest/__init__.py
index dd63ff6f..a385e45f 100644
--- a/semantica/ingest/__init__.py
+++ b/semantica/ingest/__init__.py
@@ -152,6 +152,7 @@ from .methods import (
ingest_parquet,
ingest_public_api,
ingest_repository,
+ ingest_salesforce,
ingest_stream,
ingest_web,
ingest_xml,
@@ -231,6 +232,10 @@ _LAZY_EXPORTS: Dict[str, Tuple[str, str]] = {
# XML ingestion
"XMLIngestor": (".xml_ingestor", "XMLIngestor"),
"XMLIngestionData": (".xml_ingestor", "XMLIngestionData"),
+ # Salesforce ingestion
+ "SalesforceIngestor": (".salesforce_ingestor", "SalesforceIngestor"),
+ "SalesforceData": (".salesforce_ingestor", "SalesforceData"),
+ "SalesforceConnector": (".salesforce_ingestor", "SalesforceConnector"),
}
_OPTIONAL_DEPENDENCY_MESSAGES = {
@@ -258,6 +263,11 @@ _OPTIONAL_DEPENDENCY_MESSAGES = {
"Arrow ingestion requires optional dependency 'pyarrow'. "
"Install it before importing ArrowIngestor or using ingest_arrow()."
),
+ ".salesforce_ingestor": (
+ "Salesforce ingestion requires optional dependency 'simple-salesforce'. "
+ "Install it with: pip install \"semantica[db-salesforce]\" "
+ "or: pip install simple-salesforce>=1.12.0"
+ ),
}
@@ -272,7 +282,7 @@ def __getattr__(name: str) -> Any:
except ModuleNotFoundError as exc:
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
missing_name = getattr(exc, "name", None)
- if message and missing_name in {"git", "bs4", "pyarrow"}:
+ if message and missing_name in {"git", "bs4", "pyarrow", "simple_salesforce"}:
raise ImportError(message) from exc
raise
@@ -358,6 +368,10 @@ __all__ = [
# XML ingestion
"XMLIngestor",
"XMLIngestionData",
+ # Salesforce ingestion
+ "SalesforceIngestor",
+ "SalesforceData",
+ "SalesforceConnector",
# Registry and Methods
"MethodRegistry",
"method_registry",
@@ -369,6 +383,7 @@ __all__ = [
"ingest_repository",
"ingest_email",
"ingest_database",
+ "ingest_salesforce",
"ingest_ontology",
"ingest_arrow",
"ingest_parquet",
diff --git a/semantica/ingest/config.py b/semantica/ingest/config.py
index e2cfb017..20cdbc68 100644
--- a/semantica/ingest/config.py
+++ b/semantica/ingest/config.py
@@ -186,8 +186,13 @@ class IngestConfig:
self._method_configs[method] = config
def get_method_config(self, method: str) -> Dict:
- """Get method-specific configuration."""
- return self._method_configs.get(method, {})
+ """Get method-specific configuration.
+
+ Returns a **copy** of the stored method configuration so callers can
+ safely mutate it (e.g. to merge per-call options) without poisoning the
+ global configuration for subsequent calls.
+ """
+ return dict(self._method_configs.get(method, {}))
def get_all(self) -> Dict[str, Any]:
"""Get all configuration."""
diff --git a/semantica/ingest/ingest_usage.md b/semantica/ingest/ingest_usage.md
index 78d2d36c..2354ef94 100644
--- a/semantica/ingest/ingest_usage.md
+++ b/semantica/ingest/ingest_usage.md
@@ -875,6 +875,96 @@ schema = connector.get_schema(engine)
print(f" {table_name}: {[col['name'] for col in columns]}")
```
+## Salesforce CRM Ingestion
+
+Salesforce ingestion requires `simple-salesforce`:
+
+```bash
+pip install "semantica[db-salesforce]"
+```
+
+### Basic Usage
+
+```python
+from semantica.ingest import SalesforceIngestor
+import os
+
+ingestor = SalesforceIngestor(
+ username=os.getenv("SALESFORCE_USERNAME"),
+ password=os.getenv("SALESFORCE_PASSWORD"),
+ security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
+ domain="login", # "test" for sandbox
+)
+
+# Ingest Account records
+data = ingestor.ingest_sobject(
+ "Account",
+ fields=["Id", "Name", "Industry", "BillingCity"],
+ where="Type = 'Customer'",
+ limit=5000,
+)
+print(f"Retrieved {data.row_count} of {data.total_size} matching records")
+```
+
+`SalesforceIngestor()` with no arguments reads from `SALESFORCE_USERNAME`, `SALESFORCE_PASSWORD`, `SALESFORCE_SECURITY_TOKEN`, and `SALESFORCE_DOMAIN` environment variables automatically.
+
+### Custom Objects and Raw SOQL
+
+```python
+# Custom object (API name ends in __c)
+data = ingestor.ingest_sobject("My_Custom_Object__c", fields=["Id", "Name", "Custom_Field__c"])
+
+# Raw SOQL GÇö pagination is handled automatically
+data = ingestor.ingest_query("""
+ SELECT Id, Name, StageName, Amount
+ FROM Opportunity
+ WHERE IsClosed = false
+ ORDER BY CloseDate ASC
+""")
+print(f"Open opportunities: {data.row_count}")
+```
+
+### Document Export
+
+```python
+documents = ingestor.export_as_documents(
+ data,
+ id_field="Id", # Salesforce 18-char record Id
+ text_fields=["Name", "Description"],
+)
+# Each document: {"id": "001...", "text": "...", "metadata": {"source": "salesforce", ...}}
+```
+
+### Convenience Function
+
+```python
+from semantica.ingest import ingest_salesforce
+
+# Fetch records
+data = ingest_salesforce(
+ method="sobject",
+ sobject_name="Account",
+ fields=["Id", "Name"],
+ limit=500,
+)
+
+# Ingest and export as documents in one call
+docs = ingest_salesforce(
+ method="documents",
+ sobject_name="Account",
+ text_fields=["Name", "Description"],
+)
+
+# Using the unified dispatcher
+from semantica.ingest import ingest
+result = ingest(None, source_type="salesforce", method="sobject",
+ sobject_name="Account", fields=["Id", "Name"])
+data = result["data"]
+```
+
+See [Salesforce Integration](https://docs.getsemantica.ai/integrations/salesforce) for full documentation including sandbox, schema discovery, pagination details, and troubleshooting.
+
+
## MCP Server Ingestion
**IMPORTANT**: This implementation supports **ONLY Python-based MCP servers and FastMCP servers**. Users can bring their own Python or FastMCP MCP servers via URL connections. JavaScript, TypeScript, C#, Java, and other language implementations are **NOT supported**.
diff --git a/semantica/ingest/methods.py b/semantica/ingest/methods.py
index 5ec01f17..1669a99c 100644
--- a/semantica/ingest/methods.py
+++ b/semantica/ingest/methods.py
@@ -203,6 +203,7 @@ if TYPE_CHECKING:
from .ontology_ingestor import OntologyData
from .parquet_ingestor import ParquetData
from .public_api_ingestor import PublicAPIDetection
+ from .salesforce_ingestor import SalesforceData
from .stream_ingestor import StreamProcessor
from .web_ingestor import WebContent
from .xml_ingestor import XMLIngestionData
@@ -1126,6 +1127,213 @@ def ingest_database(
raise
+def ingest_salesforce(
+ source: Optional[Dict[str, Any]] = None,
+ method: str = "sobject",
+ **kwargs,
+) -> Union["SalesforceData", List[Dict[str, Any]], Dict[str, Any]]:
+ """Ingest data from Salesforce CRM (convenience function).
+
+ A user-friendly wrapper around :class:`~semantica.ingest.SalesforceIngestor`
+ that connects, ingests, and returns data in a single call.
+
+ Args:
+ source: Optional credential/configuration dictionary. Keys mirror the
+ :class:`~semantica.ingest.SalesforceConnector` constructor:
+ ``username``, ``password``, ``security_token``, ``domain``
+ (``"login"`` for production, ``"test"`` for sandbox),
+ ``instance_url``, ``session_id``, ``api_version``.
+ When ``None``, credentials are read from environment variables
+ (``SALESFORCE_USERNAME`` / ``SALESFORCE_PASSWORD`` /
+ ``SALESFORCE_SECURITY_TOKEN`` etc.).
+ method: Ingestion method:
+
+ * ``"sobject"`` *(default)* — fetch records from a named sObject
+ (requires ``sobject_name`` kwarg).
+ * ``"query"`` — execute a raw SOQL query string (requires
+ ``soql`` kwarg).
+ * ``"list_sobjects"`` — return a sorted list of accessible sObject
+ API names.
+ * ``"schema"`` — return field metadata for a named sObject
+ (requires ``sobject_name`` kwarg).
+ * ``"documents"`` — ingest an sObject and convert to the Semantica
+ document format in one step (requires ``sobject_name`` kwarg;
+ optional ``text_fields`` and ``id_field`` kwargs).
+
+ **kwargs: Additional options forwarded to the ingestor method.
+ Common kwargs for ``"sobject"`` / ``"documents"``:
+
+ * ``sobject_name`` — Salesforce sObject API name (e.g.
+ ``"Account"``, ``"My_Custom__c"``).
+ * ``fields`` — list of field API names to select. When omitted
+ all selectable fields are fetched via ``describe()``.
+ * ``where`` — SOQL ``WHERE`` clause fragment (trusted input only).
+ * ``order_by`` — SOQL ``ORDER BY`` clause fragment.
+ * ``limit`` — maximum number of records.
+
+ For ``"query"``:
+
+ * ``soql`` — full SOQL query string.
+
+ For ``"schema"``:
+
+ * ``sobject_name`` — sObject to describe.
+
+ Returns:
+ * ``"sobject"`` / ``"query"`` → :class:`~semantica.ingest.SalesforceData`
+ * ``"documents"`` → ``List[Dict[str, Any]]`` (Semantica document format)
+ * ``"list_sobjects"`` → ``List[str]``
+ * ``"schema"`` → ``Dict[str, Any]``
+
+ Raises:
+ :class:`~semantica.utils.exceptions.ConfigurationError`: If
+ ``simple-salesforce`` is not installed.
+ :class:`~semantica.utils.exceptions.ValidationError`: If credentials
+ are incomplete or an sObject / field name is invalid.
+ :class:`~semantica.utils.exceptions.ProcessingError`: If the
+ Salesforce API call fails.
+
+ Examples::
+
+ >>> from semantica.ingest import ingest_salesforce
+
+ >>> # Fetch Account records (credentials from env vars)
+ >>> data = ingest_salesforce(
+ ... method="sobject",
+ ... sobject_name="Account",
+ ... fields=["Id", "Name", "Industry"],
+ ... limit=500,
+ ... )
+
+ >>> # Execute a raw SOQL query (credentials from environment variables)
+ >>> data = ingest_salesforce(
+ ... method="query",
+ ... soql="SELECT Id, Name FROM Contact WHERE IsActive = true",
+ ... )
+
+ >>> # Ingest and export as documents for GraphBuilder in one step
+ >>> docs = ingest_salesforce(
+ ... method="documents",
+ ... sobject_name="Account",
+ ... text_fields=["Name", "Description"],
+ ... limit=1000,
+ ... )
+
+ >>> # List all accessible sObjects in the connected org
+ >>> sobject_names = ingest_salesforce(method="list_sobjects")
+
+ >>> # Use sandbox org
+ >>> data = ingest_salesforce(
+ ... method="sobject",
+ ... sobject_name="Account",
+ ... ) # set SALESFORCE_DOMAIN=test in environment for sandbox
+ """
+ # Registry hook — allows callers to register a custom "salesforce" method
+ custom_method = method_registry.get("salesforce", method)
+ if custom_method and custom_method != ingest_salesforce:
+ fallback = kwargs.pop("fallback_on_custom_error", False)
+ result = call_custom_method(
+ logger, method, custom_method, source,
+ fallback_on_custom_error=fallback, **kwargs,
+ )
+ if result is not CUSTOM_METHOD_FELL_BACK:
+ return result
+
+ try:
+ from .salesforce_ingestor import SalesforceIngestor
+ except ModuleNotFoundError as exc:
+ if _is_missing_dependency(exc, "simple_salesforce"):
+ raise _missing_optional_dependency(
+ "Salesforce ingestion", "simple-salesforce"
+ ) from exc
+ raise
+
+ # Unpack credential dict (if given); everything else stays in kwargs.
+ creds: Dict[str, Any] = {}
+ if source is not None:
+ if not isinstance(source, dict):
+ raise ProcessingError(
+ "ingest_salesforce() source must be a credential dict or None. "
+ "Pass sobject_name / soql as keyword arguments."
+ )
+ creds = dict(source)
+
+ # Merge any ingest_config method config under "salesforce".
+ # get_method_config() now returns a copy, so this dict is safe to mutate.
+ # We build the final connector config in order of increasing priority:
+ # 1. base method config (lowest — global defaults set by operator)
+ # 2. per-call credential dict supplied via `source`
+ # 3. per-call connector params supplied as kwargs
+ # Credentials are extracted from kwargs and removed so they don't also
+ # flow into the ingest method call (which doesn't understand them).
+ _CONNECTOR_PARAMS = frozenset({
+ "username", "password", "security_token", "domain",
+ "instance_url", "session_id", "api_version",
+ })
+ connector_kwargs = {k: v for k, v in kwargs.items() if k in _CONNECTOR_PARAMS}
+ for k in _CONNECTOR_PARAMS:
+ kwargs.pop(k, None)
+
+ # Build a fresh per-call config dict — never mutate the global store.
+ config: Dict[str, Any] = {
+ **ingest_config.get_method_config("salesforce"), # base (already a copy)
+ **creds, # source dict credentials
+ **connector_kwargs, # kwarg credentials
+ }
+
+ ingestor = SalesforceIngestor(**config)
+
+ if method == "sobject":
+ sobject_name = kwargs.pop("sobject_name", None)
+ if not sobject_name:
+ raise ProcessingError(
+ "ingest_salesforce() with method='sobject' requires "
+ "sobject_name keyword argument."
+ )
+ return ingestor.ingest_sobject(sobject_name, **kwargs)
+
+ elif method == "query":
+ soql = kwargs.pop("soql", None)
+ if not soql:
+ raise ProcessingError(
+ "ingest_salesforce() with method='query' requires "
+ "soql keyword argument."
+ )
+ return ingestor.ingest_query(soql, **kwargs)
+
+ elif method == "list_sobjects":
+ return ingestor.list_sobjects()
+
+ elif method == "schema":
+ sobject_name = kwargs.pop("sobject_name", None)
+ if not sobject_name:
+ raise ProcessingError(
+ "ingest_salesforce() with method='schema' requires "
+ "sobject_name keyword argument."
+ )
+ return ingestor.get_sobject_schema(sobject_name)
+
+ elif method == "documents":
+ sobject_name = kwargs.pop("sobject_name", None)
+ if not sobject_name:
+ raise ProcessingError(
+ "ingest_salesforce() with method='documents' requires "
+ "sobject_name keyword argument."
+ )
+ id_field = kwargs.pop("id_field", "Id")
+ text_fields = kwargs.pop("text_fields", None)
+ data = ingestor.ingest_sobject(sobject_name, **kwargs)
+ return ingestor.export_as_documents(data, id_field=id_field,
+ text_fields=text_fields)
+
+ else:
+ raise ProcessingError(
+ f"Unknown ingest_salesforce method: {method!r}. "
+ "Valid methods: 'sobject', 'query', 'list_sobjects', 'schema', "
+ "'documents'."
+ )
+
+
def ingest_mcp(
source: Union[str, Dict[str, Any]],
method: str = "resources",
@@ -1306,6 +1514,7 @@ def ingest(
- "ontology": Ontology ingestion
- "parquet": Apache Parquet file or directory ingestion
- "xml": XML file or directory ingestion
+ - "salesforce": Salesforce CRM ingestion (pass credentials via kwargs)
method: Optional specific ingestion method
**kwargs: Additional options passed to ingestor
@@ -1428,6 +1637,9 @@ def ingest(
return {"ontology": ingest_ontology(sources, method=method or "file", **kwargs)}
elif source_type == "mcp":
return {"data": ingest_mcp(sources, method=method or "resources", **kwargs)}
+ elif source_type == "salesforce":
+ return {"data": ingest_salesforce(sources if isinstance(sources, dict) else None,
+ method=method or "sobject", **kwargs)}
else:
raise ProcessingError(f"Unknown source type: {source_type}")
@@ -1540,3 +1752,9 @@ method_registry.register("ontology", "file", ingest_ontology)
method_registry.register("ontology", "directory", ingest_ontology)
method_registry.register("ingest", "default", ingest)
method_registry.register("ingest", "unified", ingest)
+method_registry.register("salesforce", "default", ingest_salesforce)
+method_registry.register("salesforce", "sobject", ingest_salesforce)
+method_registry.register("salesforce", "query", ingest_salesforce)
+method_registry.register("salesforce", "list_sobjects", ingest_salesforce)
+method_registry.register("salesforce", "schema", ingest_salesforce)
+method_registry.register("salesforce", "documents", ingest_salesforce)
diff --git a/semantica/ingest/registry.py b/semantica/ingest/registry.py
index 1bf0d585..6a6022fd 100644
--- a/semantica/ingest/registry.py
+++ b/semantica/ingest/registry.py
@@ -66,6 +66,7 @@ class MethodRegistry:
"parquet": {},
"arrow": {},
"xml": {},
+ "salesforce": {},
"ingest": {},
}
diff --git a/semantica/ingest/salesforce_ingestor.py b/semantica/ingest/salesforce_ingestor.py
new file mode 100644
index 00000000..6c56097d
--- /dev/null
+++ b/semantica/ingest/salesforce_ingestor.py
@@ -0,0 +1,1415 @@
+"""
+Salesforce Ingestion Module
+
+This module provides Salesforce CRM data ingestion capabilities for the
+Semantica framework, enabling extraction of records from Salesforce
+sObjects (standard and custom) via the Salesforce REST API.
+
+Key Features:
+ - Username/password/security-token authentication
+ - Session-ID + instance-URL authentication (for pre-existing OAuth sessions)
+ - Sandbox and production domain support
+ - Configurable API version
+ - SOQL-based and sObject-based record ingestion
+ - Automatic nextRecordsUrl pagination
+ - sObject schema discovery
+ - Progress tracking and structured error handling
+ - Connection management with context-manager support
+
+Main Classes:
+ - SalesforceConnector: Manages the simple-salesforce client lifecycle
+ - SalesforceData: Dataclass representing ingested Salesforce records
+ - SalesforceIngestor: Orchestrates ingestion operations
+
+Optional Dependency:
+ Install via the ``db-salesforce`` extra::
+
+ pip install "semantica[db-salesforce]"
+
+ or directly::
+
+ pip install simple-salesforce>=1.12.0
+
+Example Usage::
+
+ >>> import os
+ >>> from semantica.ingest import SalesforceIngestor
+ >>> ingestor = SalesforceIngestor(
+ ... username=os.getenv("SALESFORCE_USERNAME"),
+ ... password=os.getenv("SALESFORCE_PASSWORD"),
+ ... security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
+ ... domain="login", # "test" for sandbox
+ ... )
+
+Author: Semantica Contributors
+License: MIT
+"""
+
+import os
+import re
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
+
+from ..utils.exceptions import ProcessingError, ValidationError
+from ..utils.logging import get_logger
+from ..utils.progress_tracker import get_progress_tracker
+
+# ---------------------------------------------------------------------------
+# Optional-dependency guard — mirrors the pattern in snowflake_ingestor and
+# databricks_ingestor exactly. The module always imports cleanly; the guard
+# fires at instantiation time via the SALESFORCE_AVAILABLE check in __init__.
+# ---------------------------------------------------------------------------
+try:
+ from simple_salesforce import Salesforce as _SimpleSalesforce
+ from simple_salesforce.exceptions import (
+ SalesforceAuthenticationFailed as _SalesforceAuthenticationFailed,
+ SalesforceError as _SalesforceError,
+ SalesforceExpiredSession as _SalesforceExpiredSession,
+ SalesforceGeneralError as _SalesforceGeneralError,
+ SalesforceMalformedRequest as _SalesforceMalformedRequest,
+ SalesforceRefusedRequest as _SalesforceRefusedRequest,
+ SalesforceResourceNotFound as _SalesforceResourceNotFound,
+ )
+
+ SALESFORCE_AVAILABLE = True
+except (ImportError, OSError):
+ _SimpleSalesforce = None # type: ignore[assignment,misc]
+ _SalesforceAuthenticationFailed = None # type: ignore[assignment,misc]
+ _SalesforceError = None # type: ignore[assignment,misc]
+ _SalesforceExpiredSession = None # type: ignore[assignment,misc]
+ _SalesforceGeneralError = None # type: ignore[assignment,misc]
+ _SalesforceMalformedRequest = None # type: ignore[assignment,misc]
+ _SalesforceRefusedRequest = None # type: ignore[assignment,misc]
+ _SalesforceResourceNotFound = None # type: ignore[assignment,misc]
+ SALESFORCE_AVAILABLE = False
+
+
+# ---------------------------------------------------------------------------
+# SOQL validation helpers
+# ---------------------------------------------------------------------------
+
+# sObject API names: start with a letter, contain letters/digits/underscores,
+# and optionally end with a Salesforce namespace suffix (__c, __mdt, __e,
+# __b, __x, __ka, __kav, __r). The __r suffix is used for relationship
+# traversal fields, not objects, but we accept it here so callers who pass
+# a field-path component don't hit a false-positive error.
+_SOBJECT_RE = re.compile(
+ r"^[A-Za-z][A-Za-z0-9_]*(__c|__mdt|__e|__b|__x|__ka|__kav|__r)?$"
+)
+
+# Field API names: start with a letter, contain letters/digits/underscores.
+# Dot-notation for relationship traversal (e.g. ``Owner.Name``) is allowed;
+# each component must individually match the base pattern.
+_FIELD_COMPONENT_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_]*(__c|__r)?$")
+
+# ORDER BY: one or more comma-separated ``FieldName [ASC|DESC] [NULLS FIRST|LAST]``
+# clauses. Relationship dot-notation is supported (e.g. ``Owner.Name ASC``).
+_SAFE_ORDER_RE = re.compile(
+ r"^[A-Za-z][A-Za-z0-9_.]*(\s+(ASC|DESC))?(\s+NULLS\s+(FIRST|LAST))?"
+ r"(\s*,\s*[A-Za-z][A-Za-z0-9_.]*(\s+(ASC|DESC))?(\s+NULLS\s+(FIRST|LAST))?)*$",
+ re.IGNORECASE,
+)
+
+# WHERE-clause fragment blocklist — reused from db_ingestor's approach.
+# SOQL has no UNION, INSERT, DROP, etc., but we still block statement
+# separators, comment markers, and SQL injection primitives defensively.
+_SOQL_WHERE_BLOCKLIST_RE = re.compile(
+ r";|--|/\*|\*/|\bunion\b|\binsert\b|\bupdate\b|\bdelete\b|\bdrop\b|"
+ r"\balter\b|\bcreate\b|\bexec\b|\bexecute\b|\bgrant\b|\brevoke\b",
+ re.IGNORECASE,
+)
+
+# SOQL single-quoted string literals — mask before blocklist check so a
+# value like ``status = 'union'`` doesn't false-positive.
+# SOQL escapes a single quote by doubling it (``''``), not with a backslash,
+# so the pattern mirrors ``db_ingestor._SQL_STRING_LITERAL_RE`` exactly.
+_SOQL_STRING_LITERAL_RE = re.compile(r"'(?:[^']|'')*'")
+
+
+def _mask_soql_literals(fragment: str) -> str:
+ """Replace quoted-literal contents with ``?`` so blocklist only sees syntax."""
+ return _SOQL_STRING_LITERAL_RE.sub(
+ lambda m: "'" + "?" * (len(m.group(0)) - 2) + "'", fragment
+ )
+
+
+def _validate_sobject_name(name: str) -> str:
+ """Validate a Salesforce sObject API name used in SOQL interpolation.
+
+ Args:
+ name: The sObject API name to validate.
+
+ Returns:
+ The unchanged name if valid.
+
+ Raises:
+ ValidationError: If *name* contains characters that could escape
+ a SOQL identifier context.
+ """
+ if not isinstance(name, str) or not _SOBJECT_RE.match(name):
+ raise ValidationError(
+ f"Invalid Salesforce sObject name: {name!r}. "
+ "Must start with a letter, contain only letters, digits, and "
+ "underscores, and may optionally end with a Salesforce suffix "
+ "such as __c, __mdt, or __e."
+ )
+ return name
+
+
+def _validate_field_name(name: str) -> str:
+ """Validate a single Salesforce field API name (dot-notation allowed).
+
+ Args:
+ name: Field name or dot-separated relationship path, e.g. ``Owner.Name``.
+
+ Returns:
+ The unchanged name if valid.
+
+ Raises:
+ ValidationError: If any component is not a valid identifier.
+ """
+ if not isinstance(name, str) or not name:
+ raise ValidationError(f"Field name must be a non-empty string; got {name!r}.")
+ for component in name.split("."):
+ if not _FIELD_COMPONENT_RE.match(component):
+ raise ValidationError(
+ f"Invalid Salesforce field name component: {component!r} "
+ f"(in {name!r}). Each component must start with a letter "
+ "and contain only letters, digits, and underscores."
+ )
+ return name
+
+
+def _validate_soql_where(fragment: str) -> str:
+ """Block known-dangerous constructs in a SOQL WHERE fragment.
+
+ This is a blocklist, not a grammar parser. It catches the common SQL
+ injection primitives (statement separators, comment sequences, and DML /
+ DDL keywords) without attempting to prove the fragment is fully safe.
+ Callers should treat ``where`` as trusted/operator input and not pass
+ raw end-user text here.
+
+ Args:
+ fragment: The raw WHERE clause fragment (without the ``WHERE`` keyword).
+
+ Returns:
+ The unchanged fragment if no blocked construct is found.
+
+ Raises:
+ ValidationError: If a blocked construct is detected.
+ """
+ if not isinstance(fragment, str):
+ raise ValidationError("WHERE clause must be a string.")
+ masked = _mask_soql_literals(fragment)
+ if _SOQL_WHERE_BLOCKLIST_RE.search(masked):
+ raise ValidationError(
+ f"WHERE clause contains a disallowed keyword or character: "
+ f"{fragment!r}. Do not pass untrusted user input as a WHERE clause."
+ )
+ return fragment
+
+
+def _validate_order_by(fragment: str) -> str:
+ """Validate a SOQL ORDER BY clause fragment.
+
+ Args:
+ fragment: The ORDER BY expression (without the ``ORDER BY`` keyword),
+ e.g. ``Name ASC, CreatedDate DESC NULLS LAST``.
+
+ Returns:
+ The unchanged fragment if valid.
+
+ Raises:
+ ValidationError: If the fragment contains unexpected characters or
+ keywords.
+ """
+ if not isinstance(fragment, str) or not fragment.strip():
+ raise ValidationError("ORDER BY clause must be a non-empty string.")
+ if not _SAFE_ORDER_RE.match(fragment.strip()):
+ raise ValidationError(
+ f"Invalid ORDER BY clause: {fragment!r}. "
+ "Only field names (with optional dot-notation), ASC/DESC, and "
+ "NULLS FIRST/LAST are permitted."
+ )
+ return fragment
+
+
+# ---------------------------------------------------------------------------
+# SalesforceData
+# ---------------------------------------------------------------------------
+
+@dataclass
+class SalesforceData:
+ """Records ingested from a Salesforce sObject or SOQL query.
+
+ Attributes:
+ data: List of cleaned record dictionaries. The ``attributes`` key
+ that ``simple-salesforce`` injects into every raw record has been
+ stripped, along with any ``attributes`` keys on nested relationship
+ sub-objects.
+ row_count: Number of records in ``data`` — i.e. ``len(data)``. When
+ a *limit* was applied this is the number of records actually
+ returned, not the total matching the query. See ``total_size``
+ for the unfiltered count.
+ columns: Ordered list of field names present across all records in
+ ``data``.
+ sobject: Salesforce sObject API name (e.g. ``"Account"``,
+ ``"My_Custom__c"``), or ``None`` for raw SOQL queries that span
+ multiple objects.
+ query: The SOQL query string that produced these records, or ``None``
+ when not applicable.
+ instance_url: Salesforce instance base URL used for the ingestion
+ (e.g. ``"https://myorg.my.salesforce.com"``).
+ total_size: The ``totalSize`` field from the Salesforce REST API
+ response — the total number of records matching the query
+ *before* any client-side ``limit`` or pagination truncation.
+ ``None`` when not available (e.g. for schema-only calls).
+ To check whether all records were retrieved: compare
+ ``row_count == total_size``.
+ metadata: Arbitrary extra metadata. Ingestion methods populate
+ ``metadata["query"]`` with the SOQL string used (defaults to
+ ``{}``).
+ ingested_at: Timestamp recorded when this object was created.
+ """
+
+ data: List[Dict[str, Any]]
+ row_count: int
+ columns: List[str]
+ sobject: Optional[str] = None
+ query: Optional[str] = None
+ instance_url: Optional[str] = None
+ total_size: Optional[int] = None
+ metadata: Dict[str, Any] = field(default_factory=dict)
+ ingested_at: datetime = field(default_factory=datetime.now)
+
+
+# ---------------------------------------------------------------------------
+# SalesforceConnector
+# ---------------------------------------------------------------------------
+
+class SalesforceConnector:
+ """Manages the ``simple-salesforce`` client lifecycle.
+
+ Responsibilities:
+
+ * Reads credentials from constructor arguments, falling back to
+ environment variables in the same way as ``SnowflakeConnector`` and
+ ``DatabricksConnector``.
+ * Validates that a usable authentication path is configured *before* any
+ network call is made.
+ * Exposes :py:meth:`connect`, :py:meth:`disconnect`, and
+ :py:meth:`test_connection` so the ingestor (and tests) can control the
+ client lifecycle explicitly.
+ * **Connection reuse**: if a client is already open :py:meth:`connect`
+ returns it immediately, preventing redundant authentication calls and
+ resource leaks when the ingestor is used as a context manager.
+
+ Supported Authentication Modes
+ --------------------------------
+ **Username / Password / Security Token** (standard server-side flow)::
+
+ import os
+ from semantica.ingest import SalesforceConnector
+
+ connector = SalesforceConnector(
+ username=os.getenv("SALESFORCE_USERNAME"),
+ password=os.getenv("SALESFORCE_PASSWORD"),
+ security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
+ domain="login", # or "test" for sandbox
+ )
+
+ **Session ID + Instance URL** (pre-authenticated OAuth session)::
+
+ import os
+ from semantica.ingest import SalesforceConnector
+
+ SalesforceConnector(
+ session_id=os.getenv("SALESFORCE_SESSION_ID"),
+ instance_url=os.getenv("SALESFORCE_INSTANCE_URL"),
+ )
+
+ .. note::
+ **JWT Bearer authentication** (``consumer_key`` + ``privatekey`` /
+ ``privatekey_file``) is supported by ``simple-salesforce`` but is not
+ implemented in this connector. It requires a separate key-management
+ flow and will be added in a later stage when the full lifecycle
+ (key loading, passphrase handling, token refresh) can be validated
+ end-to-end. Until then, pass ``session_id`` + ``instance_url`` if
+ you already hold a JWT-derived token from your own auth layer.
+
+ Environment Variables
+ ----------------------
+ Every constructor parameter has an environment-variable fallback:
+
+ * ``SALESFORCE_USERNAME``
+ * ``SALESFORCE_PASSWORD``
+ * ``SALESFORCE_SECURITY_TOKEN``
+ * ``SALESFORCE_DOMAIN`` (default: ``"login"``)
+ * ``SALESFORCE_INSTANCE_URL``
+ * ``SALESFORCE_SESSION_ID``
+ * ``SALESFORCE_API_VERSION``
+
+ Raises:
+ ImportError: If ``simple-salesforce`` is not installed.
+ ValidationError: If no usable credential set is provided.
+ """
+
+ def __init__(
+ self,
+ username: Optional[str] = None,
+ password: Optional[str] = None,
+ security_token: Optional[str] = None,
+ domain: Optional[str] = None,
+ instance_url: Optional[str] = None,
+ session_id: Optional[str] = None,
+ api_version: Optional[str] = None,
+ **config: Any,
+ ) -> None:
+ """Initialise the connector and validate credentials.
+
+ Args:
+ username: Salesforce login username.
+ password: Salesforce login password.
+ security_token: Salesforce security token appended to the password
+ during SOAP authentication.
+ domain: Login domain — ``"login"`` for production (default),
+ ``"test"`` for sandbox, or a custom My Domain value.
+ instance_url: Full Salesforce instance URL for session-based auth
+ (e.g. ``"https://myorg.my.salesforce.com"``).
+ session_id: Pre-existing Salesforce session / OAuth access token.
+ api_version: Salesforce REST API version string, e.g. ``"59.0"``.
+ Defaults to ``simple-salesforce``'s built-in default (currently
+ ``"59.0"``).
+ **config: Extra keyword arguments forwarded verbatim to
+ ``simple_salesforce.Salesforce()`` (e.g. ``proxies``,
+ ``session``).
+
+ Raises:
+ ImportError: If ``simple-salesforce`` is not installed.
+ ValidationError: If the provided credentials are insufficient for
+ any supported authentication mode.
+ """
+ if not SALESFORCE_AVAILABLE:
+ raise ImportError(
+ "simple-salesforce is required for SalesforceConnector. "
+ 'Install it with: pip install "semantica[db-salesforce]" '
+ "or: pip install simple-salesforce>=1.12.0"
+ )
+
+ self.logger = get_logger("salesforce_connector")
+
+ # ------------------------------------------------------------------
+ # Resolve credentials: explicit args take precedence over env vars.
+ # Passwords, tokens, and session IDs are stored on the instance so
+ # they can be forwarded to simple-salesforce on connect(). They are
+ # NEVER included in log messages or exception messages raised here.
+ # ------------------------------------------------------------------
+ self.username: Optional[str] = username or os.getenv("SALESFORCE_USERNAME")
+ self._password: Optional[str] = password or os.getenv("SALESFORCE_PASSWORD")
+ self._security_token: Optional[str] = (
+ security_token or os.getenv("SALESFORCE_SECURITY_TOKEN")
+ )
+ self.domain: str = (
+ domain or os.getenv("SALESFORCE_DOMAIN") or "login"
+ )
+ self.instance_url: Optional[str] = (
+ instance_url or os.getenv("SALESFORCE_INSTANCE_URL")
+ )
+ self._session_id: Optional[str] = (
+ session_id or os.getenv("SALESFORCE_SESSION_ID")
+ )
+ self.api_version: Optional[str] = (
+ api_version or os.getenv("SALESFORCE_API_VERSION")
+ )
+
+ # Extra kwargs forwarded to simple-salesforce (e.g. proxies, session)
+ self._extra_config: Dict[str, Any] = config
+
+ # Internal client reference — None until connect() is called.
+ self._client: Optional[Any] = None
+
+ # Validate that at least one viable auth path is fully configured.
+ self._validate_auth()
+
+ # Safe to log: username (not a secret) and domain.
+ self.logger.debug(
+ "Salesforce connector initialised: username=%s domain=%s",
+ self.username or "",
+ self.domain,
+ )
+
+ # ------------------------------------------------------------------
+ # Internal helpers
+ # ------------------------------------------------------------------
+
+ def _validate_auth(self) -> None:
+ """Raise :class:`~semantica.utils.exceptions.ValidationError` if no
+ usable authentication path is present.
+
+ Two valid modes are recognised:
+
+ 1. **Username / password / security token** — all three required.
+ 2. **Session ID + instance URL** — both required.
+
+ Raises:
+ ValidationError: When neither mode has all required fields.
+ """
+ has_upw = bool(
+ self.username and self._password and self._security_token
+ )
+ has_session = bool(self._session_id and self.instance_url)
+
+ if not has_upw and not has_session:
+ raise ValidationError(
+ "Salesforce authentication is required. Provide either:\n"
+ " (a) username + password + security_token "
+ "(env: SALESFORCE_USERNAME / SALESFORCE_PASSWORD / "
+ "SALESFORCE_SECURITY_TOKEN), or\n"
+ " (b) session_id + instance_url "
+ "(env: SALESFORCE_SESSION_ID / SALESFORCE_INSTANCE_URL)."
+ )
+
+ def _build_client_kwargs(self) -> Dict[str, Any]:
+ """Assemble the keyword arguments for ``simple_salesforce.Salesforce()``.
+
+ Credentials are passed to the underlying library here and nowhere
+ else. They are not logged.
+
+ Returns:
+ Keyword argument dictionary ready to pass to
+ ``simple_salesforce.Salesforce(**kwargs)``.
+ """
+ kwargs: Dict[str, Any] = {}
+
+ # API version maps to simple-salesforce's ``version`` parameter.
+ if self.api_version:
+ kwargs["version"] = self.api_version
+
+ if self._session_id and self.instance_url:
+ # Direct / pre-authenticated session — no network call during init.
+ kwargs["session_id"] = self._session_id
+ kwargs["instance_url"] = self.instance_url
+ else:
+ # Username + password + security token (SOAP login).
+ kwargs["username"] = self.username
+ kwargs["password"] = self._password
+ kwargs["security_token"] = self._security_token
+ kwargs["domain"] = self.domain
+
+ # Forward any extra config (proxies, custom requests.Session, etc.)
+ kwargs.update(self._extra_config)
+ return kwargs
+
+ # ------------------------------------------------------------------
+ # Public properties
+ # ------------------------------------------------------------------
+
+ @property
+ def client(self) -> Optional[Any]:
+ """The active ``simple_salesforce.Salesforce`` instance, or ``None``."""
+ return self._client
+
+ @property
+ def connection(self) -> Optional[Any]:
+ """Alias for :attr:`client` — used by tests that follow the
+ Databricks/Snowflake ``connector.connection`` naming convention."""
+ return self._client
+
+ # ------------------------------------------------------------------
+ # Lifecycle
+ # ------------------------------------------------------------------
+
+ def connect(self) -> Any:
+ """Create and return the ``simple_salesforce.Salesforce`` client.
+
+ If a client is already open (e.g. because the ingestor is being used
+ as a context manager) the existing instance is returned without
+ re-authenticating, preventing duplicate SOAP/OAuth round-trips and
+ resource leaks.
+
+ .. important::
+ For username/password/security-token authentication,
+ ``simple-salesforce`` performs a live SOAP login call inside its
+ constructor. Any network or authentication failure will therefore
+ be raised here, not at ``SalesforceConnector.__init__`` time.
+
+ Returns:
+ The ``simple_salesforce.Salesforce`` client instance.
+
+ Raises:
+ ProcessingError: If the connection attempt fails — wraps
+ ``SalesforceAuthenticationFailed``, ``SalesforceError``,
+ ``TypeError`` (invalid credential combination), or any
+ other unexpected exception, preserving the original as the
+ chained cause.
+ """
+ if self._client is not None:
+ return self._client
+
+ try:
+ kwargs = self._build_client_kwargs()
+ self._client = _SimpleSalesforce(**kwargs)
+
+ # Resolve a clean instance URL from the connected client so that
+ # SalesforceData and callers can reference it without re-reading
+ # config. simple-salesforce stores the hostname in sf_instance;
+ # we reconstruct the full HTTPS URL from it.
+ if self.instance_url is None:
+ sf_instance = getattr(self._client, "sf_instance", None)
+ if sf_instance:
+ self.instance_url = f"https://{sf_instance}"
+
+ # Log only non-sensitive information.
+ self.logger.info(
+ "Connected to Salesforce: instance=%s",
+ self.instance_url or "",
+ )
+ return self._client
+
+ except (_SalesforceAuthenticationFailed,) if SALESFORCE_AVAILABLE else ():
+ # Authentication failure — safe to surface the error code /
+ # server message; the password is never included in SF's response.
+ raise ProcessingError(
+ "Salesforce authentication failed. "
+ "Check username, password, security token, and domain."
+ ) from None # suppress the original — it may contain the username
+
+ except (_SalesforceError,) if SALESFORCE_AVAILABLE else ():
+ # Other Salesforce API errors during initial connection.
+ raise ProcessingError(
+ "Salesforce connection error. "
+ "Verify the instance URL and API access permissions."
+ ) from None
+
+ except TypeError as exc:
+ # simple-salesforce raises TypeError when no valid credential
+ # combination is recognised — should not happen if _validate_auth
+ # passed, but guard defensively.
+ raise ProcessingError(
+ "Salesforce connection failed: invalid credential combination."
+ ) from exc
+
+ except Exception as exc:
+ # Unexpected error (network timeout, DNS failure, etc.).
+ # Log without credentials; re-raise with a generic message.
+ self.logger.error(
+ "Unexpected error connecting to Salesforce: %s",
+ type(exc).__name__,
+ )
+ raise ProcessingError(
+ f"Failed to connect to Salesforce: {type(exc).__name__}"
+ ) from exc
+
+ def disconnect(self) -> None:
+ """Release the Salesforce client and its underlying requests session.
+
+ ``simple-salesforce`` uses a ``requests.Session`` internally. Setting
+ the reference to ``None`` allows the session to be garbage-collected.
+ There is no "logout" endpoint in the Salesforce REST API for
+ username/password flows; session-ID tokens can be revoked server-side
+ separately if required.
+ """
+ if self._client is not None:
+ # Release the requests.Session inside the SF client.
+ session = getattr(self._client, "session", None)
+ if session is not None:
+ try:
+ session.close()
+ except Exception: # noqa: BLE001
+ pass
+ self._client = None
+ self.logger.info("Disconnected from Salesforce.")
+
+ def test_connection(self) -> bool:
+ """Verify connectivity by authenticating and calling ``/limits/``.
+
+ Opens a transient connection (or reuses an existing one), calls the
+ lightweight ``limits`` endpoint that requires valid authentication but
+ reads no CRM data, then closes the connection if it was opened here.
+
+ Returns:
+ ``True`` if authentication and the API call succeed, ``False``
+ for any failure (auth error, network error, etc.).
+ """
+ already_connected = self._client is not None
+ try:
+ client = self.connect()
+ # ``limits()`` is a cheap, read-only call that proves the
+ # session is valid without touching any CRM data.
+ client.limits()
+ return True
+ except Exception as exc:
+ # Log the exception type only — no credentials in the message.
+ self.logger.debug(
+ "Salesforce connection test failed: %s", type(exc).__name__
+ )
+ return False
+ finally:
+ # Close only if we opened the connection in this call.
+ if not already_connected:
+ self.disconnect()
+
+
+# ---------------------------------------------------------------------------
+# SalesforceIngestor
+# ---------------------------------------------------------------------------
+
+class SalesforceIngestor:
+ """Salesforce CRM data ingestor for the Semantica framework.
+
+ Wraps :class:`SalesforceConnector` and will provide high-level methods
+ for pulling records from sObjects and SOQL queries into the Semantica
+ ingestion pipeline.
+
+ Full ingestion methods (``ingest_sobject``, ``ingest_query``,
+ ``list_sobjects``, ``get_sobject_schema``, ``export_as_documents``) are
+ implemented in this class.
+
+ Args:
+ username: Salesforce login username.
+ password: Salesforce login password.
+ security_token: Salesforce security token.
+ domain: Login domain — ``"login"`` (production, default) or ``"test"``
+ (sandbox).
+ instance_url: Instance URL for session-based authentication.
+ session_id: Pre-existing Salesforce session / access token.
+ api_version: REST API version string (e.g. ``"59.0"``).
+ config: Optional extra configuration forwarded to the connector.
+ **kwargs: Additional keyword arguments merged into ``config``.
+
+ Example::
+
+ import os
+ from semantica.ingest import SalesforceIngestor
+
+ # Standalone
+ ingestor = SalesforceIngestor(
+ username=os.getenv("SALESFORCE_USERNAME"),
+ password=os.getenv("SALESFORCE_PASSWORD"),
+ security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
+ )
+
+ # Context manager — preferred for long-running jobs
+ with SalesforceIngestor(
+ username=os.getenv("SALESFORCE_USERNAME"),
+ password=os.getenv("SALESFORCE_PASSWORD"),
+ security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
+ domain="test", # sandbox
+ ) as sf:
+ pass # ingest_sobject() etc. available in Stage 2
+ """
+
+ def __init__(
+ self,
+ username: Optional[str] = None,
+ password: Optional[str] = None,
+ security_token: Optional[str] = None,
+ domain: Optional[str] = None,
+ instance_url: Optional[str] = None,
+ session_id: Optional[str] = None,
+ api_version: Optional[str] = None,
+ config: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> None:
+ self.logger = get_logger("salesforce_ingestor")
+
+ self.config: Dict[str, Any] = config or {}
+ self.config.update(kwargs)
+
+ # Instantiate the connector — raises ImportError if simple-salesforce
+ # is absent, or ValidationError if credentials are incomplete.
+ self.connector = SalesforceConnector(
+ username=username,
+ password=password,
+ security_token=security_token,
+ domain=domain,
+ instance_url=instance_url,
+ session_id=session_id,
+ api_version=api_version,
+ **self.config,
+ )
+
+ # Progress tracker — consistent with all other ingestors.
+ self.progress_tracker = get_progress_tracker()
+ if not self.progress_tracker.enabled:
+ self.progress_tracker.enabled = True
+
+ self.logger.debug("Salesforce ingestor initialised.")
+
+ # ------------------------------------------------------------------
+ # Context-manager support
+ # ------------------------------------------------------------------
+
+ def __enter__(self) -> "SalesforceIngestor":
+ """Open the Salesforce connection on context entry."""
+ self.connector.connect()
+ return self
+
+ def __exit__(
+ self,
+ exc_type: Any,
+ exc_val: Any,
+ exc_tb: Any,
+ ) -> None:
+ """Close the Salesforce connection on context exit."""
+ self.close()
+
+ # ------------------------------------------------------------------
+ # Lifecycle
+ # ------------------------------------------------------------------
+
+ def close(self) -> None:
+ """Disconnect from Salesforce and release the client."""
+ self.connector.disconnect()
+
+ # ------------------------------------------------------------------
+ # Ingestion methods
+ # ------------------------------------------------------------------
+
+ def ingest_sobject(
+ self,
+ sobject_name: str,
+ fields: Optional[List[str]] = None,
+ where: Optional[str] = None,
+ order_by: Optional[str] = None,
+ limit: Optional[int] = None,
+ **options: Any,
+ ) -> "SalesforceData":
+ """Fetch records from a Salesforce sObject via SOQL.
+
+ Builds a ``SELECT ... FROM [WHERE ...] [ORDER BY ...]
+ [LIMIT ...]`` query, executes it, and follows all ``nextRecordsUrl``
+ pagination links until every matching record (up to *limit*) has been
+ collected.
+
+ Args:
+ sobject_name: Salesforce sObject API name, e.g. ``"Account"``,
+ ``"Contact"``, ``"My_Custom__c"``.
+ fields: Field API names to retrieve. Dot-notation for relationship
+ traversal is supported (e.g. ``["Id", "Name", "Owner.Name"]``).
+ When ``None``, all fields from the object's ``describe()``
+ response are used (one additional API call).
+ where: SOQL ``WHERE`` clause fragment without the ``WHERE`` keyword,
+ e.g. ``"Type = 'Customer' AND AnnualRevenue > 1000000"``.
+ **Trusted input only** — do not pass raw end-user text here.
+ order_by: SOQL ``ORDER BY`` clause fragment without the keyword,
+ e.g. ``"Name ASC, CreatedDate DESC NULLS LAST"``.
+ **Trusted input only.**
+ limit: Maximum number of records to return across all pages. When
+ ``None``, all matching records are returned (use with care on
+ large objects).
+ **options: Reserved for future use.
+
+ Returns:
+ :class:`SalesforceData` with ``data``, ``row_count``, ``columns``,
+ ``sobject``, ``query``, ``instance_url``, and ``total_size``
+ populated.
+
+ Raises:
+ ValidationError: If *sobject_name*, any field name, *where*, or
+ *order_by* fails the injection-safety check.
+ ProcessingError: If the Salesforce API call fails.
+ """
+ _validate_sobject_name(sobject_name)
+
+ # Validate WHERE and ORDER BY early — before connecting — so bad
+ # input raises ValidationError without making any network call.
+ if where:
+ _validate_soql_where(where)
+ if order_by:
+ _validate_order_by(order_by)
+
+ # limit=0 is a valid, well-defined request: "give me no records".
+ # Returning immediately avoids generating an invalid ``LIMIT 0`` SOQL
+ # clause (Salesforce requires LIMIT ≥ 1) and saves a round-trip.
+ if limit is not None and limit <= 0:
+ self.logger.debug(
+ "ingest_sobject called with limit=%d — returning empty result.", limit
+ )
+ return SalesforceData(
+ data=[],
+ row_count=0,
+ columns=[],
+ sobject=sobject_name,
+ instance_url=self.connector.instance_url,
+ total_size=0,
+ )
+
+ tracking_id = self.progress_tracker.start_tracking(
+ file=sobject_name,
+ module="ingest",
+ submodule="SalesforceIngestor",
+ message=f"sObject: {sobject_name}",
+ )
+
+ try:
+ already_connected = self.connector._client is not None
+ client = self.connector.connect()
+
+ try:
+ # Resolve field list — fetch from describe() when not provided.
+ if fields is None:
+ self.progress_tracker.update_tracking(
+ tracking_id, message="Fetching sObject schema…"
+ )
+ fields = self._get_all_field_names(client, sobject_name)
+ else:
+ for f in fields:
+ _validate_field_name(f)
+
+ soql = self._build_soql(
+ sobject_name=sobject_name,
+ fields=fields,
+ where=where,
+ order_by=order_by,
+ limit=limit,
+ )
+
+ self.progress_tracker.update_tracking(
+ tracking_id, message="Executing SOQL query…"
+ )
+
+ records, total_size = self._query_all(client, soql, limit)
+
+ self.progress_tracker.update_tracking(
+ tracking_id, message=f"Fetched {len(records)} records…"
+ )
+
+ data = self._convert_rows(records)
+ columns = list(dict.fromkeys(
+ k for row in data for k in row
+ ))
+
+ self.progress_tracker.stop_tracking(
+ tracking_id,
+ status="completed",
+ message=f"Ingested {len(data)} records",
+ )
+ self.logger.info(
+ "sObject ingestion completed: %s — %d record(s)",
+ sobject_name,
+ len(data),
+ )
+
+ return SalesforceData(
+ data=data,
+ row_count=len(data),
+ columns=columns,
+ sobject=sobject_name,
+ query=soql,
+ instance_url=self.connector.instance_url,
+ total_size=total_size,
+ metadata={"query": soql},
+ )
+
+ finally:
+ if not already_connected:
+ self.connector.disconnect()
+
+ except (ValidationError, ProcessingError):
+ self.progress_tracker.stop_tracking(
+ tracking_id, status="failed", message="Query failed"
+ )
+ raise
+ except Exception as exc:
+ self.progress_tracker.stop_tracking(
+ tracking_id, status="failed", message=str(exc)
+ )
+ self.logger.error(
+ "Failed to ingest sObject %s: %s", sobject_name, type(exc).__name__
+ )
+ raise ProcessingError(
+ f"Failed to ingest Salesforce sObject '{sobject_name}': "
+ f"{type(exc).__name__}"
+ ) from exc
+
+ def ingest_query(
+ self,
+ soql: str,
+ batch_size: Optional[int] = None, # noqa: ARG002 — reserved for future chunked progress
+ **options: Any,
+ ) -> "SalesforceData":
+ """Execute a raw SOQL query and return all matching records.
+
+ Follows ``nextRecordsUrl`` pagination automatically until ``done``
+ is ``True`` or the result set is exhausted.
+
+ Args:
+ soql: A complete, valid SOQL query string, e.g.
+ ``"SELECT Id, Name FROM Account WHERE Type = 'Customer'"``.
+ The query is passed verbatim to the Salesforce REST API — the
+ caller is responsible for correctness and safety.
+ batch_size: Accepted for API compatibility but currently unused;
+ Salesforce controls the page size. Reserved for future
+ progress-reporting granularity.
+ **options: Reserved for future use.
+
+ Returns:
+ :class:`SalesforceData` with all records collected across pages.
+
+ Raises:
+ ProcessingError: If the Salesforce API call fails.
+ """
+ tracking_id = self.progress_tracker.start_tracking(
+ file="soql_query",
+ module="ingest",
+ submodule="SalesforceIngestor",
+ message="Executing SOQL query…",
+ )
+
+ try:
+ already_connected = self.connector._client is not None
+ client = self.connector.connect()
+
+ try:
+ records, total_size = self._query_all(client, soql, limit=None)
+
+ self.progress_tracker.update_tracking(
+ tracking_id, message=f"Fetched {len(records)} records…"
+ )
+
+ data = self._convert_rows(records)
+ columns = list(dict.fromkeys(
+ k for row in data for k in row
+ ))
+
+ self.progress_tracker.stop_tracking(
+ tracking_id,
+ status="completed",
+ message=f"Query returned {len(data)} records",
+ )
+ self.logger.info(
+ "SOQL query completed: %d record(s)", len(data)
+ )
+
+ return SalesforceData(
+ data=data,
+ row_count=len(data),
+ columns=columns,
+ query=soql,
+ instance_url=self.connector.instance_url,
+ total_size=total_size,
+ metadata={"query": soql},
+ )
+
+ finally:
+ if not already_connected:
+ self.connector.disconnect()
+
+ except (ValidationError, ProcessingError):
+ self.progress_tracker.stop_tracking(
+ tracking_id, status="failed", message="Query failed"
+ )
+ raise
+ except Exception as exc:
+ self.progress_tracker.stop_tracking(
+ tracking_id, status="failed", message=str(exc)
+ )
+ self.logger.error(
+ "Failed to execute SOQL query: %s", type(exc).__name__
+ )
+ raise ProcessingError(
+ f"Failed to execute SOQL query: {type(exc).__name__}"
+ ) from exc
+
+ def list_sobjects(self) -> List[str]:
+ """Return the API names of all accessible sObjects in the connected org.
+
+ Uses ``sf.describe()`` (``GET /services/data/vXX.0/sobjects``) which
+ returns global metadata for every sObject the current user can access.
+
+ Returns:
+ Sorted list of sObject API name strings (e.g.
+ ``["Account", "Contact", "My_Custom__c", ...]``).
+
+ Raises:
+ ProcessingError: If the Salesforce API call fails.
+ """
+ try:
+ already_connected = self.connector._client is not None
+ client = self.connector.connect()
+
+ try:
+ result = client.describe()
+ sobjects = [
+ obj["name"]
+ for obj in (result.get("sobjects") or [])
+ if obj.get("name")
+ ]
+ sobjects.sort()
+ self.logger.debug(
+ "list_sobjects: found %d sObjects", len(sobjects)
+ )
+ return sobjects
+
+ finally:
+ if not already_connected:
+ self.connector.disconnect()
+
+ except (ProcessingError, ValidationError):
+ raise
+ except Exception as exc:
+ self.logger.error(
+ "Failed to list sObjects: %s", type(exc).__name__
+ )
+ raise ProcessingError(
+ f"Failed to list Salesforce sObjects: {type(exc).__name__}"
+ ) from exc
+
+ def get_sobject_schema(self, sobject_name: str) -> Dict[str, Any]:
+ """Return field metadata for a Salesforce sObject.
+
+ Calls ``sf..describe()`` (``GET
+ /services/data/vXX.0/sobjects//describe``) and returns a
+ normalised schema dictionary mirroring the structure used by
+ ``SnowflakeIngestor.get_table_schema()``.
+
+ Args:
+ sobject_name: sObject API name to introspect, e.g. ``"Account"``.
+
+ Returns:
+ Dictionary with the following keys:
+
+ ``"name"``
+ sObject API name.
+ ``"label"``
+ Human-readable label.
+ ``"fields"``
+ List of field dictionaries, each with ``"name"``, ``"type"``,
+ ``"label"``, ``"nillable"``, and ``"length"``.
+ ``"queryable"``
+ Whether the sObject supports SOQL queries.
+
+ Raises:
+ ValidationError: If *sobject_name* is not a valid identifier.
+ ProcessingError: If the Salesforce API call fails.
+ """
+ _validate_sobject_name(sobject_name)
+
+ try:
+ already_connected = self.connector._client is not None
+ client = self.connector.connect()
+
+ try:
+ sftype = getattr(client, sobject_name)
+ result = sftype.describe()
+
+ fields = [
+ {
+ "name": f.get("name"),
+ "type": f.get("type"),
+ "label": f.get("label"),
+ "nillable": f.get("nillable", True),
+ "length": f.get("length"),
+ }
+ for f in (result.get("fields") or [])
+ ]
+
+ self.logger.debug(
+ "get_sobject_schema: %s — %d field(s)",
+ sobject_name,
+ len(fields),
+ )
+
+ return {
+ "name": result.get("name", sobject_name),
+ "label": result.get("label", sobject_name),
+ "fields": fields,
+ "queryable": result.get("queryable", True),
+ }
+
+ finally:
+ if not already_connected:
+ self.connector.disconnect()
+
+ except (ValidationError, ProcessingError):
+ raise
+ except Exception as exc:
+ self.logger.error(
+ "Failed to get schema for %s: %s", sobject_name, type(exc).__name__
+ )
+ raise ProcessingError(
+ f"Failed to get Salesforce sObject schema for "
+ f"'{sobject_name}': {type(exc).__name__}"
+ ) from exc
+
+ def export_as_documents(
+ self,
+ data: "SalesforceData",
+ id_field: str = "Id",
+ text_fields: Optional[List[str]] = None,
+ ) -> List[Dict[str, Any]]:
+ """Convert :class:`SalesforceData` to the Semantica document format.
+
+ Produces the same ``{"id", "text", "metadata"}`` shape used by
+ ``SnowflakeIngestor.export_as_documents`` and
+ ``DatabricksIngestor.export_as_documents``, making
+ :class:`SalesforceData` directly usable with ``GraphBuilder``.
+
+ Args:
+ data: A :class:`SalesforceData` object returned by
+ :py:meth:`ingest_sobject` or :py:meth:`ingest_query`.
+ id_field: Record field to use as the document ``"id"``. Defaults
+ to ``"Id"`` — Salesforce's canonical 18-character record ID.
+ text_fields: List of field names whose string values are
+ space-joined to form the document ``"text"`` key. When
+ ``None``, all non-``None`` string-valued fields are joined.
+
+ Returns:
+ List of document dictionaries::
+
+ [
+ {
+ "id": str,
+ "text": str,
+ "metadata": {
+ "source": "salesforce",
+ "sobject": "Account",
+ "instance_url": "https://myorg.salesforce.com",
+ "row_data": {...},
+ },
+ },
+ ...
+ ]
+ """
+ documents = []
+
+ for idx, row in enumerate(data.data):
+ doc_id = str(row.get(id_field, idx))
+
+ if text_fields:
+ text_parts = [
+ str(row[f])
+ for f in text_fields
+ if f in row and row[f] is not None
+ ]
+ else:
+ text_parts = [
+ str(v)
+ for v in row.values()
+ if isinstance(v, str) and v
+ ]
+
+ documents.append({
+ "id": doc_id,
+ "text": " ".join(text_parts),
+ "metadata": {
+ "source": "salesforce",
+ "sobject": data.sobject,
+ "instance_url": data.instance_url,
+ "row_data": row,
+ },
+ })
+
+ self.logger.debug(
+ "export_as_documents: exported %d document(s)", len(documents)
+ )
+ return documents
+
+ # ------------------------------------------------------------------
+ # Internal helpers
+ # ------------------------------------------------------------------
+
+ def _build_soql(
+ self,
+ sobject_name: str,
+ fields: List[str],
+ where: Optional[str],
+ order_by: Optional[str],
+ limit: Optional[int],
+ ) -> str:
+ """Build a SOQL SELECT statement from validated components.
+
+ All identifiers have already been validated by the caller. This
+ method only assembles the string — it does not validate.
+
+ Args:
+ sobject_name: Validated sObject API name.
+ fields: List of validated field API names.
+ where: Optional validated WHERE clause fragment (no ``WHERE``
+ keyword).
+ order_by: Optional validated ORDER BY fragment (no keyword).
+ limit: Optional integer row cap. When *limit* is ``None``,
+ no ``LIMIT`` clause is emitted (Salesforce paginates
+ automatically). When *limit* ≤ 2000, it is safe to embed
+ directly in SOQL; for larger values the pagination loop in
+ :py:meth:`_query_all` will stop early.
+
+ Returns:
+ A complete SOQL query string.
+ """
+ field_list = ", ".join(fields)
+ soql = f"SELECT {field_list} FROM {sobject_name}"
+
+ if where:
+ soql += f" WHERE {where}"
+
+ if order_by:
+ soql += f" ORDER BY {order_by}"
+
+ # Embed LIMIT in SOQL only when 1 ≤ limit ≤ 2000.
+ # - limit=0 is handled upstream (returns early before _build_soql is called).
+ # - limit > 2000: no LIMIT clause; _query_all enforces the cap via slicing.
+ if limit is not None and 1 <= limit <= 2000:
+ soql += f" LIMIT {int(limit)}"
+
+ return soql
+
+ def _query_all(
+ self,
+ client: Any,
+ soql: str,
+ limit: Optional[int],
+ ) -> Tuple[List[Dict[str, Any]], Optional[int]]:
+ """Execute *soql* and follow all ``nextRecordsUrl`` pagination links.
+
+ Args:
+ client: An authenticated ``simple_salesforce.Salesforce`` instance.
+ soql: Complete SOQL query string.
+ limit: Maximum number of raw records to collect. ``None`` means
+ collect everything.
+
+ Returns:
+ A ``(records, total_size)`` tuple where *records* is the full
+ list of raw record dicts (``attributes`` still present at this
+ stage — they are stripped by :py:meth:`_convert_rows`) and
+ *total_size* is the ``totalSize`` value from the first response
+ (the number of records matching the query before any limit).
+
+ Raises:
+ ProcessingError: If the Salesforce API raises any exception
+ during pagination.
+ """
+ try:
+ result = client.query(soql)
+ except Exception as exc:
+ raise ProcessingError(
+ f"Salesforce SOQL query failed: {type(exc).__name__}"
+ ) from exc
+
+ total_size: Optional[int] = result.get("totalSize")
+ records: List[Dict[str, Any]] = list(result.get("records") or [])
+
+ # Follow nextRecordsUrl pages until done or limit reached.
+ while not result.get("done", True):
+ if limit is not None and len(records) >= limit:
+ break
+
+ next_url = result.get("nextRecordsUrl")
+ if not next_url:
+ break
+
+ self.logger.debug(
+ "_query_all: fetching next page (%d records so far)…",
+ len(records),
+ )
+
+ try:
+ # identifier_is_url=True: pass the full path from nextRecordsUrl
+ result = client.query_more(next_url, identifier_is_url=True)
+ except Exception as exc:
+ raise ProcessingError(
+ f"Salesforce pagination (query_more) failed: "
+ f"{type(exc).__name__}"
+ ) from exc
+
+ records.extend(result.get("records") or [])
+
+ # Apply client-side limit cap (covers the limit > 2000 case where we
+ # did not embed LIMIT in SOQL).
+ if limit is not None:
+ records = records[:limit]
+
+ return records, total_size
+
+ def _get_all_field_names(self, client: Any, sobject_name: str) -> List[str]:
+ """Return all *selectable* field API names for *sobject_name* via describe().
+
+ Compound field types ``address`` and ``location`` are excluded because
+ Salesforce rejects them in a ``SELECT`` clause with ``INVALID_FIELD``
+ — their component fields (e.g. ``BillingStreet``, ``BillingCity``) are
+ returned separately and are individually selectable.
+
+ Args:
+ client: Connected simple_salesforce client.
+ sobject_name: Already-validated sObject API name.
+
+ Returns:
+ List of selectable field name strings.
+
+ Raises:
+ ProcessingError: If the describe call fails.
+ """
+ # Compound field types that Salesforce rejects when placed in SELECT.
+ _NON_SELECTABLE_TYPES = frozenset({"address", "location"})
+
+ try:
+ sftype = getattr(client, sobject_name)
+ result = sftype.describe()
+ return [
+ f["name"]
+ for f in (result.get("fields") or [])
+ if f.get("type") not in _NON_SELECTABLE_TYPES
+ ]
+ except Exception as exc:
+ raise ProcessingError(
+ f"Failed to describe Salesforce sObject '{sobject_name}': "
+ f"{type(exc).__name__}"
+ ) from exc
+
+ def _convert_rows(
+ self, rows: List[Dict[str, Any]]
+ ) -> List[Dict[str, Any]]:
+ """Normalise raw Salesforce records to JSON-serialisable dicts.
+
+ Performs three transformations:
+
+ 1. Strips the ``attributes`` key that ``simple-salesforce`` injects
+ into every record (and into nested relationship sub-objects).
+ 2. Recursively flattens nested relationship objects — e.g.
+ ``{"Owner": {"attributes": {...}, "Name": "Alice"}}`` becomes
+ ``{"Owner": {"Name": "Alice"}}``.
+ 3. Converts ``datetime`` objects to ISO-8601 strings; other
+ non-serialisable types are converted via ``str()``.
+
+ Args:
+ rows: Raw record dicts as returned by ``sf.query()`` /
+ ``sf.query_more()``.
+
+ Returns:
+ Cleaned list of record dicts ready for :class:`SalesforceData`.
+ """
+ converted = []
+ for row in rows:
+ converted.append(self._clean_record(row))
+ return converted
+
+ def _clean_record(self, record: Any) -> Any:
+ """Recursively clean a single record or nested value.
+
+ Args:
+ record: A raw value from a Salesforce API response — may be a
+ dict (record or sub-object), a list, a scalar, or ``None``.
+
+ Returns:
+ The cleaned value.
+ """
+ if isinstance(record, dict):
+ cleaned: Dict[str, Any] = {}
+ for key, value in record.items():
+ if key == "attributes":
+ # Drop the simple-salesforce internal metadata dict.
+ continue
+ cleaned[key] = self._clean_record(value)
+ return cleaned
+
+ if isinstance(record, list):
+ return [self._clean_record(item) for item in record]
+
+ if isinstance(record, datetime):
+ return record.isoformat()
+
+ # simple-salesforce returns Salesforce datetime strings as Python
+ # strings already; other numeric/bool/None scalars pass through.
+ return record
diff --git a/tests/test_salesforce_ingestor.py b/tests/test_salesforce_ingestor.py
new file mode 100644
index 00000000..d4e7fff0
--- /dev/null
+++ b/tests/test_salesforce_ingestor.py
@@ -0,0 +1,3318 @@
+"""
+Unit tests for SalesforceConnector and SalesforceIngestor.
+
+All Salesforce API calls are mocked — no real Salesforce account is required.
+
+Test structure mirrors tests/test_snowflake_ingestor.py and
+tests/test_databricks_ingestor.py:
+ - autouse fixture mocks simple-salesforce when not installed
+ - @patch("...SALESFORCE_AVAILABLE", True) guards every test that needs
+ the library to appear installed
+ - credentials are always supplied so secrets are never logged or
+ embedded in assertions
+"""
+
+import os
+from datetime import datetime
+from unittest.mock import MagicMock, Mock, patch
+
+import pytest
+
+# Check whether simple-salesforce is available in this environment.
+try:
+ import simple_salesforce # noqa: F401
+ SALESFORCE_LIB_AVAILABLE = True
+except ImportError:
+ SALESFORCE_LIB_AVAILABLE = False
+
+
+# ---------------------------------------------------------------------------
+# autouse fixture: mock simple_salesforce when not installed
+# ---------------------------------------------------------------------------
+
+@pytest.fixture(autouse=True)
+def _mock_simple_salesforce_if_needed():
+ """If simple-salesforce is absent, inject a minimal stub so imports work."""
+ if not SALESFORCE_LIB_AVAILABLE:
+ sf_mod = MagicMock()
+ sf_exc_mod = MagicMock()
+
+ # Provide real exception classes so isinstance checks work in tests.
+ class _SFError(Exception):
+ pass
+
+ class _SFAuthFailed(_SFError):
+ def __init__(self, code, message):
+ super().__init__(message)
+ self.code = code
+ self.auth_message = message
+
+ class _SFExpired(_SFError):
+ pass
+
+ class _SFGeneral(_SFError):
+ pass
+
+ class _SFMalformed(_SFError):
+ pass
+
+ class _SFRefused(_SFError):
+ pass
+
+ class _SFNotFound(_SFError):
+ pass
+
+ sf_exc_mod.SalesforceError = _SFError
+ sf_exc_mod.SalesforceAuthenticationFailed = _SFAuthFailed
+ sf_exc_mod.SalesforceExpiredSession = _SFExpired
+ sf_exc_mod.SalesforceGeneralError = _SFGeneral
+ sf_exc_mod.SalesforceMalformedRequest = _SFMalformed
+ sf_exc_mod.SalesforceRefusedRequest = _SFRefused
+ sf_exc_mod.SalesforceResourceNotFound = _SFNotFound
+
+ sf_mod.Salesforce = MagicMock()
+ sf_mod.exceptions = sf_exc_mod
+
+ with patch.dict(
+ "sys.modules",
+ {
+ "simple_salesforce": sf_mod,
+ "simple_salesforce.exceptions": sf_exc_mod,
+ },
+ ):
+ yield
+ else:
+ yield
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _make_mock_sf_client(instance="myorg.salesforce.com"):
+ """Return a Mock that looks like a connected simple_salesforce.Salesforce."""
+ mock_client = Mock()
+ mock_client.sf_instance = instance
+ mock_client.base_url = f"https://{instance}/services/data/v59.0/"
+ mock_client.auth_type = "password"
+ mock_client.session = Mock()
+ mock_client.limits = Mock(return_value={"DailyApiRequests": {"Max": 15000, "Remaining": 14999}})
+ return mock_client
+
+
+# ---------------------------------------------------------------------------
+# TestSalesforceConnector — initialisation
+# ---------------------------------------------------------------------------
+
+class TestSalesforceConnectorInit:
+ """Tests for SalesforceConnector.__init__ and credential validation."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_init_with_username_password_token(self):
+ """Connector accepts full username/password/token credentials."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ conn = SalesforceConnector(
+ username="user@org.com",
+ password="s3cr3t",
+ security_token="TOKEN123",
+ )
+
+ assert conn.username == "user@org.com"
+ # Passwords and tokens must be stored but not exposed as plain attrs.
+ assert conn._password == "s3cr3t"
+ assert conn._security_token == "TOKEN123"
+ assert conn.domain == "login" # default
+ assert conn._client is None # not connected yet
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_init_default_domain_is_login(self):
+ """Domain defaults to 'login' when not specified."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ conn = SalesforceConnector(
+ username="u", password="p", security_token="t"
+ )
+ assert conn.domain == "login"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_init_sandbox_domain(self):
+ """domain='test' is stored and will be forwarded to simple-salesforce."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ conn = SalesforceConnector(
+ username="u", password="p", security_token="t", domain="test"
+ )
+ assert conn.domain == "test"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_init_with_session_id_and_instance_url(self):
+ """Connector accepts session_id + instance_url authentication."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ conn = SalesforceConnector(
+ session_id="00D...",
+ instance_url="https://myorg.my.salesforce.com",
+ )
+
+ assert conn._session_id == "00D..."
+ assert conn.instance_url == "https://myorg.my.salesforce.com"
+ assert conn._client is None
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_init_api_version_stored(self):
+ """api_version is stored for forwarding to simple-salesforce."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ conn = SalesforceConnector(
+ username="u", password="p", security_token="t", api_version="58.0"
+ )
+ assert conn.api_version == "58.0"
+
+
+# ---------------------------------------------------------------------------
+# TestSalesforceConnectorInit — environment variable fallback
+# ---------------------------------------------------------------------------
+
+class TestSalesforceConnectorEnvVars:
+ """Tests that env-var fallbacks work correctly."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_credentials_from_env_vars(self):
+ """All three credential env vars are read when args are omitted."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ env = {
+ "SALESFORCE_USERNAME": "env_user@org.com",
+ "SALESFORCE_PASSWORD": "env_pass",
+ "SALESFORCE_SECURITY_TOKEN": "env_token",
+ }
+ with patch.dict(os.environ, env):
+ conn = SalesforceConnector()
+
+ assert conn.username == "env_user@org.com"
+ assert conn._password == "env_pass"
+ assert conn._security_token == "env_token"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_domain_from_env_var(self):
+ """SALESFORCE_DOMAIN env var sets the domain."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ with patch.dict(
+ os.environ,
+ {
+ "SALESFORCE_USERNAME": "u",
+ "SALESFORCE_PASSWORD": "p",
+ "SALESFORCE_SECURITY_TOKEN": "t",
+ "SALESFORCE_DOMAIN": "test",
+ },
+ ):
+ conn = SalesforceConnector()
+
+ assert conn.domain == "test"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_api_version_from_env_var(self):
+ """SALESFORCE_API_VERSION env var is honoured."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ with patch.dict(
+ os.environ,
+ {
+ "SALESFORCE_USERNAME": "u",
+ "SALESFORCE_PASSWORD": "p",
+ "SALESFORCE_SECURITY_TOKEN": "t",
+ "SALESFORCE_API_VERSION": "57.0",
+ },
+ ):
+ conn = SalesforceConnector()
+
+ assert conn.api_version == "57.0"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_constructor_arg_overrides_env_var(self):
+ """Explicit constructor args take precedence over env vars."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ with patch.dict(
+ os.environ,
+ {
+ "SALESFORCE_USERNAME": "env_user",
+ "SALESFORCE_PASSWORD": "env_pass",
+ "SALESFORCE_SECURITY_TOKEN": "env_token",
+ },
+ ):
+ conn = SalesforceConnector(
+ username="explicit_user",
+ password="explicit_pass",
+ security_token="explicit_token",
+ )
+
+ assert conn.username == "explicit_user"
+ assert conn._password == "explicit_pass"
+ assert conn._security_token == "explicit_token"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_session_and_instance_from_env_vars(self):
+ """SALESFORCE_SESSION_ID + SALESFORCE_INSTANCE_URL env vars work."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ with patch.dict(
+ os.environ,
+ {
+ "SALESFORCE_SESSION_ID": "env_sid",
+ "SALESFORCE_INSTANCE_URL": "https://env.salesforce.com",
+ },
+ ):
+ conn = SalesforceConnector()
+
+ assert conn._session_id == "env_sid"
+ assert conn.instance_url == "https://env.salesforce.com"
+
+
+# ---------------------------------------------------------------------------
+# TestSalesforceConnectorValidation — missing credentials
+# ---------------------------------------------------------------------------
+
+class TestSalesforceConnectorValidation:
+ """Tests that ValidationError is raised for incomplete credentials."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_no_credentials_raises_validation_error(self):
+ """No credentials at all raises ValidationError."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+ from semantica.utils.exceptions import ValidationError
+
+ with pytest.raises(ValidationError):
+ SalesforceConnector()
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_username_only_raises_validation_error(self):
+ """Username without password/token raises ValidationError."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+ from semantica.utils.exceptions import ValidationError
+
+ with pytest.raises(ValidationError):
+ SalesforceConnector(username="only_user")
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_username_password_without_token_raises(self):
+ """Username + password without security_token raises ValidationError."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+ from semantica.utils.exceptions import ValidationError
+
+ with pytest.raises(ValidationError):
+ SalesforceConnector(username="u", password="p")
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_session_id_without_instance_url_raises(self):
+ """session_id without instance_url raises ValidationError."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+ from semantica.utils.exceptions import ValidationError
+
+ with pytest.raises(ValidationError):
+ SalesforceConnector(session_id="00D...")
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_instance_url_without_session_id_raises(self):
+ """instance_url without session_id raises ValidationError."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+ from semantica.utils.exceptions import ValidationError
+
+ with pytest.raises(ValidationError):
+ SalesforceConnector(instance_url="https://myorg.salesforce.com")
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_validation_error_message_does_not_contain_password(self):
+ """The ValidationError message must not expose credential values."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+ from semantica.utils.exceptions import ValidationError
+
+ with pytest.raises(ValidationError) as exc_info:
+ SalesforceConnector(username="u", password="super_secret")
+
+ assert "super_secret" not in str(exc_info.value)
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_missing_lib_raises_import_error(self):
+ """ImportError with install instructions when lib absent."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ with patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", False):
+ with pytest.raises(ImportError) as exc_info:
+ SalesforceConnector(username="u", password="p", security_token="t")
+
+ msg = str(exc_info.value)
+ assert "simple-salesforce" in msg
+ assert "db-salesforce" in msg
+
+
+# ---------------------------------------------------------------------------
+# TestSalesforceConnectorConnect — successful connection
+# ---------------------------------------------------------------------------
+
+class TestSalesforceConnectorConnect:
+ """Tests for connect() with username/password and session-id auth."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_connect_password_auth_calls_simple_salesforce(self, mock_sf_cls):
+ """connect() constructs a Salesforce client with correct kwargs."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+
+ conn = SalesforceConnector(
+ username="user@org.com",
+ password="p4ss",
+ security_token="TOK",
+ )
+ client = conn.connect()
+
+ assert client is mock_sf_cls.return_value
+ call_kwargs = mock_sf_cls.call_args[1]
+ assert call_kwargs["username"] == "user@org.com"
+ assert call_kwargs["password"] == "p4ss"
+ assert call_kwargs["security_token"] == "TOK"
+ assert call_kwargs["domain"] == "login"
+ assert "session_id" not in call_kwargs
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_connect_sandbox_domain_forwarded(self, mock_sf_cls):
+ """domain='test' is forwarded to simple-salesforce."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+
+ conn = SalesforceConnector(
+ username="u", password="p", security_token="t", domain="test"
+ )
+ conn.connect()
+
+ call_kwargs = mock_sf_cls.call_args[1]
+ assert call_kwargs["domain"] == "test"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_connect_api_version_forwarded_as_version(self, mock_sf_cls):
+ """api_version is forwarded as 'version' (simple-salesforce's kwarg name)."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+
+ conn = SalesforceConnector(
+ username="u", password="p", security_token="t", api_version="57.0"
+ )
+ conn.connect()
+
+ call_kwargs = mock_sf_cls.call_args[1]
+ assert call_kwargs["version"] == "57.0"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_connect_session_id_auth(self, mock_sf_cls):
+ """connect() uses session_id + instance_url when provided."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_client = _make_mock_sf_client("myorg.salesforce.com")
+ mock_sf_cls.return_value = mock_client
+
+ conn = SalesforceConnector(
+ session_id="00D_SESSION",
+ instance_url="https://myorg.salesforce.com",
+ )
+ client = conn.connect()
+
+ assert client is mock_client
+ call_kwargs = mock_sf_cls.call_args[1]
+ assert call_kwargs["session_id"] == "00D_SESSION"
+ assert call_kwargs["instance_url"] == "https://myorg.salesforce.com"
+ assert "password" not in call_kwargs
+ assert "security_token" not in call_kwargs
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_connect_populates_instance_url(self, mock_sf_cls):
+ """connect() resolves instance_url from sf_instance after login."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_client = _make_mock_sf_client("na1.salesforce.com")
+ mock_sf_cls.return_value = mock_client
+
+ conn = SalesforceConnector(
+ username="u", password="p", security_token="t"
+ )
+ # instance_url not provided — should be set after connect
+ assert conn.instance_url is None
+ conn.connect()
+ assert conn.instance_url == "https://na1.salesforce.com"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_connect_reuses_existing_client(self, mock_sf_cls):
+ """Second call to connect() returns the same client without re-authenticating."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+
+ conn = SalesforceConnector(
+ username="u", password="p", security_token="t"
+ )
+ client1 = conn.connect()
+ client2 = conn.connect()
+
+ assert client1 is client2
+ # Constructor called exactly once — no re-authentication.
+ mock_sf_cls.assert_called_once()
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_connect_client_property(self, mock_sf_cls):
+ """connector.client returns None before connect and the client after."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+ conn = SalesforceConnector(username="u", password="p", security_token="t")
+
+ assert conn.client is None
+ conn.connect()
+ assert conn.client is not None
+
+
+# ---------------------------------------------------------------------------
+# TestSalesforceConnectorConnect — connection failures
+# ---------------------------------------------------------------------------
+
+class TestSalesforceConnectorFailures:
+ """Tests that connect() raises ProcessingError on failures."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_connect_auth_failure_raises_processing_error(self, mock_sf_cls):
+ """SalesforceAuthenticationFailed is wrapped as ProcessingError."""
+ from semantica.ingest.salesforce_ingestor import (
+ SalesforceConnector,
+ _SalesforceAuthenticationFailed,
+ )
+ from semantica.utils.exceptions import ProcessingError
+
+ mock_sf_cls.side_effect = _SalesforceAuthenticationFailed(
+ "INVALID_LOGIN", "authentication failure"
+ )
+
+ conn = SalesforceConnector(username="u", password="bad", security_token="t")
+
+ with pytest.raises(ProcessingError) as exc_info:
+ conn.connect()
+
+ # The password must not appear in the raised message.
+ assert "bad" not in str(exc_info.value)
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_connect_general_sf_error_raises_processing_error(self, mock_sf_cls):
+ """SalesforceError is wrapped as ProcessingError."""
+ from semantica.ingest.salesforce_ingestor import (
+ SalesforceConnector,
+ _SalesforceError,
+ )
+ from semantica.utils.exceptions import ProcessingError
+
+ mock_sf_cls.side_effect = _SalesforceError(
+ "https://login.salesforce.com", 500, "Salesforce", b"Generic SF error"
+ )
+
+ conn = SalesforceConnector(username="u", password="p", security_token="t")
+
+ with pytest.raises(ProcessingError):
+ conn.connect()
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_connect_network_error_raises_processing_error(self, mock_sf_cls):
+ """Network errors (ConnectionError) are wrapped as ProcessingError."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+ from semantica.utils.exceptions import ProcessingError
+
+ mock_sf_cls.side_effect = ConnectionError("DNS resolution failed")
+
+ conn = SalesforceConnector(username="u", password="p", security_token="t")
+
+ with pytest.raises(ProcessingError) as exc_info:
+ conn.connect()
+
+ # Exception message should not contain the password.
+ assert "p" not in str(exc_info.value)
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_connect_leaves_client_none_on_failure(self, mock_sf_cls):
+ """A failed connect() must not leave a partial client reference."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+ from semantica.utils.exceptions import ProcessingError
+
+ mock_sf_cls.side_effect = RuntimeError("unexpected")
+
+ conn = SalesforceConnector(username="u", password="p", security_token="t")
+
+ with pytest.raises(ProcessingError):
+ conn.connect()
+
+ assert conn._client is None
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_connect_error_message_does_not_contain_password(self, mock_sf_cls):
+ """ProcessingError raised by connect() must not echo the password."""
+ from semantica.ingest.salesforce_ingestor import (
+ SalesforceConnector,
+ _SalesforceAuthenticationFailed,
+ )
+ from semantica.utils.exceptions import ProcessingError
+
+ mock_sf_cls.side_effect = _SalesforceAuthenticationFailed(
+ "INVALID_LOGIN", "bad credentials"
+ )
+
+ conn = SalesforceConnector(
+ username="u", password="secret_pass", security_token="secret_tok"
+ )
+
+ with pytest.raises(ProcessingError) as exc_info:
+ conn.connect()
+
+ assert "secret_pass" not in str(exc_info.value)
+ assert "secret_tok" not in str(exc_info.value)
+
+
+# ---------------------------------------------------------------------------
+# TestSalesforceConnectorDisconnect
+# ---------------------------------------------------------------------------
+
+class TestSalesforceConnectorDisconnect:
+ """Tests for disconnect() / close() behaviour."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_disconnect_closes_session_and_clears_client(self, mock_sf_cls):
+ """disconnect() closes the requests.Session and sets _client to None."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ conn = SalesforceConnector(username="u", password="p", security_token="t")
+ conn.connect()
+ assert conn._client is not None
+
+ conn.disconnect()
+
+ mock_client.session.close.assert_called_once()
+ assert conn._client is None
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_disconnect_is_idempotent(self, mock_sf_cls):
+ """Calling disconnect() twice does not raise."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+ conn = SalesforceConnector(username="u", password="p", security_token="t")
+ conn.connect()
+
+ conn.disconnect()
+ conn.disconnect() # second call — must not raise
+
+ assert conn._client is None
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_disconnect_without_connect_is_safe(self):
+ """disconnect() before connect() must not raise."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ conn = SalesforceConnector(username="u", password="p", security_token="t")
+ conn.disconnect() # must not raise
+ assert conn._client is None
+
+
+# ---------------------------------------------------------------------------
+# TestSalesforceConnectorTestConnection
+# ---------------------------------------------------------------------------
+
+class TestSalesforceConnectorTestConnection:
+ """Tests for test_connection()."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_test_connection_success_returns_true(self, mock_sf_cls):
+ """test_connection() returns True when limits() succeeds."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ conn = SalesforceConnector(username="u", password="p", security_token="t")
+ result = conn.test_connection()
+
+ assert result is True
+ mock_client.limits.assert_called_once()
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_test_connection_closes_transient_connection(self, mock_sf_cls):
+ """test_connection() closes the connection it opens (no leak)."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ conn = SalesforceConnector(username="u", password="p", security_token="t")
+ conn.test_connection()
+
+ # Connection must be closed after the test.
+ assert conn._client is None
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_test_connection_failure_returns_false(self, mock_sf_cls):
+ """test_connection() returns False when connect() fails."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_sf_cls.side_effect = Exception("Connection refused")
+
+ conn = SalesforceConnector(username="u", password="p", security_token="t")
+ result = conn.test_connection()
+
+ assert result is False
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_test_connection_limits_failure_returns_false(self, mock_sf_cls):
+ """test_connection() returns False when limits() raises."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_client = _make_mock_sf_client()
+ mock_client.limits.side_effect = Exception("Rate limit exceeded")
+ mock_sf_cls.return_value = mock_client
+
+ conn = SalesforceConnector(username="u", password="p", security_token="t")
+ result = conn.test_connection()
+
+ assert result is False
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_test_connection_does_not_close_pre_existing_connection(self, mock_sf_cls):
+ """test_connection() must not close a connection opened before the call."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ conn = SalesforceConnector(username="u", password="p", security_token="t")
+ conn.connect() # open before test_connection
+ assert conn._client is not None
+
+ conn.test_connection()
+
+ # Connection opened externally must still be alive.
+ assert conn._client is not None
+
+
+# ---------------------------------------------------------------------------
+# TestSalesforceConnectorSecrets
+# ---------------------------------------------------------------------------
+
+class TestSalesforceConnectorSecrets:
+ """Verify that no secret values are exposed in logs or exception messages."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_password_not_in_repr(self, mock_sf_cls):
+ """repr(connector) must not expose password."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+ conn = SalesforceConnector(
+ username="u", password="my_secret_password", security_token="tok"
+ )
+ assert "my_secret_password" not in repr(conn)
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_token_not_in_repr(self, mock_sf_cls):
+ """repr(connector) must not expose the security token."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+ conn = SalesforceConnector(
+ username="u", password="p", security_token="TOP_SECRET_TOKEN"
+ )
+ assert "TOP_SECRET_TOKEN" not in repr(conn)
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_session_id_not_in_repr(self, mock_sf_cls):
+ """repr(connector) must not expose the session ID."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector
+
+ conn = SalesforceConnector(
+ session_id="00D_VERY_SECRET_SID",
+ instance_url="https://myorg.salesforce.com",
+ )
+ assert "00D_VERY_SECRET_SID" not in repr(conn)
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_auth_error_does_not_leak_password(self, mock_sf_cls):
+ """ProcessingError from connect() must not contain password text."""
+ from semantica.ingest.salesforce_ingestor import (
+ SalesforceConnector,
+ _SalesforceAuthenticationFailed,
+ )
+ from semantica.utils.exceptions import ProcessingError
+
+ mock_sf_cls.side_effect = _SalesforceAuthenticationFailed(
+ "INVALID_LOGIN", "some server message"
+ )
+
+ conn = SalesforceConnector(
+ username="u", password="hunter2", security_token="s3cr3t"
+ )
+ with pytest.raises(ProcessingError) as exc_info:
+ conn.connect()
+
+ assert "hunter2" not in str(exc_info.value)
+ assert "s3cr3t" not in str(exc_info.value)
+
+
+# ---------------------------------------------------------------------------
+# TestSalesforceIngestor
+# ---------------------------------------------------------------------------
+
+class TestSalesforceIngestor:
+ """Tests for the SalesforceIngestor wrapper."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingestor_creates_connector(self, mock_sf_cls):
+ """SalesforceIngestor creates a SalesforceConnector on init."""
+ from semantica.ingest.salesforce_ingestor import SalesforceConnector, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(
+ username="u", password="p", security_token="t"
+ )
+
+ assert isinstance(ingestor.connector, SalesforceConnector)
+ assert ingestor.connector.username == "u"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingestor_missing_credentials_raises_validation_error(self, mock_sf_cls):
+ """ValidationError propagates when no credentials are given."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+ from semantica.utils.exceptions import ValidationError
+
+ with pytest.raises(ValidationError):
+ SalesforceIngestor()
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_context_manager_connects_and_disconnects(self, mock_sf_cls):
+ """__enter__ opens connection; __exit__ closes it."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ with SalesforceIngestor(
+ username="u", password="p", security_token="t"
+ ) as sf:
+ # Client should be live inside the context.
+ assert sf.connector._client is mock_client
+
+ # Client must be released on exit.
+ assert sf.connector._client is None
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_context_manager_connects_only_once(self, mock_sf_cls):
+ """Multiple operations inside context manager reuse a single connection."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ with SalesforceIngestor(
+ username="u", password="p", security_token="t"
+ ) as sf:
+ # Simulate ingest methods calling connector.connect() internally.
+ sf.connector.connect()
+ sf.connector.connect()
+
+ # simple-salesforce Salesforce() should have been called exactly once
+ # (by __enter__); the subsequent connect() calls reused the client.
+ mock_sf_cls.assert_called_once()
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_close_delegates_to_connector(self, mock_sf_cls):
+ """close() disconnects the connector."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ ingestor = SalesforceIngestor(
+ username="u", password="p", security_token="t"
+ )
+ ingestor.connector.connect()
+ ingestor.close()
+
+ assert ingestor.connector._client is None
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_context_manager_closes_on_exception(self, mock_sf_cls):
+ """__exit__ closes the connection even when an exception is raised inside."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ with pytest.raises(ValueError):
+ with SalesforceIngestor(
+ username="u", password="p", security_token="t"
+ ) as sf:
+ raise ValueError("something went wrong")
+
+ assert sf.connector._client is None
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_ingestor_missing_lib_raises_import_error(self):
+ """ImportError propagates when simple-salesforce is absent."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ with patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", False):
+ with pytest.raises(ImportError) as exc_info:
+ SalesforceIngestor(username="u", password="p", security_token="t")
+
+ assert "simple-salesforce" in str(exc_info.value)
+
+
+# ---------------------------------------------------------------------------
+# TestSalesforceData
+# ---------------------------------------------------------------------------
+
+class TestSalesforceData:
+ """Tests for the SalesforceData dataclass."""
+
+ def test_creation_with_required_fields(self):
+ """SalesforceData can be created with minimal required fields."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData
+
+ data = SalesforceData(data=[], row_count=0, columns=[])
+
+ assert data.row_count == 0
+ assert data.data == []
+ assert data.columns == []
+ assert data.sobject is None
+ assert data.query is None
+ assert data.instance_url is None
+ assert data.total_size is None
+ assert data.metadata == {}
+ assert isinstance(data.ingested_at, datetime)
+
+ def test_creation_with_all_fields(self):
+ """SalesforceData accepts all optional fields."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData
+
+ records = [{"Id": "001", "Name": "Acme"}]
+ data = SalesforceData(
+ data=records,
+ row_count=1,
+ columns=["Id", "Name"],
+ sobject="Account",
+ query="SELECT Id, Name FROM Account",
+ instance_url="https://myorg.salesforce.com",
+ total_size=1,
+ metadata={"custom": "value"},
+ )
+
+ assert data.row_count == 1
+ assert data.sobject == "Account"
+ assert data.query == "SELECT Id, Name FROM Account"
+ assert data.instance_url == "https://myorg.salesforce.com"
+ assert data.total_size == 1
+ assert data.metadata["custom"] == "value"
+
+ def test_no_optional_dependency_required(self):
+ """SalesforceData is usable even when simple-salesforce is not installed."""
+ # This test intentionally does NOT patch SALESFORCE_AVAILABLE.
+ from semantica.ingest.salesforce_ingestor import SalesforceData
+
+ data = SalesforceData(data=[{"Id": "x"}], row_count=1, columns=["Id"])
+ assert data.row_count == 1
+
+
+# ---------------------------------------------------------------------------
+# TestImportBehaviourWithoutLib
+# ---------------------------------------------------------------------------
+
+class TestImportBehaviourWithoutLib:
+ """Verify graceful degradation when simple-salesforce is absent."""
+
+ def test_salesforce_available_is_false_without_lib(self):
+ """SALESFORCE_AVAILABLE reflects library presence."""
+ from semantica.ingest.salesforce_ingestor import SALESFORCE_AVAILABLE
+ # SALESFORCE_AVAILABLE should match whether the lib is actually installed.
+ assert SALESFORCE_AVAILABLE is SALESFORCE_LIB_AVAILABLE
+
+ def test_semantica_ingest_imports_cleanly_without_lib(self):
+ """semantica.ingest imports successfully even without simple-salesforce."""
+ import semantica.ingest as pkg # noqa: F401 — import must not raise
+ assert hasattr(pkg, "SalesforceIngestor")
+ assert hasattr(pkg, "SalesforceConnector")
+ assert hasattr(pkg, "SalesforceData")
+
+ def test_all_contains_salesforce_names(self):
+ """All three Salesforce symbols appear in semantica.ingest.__all__."""
+ import semantica.ingest as pkg
+
+ for name in ("SalesforceIngestor", "SalesforceConnector", "SalesforceData"):
+ assert name in pkg.__all__, f"{name} missing from __all__"
+
+
+# ===========================================================================
+# Stage 3 — ingestion methods, validators, pagination, schema, export
+# ===========================================================================
+
+# ---------------------------------------------------------------------------
+# Helpers shared by Stage 3 tests
+# ---------------------------------------------------------------------------
+
+def _make_query_result(records, total_size=None, done=True, next_url=None):
+ """Build a mock sf.query() / sf.query_more() response dict."""
+ return {
+ "totalSize": total_size if total_size is not None else len(records),
+ "done": done,
+ "nextRecordsUrl": next_url,
+ "records": records,
+ }
+
+
+def _sf_record(sobject, **fields):
+ """Build a raw Salesforce record dict (with attributes, like the real API)."""
+ rec = {
+ "attributes": {
+ "type": sobject,
+ "url": f"/services/data/v59.0/sobjects/{sobject}/001",
+ }
+ }
+ rec.update(fields)
+ return rec
+
+
+def _make_describe_result(sobject, field_names):
+ """Build a minimal sf.SObjectType.describe() response."""
+ return {
+ "name": sobject,
+ "label": sobject,
+ "queryable": True,
+ "fields": [
+ {"name": f, "type": "string", "label": f, "nillable": True, "length": 255}
+ for f in field_names
+ ],
+ }
+
+
+def _make_global_describe(sobjects):
+ """Build a minimal sf.describe() (global describe) response."""
+ return {
+ "sobjects": [
+ {"name": s, "label": s, "queryable": True}
+ for s in sobjects
+ ]
+ }
+
+
+# ---------------------------------------------------------------------------
+# TestSOQLValidators
+# ---------------------------------------------------------------------------
+
+class TestSOQLValidators:
+ """Unit tests for the SOQL injection-safety validators."""
+
+ # --- _validate_sobject_name ---
+
+ def test_valid_standard_sobject(self):
+ from semantica.ingest.salesforce_ingestor import _validate_sobject_name
+ assert _validate_sobject_name("Account") == "Account"
+
+ def test_valid_custom_sobject(self):
+ from semantica.ingest.salesforce_ingestor import _validate_sobject_name
+ assert _validate_sobject_name("My_Object__c") == "My_Object__c"
+
+ def test_valid_metadata_type(self):
+ from semantica.ingest.salesforce_ingestor import _validate_sobject_name
+ assert _validate_sobject_name("My_Setting__mdt") == "My_Setting__mdt"
+
+ def test_valid_platform_event(self):
+ from semantica.ingest.salesforce_ingestor import _validate_sobject_name
+ assert _validate_sobject_name("Order_Event__e") == "Order_Event__e"
+
+ def test_invalid_sobject_with_semicolon(self):
+ from semantica.ingest.salesforce_ingestor import _validate_sobject_name
+ from semantica.utils.exceptions import ValidationError
+ with pytest.raises(ValidationError):
+ _validate_sobject_name("Account; DROP TABLE")
+
+ def test_invalid_sobject_starts_with_digit(self):
+ from semantica.ingest.salesforce_ingestor import _validate_sobject_name
+ from semantica.utils.exceptions import ValidationError
+ with pytest.raises(ValidationError):
+ _validate_sobject_name("1Account")
+
+ def test_invalid_sobject_with_space(self):
+ from semantica.ingest.salesforce_ingestor import _validate_sobject_name
+ from semantica.utils.exceptions import ValidationError
+ with pytest.raises(ValidationError):
+ _validate_sobject_name("My Object")
+
+ def test_invalid_sobject_empty_string(self):
+ from semantica.ingest.salesforce_ingestor import _validate_sobject_name
+ from semantica.utils.exceptions import ValidationError
+ with pytest.raises(ValidationError):
+ _validate_sobject_name("")
+
+ # --- _validate_field_name ---
+
+ def test_valid_simple_field(self):
+ from semantica.ingest.salesforce_ingestor import _validate_field_name
+ assert _validate_field_name("Name") == "Name"
+
+ def test_valid_custom_field(self):
+ from semantica.ingest.salesforce_ingestor import _validate_field_name
+ assert _validate_field_name("My_Field__c") == "My_Field__c"
+
+ def test_valid_relationship_dot_notation(self):
+ from semantica.ingest.salesforce_ingestor import _validate_field_name
+ assert _validate_field_name("Owner.Name") == "Owner.Name"
+
+ def test_valid_deep_relationship(self):
+ from semantica.ingest.salesforce_ingestor import _validate_field_name
+ assert _validate_field_name("Account.Owner.Name") == "Account.Owner.Name"
+
+ def test_invalid_field_with_injection(self):
+ from semantica.ingest.salesforce_ingestor import _validate_field_name
+ from semantica.utils.exceptions import ValidationError
+ with pytest.raises(ValidationError):
+ _validate_field_name("Name; DROP")
+
+ def test_invalid_field_empty(self):
+ from semantica.ingest.salesforce_ingestor import _validate_field_name
+ from semantica.utils.exceptions import ValidationError
+ with pytest.raises(ValidationError):
+ _validate_field_name("")
+
+ # --- _validate_soql_where ---
+
+ def test_valid_where_clause(self):
+ from semantica.ingest.salesforce_ingestor import _validate_soql_where
+ assert _validate_soql_where("Type = 'Customer'") == "Type = 'Customer'"
+
+ def test_valid_where_with_and(self):
+ from semantica.ingest.salesforce_ingestor import _validate_soql_where
+ result = _validate_soql_where("Type = 'Customer' AND AnnualRevenue > 1000")
+ assert result == "Type = 'Customer' AND AnnualRevenue > 1000"
+
+ def test_valid_where_with_union_as_data(self):
+ """The word 'union' inside a string literal must not be blocked."""
+ from semantica.ingest.salesforce_ingestor import _validate_soql_where
+ # 'union' inside quotes is data, not SQL syntax
+ result = _validate_soql_where("Name = 'Credit Union'")
+ assert "Credit Union" in result
+
+ def test_invalid_where_with_semicolon(self):
+ from semantica.ingest.salesforce_ingestor import _validate_soql_where
+ from semantica.utils.exceptions import ValidationError
+ with pytest.raises(ValidationError):
+ _validate_soql_where("Type = 'X'; DELETE FROM Account")
+
+ def test_invalid_where_with_comment(self):
+ from semantica.ingest.salesforce_ingestor import _validate_soql_where
+ from semantica.utils.exceptions import ValidationError
+ with pytest.raises(ValidationError):
+ _validate_soql_where("Type = 'X' -- comment")
+
+ def test_invalid_where_with_bare_union(self):
+ from semantica.ingest.salesforce_ingestor import _validate_soql_where
+ from semantica.utils.exceptions import ValidationError
+ with pytest.raises(ValidationError):
+ _validate_soql_where("1=1 UNION SELECT Id FROM Contact")
+
+ # --- _validate_order_by ---
+
+ def test_valid_order_by_simple(self):
+ from semantica.ingest.salesforce_ingestor import _validate_order_by
+ assert _validate_order_by("Name ASC") == "Name ASC"
+
+ def test_valid_order_by_multi_column(self):
+ from semantica.ingest.salesforce_ingestor import _validate_order_by
+ result = _validate_order_by("Name ASC, CreatedDate DESC")
+ assert result == "Name ASC, CreatedDate DESC"
+
+ def test_valid_order_by_nulls_last(self):
+ from semantica.ingest.salesforce_ingestor import _validate_order_by
+ result = _validate_order_by("AnnualRevenue DESC NULLS LAST")
+ assert "NULLS LAST" in result
+
+ def test_valid_order_by_relationship(self):
+ from semantica.ingest.salesforce_ingestor import _validate_order_by
+ result = _validate_order_by("Owner.Name ASC")
+ assert result == "Owner.Name ASC"
+
+ def test_invalid_order_by_with_injection(self):
+ from semantica.ingest.salesforce_ingestor import _validate_order_by
+ from semantica.utils.exceptions import ValidationError
+ with pytest.raises(ValidationError):
+ _validate_order_by("Name; DROP TABLE Account")
+
+ def test_invalid_order_by_empty(self):
+ from semantica.ingest.salesforce_ingestor import _validate_order_by
+ from semantica.utils.exceptions import ValidationError
+ with pytest.raises(ValidationError):
+ _validate_order_by("")
+
+
+# ---------------------------------------------------------------------------
+# TestConvertRows
+# ---------------------------------------------------------------------------
+
+class TestConvertRows:
+ """Tests for SalesforceIngestor._convert_rows / _clean_record."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_strips_attributes(self):
+ """attributes key is removed from each record."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ raw = [_sf_record("Account", Id="001", Name="Acme")]
+ result = ingestor._convert_rows(raw)
+
+ assert "attributes" not in result[0]
+ assert result[0]["Id"] == "001"
+ assert result[0]["Name"] == "Acme"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_strips_nested_attributes(self):
+ """attributes is removed from nested relationship sub-objects too."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ raw = [
+ {
+ "attributes": {"type": "Account"},
+ "Id": "001",
+ "Owner": {
+ "attributes": {"type": "User"},
+ "Name": "Alice",
+ },
+ }
+ ]
+ result = ingestor._convert_rows(raw)
+
+ assert "attributes" not in result[0]
+ assert "attributes" not in result[0]["Owner"]
+ assert result[0]["Owner"]["Name"] == "Alice"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_none_values_preserved(self):
+ """None (null) field values are kept as None."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ raw = [_sf_record("Account", Id="001", BillingCity=None)]
+ result = ingestor._convert_rows(raw)
+
+ assert result[0]["BillingCity"] is None
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_datetime_converted_to_iso_string(self):
+ """Python datetime objects are converted to ISO-8601 strings."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ dt = datetime(2024, 6, 15, 12, 0, 0)
+ raw = [{"attributes": {"type": "Account"}, "Id": "001", "CreatedDate": dt}]
+ result = ingestor._convert_rows(raw)
+
+ assert result[0]["CreatedDate"] == "2024-06-15T12:00:00"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_list_values_cleaned(self):
+ """List-valued fields are recursively cleaned."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ raw = [
+ {
+ "attributes": {"type": "Account"},
+ "Id": "001",
+ "Items": [
+ {"attributes": {"type": "Item"}, "Name": "Widget"},
+ ],
+ }
+ ]
+ result = ingestor._convert_rows(raw)
+
+ assert "attributes" not in result[0]["Items"][0]
+ assert result[0]["Items"][0]["Name"] == "Widget"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_string_values_pass_through(self):
+ """String values are kept unchanged."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ raw = [_sf_record("Contact", Id="003", Email="bob@example.com")]
+ result = ingestor._convert_rows(raw)
+
+ assert result[0]["Email"] == "bob@example.com"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_empty_list_returns_empty(self):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ assert ingestor._convert_rows([]) == []
+
+
+# ---------------------------------------------------------------------------
+# TestBuildSOQL
+# ---------------------------------------------------------------------------
+
+class TestBuildSOQL:
+ """Tests for SalesforceIngestor._build_soql."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_basic_select(self):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ soql = ingestor._build_soql("Account", ["Id", "Name"], None, None, None)
+ assert soql == "SELECT Id, Name FROM Account"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_with_where(self):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ soql = ingestor._build_soql("Account", ["Id"], "Type = 'Customer'", None, None)
+ assert "WHERE Type = 'Customer'" in soql
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_with_order_by(self):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ soql = ingestor._build_soql("Account", ["Id"], None, "Name ASC", None)
+ assert "ORDER BY Name ASC" in soql
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_limit_embedded_when_le_2000(self):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ soql = ingestor._build_soql("Account", ["Id"], None, None, 500)
+ assert "LIMIT 500" in soql
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_limit_not_embedded_when_gt_2000(self):
+ """For limit > 2000 we let pagination handle the cap."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ soql = ingestor._build_soql("Account", ["Id"], None, None, 5000)
+ assert "LIMIT" not in soql
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_no_limit_clause_when_none(self):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ soql = ingestor._build_soql("Account", ["Id"], None, None, None)
+ assert "LIMIT" not in soql
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_all_clauses_order(self):
+ """WHERE comes before ORDER BY before LIMIT."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ soql = ingestor._build_soql(
+ "Contact", ["Id", "Name"], "Active = true", "Name ASC", 100
+ )
+ where_pos = soql.index("WHERE")
+ order_pos = soql.index("ORDER BY")
+ limit_pos = soql.index("LIMIT")
+ assert where_pos < order_pos < limit_pos
+
+
+# ---------------------------------------------------------------------------
+# TestIngestSobject
+# ---------------------------------------------------------------------------
+
+class TestIngestSobject:
+ """Tests for SalesforceIngestor.ingest_sobject()."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_basic_ingest_with_explicit_fields(self, mock_sf_cls):
+ """ingest_sobject returns SalesforceData with correct metadata."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([
+ _sf_record("Account", Id="001", Name="Acme"),
+ _sf_record("Account", Id="002", Name="Beta"),
+ ], total_size=2)
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = ingestor.ingest_sobject("Account", fields=["Id", "Name"])
+
+ assert isinstance(data, SalesforceData)
+ assert data.sobject == "Account"
+ assert data.row_count == 2
+ assert data.total_size == 2
+ assert "Id" in data.columns
+ assert "Name" in data.columns
+ # attributes must be stripped
+ assert all("attributes" not in row for row in data.data)
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_calls_describe_when_no_fields(self, mock_sf_cls):
+ """When fields=None, describe() is called to get field list."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ mock_sftype = Mock()
+ mock_sftype.describe.return_value = _make_describe_result(
+ "Account", ["Id", "Name", "BillingCity"]
+ )
+ mock_client.Account = mock_sftype
+
+ mock_client.query.return_value = _make_query_result([
+ _sf_record("Account", Id="001", Name="Acme", BillingCity="SF")
+ ], total_size=1)
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = ingestor.ingest_sobject("Account") # no fields arg
+
+ mock_sftype.describe.assert_called_once()
+ executed_soql = mock_client.query.call_args[0][0]
+ assert "Id" in executed_soql
+ assert "Name" in executed_soql
+ assert data.row_count == 1
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_where_clause_included_in_soql(self, mock_sf_cls):
+ """WHERE fragment is passed to SOQL correctly."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ ingestor.ingest_sobject(
+ "Account", fields=["Id"], where="Type = 'Customer'"
+ )
+
+ soql = mock_client.query.call_args[0][0]
+ assert "WHERE Type = 'Customer'" in soql
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_order_by_included_in_soql(self, mock_sf_cls):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ ingestor.ingest_sobject("Account", fields=["Id"], order_by="Name ASC")
+
+ soql = mock_client.query.call_args[0][0]
+ assert "ORDER BY Name ASC" in soql
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_limit_le_2000_embedded_in_soql(self, mock_sf_cls):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ ingestor.ingest_sobject("Account", fields=["Id"], limit=100)
+
+ soql = mock_client.query.call_args[0][0]
+ assert "LIMIT 100" in soql
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_invalid_sobject_name_raises_validation_error(self, mock_sf_cls):
+ """ingest_sobject raises ValidationError for bad sObject names."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+ from semantica.utils.exceptions import ValidationError
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ with pytest.raises(ValidationError):
+ ingestor.ingest_sobject("Account; DROP TABLE", fields=["Id"])
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_invalid_field_name_raises_validation_error(self, mock_sf_cls):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+ from semantica.utils.exceptions import ValidationError
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ with pytest.raises(ValidationError):
+ ingestor.ingest_sobject("Account", fields=["Id", "Name; DROP"])
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_invalid_where_raises_validation_error(self, mock_sf_cls):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+ from semantica.utils.exceptions import ValidationError
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ with pytest.raises(ValidationError):
+ ingestor.ingest_sobject(
+ "Account", fields=["Id"], where="1=1; DELETE FROM Account"
+ )
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_api_error_wrapped_as_processing_error(self, mock_sf_cls):
+ """Salesforce API errors during query become ProcessingError."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+ from semantica.utils.exceptions import ProcessingError
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.side_effect = RuntimeError("SOQL error")
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ with pytest.raises(ProcessingError):
+ ingestor.ingest_sobject("Account", fields=["Id"])
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_instance_url_populated_on_result(self, mock_sf_cls):
+ """SalesforceData.instance_url comes from the connector."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client("myorg.salesforce.com")
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = ingestor.ingest_sobject("Account", fields=["Id"])
+
+ assert data.instance_url == "https://myorg.salesforce.com"
+
+
+# ---------------------------------------------------------------------------
+# TestPagination
+# ---------------------------------------------------------------------------
+
+class TestPagination:
+ """Tests for _query_all pagination logic."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_single_page_done_true(self, mock_sf_cls):
+ """No query_more call when first response is done=True."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result(
+ [_sf_record("Account", Id="001")], total_size=1, done=True
+ )
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ records, total_size = ingestor._query_all(mock_client, "SELECT Id FROM Account", None)
+
+ assert len(records) == 1
+ assert total_size == 1
+ mock_client.query_more.assert_not_called()
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_two_page_pagination(self, mock_sf_cls):
+ """query_more is called once when done=False on first page."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ page1 = _make_query_result(
+ [_sf_record("Account", Id="001"), _sf_record("Account", Id="002")],
+ total_size=3,
+ done=False,
+ next_url="/services/data/v59.0/query/01g-next",
+ )
+ page2 = _make_query_result(
+ [_sf_record("Account", Id="003")],
+ total_size=3,
+ done=True,
+ )
+
+ mock_client.query.return_value = page1
+ mock_client.query_more.return_value = page2
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ records, total_size = ingestor._query_all(
+ mock_client, "SELECT Id FROM Account", None
+ )
+
+ assert len(records) == 3
+ assert total_size == 3
+ mock_client.query_more.assert_called_once_with(
+ "/services/data/v59.0/query/01g-next", identifier_is_url=True
+ )
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_pagination_stops_at_limit(self, mock_sf_cls):
+ """Pagination stops as soon as limit records are collected."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ # Page 1: 2 records, more available
+ page1 = _make_query_result(
+ [_sf_record("Account", Id="001"), _sf_record("Account", Id="002")],
+ total_size=10,
+ done=False,
+ next_url="/services/data/v59.0/query/01g-next",
+ )
+ mock_client.query.return_value = page1
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ records, _ = ingestor._query_all(mock_client, "SELECT Id FROM Account", limit=2)
+
+ # Should have stopped after page 1 — limit already reached
+ assert len(records) == 2
+ mock_client.query_more.assert_not_called()
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_limit_trims_excess_records(self, mock_sf_cls):
+ """Result is sliced to limit even if a page overshoots."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ # 5 records in one page, limit=3
+ page1 = _make_query_result(
+ [_sf_record("Account", Id=str(i)) for i in range(5)],
+ total_size=5,
+ done=True,
+ )
+ mock_client.query.return_value = page1
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ records, _ = ingestor._query_all(mock_client, "SELECT Id FROM Account", limit=3)
+
+ assert len(records) == 3
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_query_more_failure_raises_processing_error(self, mock_sf_cls):
+ """query_more errors are wrapped as ProcessingError."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+ from semantica.utils.exceptions import ProcessingError
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ page1 = _make_query_result(
+ [_sf_record("Account", Id="001")],
+ total_size=2,
+ done=False,
+ next_url="/services/data/v59.0/query/01g-next",
+ )
+ mock_client.query.return_value = page1
+ mock_client.query_more.side_effect = RuntimeError("Connection reset")
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ with pytest.raises(ProcessingError):
+ ingestor._query_all(mock_client, "SELECT Id FROM Account", None)
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_sobject_with_large_limit_uses_pagination(self, mock_sf_cls):
+ """limit > 2000 doesn't embed LIMIT in SOQL, collects via pagination."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ # Return enough records to satisfy limit=3000 across two pages
+ page1 = _make_query_result(
+ [_sf_record("Account", Id=str(i)) for i in range(2000)],
+ total_size=3000,
+ done=False,
+ next_url="/next",
+ )
+ page2 = _make_query_result(
+ [_sf_record("Account", Id=str(i)) for i in range(2000, 3000)],
+ total_size=3000,
+ done=True,
+ )
+ mock_client.query.return_value = page1
+ mock_client.query_more.return_value = page2
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = ingestor.ingest_sobject("Account", fields=["Id"], limit=3000)
+
+ soql = mock_client.query.call_args[0][0]
+ assert "LIMIT" not in soql # not embedded in SOQL
+ assert data.row_count == 3000 # collected via pagination
+
+
+# ---------------------------------------------------------------------------
+# TestIngestQuery
+# ---------------------------------------------------------------------------
+
+class TestIngestQuery:
+ """Tests for SalesforceIngestor.ingest_query()."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_basic_ingest_query(self, mock_sf_cls):
+ """ingest_query passes SOQL verbatim and returns SalesforceData."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result(
+ [_sf_record("Account", Id="001", Name="Acme")], total_size=1
+ )
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ soql = "SELECT Id, Name FROM Account LIMIT 1"
+ data = ingestor.ingest_query(soql)
+
+ assert isinstance(data, SalesforceData)
+ assert data.query == soql
+ assert data.row_count == 1
+ mock_client.query.assert_called_once_with(soql)
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_query_follows_pagination(self, mock_sf_cls):
+ """ingest_query collects all pages."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ page1 = _make_query_result(
+ [_sf_record("Contact", Id="001")],
+ total_size=2,
+ done=False,
+ next_url="/next",
+ )
+ page2 = _make_query_result(
+ [_sf_record("Contact", Id="002")],
+ total_size=2,
+ done=True,
+ )
+ mock_client.query.return_value = page1
+ mock_client.query_more.return_value = page2
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = ingestor.ingest_query("SELECT Id FROM Contact")
+
+ assert data.row_count == 2
+ assert data.total_size == 2
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_query_api_error_raises_processing_error(self, mock_sf_cls):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+ from semantica.utils.exceptions import ProcessingError
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.side_effect = RuntimeError("Malformed SOQL")
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ with pytest.raises(ProcessingError):
+ ingestor.ingest_query("SELECT FROM Account") # intentionally bad
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_query_closes_transient_connection(self, mock_sf_cls):
+ """ingest_query disconnects when called outside a context manager."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ assert ingestor.connector._client is None
+ ingestor.ingest_query("SELECT Id FROM Account")
+ assert ingestor.connector._client is None
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_query_reuses_context_manager_connection(self, mock_sf_cls):
+ """ingest_query does not close a context-manager connection."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ with SalesforceIngestor(
+ username="u", password="p", security_token="t"
+ ) as sf:
+ sf.ingest_query("SELECT Id FROM Account")
+ # Connection must still be open
+ assert sf.connector._client is mock_client
+
+ # Closed only on __exit__
+ assert sf.connector._client is None
+
+
+# ---------------------------------------------------------------------------
+# TestListSobjects
+# ---------------------------------------------------------------------------
+
+class TestListSobjects:
+ """Tests for SalesforceIngestor.list_sobjects()."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_returns_sorted_list(self, mock_sf_cls):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.describe.return_value = _make_global_describe(
+ ["Contact", "Account", "Opportunity"]
+ )
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ result = ingestor.list_sobjects()
+
+ assert result == ["Account", "Contact", "Opportunity"]
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_api_error_raises_processing_error(self, mock_sf_cls):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+ from semantica.utils.exceptions import ProcessingError
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.describe.side_effect = RuntimeError("Timeout")
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ with pytest.raises(ProcessingError):
+ ingestor.list_sobjects()
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_closes_transient_connection(self, mock_sf_cls):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.describe.return_value = _make_global_describe(["Account"])
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ ingestor.list_sobjects()
+ assert ingestor.connector._client is None
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_empty_org_returns_empty_list(self, mock_sf_cls):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.describe.return_value = {"sobjects": []}
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ assert ingestor.list_sobjects() == []
+
+
+# ---------------------------------------------------------------------------
+# TestGetSobjectSchema
+# ---------------------------------------------------------------------------
+
+class TestGetSobjectSchema:
+ """Tests for SalesforceIngestor.get_sobject_schema()."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_returns_schema_dict(self, mock_sf_cls):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ mock_sftype = Mock()
+ mock_sftype.describe.return_value = _make_describe_result(
+ "Account", ["Id", "Name", "BillingCity"]
+ )
+ mock_client.Account = mock_sftype
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ schema = ingestor.get_sobject_schema("Account")
+
+ assert schema["name"] == "Account"
+ assert schema["queryable"] is True
+ assert len(schema["fields"]) == 3
+ assert schema["fields"][0]["name"] == "Id"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_field_dict_has_required_keys(self, mock_sf_cls):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ mock_sftype = Mock()
+ mock_sftype.describe.return_value = _make_describe_result("Account", ["Id"])
+ mock_client.Account = mock_sftype
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ schema = ingestor.get_sobject_schema("Account")
+
+ field = schema["fields"][0]
+ for key in ("name", "type", "label", "nillable", "length"):
+ assert key in field, f"Missing key: {key}"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_invalid_sobject_name_raises_validation_error(self, mock_sf_cls):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+ from semantica.utils.exceptions import ValidationError
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ with pytest.raises(ValidationError):
+ ingestor.get_sobject_schema("Bad Name!")
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_api_error_raises_processing_error(self, mock_sf_cls):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+ from semantica.utils.exceptions import ProcessingError
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ mock_sftype = Mock()
+ mock_sftype.describe.side_effect = RuntimeError("Not found")
+ mock_client.Account = mock_sftype
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ with pytest.raises(ProcessingError):
+ ingestor.get_sobject_schema("Account")
+
+
+# ---------------------------------------------------------------------------
+# TestExportAsDocuments
+# ---------------------------------------------------------------------------
+
+class TestExportAsDocuments:
+ """Tests for SalesforceIngestor.export_as_documents()."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_basic_export(self):
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = SalesforceData(
+ data=[
+ {"Id": "001", "Name": "Acme", "Industry": "Tech"},
+ {"Id": "002", "Name": "Beta", "Industry": "Finance"},
+ ],
+ row_count=2,
+ columns=["Id", "Name", "Industry"],
+ sobject="Account",
+ instance_url="https://myorg.salesforce.com",
+ )
+
+ docs = ingestor.export_as_documents(data, text_fields=["Name", "Industry"])
+
+ assert len(docs) == 2
+ assert docs[0]["id"] == "001"
+ assert docs[0]["text"] == "Acme Tech"
+ assert docs[0]["metadata"]["source"] == "salesforce"
+ assert docs[0]["metadata"]["sobject"] == "Account"
+ assert docs[0]["metadata"]["instance_url"] == "https://myorg.salesforce.com"
+ assert docs[0]["metadata"]["row_data"]["Name"] == "Acme"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_default_text_uses_all_string_fields(self):
+ """When text_fields is None, all non-None string values are joined."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = SalesforceData(
+ data=[{"Id": "001", "Name": "Acme", "AnnualRevenue": 1000000}],
+ row_count=1,
+ columns=["Id", "Name", "AnnualRevenue"],
+ sobject="Account",
+ instance_url=None,
+ )
+
+ docs = ingestor.export_as_documents(data)
+
+ # text should join string fields Id and Name (AnnualRevenue is int)
+ assert "Acme" in docs[0]["text"]
+ assert "001" in docs[0]["text"]
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_id_field_default_is_Id(self):
+ """Default id_field is 'Id' (Salesforce canonical)."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = SalesforceData(
+ data=[{"Id": "003RECORD", "Name": "Alice"}],
+ row_count=1,
+ columns=["Id", "Name"],
+ sobject="Contact",
+ instance_url=None,
+ )
+
+ docs = ingestor.export_as_documents(data)
+ assert docs[0]["id"] == "003RECORD"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_custom_id_field(self):
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = SalesforceData(
+ data=[{"External_Id__c": "EXT-001", "Name": "Widget"}],
+ row_count=1,
+ columns=["External_Id__c", "Name"],
+ sobject="Product__c",
+ instance_url=None,
+ )
+
+ docs = ingestor.export_as_documents(data, id_field="External_Id__c")
+ assert docs[0]["id"] == "EXT-001"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_missing_id_field_falls_back_to_index(self):
+ """If the id_field key is absent, the row index is used."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = SalesforceData(
+ data=[{"Name": "No-ID record"}],
+ row_count=1,
+ columns=["Name"],
+ sobject="Account",
+ instance_url=None,
+ )
+
+ docs = ingestor.export_as_documents(data)
+ assert docs[0]["id"] == "0" # index 0 as string
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_none_text_fields_skipped(self):
+ """None values in text_fields are not included in text."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = SalesforceData(
+ data=[{"Id": "001", "Name": "Acme", "Description": None}],
+ row_count=1,
+ columns=["Id", "Name", "Description"],
+ sobject="Account",
+ instance_url=None,
+ )
+
+ docs = ingestor.export_as_documents(data, text_fields=["Name", "Description"])
+ assert docs[0]["text"] == "Acme" # None Description excluded
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_empty_data_returns_empty_list(self):
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = SalesforceData(data=[], row_count=0, columns=[], sobject="Account")
+ docs = ingestor.export_as_documents(data)
+ assert docs == []
+
+
+# ---------------------------------------------------------------------------
+# TestConnectionLifecycleWithIngestion
+# ---------------------------------------------------------------------------
+
+class TestConnectionLifecycleWithIngestion:
+ """Connection-reuse and lifecycle tests for ingestion methods."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_multiple_calls_inside_context_manager_reuse_connection(self, mock_sf_cls):
+ """Multiple ingest calls inside a context manager share one connection."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+ mock_client.describe.return_value = _make_global_describe(["Account"])
+
+ with SalesforceIngestor(
+ username="u", password="p", security_token="t"
+ ) as sf:
+ sf.ingest_query("SELECT Id FROM Account")
+ sf.list_sobjects()
+ sf.ingest_query("SELECT Id FROM Contact")
+
+ # Simple Salesforce constructor called exactly once — __enter__ only.
+ mock_sf_cls.assert_called_once()
+ # Connection released on exit.
+ assert sf.connector._client is None
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_standalone_ingest_closes_connection(self, mock_sf_cls):
+ """Standalone ingest_query opens and closes its own connection."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ ingestor.ingest_query("SELECT Id FROM Account")
+
+ assert ingestor.connector._client is None
+
+
+# ===========================================================================
+# Review-stage regression tests — bugs fixed during code review
+# ===========================================================================
+
+class TestReviewFixes:
+ """Regression tests for bugs found and fixed during the Stage 3 review."""
+
+ # ------------------------------------------------------------------
+ # Bug 1: limit=0 must not generate LIMIT 0 in SOQL
+ # ------------------------------------------------------------------
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_limit_zero_returns_empty_without_api_call(self, mock_sf_cls):
+ """limit=0 short-circuits before connecting and returns empty SalesforceData."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = ingestor.ingest_sobject("Account", fields=["Id"], limit=0)
+
+ assert isinstance(data, SalesforceData)
+ assert data.row_count == 0
+ assert data.data == []
+ assert data.sobject == "Account"
+ # No network call should have been made
+ mock_sf_cls.assert_not_called()
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_limit_negative_returns_empty_without_api_call(self, mock_sf_cls):
+ """Negative limit also short-circuits — defence in depth."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = ingestor.ingest_sobject("Contact", fields=["Id"], limit=-1)
+
+ assert data.row_count == 0
+ mock_sf_cls.assert_not_called()
+
+ def test_build_soql_limit_zero_not_embedded(self):
+ """_build_soql must not embed LIMIT 0 — caller handles limit=0 upstream."""
+ with patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ # Calling _build_soql directly with limit=0 should not emit LIMIT 0
+ soql = ingestor._build_soql("Account", ["Id"], None, None, 0)
+ assert "LIMIT 0" not in soql, f"LIMIT 0 found in SOQL: {soql!r}"
+
+ def test_build_soql_limit_one_embedded(self):
+ """limit=1 is the smallest valid SOQL LIMIT — must be embedded."""
+ with patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ soql = ingestor._build_soql("Account", ["Id"], None, None, 1)
+ assert "LIMIT 1" in soql
+
+ def test_build_soql_limit_2000_embedded(self):
+ """limit=2000 is at the boundary — must still be embedded in SOQL."""
+ with patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ soql = ingestor._build_soql("Account", ["Id"], None, None, 2000)
+ assert "LIMIT 2000" in soql
+
+ def test_build_soql_limit_2001_not_embedded(self):
+ """limit=2001 crosses the boundary — must not embed LIMIT in SOQL."""
+ with patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True):
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ soql = ingestor._build_soql("Account", ["Id"], None, None, 2001)
+ assert "LIMIT" not in soql
+
+ # ------------------------------------------------------------------
+ # Bug 2: compound address/location fields must be excluded from auto-describe
+ # ------------------------------------------------------------------
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_get_all_field_names_excludes_address_type(self, mock_sf_cls):
+ """_get_all_field_names filters out compound address fields."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ mock_sftype = Mock()
+ mock_sftype.describe.return_value = {
+ "name": "Account",
+ "fields": [
+ {"name": "Id", "type": "id"},
+ {"name": "Name", "type": "string"},
+ {"name": "BillingAddress", "type": "address"}, # compound — excluded
+ {"name": "BillingStreet", "type": "string"}, # component — included
+ {"name": "BillingCity", "type": "string"}, # component — included
+ ],
+ }
+ mock_client.Account = mock_sftype
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ conn = ingestor.connector.connect()
+ field_names = ingestor._get_all_field_names(conn, "Account")
+
+ assert "BillingAddress" not in field_names, "Compound address field must be excluded"
+ assert "Id" in field_names
+ assert "Name" in field_names
+ assert "BillingStreet" in field_names
+ assert "BillingCity" in field_names
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_get_all_field_names_excludes_location_type(self, mock_sf_cls):
+ """_get_all_field_names filters out compound geolocation fields."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ mock_sftype = Mock()
+ mock_sftype.describe.return_value = {
+ "name": "MyObj__c",
+ "fields": [
+ {"name": "Id", "type": "id"},
+ {"name": "Location__c", "type": "location"}, # compound — excluded
+ {"name": "Location__Latitude__s", "type": "double"}, # component — included
+ {"name": "Location__Longitude__s", "type": "double"}, # component — included
+ ],
+ }
+ mock_client.MyObj__c = mock_sftype
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ conn = ingestor.connector.connect()
+ field_names = ingestor._get_all_field_names(conn, "MyObj__c")
+
+ assert "Location__c" not in field_names
+ assert "Location__Latitude__s" in field_names
+ assert "Location__Longitude__s" in field_names
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_sobject_with_no_fields_excludes_compound_fields(self, mock_sf_cls):
+ """ingest_sobject fields=None must not include address/location in SOQL."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ mock_sftype = Mock()
+ mock_sftype.describe.return_value = {
+ "name": "Account",
+ "fields": [
+ {"name": "Id", "type": "id"},
+ {"name": "Name", "type": "string"},
+ {"name": "BillingAddress", "type": "address"},
+ {"name": "BillingStreet", "type": "string"},
+ ],
+ }
+ mock_client.Account = mock_sftype
+ mock_client.query.return_value = _make_query_result(
+ [_sf_record("Account", Id="001", Name="Acme", BillingStreet="123 Main")],
+ total_size=1,
+ )
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = ingestor.ingest_sobject("Account") # fields=None
+
+ executed_soql = mock_client.query.call_args[0][0]
+ assert "BillingAddress" not in executed_soql, (
+ f"Compound address field found in SOQL: {executed_soql!r}"
+ )
+ assert "BillingStreet" in executed_soql
+
+ # ------------------------------------------------------------------
+ # Bug 3: SOQL string literal regex must use '' not backslash escaping
+ # ------------------------------------------------------------------
+
+ def test_soql_string_literal_regex_matches_sql_style_escaped_quote(self):
+ """_SOQL_STRING_LITERAL_RE must treat '' (two single quotes) as an escaped quote."""
+ from semantica.ingest.salesforce_ingestor import _SOQL_STRING_LITERAL_RE
+
+ # In SOQL, O'Brien is written as 'O''Brien' (two consecutive single quotes)
+ text = "Name = 'O''Brien'"
+ matches = _SOQL_STRING_LITERAL_RE.findall(text)
+
+ # The whole 'O''Brien' must be a single match, not two separate matches
+ assert len(matches) == 1, (
+ f"Expected 1 match for SOQL-style escaped quote, got {len(matches)}: {matches}"
+ )
+ assert matches[0] == "'O''Brien'", (
+ f"Matched wrong literal: {matches[0]!r}"
+ )
+
+ def test_soql_where_union_inside_escaped_quote_literal_not_blocked(self):
+ """'union' inside a SOQL-style ''quoted'' literal must not be blocked."""
+ from semantica.ingest.salesforce_ingestor import _validate_soql_where
+ # SOQL literal containing 'union' as data (properly quoted)
+ # should not raise ValidationError
+ _validate_soql_where("Industry = 'credit union'") # must not raise
+
+ def test_mask_soql_literals_handles_double_quote_escape(self):
+ """_mask_soql_literals must correctly mask O''Brien as a single literal."""
+ from semantica.ingest.salesforce_ingestor import _mask_soql_literals
+
+ result = _mask_soql_literals("Name = 'O''Brien'")
+ # The masked result should have no unmasked 'union'-style tokens
+ # and the literal should be fully replaced
+ assert "O''Brien" not in result, (
+ "Literal content 'O''Brien' should have been masked"
+ )
+ # The quotes and replacement characters should be present
+ assert "'" in result # opening and closing quotes remain
+
+ def test_soql_literal_regex_no_backslash_escape(self):
+ """Backslash is NOT a SOQL quote escape — must not be treated as one."""
+ from semantica.ingest.salesforce_ingestor import _SOQL_STRING_LITERAL_RE
+
+ # In SOQL, backslash has no special meaning inside a string literal
+ # A string ending with backslash before the closing quote is still valid
+ # (backslash is just a literal backslash character)
+ text = r"Name = 'test\value'"
+ matches = _SOQL_STRING_LITERAL_RE.findall(text)
+ # Should match 'test\value' as one literal (backslash is literal)
+ assert len(matches) == 1
+
+
+# ===========================================================================
+# Stage 4 — focused tests per task specification
+# ===========================================================================
+
+# ---------------------------------------------------------------------------
+# TestSalesforceDataConventions
+# ---------------------------------------------------------------------------
+
+class TestSalesforceDataConventions:
+ """Verify SalesforceData field semantics, defaults, and row_count / total_size
+ distinction against the Snowflake/Databricks sibling conventions."""
+
+ def test_row_count_reflects_data_length(self):
+ """row_count must equal len(data), not total_size."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData
+
+ records = [{"Id": "001", "Name": "Acme"}, {"Id": "002", "Name": "Beta"}]
+ data = SalesforceData(
+ data=records,
+ row_count=2,
+ columns=["Id", "Name"],
+ total_size=9999, # many more records match the query
+ )
+
+ assert data.row_count == 2 # records actually in data
+ assert data.total_size == 9999 # records matching the query before limit
+ assert data.row_count != data.total_size # clear distinction
+
+ def test_total_size_none_when_unknown(self):
+ """total_size may be None when the information is not available."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData
+
+ data = SalesforceData(data=[], row_count=0, columns=[])
+ assert data.total_size is None
+
+ def test_metadata_defaults_to_empty_dict(self):
+ """metadata defaults to a fresh empty dict (not a shared instance)."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData
+
+ d1 = SalesforceData(data=[], row_count=0, columns=[])
+ d2 = SalesforceData(data=[], row_count=0, columns=[])
+ assert d1.metadata == {}
+ assert d2.metadata == {}
+ # Mutable default: must be separate instances
+ d1.metadata["x"] = 1
+ assert "x" not in d2.metadata
+
+ def test_ingested_at_is_a_datetime(self):
+ """ingested_at is auto-populated with a datetime on creation."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData
+
+ data = SalesforceData(data=[], row_count=0, columns=[])
+ assert isinstance(data.ingested_at, datetime)
+
+ def test_sobject_none_for_raw_soql(self):
+ """sobject is None when data comes from a raw SOQL query (may span objects)."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData
+
+ data = SalesforceData(
+ data=[{"Id": "001"}],
+ row_count=1,
+ columns=["Id"],
+ query="SELECT Id FROM Account",
+ # sobject intentionally not set
+ )
+ assert data.sobject is None
+
+ def test_sobject_set_for_ingest_sobject_result(self):
+ """sobject is set when data comes from ingest_sobject()."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData
+
+ data = SalesforceData(
+ data=[],
+ row_count=0,
+ columns=[],
+ sobject="Account",
+ )
+ assert data.sobject == "Account"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_sobject_metadata_uses_query_key(self, mock_sf_cls):
+ """ingest_sobject metadata must use 'query' (not 'soql') — consistent with
+ Snowflake/Databricks metadata={'query': query}."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result(
+ [_sf_record("Account", Id="001")], total_size=1
+ )
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = ingestor.ingest_sobject("Account", fields=["Id"])
+
+ assert "query" in data.metadata, (
+ "metadata must contain 'query' key to match Snowflake/Databricks convention"
+ )
+ assert "soql" not in data.metadata, (
+ "'soql' is the old key name — must be 'query'"
+ )
+ # The metadata['query'] value should be the executed SOQL
+ assert "SELECT" in data.metadata["query"]
+ assert "Account" in data.metadata["query"]
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_query_metadata_uses_query_key(self, mock_sf_cls):
+ """ingest_query metadata must contain 'query' key with the executed SOQL."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ soql = "SELECT Id, Name FROM Contact WHERE IsActive = true"
+ data = ingestor.ingest_query(soql)
+
+ assert "query" in data.metadata
+ assert data.metadata["query"] == soql
+
+
+# ---------------------------------------------------------------------------
+# TestObjectDiscoveryCustomObjects
+# ---------------------------------------------------------------------------
+
+class TestObjectDiscoveryCustomObjects:
+ """Verify list_sobjects and get_sobject_schema work with custom objects."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_list_sobjects_includes_custom_objects(self, mock_sf_cls):
+ """list_sobjects returns custom object names ending in __c."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.describe.return_value = _make_global_describe(
+ ["Account", "My_Custom__c", "Another_Object__c", "Contact"]
+ )
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ result = ingestor.list_sobjects()
+
+ assert "My_Custom__c" in result
+ assert "Another_Object__c" in result
+ assert result == sorted(result) # always sorted
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_list_sobjects_includes_metadata_types(self, mock_sf_cls):
+ """list_sobjects includes metadata types ending in __mdt."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.describe.return_value = _make_global_describe(
+ ["Account", "My_Setting__mdt"]
+ )
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ result = ingestor.list_sobjects()
+
+ assert "My_Setting__mdt" in result
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_get_sobject_schema_custom_object(self, mock_sf_cls):
+ """get_sobject_schema works for custom objects (My_Object__c)."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ mock_sftype = Mock()
+ mock_sftype.describe.return_value = {
+ "name": "My_Object__c",
+ "label": "My Object",
+ "queryable": True,
+ "fields": [
+ {"name": "Id", "type": "id", "label": "Record ID",
+ "nillable": False, "length": 18},
+ {"name": "Name", "type": "string", "label": "Name",
+ "nillable": True, "length": 255},
+ {"name": "Custom_Field__c", "type": "string", "label": "Custom",
+ "nillable": True, "length": 100},
+ ],
+ }
+ # simple-salesforce accesses custom objects via attribute access
+ mock_client.My_Object__c = mock_sftype
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ schema = ingestor.get_sobject_schema("My_Object__c")
+
+ assert schema["name"] == "My_Object__c"
+ assert schema["label"] == "My Object"
+ assert schema["queryable"] is True
+ assert len(schema["fields"]) == 3
+ # Custom field present
+ custom_field = next(f for f in schema["fields"] if f["name"] == "Custom_Field__c")
+ assert custom_field["type"] == "string"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_sobject_custom_object(self, mock_sf_cls):
+ """ingest_sobject fetches records from a custom object."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([
+ _sf_record("My_Object__c", Id="a01", Name="Rec1", Custom_Field__c="val1"),
+ _sf_record("My_Object__c", Id="a02", Name="Rec2", Custom_Field__c="val2"),
+ ], total_size=2)
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = ingestor.ingest_sobject(
+ "My_Object__c", fields=["Id", "Name", "Custom_Field__c"]
+ )
+
+ assert data.sobject == "My_Object__c"
+ assert data.row_count == 2
+ assert "Custom_Field__c" in data.columns
+ soql = mock_client.query.call_args[0][0]
+ assert "My_Object__c" in soql
+ assert "Custom_Field__c" in soql
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_invalid_custom_object_name_rejected(self):
+ """Custom object names with injection characters are rejected."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+ from semantica.utils.exceptions import ValidationError
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ with pytest.raises(ValidationError):
+ ingestor.ingest_sobject("My Object__c; DROP", fields=["Id"])
+
+
+# ---------------------------------------------------------------------------
+# TestExportAsDocumentsDetailedCoverage
+# ---------------------------------------------------------------------------
+
+class TestExportAsDocumentsDetailedCoverage:
+ """Detailed export_as_documents tests covering IDs, nested records,
+ document text composition, and metadata completeness."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_salesforce_id_is_18_char_string_in_document_id(self):
+ """Salesforce 18-character Ids are preserved exactly as strings."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ sf_id = "001xx000003GYk2AAG" # realistic 18-char SF Id
+ data = SalesforceData(
+ data=[{"Id": sf_id, "Name": "Acme"}],
+ row_count=1, columns=["Id", "Name"],
+ sobject="Account", instance_url="https://myorg.salesforce.com",
+ )
+
+ docs = ingestor.export_as_documents(data)
+ assert docs[0]["id"] == sf_id # exact 18-char string preserved
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_nested_relationship_fields_in_row_data(self):
+ """Nested relationship sub-objects are accessible in row_data metadata."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ # Record with a cleaned nested relationship (attributes already stripped)
+ record = {
+ "Id": "001",
+ "Name": "Acme",
+ "Owner": {"Name": "Alice", "Id": "005"}, # cleaned sub-object
+ }
+ data = SalesforceData(
+ data=[record], row_count=1, columns=["Id", "Name", "Owner"],
+ sobject="Account", instance_url="https://myorg.salesforce.com",
+ )
+
+ docs = ingestor.export_as_documents(data, text_fields=["Name"])
+ assert docs[0]["metadata"]["row_data"]["Owner"]["Name"] == "Alice"
+ assert docs[0]["metadata"]["row_data"]["Owner"]["Id"] == "005"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_nested_dict_does_not_leak_into_text(self):
+ """When text_fields=None, nested relationship dicts must not appear in text."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ record = {
+ "Id": "001",
+ "Name": "Acme",
+ "Owner": {"Name": "Alice", "Id": "005"}, # dict, not str
+ }
+ data = SalesforceData(
+ data=[record], row_count=1, columns=["Id", "Name", "Owner"],
+ sobject="Account", instance_url=None,
+ )
+
+ docs = ingestor.export_as_documents(data) # text_fields=None
+ # The Owner field is a dict, not a string — must not appear as str(dict)
+ assert "{'Name': 'Alice'" not in docs[0]["text"]
+ assert "OrderedDict" not in docs[0]["text"]
+ # Name and Id (strings) should appear
+ assert "Acme" in docs[0]["text"]
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_text_fields_explicit_excludes_unwanted_fields(self):
+ """Explicit text_fields limits what goes into the text key."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = SalesforceData(
+ data=[{"Id": "001", "Name": "Acme Corp", "Website": "https://acme.com",
+ "Description": "Enterprise software"}],
+ row_count=1, columns=["Id", "Name", "Website", "Description"],
+ sobject="Account", instance_url=None,
+ )
+
+ docs = ingestor.export_as_documents(data, text_fields=["Name", "Description"])
+
+ assert docs[0]["text"] == "Acme Corp Enterprise software"
+ assert "https://acme.com" not in docs[0]["text"]
+ assert "001" not in docs[0]["text"]
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_metadata_source_is_salesforce(self):
+ """Every exported document has metadata.source == 'salesforce'."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = SalesforceData(
+ data=[{"Id": "001", "Name": "X"}, {"Id": "002", "Name": "Y"}],
+ row_count=2, columns=["Id", "Name"],
+ sobject="Contact", instance_url="https://myorg.salesforce.com",
+ )
+
+ docs = ingestor.export_as_documents(data)
+ assert all(d["metadata"]["source"] == "salesforce" for d in docs)
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_metadata_sobject_propagated_to_all_documents(self):
+ """metadata.sobject is set on every document from ingest_sobject data."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = SalesforceData(
+ data=[{"Id": "001"}, {"Id": "002"}],
+ row_count=2, columns=["Id"],
+ sobject="Opportunity", instance_url="https://myorg.salesforce.com",
+ )
+
+ docs = ingestor.export_as_documents(data)
+ assert all(d["metadata"]["sobject"] == "Opportunity" for d in docs)
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_export_from_ingest_query_has_none_sobject(self):
+ """Documents from a raw ingest_query have sobject=None (query may span objects)."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ data = SalesforceData(
+ data=[{"Id": "001", "Name": "Acme"}],
+ row_count=1, columns=["Id", "Name"],
+ sobject=None, # raw SOQL — no sobject
+ query="SELECT Id, Name FROM Account",
+ instance_url="https://myorg.salesforce.com",
+ )
+
+ docs = ingestor.export_as_documents(data)
+ assert docs[0]["metadata"]["sobject"] is None
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_row_data_contains_all_cleaned_fields(self):
+ """metadata.row_data must contain all fields from the cleaned record."""
+ from semantica.ingest.salesforce_ingestor import SalesforceData, SalesforceIngestor
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ record = {
+ "Id": "001",
+ "Name": "Acme",
+ "Phone": "+1-555-0100",
+ "AnnualRevenue": 5000000,
+ "IsActive": True,
+ }
+ data = SalesforceData(
+ data=[record], row_count=1, columns=list(record.keys()),
+ sobject="Account", instance_url=None,
+ )
+
+ docs = ingestor.export_as_documents(data, text_fields=["Name"])
+ row_data = docs[0]["metadata"]["row_data"]
+
+ # All fields preserved in row_data
+ assert row_data["Id"] == "001"
+ assert row_data["Name"] == "Acme"
+ assert row_data["Phone"] == "+1-555-0100"
+ assert row_data["AnnualRevenue"] == 5000000
+ assert row_data["IsActive"] is True
+
+
+# ---------------------------------------------------------------------------
+# TestConnectionReuseWithIngestion
+# ---------------------------------------------------------------------------
+
+class TestConnectionReuseWithIngestion:
+ """Verify connection lifecycle consistency between standalone and
+ context-manager usage across all ingestion methods."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_sobject_standalone_opens_and_closes(self, mock_sf_cls):
+ """Standalone ingest_sobject opens a transient connection and closes it."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ assert ingestor.connector._client is None
+
+ ingestor.ingest_sobject("Account", fields=["Id"])
+
+ assert ingestor.connector._client is None # closed after call
+ mock_sf_cls.assert_called_once() # connected exactly once
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_get_sobject_schema_standalone_opens_and_closes(self, mock_sf_cls):
+ """Standalone get_sobject_schema opens a transient connection and closes it."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+
+ mock_sftype = Mock()
+ mock_sftype.describe.return_value = _make_describe_result("Account", ["Id"])
+ mock_client.Account = mock_sftype
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ ingestor.get_sobject_schema("Account")
+
+ assert ingestor.connector._client is None
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_context_manager_all_methods_reuse_connection(self, mock_sf_cls):
+ """All four ingestion methods inside a context manager reuse the single
+ connection opened by __enter__."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result(
+ [_sf_record("Account", Id="001")], total_size=1
+ )
+ mock_client.describe.return_value = _make_global_describe(["Account"])
+
+ mock_sftype = Mock()
+ mock_sftype.describe.return_value = _make_describe_result("Account", ["Id"])
+ mock_client.Account = mock_sftype
+
+ with SalesforceIngestor(
+ username="u", password="p", security_token="t"
+ ) as sf:
+ # All four methods in one context manager
+ sf.ingest_sobject("Account", fields=["Id"])
+ sf.ingest_query("SELECT Id FROM Account")
+ sf.list_sobjects()
+ sf.get_sobject_schema("Account")
+
+ # Connection must still be live throughout
+ assert sf.connector._client is mock_client
+
+ # Salesforce() constructor called exactly once (__enter__)
+ mock_sf_cls.assert_called_once()
+ # Released only on __exit__
+ assert sf.connector._client is None
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_standalone_ingest_sobject_does_not_close_pre_opened_connection(
+ self, mock_sf_cls
+ ):
+ """If the caller has already opened a connection manually, ingest_sobject
+ must not close it after the call."""
+ from semantica.ingest.salesforce_ingestor import SalesforceIngestor
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ ingestor = SalesforceIngestor(username="u", password="p", security_token="t")
+ ingestor.connector.connect() # manually open
+ assert ingestor.connector._client is mock_client
+
+ ingestor.ingest_sobject("Account", fields=["Id"])
+
+ # Still connected — we opened it, ingest_sobject must not close it
+ assert ingestor.connector._client is mock_client
+
+
+# ===========================================================================
+# Stage 5 — ingest_salesforce() convenience function tests
+# ===========================================================================
+
+class TestIngestSalesforceConvenienceFunction:
+ """Tests for the ingest_salesforce() public-API convenience wrapper."""
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_salesforce_sobject_method(self, mock_sf_cls):
+ """ingest_salesforce(method='sobject') returns SalesforceData."""
+ from semantica.ingest import ingest_salesforce
+ from semantica.ingest.salesforce_ingestor import SalesforceData
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result(
+ [_sf_record("Account", Id="001", Name="Acme")], total_size=1
+ )
+
+ data = ingest_salesforce(
+ {"username": "u", "password": "p", "security_token": "t"},
+ method="sobject",
+ sobject_name="Account",
+ fields=["Id", "Name"],
+ )
+
+ assert isinstance(data, SalesforceData)
+ assert data.sobject == "Account"
+ assert data.row_count == 1
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_salesforce_query_method(self, mock_sf_cls):
+ """ingest_salesforce(method='query') executes raw SOQL."""
+ from semantica.ingest import ingest_salesforce
+ from semantica.ingest.salesforce_ingestor import SalesforceData
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ soql = "SELECT Id, Name FROM Contact LIMIT 10"
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ data = ingest_salesforce(
+ {"username": "u", "password": "p", "security_token": "t"},
+ method="query",
+ soql=soql,
+ )
+
+ assert isinstance(data, SalesforceData)
+ mock_client.query.assert_called_once_with(soql)
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_salesforce_list_sobjects_method(self, mock_sf_cls):
+ """ingest_salesforce(method='list_sobjects') returns a sorted list."""
+ from semantica.ingest import ingest_salesforce
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.describe.return_value = _make_global_describe(
+ ["Contact", "Account", "Lead"]
+ )
+
+ result = ingest_salesforce(
+ {"username": "u", "password": "p", "security_token": "t"},
+ method="list_sobjects",
+ )
+
+ assert isinstance(result, list)
+ assert result == ["Account", "Contact", "Lead"]
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_salesforce_schema_method(self, mock_sf_cls):
+ """ingest_salesforce(method='schema') returns sObject field metadata."""
+ from semantica.ingest import ingest_salesforce
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_sftype = Mock()
+ mock_sftype.describe.return_value = _make_describe_result("Account", ["Id", "Name"])
+ mock_client.Account = mock_sftype
+
+ result = ingest_salesforce(
+ {"username": "u", "password": "p", "security_token": "t"},
+ method="schema",
+ sobject_name="Account",
+ )
+
+ assert isinstance(result, dict)
+ assert result["name"] == "Account"
+ assert len(result["fields"]) == 2
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_salesforce_documents_method(self, mock_sf_cls):
+ """ingest_salesforce(method='documents') returns Semantica document list."""
+ from semantica.ingest import ingest_salesforce
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([
+ _sf_record("Account", Id="001", Name="Acme", Industry="Tech"),
+ ], total_size=1)
+
+ docs = ingest_salesforce(
+ {"username": "u", "password": "p", "security_token": "t"},
+ method="documents",
+ sobject_name="Account",
+ fields=["Id", "Name", "Industry"],
+ text_fields=["Name", "Industry"],
+ )
+
+ assert isinstance(docs, list)
+ assert len(docs) == 1
+ assert docs[0]["id"] == "001"
+ assert "Acme" in docs[0]["text"]
+ assert docs[0]["metadata"]["source"] == "salesforce"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_salesforce_credentials_from_env_vars(self, mock_sf_cls):
+ """When source is None, credentials come from environment variables."""
+ from semantica.ingest import ingest_salesforce
+ from semantica.ingest.salesforce_ingestor import SalesforceData
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ with patch.dict(
+ os.environ,
+ {
+ "SALESFORCE_USERNAME": "env_user",
+ "SALESFORCE_PASSWORD": "env_pass",
+ "SALESFORCE_SECURITY_TOKEN": "env_token",
+ },
+ ):
+ data = ingest_salesforce(
+ method="sobject",
+ sobject_name="Contact",
+ fields=["Id"],
+ )
+
+ assert isinstance(data, SalesforceData)
+ # Credentials must have come from env vars
+ call_kwargs = mock_sf_cls.call_args[1]
+ assert call_kwargs["username"] == "env_user"
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_salesforce_missing_sobject_name_raises(self, mock_sf_cls):
+ """method='sobject' without sobject_name raises ProcessingError."""
+ from semantica.ingest import ingest_salesforce
+ from semantica.utils.exceptions import ProcessingError
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+
+ with pytest.raises(ProcessingError, match="sobject_name"):
+ ingest_salesforce(
+ {"username": "u", "password": "p", "security_token": "t"},
+ method="sobject",
+ # sobject_name intentionally omitted
+ )
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_salesforce_missing_soql_raises(self, mock_sf_cls):
+ """method='query' without soql raises ProcessingError."""
+ from semantica.ingest import ingest_salesforce
+ from semantica.utils.exceptions import ProcessingError
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+
+ with pytest.raises(ProcessingError, match="soql"):
+ ingest_salesforce(
+ {"username": "u", "password": "p", "security_token": "t"},
+ method="query",
+ # soql intentionally omitted
+ )
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_salesforce_unknown_method_raises(self, mock_sf_cls):
+ """Unknown method name raises ProcessingError."""
+ from semantica.ingest import ingest_salesforce
+ from semantica.utils.exceptions import ProcessingError
+
+ mock_sf_cls.return_value = _make_mock_sf_client()
+
+ with pytest.raises(ProcessingError, match="Unknown"):
+ ingest_salesforce(
+ {"username": "u", "password": "p", "security_token": "t"},
+ method="bulk_load",
+ )
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ def test_ingest_salesforce_non_dict_source_raises(self):
+ """Non-dict, non-None source raises ProcessingError."""
+ from semantica.ingest import ingest_salesforce
+ from semantica.utils.exceptions import ProcessingError
+
+ with pytest.raises(ProcessingError):
+ ingest_salesforce("login.salesforce.com", method="sobject", sobject_name="Account")
+
+ def test_ingest_salesforce_missing_lib_raises_configuration_error(self):
+ """ConfigurationError with install hint when simple-salesforce absent."""
+ from semantica.ingest import ingest_salesforce
+ from semantica.utils.exceptions import ConfigurationError
+
+ with patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", False):
+ with pytest.raises((ConfigurationError, ImportError)):
+ ingest_salesforce(
+ {"username": "u", "password": "p", "security_token": "t"},
+ method="sobject",
+ sobject_name="Account",
+ )
+
+ def test_ingest_salesforce_in_public_all(self):
+ """ingest_salesforce is exported from semantica.ingest.__all__."""
+ import semantica.ingest as pkg
+ assert "ingest_salesforce" in pkg.__all__
+
+ def test_ingest_salesforce_accessible_from_package(self):
+ """ingest_salesforce is importable directly from semantica.ingest."""
+ from semantica.ingest import ingest_salesforce # must not raise
+ assert callable(ingest_salesforce)
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_unified_ingest_dispatches_to_salesforce(self, mock_sf_cls):
+ """ingest(source_type='salesforce') routes to ingest_salesforce."""
+ from semantica.ingest import ingest
+ from semantica.ingest.salesforce_ingestor import SalesforceData
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ result = ingest(
+ None,
+ source_type="salesforce",
+ method="sobject",
+ username="u",
+ password="p",
+ security_token="t",
+ sobject_name="Account",
+ fields=["Id"],
+ )
+
+ assert "data" in result
+ assert isinstance(result["data"], SalesforceData)
+
+
+# ===========================================================================
+# Credential isolation — regression tests for the global-config mutation bug
+# ===========================================================================
+
+class TestCredentialIsolation:
+ """Verify that ingest_salesforce() does not write per-call credentials
+ into the global IngestConfig, which would let a later call (or a concurrent
+ call) silently authenticate against the wrong Salesforce org.
+
+ Regression tests for: per-call credentials mutating get_method_config()
+ return value (config.py) and the ingest_salesforce() wrapper (methods.py).
+ """
+
+ def test_get_method_config_returns_copy(self):
+ """IngestConfig.get_method_config() must return a fresh dict each call.
+
+ Mutating the returned dict must not affect the stored method config,
+ and two successive calls must return independent objects.
+ """
+ from semantica.ingest.config import IngestConfig
+
+ cfg = IngestConfig()
+ cfg.set_method_config("salesforce", username="org_user", domain="login")
+
+ first = cfg.get_method_config("salesforce")
+ assert first["username"] == "org_user"
+
+ # Mutate the returned copy — must not affect the stored config.
+ first["username"] = "POISONED"
+ first["password"] = "LEAKED_SECRET"
+
+ second = cfg.get_method_config("salesforce")
+ assert second["username"] == "org_user", (
+ "Stored method config was mutated: credential leaked into global store"
+ )
+ assert "password" not in second, (
+ "Per-call credential 'password' leaked into global method config"
+ )
+
+ def test_get_method_config_returns_independent_copies(self):
+ """Two calls to get_method_config() must return distinct dict objects."""
+ from semantica.ingest.config import IngestConfig
+
+ cfg = IngestConfig()
+ cfg.set_method_config("salesforce", domain="login")
+
+ first = cfg.get_method_config("salesforce")
+ second = cfg.get_method_config("salesforce")
+
+ assert first is not second, (
+ "get_method_config() returned the same object twice; "
+ "callers share a mutable reference"
+ )
+
+ def test_get_method_config_empty_returns_independent_empty_dicts(self):
+ """Even the empty-fallback dict must not be shared across calls."""
+ from semantica.ingest.config import IngestConfig
+
+ cfg = IngestConfig()
+ # No "salesforce" entry registered — both calls hit the {} fallback.
+ first = cfg.get_method_config("salesforce")
+ first["leaked"] = True
+
+ second = cfg.get_method_config("salesforce")
+ assert "leaked" not in second, (
+ "Empty fallback dict is shared; mutation in one call affected another"
+ )
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_ingest_salesforce_credentials_do_not_persist_in_global_config(
+ self, mock_sf_cls
+ ):
+ """Credentials passed to ingest_salesforce() must not persist in the
+ global IngestConfig after the call returns.
+
+ This is the core multi-tenant / long-lived-process regression: a second
+ call without credentials must not silently authenticate as the first
+ caller's org.
+ """
+ from semantica.ingest import ingest_salesforce
+ from semantica.ingest.config import ingest_config
+
+ mock_client = _make_mock_sf_client("org1.salesforce.com")
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ # First call — explicit credentials supplied via source dict.
+ ingest_salesforce(
+ {
+ "username": "user@org1.com",
+ "password": "secret-org1-pass",
+ "security_token": "secret-org1-token",
+ },
+ method="sobject",
+ sobject_name="Account",
+ fields=["Id"],
+ )
+
+ # Inspect the global config store — credentials must NOT be present.
+ stored = ingest_config.get_method_config("salesforce")
+ assert "password" not in stored, (
+ f"'password' leaked into global config after call: {stored}"
+ )
+ assert "security_token" not in stored, (
+ f"'security_token' leaked into global config after call: {stored}"
+ )
+ assert "username" not in stored, (
+ f"'username' leaked into global config after call: {stored}"
+ )
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_second_call_does_not_reuse_first_call_credentials(
+ self, mock_sf_cls
+ ):
+ """A second ingest_salesforce() call with different credentials must
+ not silently inherit credentials from the first call.
+
+ Simulates the multi-tenant scenario: two different orgs called in
+ sequence; each must connect with its own credentials.
+ """
+ from semantica.ingest import ingest_salesforce
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ # First call — org 1.
+ ingest_salesforce(
+ {
+ "username": "user@org1.com",
+ "password": "pass-org1",
+ "security_token": "token-org1",
+ },
+ method="sobject",
+ sobject_name="Account",
+ fields=["Id"],
+ )
+ first_call_kwargs = mock_sf_cls.call_args[1]
+
+ mock_sf_cls.reset_mock()
+
+ # Second call — org 2 with completely different credentials.
+ ingest_salesforce(
+ {
+ "username": "user@org2.com",
+ "password": "pass-org2",
+ "security_token": "token-org2",
+ },
+ method="sobject",
+ sobject_name="Contact",
+ fields=["Id"],
+ )
+ second_call_kwargs = mock_sf_cls.call_args[1]
+
+ # Each call must have connected with its own credentials.
+ assert second_call_kwargs.get("username") == "user@org2.com", (
+ "Second call used wrong username — possible credential bleed from first call"
+ )
+ assert second_call_kwargs.get("password") == "pass-org2", (
+ "Second call used wrong password — first call's password bled into second"
+ )
+ assert second_call_kwargs.get("security_token") == "token-org2", (
+ "Second call used wrong token — first call's token bled into second"
+ )
+
+ @patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", True)
+ @patch("semantica.ingest.salesforce_ingestor._SimpleSalesforce")
+ def test_kwargs_credentials_do_not_persist_in_global_config(
+ self, mock_sf_cls
+ ):
+ """Credentials passed as kwargs (no source dict) also must not persist
+ in the global config after the call.
+ """
+ from semantica.ingest import ingest_salesforce
+ from semantica.ingest.config import ingest_config
+
+ mock_client = _make_mock_sf_client()
+ mock_sf_cls.return_value = mock_client
+ mock_client.query.return_value = _make_query_result([], total_size=0)
+
+ ingest_salesforce(
+ method="sobject",
+ sobject_name="Account",
+ fields=["Id"],
+ username="user@org.com",
+ password="kwarg-secret",
+ security_token="kwarg-token",
+ )
+
+ stored = ingest_config.get_method_config("salesforce")
+ assert "password" not in stored, (
+ f"kwarg password leaked into global config: {stored}"
+ )
+ assert "security_token" not in stored, (
+ f"kwarg security_token leaked into global config: {stored}"
+ )