mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
[SECURITY] Enhance caching security by excluding sensitive keys and using SHA-256
This commit is contained in:
@@ -14,6 +14,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- **Parallel Chunking**: Implemented parallel processing for large document chunking in `_extract_entities_chunked` and `_extract_relations_chunked`, significantly reducing latency for long-form text analysis.
|
||||
- **Thread-Safe Progress Tracking**: Enhanced `ProgressTracker` to handle concurrent updates from multiple threads without race conditions during batch processing.
|
||||
|
||||
### Security
|
||||
- **Secure Caching**:
|
||||
- Updated `ExtractionCache` to exclude sensitive parameters (e.g., `api_key`, `token`, `password`) from cache key generation, preventing secret leakage and enabling safe cache sharing.
|
||||
- Upgraded cache key hashing algorithm from MD5 to **SHA-256** for enhanced collision resistance and security.
|
||||
|
||||
### Performance
|
||||
- **Bottleneck Optimization (GitHub Issue #186)**:
|
||||
- **Resolved Bottleneck #1 (Sequential Processing)**: Replaced sequential `for` loops with parallel execution for both document-level batches and intra-document chunks.
|
||||
|
||||
@@ -72,16 +72,23 @@ class ExtractionCache:
|
||||
def _generate_key(self, text: str, **params) -> str:
|
||||
"""
|
||||
Generate a stable cache key based on text and parameters.
|
||||
|
||||
Note: Sensitive parameters like 'api_key' are excluded from the cache key
|
||||
to prevent security risks and ensure cache sharing where appropriate.
|
||||
"""
|
||||
# Filter out sensitive keys
|
||||
sensitive_keys = {'api_key', 'token', 'password', 'secret', 'auth', 'authorization'}
|
||||
filtered_params = {k: v for k, v in params.items() if k.lower() not in sensitive_keys}
|
||||
|
||||
# Create a stable string representation of params
|
||||
# Sort keys to ensure consistent ordering
|
||||
param_str = json.dumps(params, sort_keys=True, default=str)
|
||||
param_str = json.dumps(filtered_params, sort_keys=True, default=str)
|
||||
|
||||
# Combine text and params
|
||||
content = f"{text}|{param_str}"
|
||||
|
||||
# Return hash
|
||||
return hashlib.md5(content.encode('utf-8')).hexdigest()
|
||||
# Return hash (SHA-256 for better security than MD5)
|
||||
return hashlib.sha256(content.encode('utf-8')).hexdigest()
|
||||
|
||||
def get(self, namespace: str, text: str, **params) -> Optional[Any]:
|
||||
"""
|
||||
|
||||
@@ -98,6 +98,31 @@ class TestSemanticExtractImprovements(unittest.TestCase):
|
||||
self.assertEqual(mock_provider.generate_typed.call_count, 1)
|
||||
|
||||
print(" Cache hit verified for entities.")
|
||||
|
||||
def test_secure_caching(self):
|
||||
"""Test that sensitive parameters are excluded from cache keys."""
|
||||
print("\nTesting Secure Caching...")
|
||||
|
||||
text = "Security test."
|
||||
|
||||
# Mock provider
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.is_available.return_value = True
|
||||
mock_entities_response = MagicMock()
|
||||
mock_entities_response.entities = [MagicMock(text="Test", label="TEST", confidence=1.0)]
|
||||
mock_provider.generate_typed.return_value = mock_entities_response
|
||||
|
||||
with patch('semantica.semantic_extract.methods.create_provider', return_value=mock_provider):
|
||||
# First call with one API key
|
||||
extract_entities_llm(text, provider="openai", model="gpt-4", api_key="secret_key_1")
|
||||
|
||||
# Second call with DIFFERENT API key
|
||||
# If secure caching is working, this should be a CACHE HIT because api_key is ignored
|
||||
extract_entities_llm(text, provider="openai", model="gpt-4", api_key="secret_key_2")
|
||||
|
||||
# Provider should have been called ONLY ONCE
|
||||
self.assertEqual(mock_provider.generate_typed.call_count, 1)
|
||||
print(" Secure caching verified: Changing API key did not trigger new extraction.")
|
||||
|
||||
# Verify cache content
|
||||
self.assertIn("entities", _result_cache._caches)
|
||||
|
||||
Reference in New Issue
Block a user