diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7709a845..0bb6cb33 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -50,45 +50,19 @@ jobs: wait-for-processing: true continue-on-error: true - dismiss-fixed-alerts: - name: Dismiss Fixed Security Alerts - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - steps: - - name: Dismiss resolved CodeQL alerts via API - env: - GH_TOKEN: ${{ github.token }} - REPO: ${{ github.repository }} - run: | - # Patterns fixed in application code (py/path-injection, py/polynomial-redos) - # or excluded via codeql-config.yml (JS alerts in third-party cookbook bundles). - FIXED_PATTERNS=( - "py/clear-text-logging-sensitive-data" - "py/incomplete-url-substring-sanitization" - "py/path-injection" - "py/polynomial-redos" - "js/incomplete-url-substring-sanitization" - "js/insecure-randomness" - "js/prototype-pollution-utility" - "actions/missing-workflow-permissions" - ) - - # Fetch all open code scanning alerts (up to 100 per page) - ALERTS=$(gh api repos/$REPO/code-scanning/alerts \ - --jq '.[] | {number: .number, rule: .rule.id, state: .state}' \ - -X GET -f state=open -f per_page=100) - - for PATTERN in "${FIXED_PATTERNS[@]}"; do - ALERT_NUMS=$(echo "$ALERTS" | jq -r \ - "select(.rule == \"$PATTERN\") | .number") - for NUM in $ALERT_NUMS; do - echo "Dismissing alert #$NUM ($PATTERN)" - gh api repos/$REPO/code-scanning/alerts/$NUM \ - -X PATCH \ - -f state=dismissed \ - -f dismissed_reason="won't fix" \ - -f dismissed_comment="Resolved: py/path-injection and py/polynomial-redos fixed in application code (server.py, enrich.py). JS alerts (js/incomplete-url-substring-sanitization, js/insecure-randomness, js/prototype-pollution-utility) are false positives in minified third-party Plotly/MapLibre bundles excluded via .github/codeql/codeql-config.yml." \ - && echo " ✓ Alert #$NUM dismissed" \ - || echo " ⚠ Could not dismiss alert #$NUM (may already be closed)" - done - done + # NOTE: Auto-dismissal by rule-id is intentionally removed. + # Dismissing every alert that matches a rule ID would silently suppress + # future real vulnerabilities of the same type. The alerts below were + # individually triaged and dismissed manually in the security-enhancement + # PR (alerts #12–#18). New alerts must be reviewed and dismissed by hand, + # or will auto-close when the underlying code no longer triggers them. + # + # If you need to dismiss a specific known-safe alert, pin its alert NUMBER + # here and remove it once CodeQL stops reporting it naturally. Example: + # + # PINNED_ALERT_NUMBERS=(12 13 14 15 16 17 18) + # for NUM in "${PINNED_ALERT_NUMBERS[@]}"; do + # gh api repos/$REPO/code-scanning/alerts/$NUM \ + # -X PATCH -f state=dismissed -f dismissed_reason="false positive" \ + # -f dismissed_comment="" + # done diff --git a/semantica/context/agent_memory.py b/semantica/context/agent_memory.py index 50464598..1811623b 100644 --- a/semantica/context/agent_memory.py +++ b/semantica/context/agent_memory.py @@ -82,6 +82,35 @@ class MemoryItem: embedding: Optional[Any] = None memory_id: Optional[str] = None + def to_dict(self) -> Dict[str, Any]: + """Serialise to a JSON-safe dict. Embeddings are dropped (not JSON-safe).""" + return { + "content": self.content, + "timestamp": self.timestamp.isoformat(), + "metadata": self.metadata, + "entities": self.entities, + "relationships": self.relationships, + "memory_id": self.memory_id, + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "MemoryItem": + """Reconstruct a MemoryItem from a serialised dict.""" + raw_ts = data.get("timestamp") + try: + ts = datetime.fromisoformat(raw_ts) if raw_ts else datetime.utcnow() + except (ValueError, TypeError): + ts = datetime.utcnow() + return cls( + content=data.get("content", ""), + timestamp=ts, + metadata=data.get("metadata", {}), + entities=data.get("entities", []), + relationships=data.get("relationships", []), + embedding=None, # embeddings are not persisted; regenerate on demand + memory_id=data.get("memory_id"), + ) + class AgentMemory: """ @@ -147,9 +176,9 @@ class AgentMemory: os.makedirs(path, exist_ok=True) data = { - "memory_items": self.memory_items, + "memory_items": {k: v.to_dict() for k, v in self.memory_items.items()}, "memory_index": list(self.memory_index), - "short_term_memory": self.short_term_memory, + "short_term_memory": [item.to_dict() for item in self.short_term_memory], "stats": self.stats, } @@ -188,10 +217,15 @@ class AgentMemory: self.logger.warning(f"Memory file not found in: {path}") return - self.memory_items = data.get("memory_items", {}) + raw_items = data.get("memory_items", {}) + self.memory_items = { + k: MemoryItem.from_dict(v) for k, v in raw_items.items() + } raw_index = data.get("memory_index", []) self.memory_index = deque(raw_index, maxlen=self.max_memory_size) - self.short_term_memory = data.get("short_term_memory", []) + self.short_term_memory = [ + MemoryItem.from_dict(item) for item in data.get("short_term_memory", []) + ] self.stats = data.get( "stats", {"total_items": 0, "items_by_type": {}, "last_accessed": None}, diff --git a/semantica/explorer/routes/export_import.py b/semantica/explorer/routes/export_import.py index 655cecdf..e677b28f 100644 --- a/semantica/explorer/routes/export_import.py +++ b/semantica/explorer/routes/export_import.py @@ -18,7 +18,9 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Export / Import"]) _IMPORT_MAX_BYTES = 50 * 1024 * 1024 # 50 MB -_ALLOWED_IMPORT_EXTENSIONS = frozenset({".json", ".csv", ".graphml", ".gexf", ".ttl", ".rdf"}) +# Only formats that the import handler actually parses. +# Do not add extensions here unless a corresponding parsing branch exists below. +_ALLOWED_IMPORT_EXTENSIONS = frozenset({".json", ".csv"}) def _import_response(nodes_added: int, edges_added: int, message: str = "Import successful") -> ImportResponse: @@ -186,7 +188,10 @@ async def import_file( edges_added = session.add_edges(edges) return _import_response(nodes_added, edges_added) - raise HTTPException(status_code=422, detail="Unsupported file type. Upload a .json or .csv file.") + raise HTTPException( + status_code=422, + detail=f"Unsupported file type '{_os.path.splitext(filename)[1]}'. Allowed: {sorted(_ALLOWED_IMPORT_EXTENSIONS)}", + ) @router.post("/api/export") diff --git a/semantica/explorer/routes/sparql.py b/semantica/explorer/routes/sparql.py index a59c7b72..c543e02b 100644 --- a/semantica/explorer/routes/sparql.py +++ b/semantica/explorer/routes/sparql.py @@ -34,6 +34,7 @@ class SparqlResponse(BaseModel): columns: List[str] rows: List[Dict[str, Any]] total: int + truncated: bool = False # True when _SPARQL_MAX_ROWS was hit error: Optional[str] = None error_line: Optional[int] = None error_column: Optional[int] = None @@ -74,8 +75,14 @@ def _build_rdflib_graph(session: GraphSession) -> rdflib.Graph: return graph -_SPARQL_MAX_ROWS = 5_000 # hard cap on returned rows -_SPARQL_TIMEOUT_S = 30 # seconds before aborting the query thread +_SPARQL_MAX_ROWS = 5_000 # hard cap on returned rows +_SPARQL_TIMEOUT_S = 30 # seconds before abandoning the await +_SPARQL_MAX_CONCURRENT = 4 # semaphore: max simultaneous executions + +# Semaphore caps how many graph.query calls run concurrently so that +# timed-out threads (which keep running in the pool) cannot crowd out +# other requests by exhausting the default ThreadPoolExecutor workers. +_sparql_semaphore = asyncio.Semaphore(_SPARQL_MAX_CONCURRENT) @router.post("", response_model=SparqlResponse) @@ -92,38 +99,43 @@ async def execute_sparql( ) graph = await asyncio.to_thread(_build_rdflib_graph, session) - try: - query_results = await asyncio.wait_for( - asyncio.to_thread(graph.query, req.query), - timeout=_SPARQL_TIMEOUT_S, - ) - columns = [str(var) for var in query_results.vars] if query_results.vars else [] - rows: List[Dict[str, Any]] = [] - for row in query_results: - if len(rows) >= _SPARQL_MAX_ROWS: - break - row_data = {} - for index, column in enumerate(columns): - value = row[index] - row_data[column] = str(value) if value is not None else None - rows.append(row_data) - return SparqlResponse(columns=columns, rows=rows, total=len(rows)) - except asyncio.TimeoutError: - return SparqlResponse( - columns=[], - rows=[], - total=0, - error=f"Query timed out after {_SPARQL_TIMEOUT_S} seconds.", - ) - except Exception as exc: - error = str(exc) - line_match = re.search(r"line[\s:]+(\d+)", error, re.IGNORECASE) - column_match = re.search(r"col(?:umn)?[\s:]+(\d+)", error, re.IGNORECASE) - return SparqlResponse( - columns=[], - rows=[], - total=0, - error=error, - error_line=int(line_match.group(1)) if line_match else None, - error_column=int(column_match.group(1)) if column_match else None, - ) + + async with _sparql_semaphore: + try: + query_results = await asyncio.wait_for( + asyncio.to_thread(graph.query, req.query), + timeout=_SPARQL_TIMEOUT_S, + ) + except asyncio.TimeoutError: + return SparqlResponse( + columns=[], + rows=[], + total=0, + error=f"Query timed out after {_SPARQL_TIMEOUT_S} seconds.", + ) + except Exception as exc: + error = str(exc) + line_match = re.search(r"line[\s:]+(\d+)", error, re.IGNORECASE) + column_match = re.search(r"col(?:umn)?[\s:]+(\d+)", error, re.IGNORECASE) + return SparqlResponse( + columns=[], + rows=[], + total=0, + error=error, + error_line=int(line_match.group(1)) if line_match else None, + error_column=int(column_match.group(1)) if column_match else None, + ) + + columns = [str(var) for var in query_results.vars] if query_results.vars else [] + rows: List[Dict[str, Any]] = [] + for row in query_results: + if len(rows) >= _SPARQL_MAX_ROWS: + break + row_data = {} + for index, column in enumerate(columns): + value = row[index] + row_data[column] = str(value) if value is not None else None + rows.append(row_data) + + truncated = len(rows) == _SPARQL_MAX_ROWS + return SparqlResponse(columns=columns, rows=rows, total=len(rows), truncated=truncated)