mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
fix: address 4 post-review bugs from security-enhancement PR
fix(agent_memory): implement MemoryItem.to_dict() / from_dict() for safe JSON persistence — timestamps serialised via isoformat(), embeddings dropped (not JSON-safe, regenerated on demand); save() and load() now round-trip correctly without TypeError or AttributeError (Bug #1) fix(sparql): add asyncio.Semaphore(_SPARQL_MAX_CONCURRENT=4) around graph.query so timed-out threads cannot exhaust the default ThreadPoolExecutor; add `truncated: bool` field to SparqlResponse so callers know when the 5 000-row cap was hit (Bug #2) fix(export_import): trim _ALLOWED_IMPORT_EXTENSIONS to {.json, .csv} — the only formats the handler actually parses; removes .graphml/.gexf/.ttl/.rdf that passed the allowlist check but hit a hard 422 inside the handler (Bug #3) fix(codeql): remove blanket rule-ID auto-dismiss job; replace with a commented template for pinning specific alert numbers — prevents future real alerts of the same rule being silently suppressed (Bug #4) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
a16cb9c468
commit
4acdefd4b8
@@ -50,45 +50,19 @@ jobs:
|
||||
wait-for-processing: true
|
||||
continue-on-error: true
|
||||
|
||||
dismiss-fixed-alerts:
|
||||
name: Dismiss Fixed Security Alerts
|
||||
runs-on: ubuntu-latest
|
||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
steps:
|
||||
- name: Dismiss resolved CodeQL alerts via API
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
# Patterns fixed in application code (py/path-injection, py/polynomial-redos)
|
||||
# or excluded via codeql-config.yml (JS alerts in third-party cookbook bundles).
|
||||
FIXED_PATTERNS=(
|
||||
"py/clear-text-logging-sensitive-data"
|
||||
"py/incomplete-url-substring-sanitization"
|
||||
"py/path-injection"
|
||||
"py/polynomial-redos"
|
||||
"js/incomplete-url-substring-sanitization"
|
||||
"js/insecure-randomness"
|
||||
"js/prototype-pollution-utility"
|
||||
"actions/missing-workflow-permissions"
|
||||
)
|
||||
|
||||
# Fetch all open code scanning alerts (up to 100 per page)
|
||||
ALERTS=$(gh api repos/$REPO/code-scanning/alerts \
|
||||
--jq '.[] | {number: .number, rule: .rule.id, state: .state}' \
|
||||
-X GET -f state=open -f per_page=100)
|
||||
|
||||
for PATTERN in "${FIXED_PATTERNS[@]}"; do
|
||||
ALERT_NUMS=$(echo "$ALERTS" | jq -r \
|
||||
"select(.rule == \"$PATTERN\") | .number")
|
||||
for NUM in $ALERT_NUMS; do
|
||||
echo "Dismissing alert #$NUM ($PATTERN)"
|
||||
gh api repos/$REPO/code-scanning/alerts/$NUM \
|
||||
-X PATCH \
|
||||
-f state=dismissed \
|
||||
-f dismissed_reason="won't fix" \
|
||||
-f dismissed_comment="Resolved: py/path-injection and py/polynomial-redos fixed in application code (server.py, enrich.py). JS alerts (js/incomplete-url-substring-sanitization, js/insecure-randomness, js/prototype-pollution-utility) are false positives in minified third-party Plotly/MapLibre bundles excluded via .github/codeql/codeql-config.yml." \
|
||||
&& echo " ✓ Alert #$NUM dismissed" \
|
||||
|| echo " ⚠ Could not dismiss alert #$NUM (may already be closed)"
|
||||
done
|
||||
done
|
||||
# NOTE: Auto-dismissal by rule-id is intentionally removed.
|
||||
# Dismissing every alert that matches a rule ID would silently suppress
|
||||
# future real vulnerabilities of the same type. The alerts below were
|
||||
# individually triaged and dismissed manually in the security-enhancement
|
||||
# PR (alerts #12–#18). New alerts must be reviewed and dismissed by hand,
|
||||
# or will auto-close when the underlying code no longer triggers them.
|
||||
#
|
||||
# If you need to dismiss a specific known-safe alert, pin its alert NUMBER
|
||||
# here and remove it once CodeQL stops reporting it naturally. Example:
|
||||
#
|
||||
# PINNED_ALERT_NUMBERS=(12 13 14 15 16 17 18)
|
||||
# for NUM in "${PINNED_ALERT_NUMBERS[@]}"; do
|
||||
# gh api repos/$REPO/code-scanning/alerts/$NUM \
|
||||
# -X PATCH -f state=dismissed -f dismissed_reason="false positive" \
|
||||
# -f dismissed_comment="<reason>"
|
||||
# done
|
||||
|
||||
@@ -82,6 +82,35 @@ class MemoryItem:
|
||||
embedding: Optional[Any] = None
|
||||
memory_id: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Serialise to a JSON-safe dict. Embeddings are dropped (not JSON-safe)."""
|
||||
return {
|
||||
"content": self.content,
|
||||
"timestamp": self.timestamp.isoformat(),
|
||||
"metadata": self.metadata,
|
||||
"entities": self.entities,
|
||||
"relationships": self.relationships,
|
||||
"memory_id": self.memory_id,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "MemoryItem":
|
||||
"""Reconstruct a MemoryItem from a serialised dict."""
|
||||
raw_ts = data.get("timestamp")
|
||||
try:
|
||||
ts = datetime.fromisoformat(raw_ts) if raw_ts else datetime.utcnow()
|
||||
except (ValueError, TypeError):
|
||||
ts = datetime.utcnow()
|
||||
return cls(
|
||||
content=data.get("content", ""),
|
||||
timestamp=ts,
|
||||
metadata=data.get("metadata", {}),
|
||||
entities=data.get("entities", []),
|
||||
relationships=data.get("relationships", []),
|
||||
embedding=None, # embeddings are not persisted; regenerate on demand
|
||||
memory_id=data.get("memory_id"),
|
||||
)
|
||||
|
||||
|
||||
class AgentMemory:
|
||||
"""
|
||||
@@ -147,9 +176,9 @@ class AgentMemory:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
|
||||
data = {
|
||||
"memory_items": self.memory_items,
|
||||
"memory_items": {k: v.to_dict() for k, v in self.memory_items.items()},
|
||||
"memory_index": list(self.memory_index),
|
||||
"short_term_memory": self.short_term_memory,
|
||||
"short_term_memory": [item.to_dict() for item in self.short_term_memory],
|
||||
"stats": self.stats,
|
||||
}
|
||||
|
||||
@@ -188,10 +217,15 @@ class AgentMemory:
|
||||
self.logger.warning(f"Memory file not found in: {path}")
|
||||
return
|
||||
|
||||
self.memory_items = data.get("memory_items", {})
|
||||
raw_items = data.get("memory_items", {})
|
||||
self.memory_items = {
|
||||
k: MemoryItem.from_dict(v) for k, v in raw_items.items()
|
||||
}
|
||||
raw_index = data.get("memory_index", [])
|
||||
self.memory_index = deque(raw_index, maxlen=self.max_memory_size)
|
||||
self.short_term_memory = data.get("short_term_memory", [])
|
||||
self.short_term_memory = [
|
||||
MemoryItem.from_dict(item) for item in data.get("short_term_memory", [])
|
||||
]
|
||||
self.stats = data.get(
|
||||
"stats",
|
||||
{"total_items": 0, "items_by_type": {}, "last_accessed": None},
|
||||
|
||||
@@ -18,7 +18,9 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter(tags=["Export / Import"])
|
||||
|
||||
_IMPORT_MAX_BYTES = 50 * 1024 * 1024 # 50 MB
|
||||
_ALLOWED_IMPORT_EXTENSIONS = frozenset({".json", ".csv", ".graphml", ".gexf", ".ttl", ".rdf"})
|
||||
# Only formats that the import handler actually parses.
|
||||
# Do not add extensions here unless a corresponding parsing branch exists below.
|
||||
_ALLOWED_IMPORT_EXTENSIONS = frozenset({".json", ".csv"})
|
||||
|
||||
|
||||
def _import_response(nodes_added: int, edges_added: int, message: str = "Import successful") -> ImportResponse:
|
||||
@@ -186,7 +188,10 @@ async def import_file(
|
||||
edges_added = session.add_edges(edges)
|
||||
return _import_response(nodes_added, edges_added)
|
||||
|
||||
raise HTTPException(status_code=422, detail="Unsupported file type. Upload a .json or .csv file.")
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"Unsupported file type '{_os.path.splitext(filename)[1]}'. Allowed: {sorted(_ALLOWED_IMPORT_EXTENSIONS)}",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/export")
|
||||
|
||||
@@ -34,6 +34,7 @@ class SparqlResponse(BaseModel):
|
||||
columns: List[str]
|
||||
rows: List[Dict[str, Any]]
|
||||
total: int
|
||||
truncated: bool = False # True when _SPARQL_MAX_ROWS was hit
|
||||
error: Optional[str] = None
|
||||
error_line: Optional[int] = None
|
||||
error_column: Optional[int] = None
|
||||
@@ -74,8 +75,14 @@ def _build_rdflib_graph(session: GraphSession) -> rdflib.Graph:
|
||||
return graph
|
||||
|
||||
|
||||
_SPARQL_MAX_ROWS = 5_000 # hard cap on returned rows
|
||||
_SPARQL_TIMEOUT_S = 30 # seconds before aborting the query thread
|
||||
_SPARQL_MAX_ROWS = 5_000 # hard cap on returned rows
|
||||
_SPARQL_TIMEOUT_S = 30 # seconds before abandoning the await
|
||||
_SPARQL_MAX_CONCURRENT = 4 # semaphore: max simultaneous executions
|
||||
|
||||
# Semaphore caps how many graph.query calls run concurrently so that
|
||||
# timed-out threads (which keep running in the pool) cannot crowd out
|
||||
# other requests by exhausting the default ThreadPoolExecutor workers.
|
||||
_sparql_semaphore = asyncio.Semaphore(_SPARQL_MAX_CONCURRENT)
|
||||
|
||||
|
||||
@router.post("", response_model=SparqlResponse)
|
||||
@@ -92,38 +99,43 @@ async def execute_sparql(
|
||||
)
|
||||
|
||||
graph = await asyncio.to_thread(_build_rdflib_graph, session)
|
||||
try:
|
||||
query_results = await asyncio.wait_for(
|
||||
asyncio.to_thread(graph.query, req.query),
|
||||
timeout=_SPARQL_TIMEOUT_S,
|
||||
)
|
||||
columns = [str(var) for var in query_results.vars] if query_results.vars else []
|
||||
rows: List[Dict[str, Any]] = []
|
||||
for row in query_results:
|
||||
if len(rows) >= _SPARQL_MAX_ROWS:
|
||||
break
|
||||
row_data = {}
|
||||
for index, column in enumerate(columns):
|
||||
value = row[index]
|
||||
row_data[column] = str(value) if value is not None else None
|
||||
rows.append(row_data)
|
||||
return SparqlResponse(columns=columns, rows=rows, total=len(rows))
|
||||
except asyncio.TimeoutError:
|
||||
return SparqlResponse(
|
||||
columns=[],
|
||||
rows=[],
|
||||
total=0,
|
||||
error=f"Query timed out after {_SPARQL_TIMEOUT_S} seconds.",
|
||||
)
|
||||
except Exception as exc:
|
||||
error = str(exc)
|
||||
line_match = re.search(r"line[\s:]+(\d+)", error, re.IGNORECASE)
|
||||
column_match = re.search(r"col(?:umn)?[\s:]+(\d+)", error, re.IGNORECASE)
|
||||
return SparqlResponse(
|
||||
columns=[],
|
||||
rows=[],
|
||||
total=0,
|
||||
error=error,
|
||||
error_line=int(line_match.group(1)) if line_match else None,
|
||||
error_column=int(column_match.group(1)) if column_match else None,
|
||||
)
|
||||
|
||||
async with _sparql_semaphore:
|
||||
try:
|
||||
query_results = await asyncio.wait_for(
|
||||
asyncio.to_thread(graph.query, req.query),
|
||||
timeout=_SPARQL_TIMEOUT_S,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
return SparqlResponse(
|
||||
columns=[],
|
||||
rows=[],
|
||||
total=0,
|
||||
error=f"Query timed out after {_SPARQL_TIMEOUT_S} seconds.",
|
||||
)
|
||||
except Exception as exc:
|
||||
error = str(exc)
|
||||
line_match = re.search(r"line[\s:]+(\d+)", error, re.IGNORECASE)
|
||||
column_match = re.search(r"col(?:umn)?[\s:]+(\d+)", error, re.IGNORECASE)
|
||||
return SparqlResponse(
|
||||
columns=[],
|
||||
rows=[],
|
||||
total=0,
|
||||
error=error,
|
||||
error_line=int(line_match.group(1)) if line_match else None,
|
||||
error_column=int(column_match.group(1)) if column_match else None,
|
||||
)
|
||||
|
||||
columns = [str(var) for var in query_results.vars] if query_results.vars else []
|
||||
rows: List[Dict[str, Any]] = []
|
||||
for row in query_results:
|
||||
if len(rows) >= _SPARQL_MAX_ROWS:
|
||||
break
|
||||
row_data = {}
|
||||
for index, column in enumerate(columns):
|
||||
value = row[index]
|
||||
row_data[column] = str(value) if value is not None else None
|
||||
rows.append(row_data)
|
||||
|
||||
truncated = len(rows) == _SPARQL_MAX_ROWS
|
||||
return SparqlResponse(columns=columns, rows=rows, total=len(rows), truncated=truncated)
|
||||
|
||||
Reference in New Issue
Block a user