From 017a36591d7639a5203f6f94875d53836f81b930 Mon Sep 17 00:00:00 2001 From: saloni <138708065+saloni0318@users.noreply.github.com> Date: Thu, 29 Jan 2026 00:07:19 +0530 Subject: [PATCH 1/3] pandas_ingestor.py --- semantica/ingest/pandas_ingestor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/semantica/ingest/pandas_ingestor.py b/semantica/ingest/pandas_ingestor.py index b699eb91..a5c2fed5 100644 --- a/semantica/ingest/pandas_ingestor.py +++ b/semantica/ingest/pandas_ingestor.py @@ -193,10 +193,10 @@ class PandasIngestor: raise ProcessingError(f"Failed to ingest DataFrame: {e}") from e - def from_csv( + def from_csv( self, file_path: Union[str, Path], - chunksize: Optional[int] = None, + chunksize: Optional[int] = None , **pandas_options, ) -> PandasData: file_path = Path(file_path) @@ -247,7 +247,7 @@ class PandasIngestor: sep=delimiter, encoding=encoding, encoding_errors="replace", - quoting=csv.QUOTE_MINIMAL, + quoting=csv.QUOTE_NONE, header=0 if has_header else None, quotechar=quotechar, escapechar="\\", From 327792c830a5497b03d3f80ca744cad3927d01ce Mon Sep 17 00:00:00 2001 From: saloni <138708065+saloni0318@users.noreply.github.com> Date: Thu, 29 Jan 2026 00:09:12 +0530 Subject: [PATCH 2/3] test_ingest_from_csv().py --- tests/ingest/test_ingest_from_csv().py | 156 +++++++++++++++---------- 1 file changed, 94 insertions(+), 62 deletions(-) diff --git a/tests/ingest/test_ingest_from_csv().py b/tests/ingest/test_ingest_from_csv().py index a05ad271..978af615 100644 --- a/tests/ingest/test_ingest_from_csv().py +++ b/tests/ingest/test_ingest_from_csv().py @@ -1,13 +1,11 @@ import os import tempfile -import pandas as pd -import pytest from semantica.ingest.pandas_ingestor import PandasIngestor # ------------------------------------------------------- -# Helper to write temporary CSV +# Helper # ------------------------------------------------------- def write_temp_csv(content: str, encoding="utf-8"): @@ -18,75 +16,109 @@ def write_temp_csv(content: str, encoding="utf-8"): return tmp.name -# ------------------------------------------------------- -# Test 1: Encoding Detection (international entities) -# ------------------------------------------------------- - -def test_from_csv_detects_encoding(): - # Latin-1 encoded content - content = ( - "entity_name,location\n" - "Siemens,München\n" - "Telefónica,España\n" - ) +# ======================================================= +# ENCODING (5 tests) +# ======================================================= +def test_encoding_latin1(): + content = "name,city\nJosé,São Paulo\n" path = write_temp_csv(content, encoding="latin-1") - - ingestor = PandasIngestor() - data = ingestor.from_csv(path) - - assert data.row_count == 2 - assert data.columns == ["entity_name", "location"] - assert data.dataframe.iloc[0]["entity_name"] == "Siemens" - + data = PandasIngestor().from_csv(path) + assert data.dataframe.iloc[0]["name"] == "José" os.remove(path) -# ------------------------------------------------------- -# Test 2: Delimiter Detection (semicolon separated) -# ------------------------------------------------------- +def test_encoding_utf8(): + content = "user,country\n李雷,China\n" + path = write_temp_csv(content, encoding="utf-8") + data = PandasIngestor().from_csv(path) + assert data.dataframe.iloc[0]["user"] == "李雷" + os.remove(path) -def test_from_csv_detects_delimiter(): - content = ( - "company;sector\n" - "CrowdStrike;Cybersecurity\n" - "Fortinet;Network Security\n" - ) +def test_encoding_accented_text(): + content = "company,city\nRenée,Zürich\n" + path = write_temp_csv(content, encoding="latin-1") + data = PandasIngestor().from_csv(path) + assert data.row_count == 1 + os.remove(path) + + +def test_encoding_spanish(): + content = "org,country\nTelefónica,España\n" + path = write_temp_csv(content, encoding="latin-1") + data = PandasIngestor().from_csv(path) + assert data.row_count == 1 + os.remove(path) + + +def test_encoding_ansi_cp1252(): + content = "brand,city\nPeugeot,Montréal\n" + path = write_temp_csv(content, encoding="cp1252") + data = PandasIngestor().from_csv(path) + assert data.dataframe.iloc[0]["city"] == "Montréal" + os.remove(path) + + +# ======================================================= +# DELIMITERS (4 tests) +# ======================================================= + +def test_delimiter_comma(): + content = "a,b\n1,2\n" path = write_temp_csv(content) - - ingestor = PandasIngestor() - data = ingestor.from_csv(path) - - assert data.row_count == 2 - assert data.columns == ["company", "sector"] - assert data.dataframe.iloc[1]["company"] == "Fortinet" - + data = PandasIngestor().from_csv(path) + assert list(data.columns) == ["a", "b"] os.remove(path) -# ------------------------------------------------------- -# Test 3: Bad Rows Are Skipped (malformed threat feed) -# ------------------------------------------------------- - -def test_from_csv_skips_bad_rows(): - content = ( - "indicator,type\n" - "192.168.1.10,IP\n" - "this,is,too,many,columns\n" # malformed row - "evil-domain.com,Domain\n" - ) - +def test_delimiter_semicolon(): + content = "a;b\n1;2\n" path = write_temp_csv(content) - - ingestor = PandasIngestor() - data = ingestor.from_csv(path) - - # Malformed row should be skipped - assert data.row_count == 2 - assert list(data.dataframe["indicator"]) == [ - "192.168.1.10", - "evil-domain.com", - ] - + data = PandasIngestor().from_csv(path) + assert list(data.columns) == ["a", "b"] os.remove(path) + + +def test_delimiter_pipe(): + content = "a|b\n1|2\n" + path = write_temp_csv(content) + data = PandasIngestor().from_csv(path) + assert list(data.columns) == ["a", "b"] + os.remove(path) + + +def test_delimiter_tab(): + content = "a\tb\n1\t2\n" + path = write_temp_csv(content) + data = PandasIngestor().from_csv(path) + assert list(data.columns) == ["a", "b"] + os.remove(path) + + +# ======================================================= +# BAD ROWS (3 tests) +# ======================================================= + +def test_bad_row_extra_columns(): + content = "x,y\n1,2\n1,2,3,4\n5,6\n" + path = write_temp_csv(content) + data = PandasIngestor().from_csv(path) + assert data.row_count == 2 + os.remove(path) + +def test_bad_row_missing_column(): + content = "x,y\n1,2\n3\n4,5\n" + path = write_temp_csv(content) + data = PandasIngestor().from_csv(path) + assert data.row_count == 3 + assert data.dataframe["y"].isna().sum() == 1 + +def test_bad_row_unclosed_quote(): + content = "x,y\n1,2\n\"3,4\n5,6\n" + path = write_temp_csv(content) + data = PandasIngestor().from_csv(path) + assert data.row_count == 3 + + + From 979653e49818bbb07c367b59769febf2b003f86f Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Thu, 29 Jan 2026 00:32:07 +0530 Subject: [PATCH 3/3] Finalize CSV tests after conflict resolution --- tests/ingest/test_ingest_from_csv().py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/ingest/test_ingest_from_csv().py b/tests/ingest/test_ingest_from_csv().py index d2fa42e4..33e46ba5 100644 --- a/tests/ingest/test_ingest_from_csv().py +++ b/tests/ingest/test_ingest_from_csv().py @@ -227,7 +227,9 @@ def test_bad_row_unclosed_quote(): content = "x,y\n1,2\n\"3,4\n5,6\n" path = write_temp_csv(content) data = PandasIngestor().from_csv(path) - assert data.row_count == 3 + # The malformed quoted line consumes the following line; both are skipped. + # Only the first valid row remains. + assert data.row_count == 1