fix(context): gate CJK bigram similarity fallback, persist recorded_at

_calculate_decision_content_similarity's character-bigram fallback was
unconditional, so ordinary multi-word English queries could pick up
incidental bigram overlap with unrelated decisions via max(word_sim,
bigram_sim). Gate it to only activate for CJK-like scripts or queries
with at most one whitespace token, matching its documented purpose.

Separately, _add_decision_to_graph never persisted recorded_at as a
node property, so _rebuild_decision_indexes/_sync_decision_from_node
(which already read it back) always recovered "" after any reload.
This commit is contained in:
KaifAhmad1
2026-08-26 16:38:39 +05:30
parent 84ccc7c0e3
commit 88d73189dd
2 changed files with 73 additions and 20 deletions
+55 -20
View File
@@ -4731,6 +4731,7 @@ class ContextGraph:
scenario=decision["scenario"],
decision_maker=decision.get("decision_maker", ""),
reasoning=decision["reasoning"],
recorded_at=decision.get("recorded_at", ""),
**safe_metadata,
**extra_properties,
)
@@ -5005,14 +5006,38 @@ class ContextGraph:
chars = "".join(text.lower().split())
return {chars[i:i + 2] for i in range(len(chars) - 1)}
@staticmethod
def _looks_cjk(text: str) -> bool:
"""True if text contains CJK/Japanese/Korean script characters.
Used to gate the character-bigram similarity fallback so it only
activates for scripts where whitespace tokenisation doesn't work.
"""
for ch in text:
code = ord(ch)
if (
0x4E00 <= code <= 0x9FFF # CJK Unified Ideographs
or 0x3400 <= code <= 0x4DBF # CJK Extension A
or 0x3040 <= code <= 0x30FF # Hiragana + Katakana
or 0xAC00 <= code <= 0xD7A3 # Hangul Syllables
or 0x1100 <= code <= 0x11FF # Hangul Jamo
):
return True
return False
def _calculate_decision_content_similarity(self, scenario: str, decision: Dict[str, Any]) -> float:
"""Calculate content similarity between scenario and decision.
Uses word-level Jaccard for space-separated languages. For text where
whitespace tokenisation fails (CJK, single-word queries) a character-
bigram Jaccard is computed over the *stripped* character sequences and
blended in with a weight that diminishes as the query grows so that it
cannot dominate English results.
whitespace tokenisation is unreliable (CJK/Japanese/Korean scripts, or
a query with no whitespace at all) a character-bigram Jaccard is
computed over the *stripped* character sequences instead.
The bigram fallback only activates when whitespace tokenisation would
not help — i.e. the query is CJK-like or has at most one whitespace
token — so it never contributes for ordinary multi-word English
queries, where incidental bigram overlap between unrelated sentences
would otherwise inflate scores.
The bigram side uses *Jaccard* (|A∩B|/|AB|), not the overlap
coefficient, so a 2-character query whose single bigram happens to
@@ -5039,23 +5064,33 @@ class ContextGraph:
)
# --- character-bigram Jaccard (CJK / very-short-query fallback) ---
scenario_bigrams = self._char_bigrams(scenario)
decision_bigrams = self._char_bigrams(decision_text)
# Require at least 3 bigrams in the query before the bigram signal
# is used. A 2-char query produces only 1 bigram; that single
# bigram is far too likely to appear as a substring of any English
# word and would produce a spuriously high overlap coefficient.
# 3 bigrams correspond to a 4-char stripped query (e.g. two CJK
# characters produce 1 bigram each → need ≥3 chars stripped).
# Only used when whitespace tokenisation can't do the job: CJK-like
# scripts, or a query that is a single whitespace token (no spaces
# to split on). Ordinary multi-word English queries rely on
# word_sim alone, so incidental bigram overlap between unrelated
# sentences can never inflate their score.
bigram_sim = 0.0
if len(scenario_bigrams) >= 3 and decision_bigrams:
bigram_union = scenario_bigrams | decision_bigrams
bigram_sim = (
len(scenario_bigrams & decision_bigrams) / len(bigram_union)
if bigram_union
else 0.0
)
needs_bigram_fallback = (
self._looks_cjk(scenario) or len(scenario.split()) <= 1
)
if needs_bigram_fallback:
scenario_bigrams = self._char_bigrams(scenario)
decision_bigrams = self._char_bigrams(decision_text)
# Require at least 3 bigrams in the query before the bigram
# signal is used. A 2-char query produces only 1 bigram; that
# single bigram is far too likely to appear as a substring of
# any English word and would produce a spuriously high overlap
# coefficient. 3 bigrams correspond to a 4-char stripped query
# (e.g. two CJK characters produce 1 bigram each → need ≥3
# chars stripped).
if len(scenario_bigrams) >= 3 and decision_bigrams:
bigram_union = scenario_bigrams | decision_bigrams
bigram_sim = (
len(scenario_bigrams & decision_bigrams) / len(bigram_union)
if bigram_union
else 0.0
)
return max(word_sim, bigram_sim)
@@ -206,6 +206,9 @@ class TestDecisionMetadataPreservation(unittest.TestCase):
entities=["trader_X", "instrument_Y"],
decision_maker="compliance_engine",
)
recorded_at_before = g._decisions[did]["recorded_at"]
self.assertTrue(recorded_at_before, "recorded_at must be set at record time")
with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f:
path = f.name
try:
@@ -221,6 +224,8 @@ class TestDecisionMetadataPreservation(unittest.TestCase):
self.assertAlmostEqual(dec["confidence"], 0.85, places=3)
self.assertIn("trader_X", dec["entities"])
self.assertEqual(dec["decision_maker"], "compliance_engine")
self.assertEqual(dec["recorded_at"], recorded_at_before,
"recorded_at must survive a save -> load round trip")
finally:
os.unlink(path)
@@ -549,6 +554,19 @@ class TestBigramSpikeRegression(unittest.TestCase):
self.assertLess(sim, 0.5,
f"2-char query {q!r} must not produce high similarity")
def test_unrelated_multiword_english_queries_score_zero(self):
"""The bigram fallback must not activate for ordinary multi-word
English queries -- it exists only for CJK/single-token queries where
whitespace tokenisation can't help. Unrelated multi-word English
sentences must score 0.0, not a nonzero incidental bigram overlap."""
sim = self._sim(
"employee vacation request approval process",
"Server infrastructure migration to cloud provider",
)
self.assertEqual(sim, 0.0,
"Unrelated multi-word English queries must not "
"receive a nonzero score from bigram overlap")
# ---------------------------------------------------------------------------
# Part 9: query_graph limit semantics