fix(chunk-context): propagate chunk_index=None through ChunkContext
Addresses the deferred half of PR #767 review issue 2: instead of just documenting the "0/N misreport" with a logger.warning, propagate the caller's None for chunk_index through the dataclass, position markers, and response builders so callers can distinguish "unknown position" from "actually chunk 0". Changes: - ChunkContext.chunk_index: int → int | None - _insert_position_markers: chunk_index parameter is int | None; when None, renders "Chunk ?/N" instead of "Chunk 1 of N" - get_chunk_with_context: drops the effective_chunk_index local entirely. Passes chunk_index (may be None) directly into both ChunkContext and _insert_position_markers, in both the Qdrant fast path and the doc-text fallback. - Fast path: when chunk_index is None and chunk_text was retrieved via the offset lookup (notes/cards), skip the adjacent-chunk fetch. Index arithmetic from a default 0 would query the chunks at positions -1 and 1 even when the actual chunk is, say, 5/20 — silently producing wrong "before"/"after" text. Mark both sides as truncated instead. - Drop the now-redundant logger.warning in the doc-text fallback (the response correctly communicates the unknown state via chunk_index=None). Both existing response builders (`api/visualization.py:657` and `auth/viz_routes.py:717`) already serialise `chunk_context.chunk_index` unconditionally; `None` becomes JSON `null`. No route changes needed. Adds 5 regression tests: - ChunkContext.chunk_index propagates as None in fast path - ChunkContext.chunk_index propagates as None in doc-text fallback - Fast path with chunk_index renders "Chunk N of M" correctly - _insert_position_markers renders "?/N" for None chunk_index - _insert_position_markers renders explicit index when supplied Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
51c1d42ea3
commit
45da876cf5
@@ -135,3 +135,168 @@ class TestOffsetFallbackGate:
|
||||
|
||||
mock_indexed.assert_not_awaited()
|
||||
mock_offset.assert_awaited_once()
|
||||
|
||||
|
||||
class TestNullableChunkIndexPropagation:
|
||||
"""When the caller doesn't supply chunk_index, it must propagate as None
|
||||
through to ChunkContext and the position markers — distinguishing
|
||||
"unknown position" from "actually chunk 0". See PR #767 review (🟡 issue 2).
|
||||
"""
|
||||
|
||||
async def test_fast_path_without_chunk_index_returns_none_in_response(
|
||||
self, mock_nc_client
|
||||
):
|
||||
"""Note retrieved via offset fallback (chunk_index=None) → response
|
||||
chunk_index is None, markers render '?/N', and adjacent fetch is
|
||||
skipped (would otherwise produce wrong neighbours from index 0).
|
||||
"""
|
||||
with (
|
||||
patch.object(
|
||||
context_module,
|
||||
"_get_chunk_by_index_from_qdrant",
|
||||
new_callable=AsyncMock,
|
||||
return_value=None,
|
||||
) as mock_indexed,
|
||||
patch.object(
|
||||
context_module,
|
||||
"_get_chunk_from_qdrant",
|
||||
new_callable=AsyncMock,
|
||||
return_value="matched chunk text",
|
||||
),
|
||||
):
|
||||
result = await get_chunk_with_context(
|
||||
nc_client=mock_nc_client,
|
||||
user_id="alice",
|
||||
doc_id=42,
|
||||
doc_type="note",
|
||||
chunk_start=100,
|
||||
chunk_end=200,
|
||||
chunk_index=None,
|
||||
total_chunks=8,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.chunk_index is None, (
|
||||
"chunk_index must propagate as None, not default to 0"
|
||||
)
|
||||
assert "Chunk ?/8" in result.marked_text
|
||||
assert "Chunk 1 of 8" not in result.marked_text
|
||||
# Adjacent fetch must be skipped — index arithmetic from 0 would
|
||||
# query the wrong neighbours when actual position isn't 0.
|
||||
mock_indexed.assert_not_awaited()
|
||||
assert result.has_before_truncation is True
|
||||
assert result.has_after_truncation is True
|
||||
|
||||
async def test_fast_path_with_chunk_index_renders_position_correctly(
|
||||
self, mock_nc_client
|
||||
):
|
||||
"""Counter-positive: when chunk_index is supplied, response carries
|
||||
the value and markers render the explicit "Chunk N of M".
|
||||
"""
|
||||
with (
|
||||
patch.object(
|
||||
context_module,
|
||||
"_get_chunk_by_index_from_qdrant",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=[
|
||||
"current chunk text", # primary lookup
|
||||
"previous chunk text", # adjacent before
|
||||
"next chunk text", # adjacent after
|
||||
],
|
||||
),
|
||||
patch.object(
|
||||
context_module,
|
||||
"_get_chunk_from_qdrant",
|
||||
new_callable=AsyncMock,
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
result = await get_chunk_with_context(
|
||||
nc_client=mock_nc_client,
|
||||
user_id="alice",
|
||||
doc_id=42,
|
||||
doc_type="note",
|
||||
chunk_start=0,
|
||||
chunk_end=10,
|
||||
chunk_index=5,
|
||||
total_chunks=20,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.chunk_index == 5
|
||||
assert "Chunk 6 of 20" in result.marked_text
|
||||
|
||||
async def test_doc_text_fallback_without_chunk_index_returns_none(
|
||||
self, mock_nc_client
|
||||
):
|
||||
"""Doc-text fallback (Qdrant miss → re-fetch document) must also
|
||||
propagate chunk_index=None into the response so callers can tell
|
||||
the position is unknown.
|
||||
"""
|
||||
with (
|
||||
patch.object(
|
||||
context_module,
|
||||
"_get_chunk_by_index_from_qdrant",
|
||||
new_callable=AsyncMock,
|
||||
return_value=None,
|
||||
),
|
||||
patch.object(
|
||||
context_module,
|
||||
"_get_chunk_from_qdrant",
|
||||
new_callable=AsyncMock,
|
||||
return_value=None,
|
||||
),
|
||||
patch.object(
|
||||
context_module,
|
||||
"_fetch_document_text",
|
||||
new_callable=AsyncMock,
|
||||
return_value="x" * 500,
|
||||
),
|
||||
):
|
||||
result = await get_chunk_with_context(
|
||||
nc_client=mock_nc_client,
|
||||
user_id="alice",
|
||||
doc_id=42,
|
||||
doc_type="note",
|
||||
chunk_start=100,
|
||||
chunk_end=200,
|
||||
chunk_index=None,
|
||||
total_chunks=10,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.chunk_index is None
|
||||
assert "Chunk ?/10" in result.marked_text
|
||||
|
||||
|
||||
class TestPositionMarkers:
|
||||
"""Direct tests for `_insert_position_markers` rendering when chunk_index
|
||||
is None vs explicit.
|
||||
"""
|
||||
|
||||
def test_marker_renders_question_mark_when_chunk_index_is_none(self):
|
||||
text = context_module._insert_position_markers(
|
||||
before_context="",
|
||||
chunk_text="x",
|
||||
after_context="",
|
||||
page_number=None,
|
||||
chunk_index=None,
|
||||
total_chunks=12,
|
||||
has_before_truncation=False,
|
||||
has_after_truncation=False,
|
||||
)
|
||||
assert "Chunk ?/12" in text
|
||||
|
||||
def test_marker_renders_explicit_index_when_supplied(self):
|
||||
text = context_module._insert_position_markers(
|
||||
before_context="",
|
||||
chunk_text="x",
|
||||
after_context="",
|
||||
page_number=3,
|
||||
chunk_index=4,
|
||||
total_chunks=12,
|
||||
has_before_truncation=False,
|
||||
has_after_truncation=False,
|
||||
)
|
||||
assert "Page 3" in text
|
||||
assert "Chunk 5 of 12" in text
|
||||
|
||||
Reference in New Issue
Block a user