Chris CoutinhoandClaude Opus 4.7
80b27b1bc6
- Drop chunk_bbox_page from Qdrant payload — viz endpoints never read it (page_number is the canonical PDF page field). - Bump upsert BATCH_SIZE 10 → 100 now that payloads no longer carry PNGs. - compute_chunk_bboxes_batch: move doc.close() into finally, replace unused stored_page_num with _. - purge_page_images.py: switch to anyio.run() per project convention, and wrap AsyncQdrantClient in try/finally so the aiohttp session is always closed (the class doesn't implement async-context-manager). - Decorate new bbox unit tests with @pytest.mark.unit so they run under the fast-feedback selector. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
136 lines
3.9 KiB
Python
136 lines
3.9 KiB
Python
"""Unit tests for PDFHighlighter.compute_chunk_bboxes_batch (Deck #76).
|
|
|
|
Replaces the legacy `highlight_chunks_batch`-+-base64 pipeline that inflated
|
|
Qdrant payloads with per-chunk PNG screenshots. The new path returns
|
|
normalized bounding boxes only.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pymupdf
|
|
import pytest
|
|
|
|
from nextcloud_mcp_server.search.pdf_highlighter import PDFHighlighter
|
|
|
|
|
|
def _make_pdf(pages: list[str]) -> bytes:
|
|
"""Build an in-memory PDF whose pages contain the given text."""
|
|
doc = pymupdf.open()
|
|
for body in pages:
|
|
page = doc.new_page(width=595, height=842) # A4
|
|
page.insert_text((50, 50), body)
|
|
pdf_bytes = doc.tobytes()
|
|
doc.close()
|
|
return pdf_bytes
|
|
|
|
|
|
def _page_boundaries(pages: list[str]) -> tuple[list[dict], str]:
|
|
"""Build (page_boundaries, full_text) compatible with the highlighter API."""
|
|
boundaries: list[dict] = []
|
|
cursor = 0
|
|
parts: list[str] = []
|
|
for i, body in enumerate(pages, start=1):
|
|
end = cursor + len(body)
|
|
boundaries.append({"page": i, "start_offset": cursor, "end_offset": end})
|
|
parts.append(body)
|
|
cursor = end
|
|
return boundaries, "".join(parts)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_compute_chunk_bboxes_returns_normalized_rects():
|
|
"""Each returned bbox should be 4 floats in [0, 1] tagged with the page."""
|
|
pages = [
|
|
"Chapter 1: Introduction. Nextcloud is a self-hosted collaboration platform "
|
|
"covering installation, configuration and maintenance topics.",
|
|
"Chapter 2: Installation. Download the package, extract it to the web "
|
|
"server directory, and configure the database connection.",
|
|
]
|
|
pdf_bytes = _make_pdf(pages)
|
|
boundaries, full_text = _page_boundaries(pages)
|
|
|
|
chunks = [
|
|
(
|
|
0,
|
|
0,
|
|
len(pages[0]),
|
|
1,
|
|
"Chapter 1: Introduction. Nextcloud is a self-hosted collaboration platform.",
|
|
),
|
|
(
|
|
1,
|
|
len(pages[0]),
|
|
len(pages[0]) + len(pages[1]),
|
|
2,
|
|
"Chapter 2: Installation. Download the package.",
|
|
),
|
|
]
|
|
|
|
results = PDFHighlighter.compute_chunk_bboxes_batch(
|
|
pdf_bytes=pdf_bytes,
|
|
chunks=chunks,
|
|
page_boundaries=boundaries,
|
|
full_text=full_text,
|
|
)
|
|
|
|
assert set(results) == {0, 1}
|
|
|
|
bboxes_p1, page_p1 = results[0]
|
|
bboxes_p2, page_p2 = results[1]
|
|
|
|
assert page_p1 == 1
|
|
assert page_p2 == 2
|
|
|
|
for rects in (bboxes_p1, bboxes_p2):
|
|
assert len(rects) >= 1
|
|
for rect in rects:
|
|
assert len(rect) == 4
|
|
x0, y0, x1, y1 = rect
|
|
assert 0.0 <= x0 < x1 <= 1.0
|
|
assert 0.0 <= y0 < y1 <= 1.0
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_compute_chunk_bboxes_empty_input():
|
|
assert (
|
|
PDFHighlighter.compute_chunk_bboxes_batch(
|
|
pdf_bytes=b"",
|
|
chunks=[],
|
|
page_boundaries=[],
|
|
full_text="",
|
|
)
|
|
== {}
|
|
)
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("page_index", [0, 1])
|
|
def test_compute_chunk_bboxes_assigns_correct_page(page_index: int):
|
|
"""Verify the page number returned matches the page the chunk lives on."""
|
|
pages = [
|
|
"Page one talks about apples and oranges in detail.",
|
|
"Page two discusses bananas and grapes thoroughly.",
|
|
]
|
|
pdf_bytes = _make_pdf(pages)
|
|
boundaries, full_text = _page_boundaries(pages)
|
|
|
|
if page_index == 0:
|
|
chunk_text = "apples and oranges"
|
|
offsets = (0, len(pages[0]))
|
|
else:
|
|
chunk_text = "bananas and grapes"
|
|
offsets = (len(pages[0]), len(pages[0]) + len(pages[1]))
|
|
|
|
chunks = [(0, offsets[0], offsets[1], page_index + 1, chunk_text)]
|
|
|
|
results = PDFHighlighter.compute_chunk_bboxes_batch(
|
|
pdf_bytes=pdf_bytes,
|
|
chunks=chunks,
|
|
page_boundaries=boundaries,
|
|
full_text=full_text,
|
|
)
|
|
|
|
assert results, "expected a bbox for the chunk"
|
|
_, page_num = results[0]
|
|
assert page_num == page_index + 1
|