Per-chunk PDF page renders (~150–700 KB base64 PNG each) were the dominant disk consumer in production, repeatedly tripping `No space left on device: WAL buffer size exceeds available disk space` on welcomed-malamute Qdrant. Replace the inline highlighted_page_image / highlighted_page_number / highlight_count fields with a small `chunk_bbox` field: list[(x0, y0, x1, y1)] of normalized [0, 1] floats, ~32 bytes per chunk. Astrolabe (the only known consumer) renders the highlight client-side as a percentage-positioned overlay on top of the existing /api/v1/pdf-preview render-on-demand path (cbcoutinho/astrolabe#76). - pdf_highlighter: new compute_chunk_bboxes_batch() that reuses the existing _find_chunk_bbox text-search path, skipping all pixmap/PIL/PNG work. - processor: store chunk_bbox + chunk_bbox_page in the Qdrant payload, drop highlighted_page_image + friends, drop the base64 import. - visualization /api/v1/chunk-context and auth/viz_routes: read chunk_bbox instead of highlighted_page_image. - vector/__init__: stop eagerly re-exporting `processor`/`scanner` — fixes a pre-existing circular import (search.algorithms -> vector.placeholder -> vector/__init__ -> processor -> scanner -> server.semantic -> search.bm25_hybrid -> search.algorithms partial). Test suite that was broken on master (test_bm25_hybrid.py et al.) now collects and passes. - scripts/purge_page_images.py: ad-hoc, idempotent migration that delete_payload's the legacy keys from existing points. No reindex required; legacy chunks render the page with no overlay. Pairs with cbcoutinho/astrolabe#76. Frontend handles missing chunk_bbox gracefully, so this can land in either order. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
133 lines
3.9 KiB
Python
133 lines
3.9 KiB
Python
"""Unit tests for PDFHighlighter.compute_chunk_bboxes_batch (Deck #76).
|
|
|
|
Replaces the legacy `highlight_chunks_batch`-+-base64 pipeline that inflated
|
|
Qdrant payloads with per-chunk PNG screenshots. The new path returns
|
|
normalized bounding boxes only.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pymupdf
|
|
import pytest
|
|
|
|
from nextcloud_mcp_server.search.pdf_highlighter import PDFHighlighter
|
|
|
|
|
|
def _make_pdf(pages: list[str]) -> bytes:
|
|
"""Build an in-memory PDF whose pages contain the given text."""
|
|
doc = pymupdf.open()
|
|
for body in pages:
|
|
page = doc.new_page(width=595, height=842) # A4
|
|
page.insert_text((50, 50), body)
|
|
pdf_bytes = doc.tobytes()
|
|
doc.close()
|
|
return pdf_bytes
|
|
|
|
|
|
def _page_boundaries(pages: list[str]) -> tuple[list[dict], str]:
|
|
"""Build (page_boundaries, full_text) compatible with the highlighter API."""
|
|
boundaries: list[dict] = []
|
|
cursor = 0
|
|
parts: list[str] = []
|
|
for i, body in enumerate(pages, start=1):
|
|
end = cursor + len(body)
|
|
boundaries.append({"page": i, "start_offset": cursor, "end_offset": end})
|
|
parts.append(body)
|
|
cursor = end
|
|
return boundaries, "".join(parts)
|
|
|
|
|
|
def test_compute_chunk_bboxes_returns_normalized_rects():
|
|
"""Each returned bbox should be 4 floats in [0, 1] tagged with the page."""
|
|
pages = [
|
|
"Chapter 1: Introduction. Nextcloud is a self-hosted collaboration platform "
|
|
"covering installation, configuration and maintenance topics.",
|
|
"Chapter 2: Installation. Download the package, extract it to the web "
|
|
"server directory, and configure the database connection.",
|
|
]
|
|
pdf_bytes = _make_pdf(pages)
|
|
boundaries, full_text = _page_boundaries(pages)
|
|
|
|
chunks = [
|
|
(
|
|
0,
|
|
0,
|
|
len(pages[0]),
|
|
1,
|
|
"Chapter 1: Introduction. Nextcloud is a self-hosted collaboration platform.",
|
|
),
|
|
(
|
|
1,
|
|
len(pages[0]),
|
|
len(pages[0]) + len(pages[1]),
|
|
2,
|
|
"Chapter 2: Installation. Download the package.",
|
|
),
|
|
]
|
|
|
|
results = PDFHighlighter.compute_chunk_bboxes_batch(
|
|
pdf_bytes=pdf_bytes,
|
|
chunks=chunks,
|
|
page_boundaries=boundaries,
|
|
full_text=full_text,
|
|
)
|
|
|
|
assert set(results) == {0, 1}
|
|
|
|
bboxes_p1, page_p1 = results[0]
|
|
bboxes_p2, page_p2 = results[1]
|
|
|
|
assert page_p1 == 1
|
|
assert page_p2 == 2
|
|
|
|
for rects in (bboxes_p1, bboxes_p2):
|
|
assert len(rects) >= 1
|
|
for rect in rects:
|
|
assert len(rect) == 4
|
|
x0, y0, x1, y1 = rect
|
|
assert 0.0 <= x0 < x1 <= 1.0
|
|
assert 0.0 <= y0 < y1 <= 1.0
|
|
|
|
|
|
def test_compute_chunk_bboxes_empty_input():
|
|
assert (
|
|
PDFHighlighter.compute_chunk_bboxes_batch(
|
|
pdf_bytes=b"",
|
|
chunks=[],
|
|
page_boundaries=[],
|
|
full_text="",
|
|
)
|
|
== {}
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("page_index", [0, 1])
|
|
def test_compute_chunk_bboxes_assigns_correct_page(page_index: int):
|
|
"""Verify the page number returned matches the page the chunk lives on."""
|
|
pages = [
|
|
"Page one talks about apples and oranges in detail.",
|
|
"Page two discusses bananas and grapes thoroughly.",
|
|
]
|
|
pdf_bytes = _make_pdf(pages)
|
|
boundaries, full_text = _page_boundaries(pages)
|
|
|
|
if page_index == 0:
|
|
chunk_text = "apples and oranges"
|
|
offsets = (0, len(pages[0]))
|
|
else:
|
|
chunk_text = "bananas and grapes"
|
|
offsets = (len(pages[0]), len(pages[0]) + len(pages[1]))
|
|
|
|
chunks = [(0, offsets[0], offsets[1], page_index + 1, chunk_text)]
|
|
|
|
results = PDFHighlighter.compute_chunk_bboxes_batch(
|
|
pdf_bytes=pdf_bytes,
|
|
chunks=chunks,
|
|
page_boundaries=boundaries,
|
|
full_text=full_text,
|
|
)
|
|
|
|
assert results, "expected a bbox for the chunk"
|
|
_, page_num = results[0]
|
|
assert page_num == page_index + 1
|