feat(vector): replace inline page-image payloads with chunk_bbox (Deck #76)

Per-chunk PDF page renders (~150–700 KB base64 PNG each) were the dominant
disk consumer in production, repeatedly tripping `No space left on device:
WAL buffer size exceeds available disk space` on welcomed-malamute Qdrant.

Replace the inline highlighted_page_image / highlighted_page_number /
highlight_count fields with a small `chunk_bbox` field:
list[(x0, y0, x1, y1)] of normalized [0, 1] floats, ~32 bytes per chunk.
Astrolabe (the only known consumer) renders the highlight client-side as
a percentage-positioned overlay on top of the existing /api/v1/pdf-preview
render-on-demand path (cbcoutinho/astrolabe#76).

- pdf_highlighter: new compute_chunk_bboxes_batch() that reuses the
  existing _find_chunk_bbox text-search path, skipping all pixmap/PIL/PNG
  work.
- processor: store chunk_bbox + chunk_bbox_page in the Qdrant payload,
  drop highlighted_page_image + friends, drop the base64 import.
- visualization /api/v1/chunk-context and auth/viz_routes: read
  chunk_bbox instead of highlighted_page_image.
- vector/__init__: stop eagerly re-exporting `processor`/`scanner` —
  fixes a pre-existing circular import (search.algorithms ->
  vector.placeholder -> vector/__init__ -> processor -> scanner ->
  server.semantic -> search.bm25_hybrid -> search.algorithms partial).
  Test suite that was broken on master (test_bm25_hybrid.py et al.) now
  collects and passes.
- scripts/purge_page_images.py: ad-hoc, idempotent migration that
  delete_payload's the legacy keys from existing points. No reindex
  required; legacy chunks render the page with no overlay.

Pairs with cbcoutinho/astrolabe#76. Frontend handles missing chunk_bbox
gracefully, so this can land in either order.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2026-05-08 21:16:00 +02:00
co-authored by Claude Opus 4.7
parent 2e1ab99a88
commit ee402ea00e
8 changed files with 428 additions and 72 deletions
@@ -691,6 +691,111 @@ class PDFHighlighter:
f"Failed to delete temp directory {temp_pdf_path.parent}: {e}"
)
@staticmethod
def compute_chunk_bboxes_batch(
pdf_bytes: bytes,
chunks: list[tuple[int, int, int, int | None, str]],
page_boundaries: list[dict],
full_text: str,
) -> dict[int, tuple[list[tuple[float, float, float, float]], int]]:
"""Compute normalized bounding boxes for chunks without rendering.
Lightweight alternative to highlight_chunks_batch — opens the PDF,
locates each chunk on its assigned page using the same text-search
path as the highlighter (`_find_chunk_bbox`), and returns
page-normalized rectangles. Skips the get_pixmap + PIL pipeline
entirely, so no PNG bytes are produced.
Args:
pdf_bytes: PDF file bytes.
chunks: List of (chunk_index, start_offset, end_offset,
stored_page_number, chunk_text). chunk_index is the dict key.
page_boundaries: Pre-computed page boundaries from the document
processor; each entry is {"page", "start_offset", "end_offset"}.
full_text: Full document text (for cross-page chunk handling).
Returns:
dict mapping chunk_index to (normalized_bboxes, page_number).
Each bbox is (x0, y0, x1, y1) in [0, 1] relative to page width
and height, top-left origin. Chunks whose bbox cannot be located
are omitted from the result.
"""
results: dict[int, tuple[list[tuple[float, float, float, float]], int]] = {}
if not chunks:
return results
temp_pdf_path = None
try:
temp_dir = Path(tempfile.mkdtemp(prefix="pdf_bbox_batch_"))
temp_pdf_path = temp_dir / "pdf.pdf"
temp_pdf_path.write_bytes(pdf_bytes)
doc = pymupdf.open(temp_pdf_path)
for (
chunk_index,
start_offset,
end_offset,
stored_page_num,
chunk_text,
) in chunks:
chunk_page_info = PDFHighlighter.find_chunk_page(
start_offset, end_offset, page_boundaries
)
if not chunk_page_info:
logger.debug(f"Chunk {chunk_index}: not found on any page")
continue
page_num = chunk_page_info["page_num"]
page_boundary = page_boundaries[page_num - 1]
page_text_length = (
page_boundary["end_offset"] - page_boundary["start_offset"]
)
# Page-relative slice (handles chunks that span page boundaries)
chunk_start_on_page = max(start_offset, page_boundary["start_offset"])
chunk_end_on_page = min(end_offset, page_boundary["end_offset"])
page_relative_text = full_text[chunk_start_on_page:chunk_end_on_page]
page = doc[page_num - 1]
bbox = PDFHighlighter._find_chunk_bbox(
page,
page_relative_text,
chunk_page_info["page_relative_start"],
chunk_page_info["page_relative_end"],
page_text_length,
)
if bbox is None:
continue
page_rect = page.rect
w = page_rect.width or 1.0
h = page_rect.height or 1.0
normalized = (
bbox[0] / w,
bbox[1] / h,
bbox[2] / w,
bbox[3] / h,
)
results[chunk_index] = ([normalized], page_num)
doc.close()
logger.info(f"Computed bboxes for {len(results)}/{len(chunks)} chunks")
return results
except Exception as e:
logger.error(f"Error computing chunk bboxes: {e}", exc_info=True)
return results
finally:
if temp_pdf_path and temp_pdf_path.parent.exists():
try:
shutil.rmtree(temp_pdf_path.parent)
except Exception as e:
logger.warning(f"Failed to clean up temp dir: {e}")
@staticmethod
def highlight_chunks_batch(
pdf_bytes: bytes,