feat(vector): replace inline page-image payloads with chunk_bbox (Deck #76)
Per-chunk PDF page renders (~150–700 KB base64 PNG each) were the dominant disk consumer in production, repeatedly tripping `No space left on device: WAL buffer size exceeds available disk space` on welcomed-malamute Qdrant. Replace the inline highlighted_page_image / highlighted_page_number / highlight_count fields with a small `chunk_bbox` field: list[(x0, y0, x1, y1)] of normalized [0, 1] floats, ~32 bytes per chunk. Astrolabe (the only known consumer) renders the highlight client-side as a percentage-positioned overlay on top of the existing /api/v1/pdf-preview render-on-demand path (cbcoutinho/astrolabe#76). - pdf_highlighter: new compute_chunk_bboxes_batch() that reuses the existing _find_chunk_bbox text-search path, skipping all pixmap/PIL/PNG work. - processor: store chunk_bbox + chunk_bbox_page in the Qdrant payload, drop highlighted_page_image + friends, drop the base64 import. - visualization /api/v1/chunk-context and auth/viz_routes: read chunk_bbox instead of highlighted_page_image. - vector/__init__: stop eagerly re-exporting `processor`/`scanner` — fixes a pre-existing circular import (search.algorithms -> vector.placeholder -> vector/__init__ -> processor -> scanner -> server.semantic -> search.bm25_hybrid -> search.algorithms partial). Test suite that was broken on master (test_bm25_hybrid.py et al.) now collects and passes. - scripts/purge_page_images.py: ad-hoc, idempotent migration that delete_payload's the legacy keys from existing points. No reindex required; legacy chunks render the page with no overlay. Pairs with cbcoutinho/astrolabe#76. Frontend handles missing chunk_bbox gracefully, so this can land in either order. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
2e1ab99a88
commit
ee402ea00e
@@ -691,6 +691,111 @@ class PDFHighlighter:
|
||||
f"Failed to delete temp directory {temp_pdf_path.parent}: {e}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def compute_chunk_bboxes_batch(
|
||||
pdf_bytes: bytes,
|
||||
chunks: list[tuple[int, int, int, int | None, str]],
|
||||
page_boundaries: list[dict],
|
||||
full_text: str,
|
||||
) -> dict[int, tuple[list[tuple[float, float, float, float]], int]]:
|
||||
"""Compute normalized bounding boxes for chunks without rendering.
|
||||
|
||||
Lightweight alternative to highlight_chunks_batch — opens the PDF,
|
||||
locates each chunk on its assigned page using the same text-search
|
||||
path as the highlighter (`_find_chunk_bbox`), and returns
|
||||
page-normalized rectangles. Skips the get_pixmap + PIL pipeline
|
||||
entirely, so no PNG bytes are produced.
|
||||
|
||||
Args:
|
||||
pdf_bytes: PDF file bytes.
|
||||
chunks: List of (chunk_index, start_offset, end_offset,
|
||||
stored_page_number, chunk_text). chunk_index is the dict key.
|
||||
page_boundaries: Pre-computed page boundaries from the document
|
||||
processor; each entry is {"page", "start_offset", "end_offset"}.
|
||||
full_text: Full document text (for cross-page chunk handling).
|
||||
|
||||
Returns:
|
||||
dict mapping chunk_index to (normalized_bboxes, page_number).
|
||||
Each bbox is (x0, y0, x1, y1) in [0, 1] relative to page width
|
||||
and height, top-left origin. Chunks whose bbox cannot be located
|
||||
are omitted from the result.
|
||||
"""
|
||||
results: dict[int, tuple[list[tuple[float, float, float, float]], int]] = {}
|
||||
|
||||
if not chunks:
|
||||
return results
|
||||
|
||||
temp_pdf_path = None
|
||||
try:
|
||||
temp_dir = Path(tempfile.mkdtemp(prefix="pdf_bbox_batch_"))
|
||||
temp_pdf_path = temp_dir / "pdf.pdf"
|
||||
temp_pdf_path.write_bytes(pdf_bytes)
|
||||
|
||||
doc = pymupdf.open(temp_pdf_path)
|
||||
|
||||
for (
|
||||
chunk_index,
|
||||
start_offset,
|
||||
end_offset,
|
||||
stored_page_num,
|
||||
chunk_text,
|
||||
) in chunks:
|
||||
chunk_page_info = PDFHighlighter.find_chunk_page(
|
||||
start_offset, end_offset, page_boundaries
|
||||
)
|
||||
if not chunk_page_info:
|
||||
logger.debug(f"Chunk {chunk_index}: not found on any page")
|
||||
continue
|
||||
|
||||
page_num = chunk_page_info["page_num"]
|
||||
page_boundary = page_boundaries[page_num - 1]
|
||||
page_text_length = (
|
||||
page_boundary["end_offset"] - page_boundary["start_offset"]
|
||||
)
|
||||
|
||||
# Page-relative slice (handles chunks that span page boundaries)
|
||||
chunk_start_on_page = max(start_offset, page_boundary["start_offset"])
|
||||
chunk_end_on_page = min(end_offset, page_boundary["end_offset"])
|
||||
page_relative_text = full_text[chunk_start_on_page:chunk_end_on_page]
|
||||
|
||||
page = doc[page_num - 1]
|
||||
bbox = PDFHighlighter._find_chunk_bbox(
|
||||
page,
|
||||
page_relative_text,
|
||||
chunk_page_info["page_relative_start"],
|
||||
chunk_page_info["page_relative_end"],
|
||||
page_text_length,
|
||||
)
|
||||
|
||||
if bbox is None:
|
||||
continue
|
||||
|
||||
page_rect = page.rect
|
||||
w = page_rect.width or 1.0
|
||||
h = page_rect.height or 1.0
|
||||
normalized = (
|
||||
bbox[0] / w,
|
||||
bbox[1] / h,
|
||||
bbox[2] / w,
|
||||
bbox[3] / h,
|
||||
)
|
||||
results[chunk_index] = ([normalized], page_num)
|
||||
|
||||
doc.close()
|
||||
logger.info(f"Computed bboxes for {len(results)}/{len(chunks)} chunks")
|
||||
return results
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error computing chunk bboxes: {e}", exc_info=True)
|
||||
return results
|
||||
|
||||
finally:
|
||||
if temp_pdf_path and temp_pdf_path.parent.exists():
|
||||
try:
|
||||
shutil.rmtree(temp_pdf_path.parent)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to clean up temp dir: {e}")
|
||||
|
||||
@staticmethod
|
||||
def highlight_chunks_batch(
|
||||
pdf_bytes: bytes,
|
||||
|
||||
Reference in New Issue
Block a user