fix: Align PDF text extraction between indexing and context expansion
This commit fixes two critical issues with PDF processing:
1. **Text extraction mismatch (context expansion bug)**:
- Indexing used pymupdf4llm.to_markdown() producing markdown text
- Context expansion used page.get_text() producing plain text
- Different text formats caused character offset misalignment
- Search would find correct chunk, but expansion showed wrong section
- Fixed by making context.py use pymupdf4llm.to_markdown() consistently
2. **Diagnostic logging for page number assignment**:
- Added logging to verify page_boundaries exist in metadata
- Added logging to verify assign_page_numbers() assigns values
- Helps diagnose why page numbers show as null in search results
3. **mime_type storage bug**:
- Fixed incorrect field reference in processor.py:405
- Was using file_metadata.get("content_type", "")
- Should use content_type from WebDAV response
Changes:
- nextcloud_mcp_server/search/context.py: Use pymupdf4llm.to_markdown()
for PDF text extraction to match indexing method
- nextcloud_mcp_server/vector/processor.py: Add diagnostic logging for
page boundaries and assignment, fix mime_type storage
- tests/unit/client/test_webdav.py: Fix import sorting
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -247,19 +247,32 @@ async def _fetch_document_text(
|
||||
|
||||
if is_pdf:
|
||||
# Extract text from PDF using PyMuPDF
|
||||
import fitz # PyMuPDF
|
||||
# IMPORTANT: Use pymupdf4llm.to_markdown() to match indexing extraction
|
||||
# This ensures character offsets align between indexed chunks and retrieval
|
||||
import pymupdf
|
||||
import pymupdf4llm
|
||||
|
||||
logger.debug(f"Extracting text from PDF: {file_path}")
|
||||
pdf_doc = fitz.open(stream=file_content, filetype="pdf")
|
||||
pdf_doc = pymupdf.open(stream=file_content, filetype="pdf")
|
||||
text_parts = []
|
||||
for page in pdf_doc:
|
||||
text_parts.append(page.get_text())
|
||||
|
||||
# Extract each page as markdown (same as indexing)
|
||||
for page_num in range(pdf_doc.page_count):
|
||||
page_md = pymupdf4llm.to_markdown(
|
||||
pdf_doc,
|
||||
pages=[page_num],
|
||||
write_images=False, # Don't need images for context
|
||||
page_chunks=False,
|
||||
)
|
||||
text_parts.append(page_md)
|
||||
|
||||
pdf_doc.close()
|
||||
|
||||
full_text = "\n".join(text_parts)
|
||||
# Join pages (no separator - matches indexing)
|
||||
full_text = "".join(text_parts)
|
||||
logger.debug(
|
||||
f"Extracted {len(full_text)} characters from "
|
||||
f"{len(text_parts)} pages in {file_path}"
|
||||
f"{pdf_doc.page_count} pages in {file_path}"
|
||||
)
|
||||
return full_text
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user