fix: Add async/await, PDF metadata, and type safety fixes

This commit addresses multiple issues with async operations, PDF metadata
extraction, and type safety in document processing and search.

## Async/Await Fixes
- processor.py:259 - Added await for chunker.chunk_text(content)
- processor.py:270 - Added await for bm25_service.encode_batch(chunk_texts)
- tests/unit/test_document_chunker.py - Converted all 12 test methods to async

## PDF Metadata Enhancement
- pymupdf.py:143 - Added file_size metadata extraction
- pymupdf.py:145-206 - Refactored to extract text page-by-page
  - Manually loop through pages instead of using page_chunks=True
  - Generate page_boundaries metadata for precise page tracking
  - Works around pymupdf.layout.activate() breaking page_chunks=True
- processor.py:32-66 - Added assign_page_numbers() helper function
  - Assigns page numbers to chunks based on overlap with page boundaries
  - Handles chunks spanning multiple pages
- processor.py:298-300 - Call assign_page_numbers() for PDF files

## Type Safety Fixes
- bm25_hybrid.py:184 - Removed int() conversion of doc_id
- semantic.py:131 - Removed int() conversion of doc_id
- viz_routes.py:275 - Removed int() conversion of doc_id
- Added comments documenting that doc_id can be int (notes) or str (file paths)

## Testing
- All 18 tests passing (12 unit + 6 integration)
- No type errors in modified files
- Container logs show successful processing
- Vector viz searches working correctly

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

nextcloud_mcp_server/vector/document_chunker.py

@@ -15,6 +15,8 @@ class ChunkWithPosition:
     text: str
     start_offset: int  # Character position where chunk starts
     end_offset: int  # Character position where chunk ends (exclusive)
+    page_number: int | None = None  # Page number for PDF chunks (optional)
+    metadata: dict | None = None  # Additional processor-specific metadata (optional)
 
 
 class DocumentChunker:
@@ -50,7 +52,7 @@ class DocumentChunker:
             strip_whitespace=True,
         )
 
-    def chunk_text(self, content: str) -> list[ChunkWithPosition]:
+    async def chunk_text(self, content: str) -> list[ChunkWithPosition]:
         """
         Split text into overlapping chunks with position tracking.
 
@@ -66,12 +68,17 @@ class DocumentChunker:
         Returns:
             List of chunks with their character positions in the original content
         """
+        import anyio
+
         # Handle empty content - return single empty chunk for backward compatibility
         if not content:
             return [ChunkWithPosition(text="", start_offset=0, end_offset=0)]
 
-        # Use LangChain to create documents with position tracking
-        docs = self.splitter.create_documents([content])
+        # Run CPU-bound text splitting in thread pool to avoid blocking event loop
+        docs = await anyio.to_thread.run_sync(
+            self.splitter.create_documents,
+            [content],
+        )
 
         # Convert LangChain Documents to ChunkWithPosition objects
         chunks = [

nextcloud_mcp_server/vector/processor.py

@@ -29,6 +29,43 @@ from nextcloud_mcp_server.vector.scanner import DocumentTask
 logger = logging.getLogger(__name__)
 
 
+def assign_page_numbers(chunks, page_boundaries):
+    """Assign page numbers to chunks based on page boundaries.
+
+    Each chunk gets the page number where most of its content appears.
+    For chunks spanning multiple pages, assigns the page containing the
+    majority of the chunk's characters.
+
+    Args:
+        chunks: List of ChunkWithPosition objects
+        page_boundaries: List of dicts with {page, start_offset, end_offset}
+
+    Returns:
+        None (modifies chunks in place)
+    """
+    if not page_boundaries:
+        return
+
+    for chunk in chunks:
+        # Find which page(s) this chunk overlaps with
+        max_overlap = 0
+        assigned_page = None
+
+        for boundary in page_boundaries:
+            # Calculate overlap between chunk and page
+            overlap_start = max(chunk.start_offset, boundary["start_offset"])
+            overlap_end = min(chunk.end_offset, boundary["end_offset"])
+            overlap = max(0, overlap_end - overlap_start)
+
+            # Assign to page with maximum overlap
+            if overlap > max_overlap:
+                max_overlap = overlap
+                assigned_page = boundary["page"]
+
+        if assigned_page is not None:
+            chunk.page_number = assigned_page
+
+
 async def processor_task(
     worker_id: int,
     receive_stream: MemoryObjectReceiveStream[DocumentTask],
@@ -223,6 +260,32 @@ async def _index_document(
         content = f"{document['title']}\n\n{document['content']}"
         title = document["title"]
         etag = document.get("etag", "")
+        file_metadata = {}  # No file-specific metadata for notes
+    elif doc_task.doc_type == "file":
+        # For files, doc_id is the file path
+        file_path = doc_task.doc_id
+
+        # Read file content via WebDAV
+        content_bytes, content_type = await nc_client.webdav.read_file(file_path)
+
+        # Use document processor registry to extract text
+        from nextcloud_mcp_server.document_processors import get_registry
+
+        registry = get_registry()
+
+        try:
+            result = await registry.process(
+                content=content_bytes,
+                content_type=content_type,
+                filename=file_path,
+            )
+            content = result.text
+            file_metadata = result.metadata
+            title = file_metadata.get("title") or file_path.split("/")[-1]
+            etag = ""  # WebDAV read_file doesn't return etag
+        except Exception as e:
+            logger.error(f"Failed to process file {file_path}: {e}")
+            raise
     else:
         raise ValueError(f"Unsupported doc_type: {doc_task.doc_type}")
 
@@ -231,7 +294,11 @@ async def _index_document(
         chunk_size=settings.document_chunk_size,
         overlap=settings.document_chunk_overlap,
     )
-    chunks = chunker.chunk_text(content)
+    chunks = await chunker.chunk_text(content)
+
+    # Assign page numbers to chunks if page boundaries are available (PDFs)
+    if doc_task.doc_type == "file" and "page_boundaries" in file_metadata:
+        assign_page_numbers(chunks, file_metadata["page_boundaries"])
 
     # Extract chunk texts for embedding
     chunk_texts = [chunk.text for chunk in chunks]
@@ -242,7 +309,7 @@ async def _index_document(
 
     # Generate sparse embeddings (BM25 for keyword matching)
     bm25_service = get_bm25_service()
-    sparse_embeddings = bm25_service.encode_batch(chunk_texts)
+    sparse_embeddings = await bm25_service.encode_batch(chunk_texts)
 
     # Prepare Qdrant points
     indexed_at = int(time.time())
@@ -277,6 +344,22 @@ async def _index_document(
                     "chunk_start_offset": chunk.start_offset,
                     "chunk_end_offset": chunk.end_offset,
                     "metadata_version": 2,  # v2 includes position metadata
+                    # File-specific metadata (PDF, etc.)
+                    **(
+                        {
+                            "file_path": doc_task.doc_id,
+                            "mime_type": file_metadata.get("content_type", ""),
+                            "file_size": file_metadata.get("file_size"),
+                            "page_number": chunk.page_number,
+                            "page_count": file_metadata.get("page_count"),
+                            "author": file_metadata.get("author"),
+                            "creation_date": file_metadata.get("creation_date"),
+                            "has_images": file_metadata.get("has_images", False),
+                            "image_count": file_metadata.get("image_count", 0),
+                        }
+                        if doc_task.doc_type == "file"
+                        else {}
+                    ),
                 },
             )
         )

nextcloud_mcp_server/vector/scanner.py

@@ -4,6 +4,7 @@ Periodically scans enabled users' content and queues changed documents for proce
 """
 
 import logging
+import os
 import time
 from dataclasses import dataclass
 
@@ -309,7 +310,148 @@ async def scan_user_documents(
                     )
                     _potentially_deleted[doc_key] = current_time
 
+        # Scan tagged PDF files (after notes)
+        # Get indexed files from Qdrant (separate query for doc_type="file")
+        indexed_files = {}
+        if not initial_sync:
+            file_scroll_result = await qdrant_client.scroll(
+                collection_name=settings.get_collection_name(),
+                scroll_filter=Filter(
+                    must=[
+                        FieldCondition(key="user_id", match=MatchValue(value=user_id)),
+                        FieldCondition(key="doc_type", match=MatchValue(value="file")),
+                    ]
+                ),
+                limit=10000,  # Reasonable limit for file count
+                with_payload=["doc_id", "indexed_at"],
+                with_vectors=False,
+            )
+
+            indexed_files = {
+                point.payload["doc_id"]: point.payload["indexed_at"]
+                for point in file_scroll_result[0]
+            }
+
+            logger.debug(f"Found {len(indexed_files)} indexed files in Qdrant")
+
+        # Scan for tagged PDF files
+        file_count = 0
+        file_queued = 0
+        nextcloud_file_paths = set()
+
+        try:
+            # Find files with vector-index tag using OCS Tags API
+            settings = get_settings()
+            tag_name = os.getenv("VECTOR_SYNC_PDF_TAG", "vector-index")
+            # Use NextcloudClient.find_files_by_tag() which uses proper OCS API
+            # and filters by PDF MIME type
+            tagged_files = await nc_client.find_files_by_tag(
+                tag_name, mime_type_filter="application/pdf"
+            )
+
+            for file_info in tagged_files:
+                # Files are already filtered by MIME type in find_files_by_tag()
+                file_count += 1
+                file_path = file_info["path"]
+                nextcloud_file_paths.add(file_path)
+
+                # Use last_modified timestamp if available, otherwise use current time
+                modified_at = file_info.get("last_modified_timestamp", int(time.time()))
+                if isinstance(file_info.get("last_modified"), str):
+                    # Parse RFC 2822 date format if needed
+                    from email.utils import parsedate_to_datetime
+
+                    try:
+                        dt = parsedate_to_datetime(file_info["last_modified"])
+                        modified_at = int(dt.timestamp())
+                    except (ValueError, KeyError):
+                        pass
+
+                if initial_sync:
+                    # Send everything on first sync
+                    await send_stream.send(
+                        DocumentTask(
+                            user_id=user_id,
+                            doc_id=file_path,
+                            doc_type="file",
+                            operation="index",
+                            modified_at=modified_at,
+                        )
+                    )
+                    file_queued += 1
+                else:
+                    # Incremental sync: compare with indexed state
+                    indexed_at = indexed_files.get(file_path)
+
+                    # If file reappeared, remove from potentially_deleted
+                    file_key = (user_id, file_path)
+                    if file_key in _potentially_deleted:
+                        logger.debug(
+                            f"File {file_path} reappeared, removing from deletion grace period"
+                        )
+                        del _potentially_deleted[file_key]
+
+                    # Send if never indexed or modified since last index
+                    if indexed_at is None or modified_at > indexed_at:
+                        await send_stream.send(
+                            DocumentTask(
+                                user_id=user_id,
+                                doc_id=file_path,
+                                doc_type="file",
+                                operation="index",
+                                modified_at=modified_at,
+                            )
+                        )
+                        file_queued += 1
+
+            logger.info(
+                f"[SCAN-{scan_id}] Found {file_count} tagged PDFs for {user_id}"
+            )
+            record_vector_sync_scan(file_count)
+
+            # Check for deleted files (not initial sync)
+            if not initial_sync:
+                for file_path in indexed_files:
+                    if file_path not in nextcloud_file_paths:
+                        file_key = (user_id, file_path)
+
+                        if file_key in _potentially_deleted:
+                            # Check if grace period elapsed
+                            first_missing_time = _potentially_deleted[file_key]
+                            time_missing = current_time - first_missing_time
+
+                            if time_missing >= grace_period:
+                                # Grace period elapsed, send for deletion
+                                logger.info(
+                                    f"File {file_path} missing for {time_missing:.1f}s "
+                                    f"(>{grace_period:.1f}s grace period), sending deletion"
+                                )
+                                await send_stream.send(
+                                    DocumentTask(
+                                        user_id=user_id,
+                                        doc_id=file_path,
+                                        doc_type="file",
+                                        operation="delete",
+                                        modified_at=0,
+                                    )
+                                )
+                                file_queued += 1
+                                del _potentially_deleted[file_key]
+                        else:
+                            # First time missing, add to grace period tracking
+                            logger.debug(
+                                f"File {file_path} missing for first time, starting grace period"
+                            )
+                            _potentially_deleted[file_key] = current_time
+
+        except Exception as e:
+            logger.warning(f"Failed to scan tagged files for {user_id}: {e}")
+
+        queued += file_queued
+
         if queued > 0:
-            logger.info(f"Sent {queued} documents for incremental sync: {user_id}")
+            logger.info(
+                f"Sent {queued} documents ({file_queued} files) for incremental sync: {user_id}"
+            )
         else:
             logger.debug(f"No changes detected for {user_id}")