feat: Add text processing background worker for telling client about progress

2025-10-25 19:52:45 +02:00
parent 2147fc1696
commit a36038422b
13 changed files with 487 additions and 21 deletions
@@ -0,0 +1,143 @@
+"""Integration tests for document processing with progress notifications."""
+
+import io
+
+import pytest
+from PIL import Image
+
+pytestmark = pytest.mark.integration
+
+
+class TestDocumentProcessingProgress:
+    """Test document processing with progress notifications."""
+
+    async def test_unstructured_processor_with_progress_callback(self, nc_client):
+        """Test that UnstructuredProcessor calls progress callback during processing."""
+        import os
+
+        # Skip if unstructured is not enabled
+        if os.getenv("ENABLE_UNSTRUCTURED", "false").lower() != "true":
+            pytest.skip("Unstructured processor not enabled")
+
+        from nextcloud_mcp_server.document_processors.unstructured import (
+            UnstructuredProcessor,
+        )
+
+        # Track progress callback invocations
+        progress_updates = []
+
+        async def track_progress(progress: float, total: float | None, message: str):
+            progress_updates.append(
+                {"progress": progress, "total": total, "message": message}
+            )
+
+        # Create processor configured to use local unstructured service
+        processor = UnstructuredProcessor(
+            api_url=os.getenv("UNSTRUCTURED_API_URL", "http://unstructured:8000"),
+            timeout=120,
+            progress_interval=2,  # 2 second intervals for testing
+        )
+
+        # Create a simple test image (which requires OCR processing)
+        # This should take long enough to trigger at least one progress update
+        img = Image.new("RGB", (400, 200), color=(73, 109, 137))
+        buffer = io.BytesIO()
+        img.save(buffer, format="PNG")
+        test_image = buffer.getvalue()
+
+        # Process with progress callback
+        result = await processor.process(
+            content=test_image,
+            content_type="image/png",
+            filename="test.png",
+            progress_callback=track_progress,
+        )
+
+        # Verify processing succeeded
+        assert result.success is True
+        assert result.processor == "unstructured"
+        assert isinstance(result.text, str)
+
+        # Note: Progress updates may or may not occur depending on processing speed
+        # If updates occurred, verify their structure
+        if progress_updates:
+            for update in progress_updates:
+                assert isinstance(update["progress"], float)
+                assert update["total"] is None  # Unknown total
+                assert "Processing document with unstructured" in update["message"]
+                assert "elapsed" in update["message"]
+
+    async def test_webdav_read_file_sends_progress_notifications(
+        self, nc_mcp_client, nc_client
+    ):
+        """Test that reading a document via WebDAV MCP tool sends progress notifications."""
+        import os
+
+        # Skip if document processing is not enabled
+        if os.getenv("ENABLE_DOCUMENT_PROCESSING", "false").lower() != "true":
+            pytest.skip("Document processing not enabled")
+
+        # Create a test image file in Nextcloud via WebDAV
+        from PIL import Image
+
+        img = Image.new("RGB", (400, 200), color=(100, 150, 200))
+        buffer = io.BytesIO()
+        img.save(buffer, format="PNG")
+        test_image = buffer.getvalue()
+
+        # Upload test file
+        test_path = "test_progress.png"
+        await nc_client.webdav.write_file(test_path, test_image, "image/png")
+
+        try:
+            # Read file via MCP tool (which should trigger document processing)
+            # The MCP client will automatically track progress notifications
+            result = await nc_mcp_client.call_tool(
+                "nc_webdav_read_file", arguments={"path": test_path}
+            )
+
+            # Note: FastMCP progress notifications are sent automatically by ctx.report_progress
+            # We can't easily capture them in this test without mocking the MCP transport layer
+            # The important thing is that the code path is exercised without errors
+            assert result.isError is False
+
+        finally:
+            # Cleanup
+            try:
+                await nc_client.webdav.delete_resource(test_path)
+            except Exception:
+                pass  # Ignore cleanup errors
+
+    async def test_progress_callback_not_required(self, nc_client):
+        """Test that processing works without progress callback (backward compatibility)."""
+        import os
+
+        if os.getenv("ENABLE_UNSTRUCTURED", "false").lower() != "true":
+            pytest.skip("Unstructured processor not enabled")
+
+        from nextcloud_mcp_server.document_processors.unstructured import (
+            UnstructuredProcessor,
+        )
+
+        processor = UnstructuredProcessor(
+            api_url=os.getenv("UNSTRUCTURED_API_URL", "http://unstructured:8000"),
+            timeout=120,
+        )
+
+        # Create simple test image
+        img = Image.new("RGB", (200, 100), color=(50, 100, 150))
+        buffer = io.BytesIO()
+        img.save(buffer, format="PNG")
+        test_image = buffer.getvalue()
+
+        # Process WITHOUT progress callback
+        result = await processor.process(
+            content=test_image,
+            content_type="image/png",
+            filename="test.png",
+            progress_callback=None,  # Explicitly None
+        )
+
+        # Should still work
+        assert result.success is True
+        assert result.processor == "unstructured"
@@ -0,0 +1,164 @@
+"""Unit tests for progress notification system."""
+
+import time
+from unittest.mock import AsyncMock
+
+import anyio
+import pytest
+
+pytestmark = pytest.mark.unit
+
+
+class TestProgressNotification:
+    """Test progress notification in document processors."""
+
+    async def test_progress_callback_called_during_processing(self):
+        """Test that progress callback is called at intervals during processing."""
+        from nextcloud_mcp_server.document_processors.unstructured import (
+            UnstructuredProcessor,
+        )
+
+        # Mock progress callback to track calls
+        progress_callback = AsyncMock()
+
+        # Create processor with 1-second interval for faster testing
+        processor = UnstructuredProcessor(
+            api_url="http://test:8000",
+            timeout=10,
+            progress_interval=1,
+        )
+
+        # Create a mock event and start time
+        stop_event = anyio.Event()
+        start_time = time.time()
+
+        # Run the poller for 3 seconds, then stop it
+        async def stop_after_delay():
+            await anyio.sleep(3.5)
+            stop_event.set()
+
+        # Run poller and stopper concurrently
+        async with anyio.create_task_group() as tg:
+            tg.start_soon(
+                processor._run_progress_poller,
+                stop_event,
+                progress_callback,
+                start_time,
+            )
+            tg.start_soon(stop_after_delay)
+
+        # Verify progress callback was called at least 3 times (1s, 2s, 3s)
+        assert progress_callback.call_count >= 3
+
+        # Verify each call had correct structure
+        for call in progress_callback.call_args_list:
+            # Calls are made with keyword arguments
+            assert "progress" in call.kwargs
+            assert "total" in call.kwargs
+            assert "message" in call.kwargs
+
+            progress = call.kwargs["progress"]
+            total = call.kwargs["total"]
+            message = call.kwargs["message"]
+
+            assert isinstance(progress, float)
+            assert total is None  # Unknown total for unstructured
+            assert "Processing document with unstructured" in message
+            assert "elapsed" in message
+
+    async def test_progress_poller_stops_when_event_set(self):
+        """Test that progress poller stops immediately when event is set."""
+        from nextcloud_mcp_server.document_processors.unstructured import (
+            UnstructuredProcessor,
+        )
+
+        progress_callback = AsyncMock()
+        processor = UnstructuredProcessor(
+            api_url="http://test:8000",
+            timeout=10,
+            progress_interval=10,  # Long interval
+        )
+
+        stop_event = anyio.Event()
+        start_time = time.time()
+
+        # Set event immediately
+        stop_event.set()
+
+        # Run poller
+        await processor._run_progress_poller(stop_event, progress_callback, start_time)
+
+        # Should not call progress callback since event was already set
+        assert progress_callback.call_count == 0
+
+    async def test_progress_callback_exception_handled(self):
+        """Test that exceptions in progress callback don't crash the poller."""
+        from nextcloud_mcp_server.document_processors.unstructured import (
+            UnstructuredProcessor,
+        )
+
+        # Mock callback that raises exception
+        progress_callback = AsyncMock(side_effect=Exception("Callback error"))
+
+        processor = UnstructuredProcessor(
+            api_url="http://test:8000",
+            timeout=10,
+            progress_interval=1,
+        )
+
+        stop_event = anyio.Event()
+        start_time = time.time()
+
+        # Run poller for 2 seconds
+        async def stop_after_delay():
+            await anyio.sleep(2.5)
+            stop_event.set()
+
+        # Should not raise exception even though callback fails
+        async with anyio.create_task_group() as tg:
+            tg.start_soon(
+                processor._run_progress_poller,
+                stop_event,
+                progress_callback,
+                start_time,
+            )
+            tg.start_soon(stop_after_delay)
+
+        # Callback should have been called (and failed) at least twice
+        assert progress_callback.call_count >= 2
+
+    async def test_process_without_progress_callback(self):
+        """Test that processing works without progress callback (backward compatibility)."""
+        from nextcloud_mcp_server.document_processors.unstructured import (
+            UnstructuredProcessor,
+        )
+
+        processor = UnstructuredProcessor(
+            api_url="http://test:8000",
+            timeout=10,
+            progress_interval=1,
+        )
+
+        # Mock the _make_api_request method to avoid actual HTTP call
+        from unittest.mock import patch
+
+        from nextcloud_mcp_server.document_processors.base import ProcessingResult
+
+        mock_result = ProcessingResult(
+            text="Test content",
+            metadata={"test": "data"},
+            processor="unstructured",
+            success=True,
+        )
+
+        with patch.object(
+            processor, "_make_api_request", return_value=mock_result
+        ) as mock_request:
+            # Call process without progress_callback
+            result = await processor.process(
+                content=b"test", content_type="application/pdf", progress_callback=None
+            )
+
+            # Should call _make_api_request directly
+            assert result == mock_result
+            mock_request.assert_called_once()