refactor: Transform document parsing into pluggable processor architecture
Refactors PR #190's hardcoded Unstructured.io integration into a flexible, extensible plugin system supporting multiple text extraction engines. - **`DocumentProcessor` ABC**: Abstract interface for all processors - **`ProcessorRegistry`**: Central registry for discovery and routing - **`ProcessingResult`**: Standardized output format across processors - **`UnstructuredProcessor`**: Refactored from `UnstructuredClient` - **`TesseractProcessor`**: Local OCR for images (lightweight alternative) - **`CustomHTTPProcessor`**: Generic wrapper for custom HTTP APIs - New `get_document_processor_config()` returns structured config - Supports enabling/disabling individual processors - Per-processor configuration via environment variables - **Breaking Change**: `ENABLE_UNSTRUCTURED_PARSING` replaced with: - `ENABLE_DOCUMENT_PROCESSING=true/false` (master switch) - `ENABLE_UNSTRUCTURED=true/false` (per-processor) - `ENABLE_TESSERACT=true/false` - `ENABLE_CUSTOM_PROCESSOR=true/false` - `parse_document()` now uses `ProcessorRegistry` - Auto-selects appropriate processor based on MIME type - Processor priority system (Unstructured=10, Tesseract=5, Custom=1) - `initialize_document_processors()` registers processors at startup - Integrated into both BasicAuth and OAuth lifespans - Graceful degradation if processors fail to initialize ```env ENABLE_DOCUMENT_PROCESSING=false ENABLE_UNSTRUCTURED=false UNSTRUCTURED_API_URL=http://unstructured:8000 UNSTRUCTURED_STRATEGY=auto # auto|fast|hi_res UNSTRUCTURED_LANGUAGES=eng,deu ENABLE_TESSERACT=false TESSERACT_LANG=eng ENABLE_CUSTOM_PROCESSOR=false CUSTOM_PROCESSOR_URL=http://localhost:9000/process CUSTOM_PROCESSOR_TYPES=application/pdf,image/jpeg ``` - **Removed**: `tests/test_unstructured_config.py` (legacy tests) - **Added**: `tests/unit/test_document_processor_config.py` - 7 unit tests for new config system - Tests individual and multi-processor configurations - **Added**: - `nextcloud_mcp_server/document_processors/__init__.py` - `nextcloud_mcp_server/document_processors/base.py` - `nextcloud_mcp_server/document_processors/registry.py` - `nextcloud_mcp_server/document_processors/unstructured.py` - `nextcloud_mcp_server/document_processors/tesseract.py` - `nextcloud_mcp_server/document_processors/custom_http.py` - `tests/unit/test_document_processor_config.py` - **Modified**: - `nextcloud_mcp_server/config.py` - New plugin config system - `nextcloud_mcp_server/app.py` - Processor initialization - `nextcloud_mcp_server/utils/document_parser.py` - Uses registry - `nextcloud_mcp_server/server/webdav.py` - Import updates - `env.sample` - New configuration format - `docker-compose.yml` - (profile changes from previous work) - **Removed**: - `nextcloud_mcp_server/client/unstructured_client.py` - Replaced by UnstructuredProcessor - `tests/test_unstructured_config.py` - Replaced with new tests ✅ **Extensible**: Add processors without modifying core code ✅ **Testable**: Mock processors for unit tests ✅ **Configurable**: Enable only needed processors ✅ **Flexible**: Choose fast (Tesseract) vs accurate (Unstructured) ✅ **Opt-in**: Disabled by default, no mandatory dependencies Users upgrading from PR #190 need to update environment variables: ```bash ENABLE_UNSTRUCTURED_PARSING=true ENABLE_DOCUMENT_PROCESSING=true ENABLE_UNSTRUCTURED=true ``` 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
@@ -1,6 +1,6 @@
|
||||
import logging.config
|
||||
import os
|
||||
from typing import Optional
|
||||
from typing import Any
|
||||
|
||||
LOGGING_CONFIG = {
|
||||
"version": 1,
|
||||
@@ -55,86 +55,65 @@ def setup_logging():
|
||||
logging.config.dictConfig(LOGGING_CONFIG)
|
||||
|
||||
|
||||
# Document Parsing Configuration
|
||||
def get_unstructured_api_url() -> Optional[str]:
|
||||
"""Get the Unstructured API URL from environment variables.
|
||||
# Document Processing Configuration
|
||||
|
||||
|
||||
def get_document_processor_config() -> dict[str, Any]:
|
||||
"""Get document processor configuration from environment.
|
||||
|
||||
Returns:
|
||||
The Unstructured API URL if parsing is enabled, None otherwise.
|
||||
Dict with processor configs:
|
||||
{
|
||||
"enabled": bool,
|
||||
"default_processor": str,
|
||||
"processors": {
|
||||
"unstructured": {...},
|
||||
"tesseract": {...},
|
||||
"custom": {...},
|
||||
}
|
||||
}
|
||||
"""
|
||||
enabled = os.getenv("ENABLE_UNSTRUCTURED_PARSING", "true").lower() == "true"
|
||||
if not enabled:
|
||||
return None
|
||||
config: dict[str, Any] = {
|
||||
"enabled": os.getenv("ENABLE_DOCUMENT_PROCESSING", "false").lower() == "true",
|
||||
"default_processor": os.getenv("DOCUMENT_PROCESSOR", "unstructured"),
|
||||
"processors": {},
|
||||
}
|
||||
|
||||
return os.getenv("UNSTRUCTURED_API_URL", "http://unstructured:8000")
|
||||
# Unstructured configuration
|
||||
if os.getenv("ENABLE_UNSTRUCTURED", "false").lower() == "true":
|
||||
config["processors"]["unstructured"] = {
|
||||
"api_url": os.getenv("UNSTRUCTURED_API_URL", "http://unstructured:8000"),
|
||||
"timeout": int(os.getenv("UNSTRUCTURED_TIMEOUT", "120")),
|
||||
"strategy": os.getenv("UNSTRUCTURED_STRATEGY", "auto"),
|
||||
"languages": [
|
||||
lang.strip()
|
||||
for lang in os.getenv("UNSTRUCTURED_LANGUAGES", "eng,deu").split(",")
|
||||
if lang.strip()
|
||||
],
|
||||
}
|
||||
|
||||
# Tesseract configuration
|
||||
if os.getenv("ENABLE_TESSERACT", "false").lower() == "true":
|
||||
config["processors"]["tesseract"] = {
|
||||
"tesseract_cmd": os.getenv("TESSERACT_CMD"), # None = auto-detect
|
||||
"lang": os.getenv("TESSERACT_LANG", "eng"),
|
||||
}
|
||||
|
||||
def is_unstructured_parsing_enabled() -> bool:
|
||||
"""Check if unstructured document parsing is enabled.
|
||||
# Custom processor (via HTTP API)
|
||||
if os.getenv("ENABLE_CUSTOM_PROCESSOR", "false").lower() == "true":
|
||||
custom_url = os.getenv("CUSTOM_PROCESSOR_URL")
|
||||
if custom_url:
|
||||
supported_types_str = os.getenv("CUSTOM_PROCESSOR_TYPES", "application/pdf")
|
||||
supported_types = {
|
||||
t.strip() for t in supported_types_str.split(",") if t.strip()
|
||||
}
|
||||
|
||||
Returns:
|
||||
True if enabled, False otherwise.
|
||||
"""
|
||||
return os.getenv("ENABLE_UNSTRUCTURED_PARSING", "true").lower() == "true"
|
||||
config["processors"]["custom"] = {
|
||||
"name": os.getenv("CUSTOM_PROCESSOR_NAME", "custom"),
|
||||
"api_url": custom_url,
|
||||
"api_key": os.getenv("CUSTOM_PROCESSOR_API_KEY"),
|
||||
"timeout": int(os.getenv("CUSTOM_PROCESSOR_TIMEOUT", "60")),
|
||||
"supported_types": supported_types,
|
||||
}
|
||||
|
||||
|
||||
def get_unstructured_strategy() -> str:
|
||||
"""Get the parsing strategy for the Unstructured API.
|
||||
|
||||
Valid values are:
|
||||
- 'auto': Automatically choose the best strategy (default)
|
||||
- 'fast': Fast parsing without OCR
|
||||
- 'hi_res': High-resolution parsing with OCR for better accuracy
|
||||
|
||||
Returns:
|
||||
The parsing strategy to use.
|
||||
"""
|
||||
strategy = os.getenv("UNSTRUCTURED_STRATEGY", "auto").lower()
|
||||
valid_strategies = ["auto", "fast", "hi_res"]
|
||||
|
||||
if strategy not in valid_strategies:
|
||||
logging.warning(
|
||||
f"Invalid UNSTRUCTURED_STRATEGY '{strategy}'. Using 'hi_res'. "
|
||||
f"Valid options: {', '.join(valid_strategies)}"
|
||||
)
|
||||
return "hi_res"
|
||||
|
||||
return strategy
|
||||
|
||||
|
||||
def get_unstructured_languages() -> list[str]:
|
||||
"""Get the OCR languages for the Unstructured API.
|
||||
|
||||
Languages should be specified as ISO 639-3 codes (e.g., 'eng', 'deu', 'fra').
|
||||
Multiple languages can be specified separated by commas.
|
||||
|
||||
Default languages: English (eng) and German (deu)
|
||||
|
||||
Common language codes:
|
||||
- eng: English
|
||||
- deu: German
|
||||
- fra: French
|
||||
- spa: Spanish
|
||||
- ita: Italian
|
||||
- por: Portuguese
|
||||
- rus: Russian
|
||||
- ara: Arabic
|
||||
- zho: Chinese
|
||||
- jpn: Japanese
|
||||
- kor: Korean
|
||||
|
||||
Returns:
|
||||
List of language codes for OCR processing.
|
||||
"""
|
||||
languages_str = os.getenv("UNSTRUCTURED_LANGUAGES", "eng,deu")
|
||||
|
||||
# Split by comma and clean up whitespace
|
||||
languages = [lang.strip() for lang in languages_str.split(",") if lang.strip()]
|
||||
|
||||
if not languages:
|
||||
logging.warning(
|
||||
"No languages specified in UNSTRUCTURED_LANGUAGES. Using default: eng,deu"
|
||||
)
|
||||
return ["eng", "deu"]
|
||||
|
||||
return languages
|
||||
return config
|
||||
|
||||
Reference in New Issue
Block a user