fix(vector): address PR review round 12 — bool guard + strict doc_id validation
- _group_int_doc_ids: use type(value) is not int instead of isinstance, since bool is an int subclass and would otherwise stringify to "True"/"False" and corrupt legacy payloads on backfill. - Replace doc_id.isdigit() guards in 5 boundary sites (api/visualization, auth/viz_routes, search/context note/news_item/ deck_card branches) with a shared is_valid_nextcloud_doc_id helper that rejects "0", leading zeros, and Unicode digit classes (superscripts, Arabic-Indic, Devanagari) which pass isdigit() but cannot be valid MySQL AUTO_INCREMENT IDs. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
f3ce46da0f
commit
f9ad7dc52e
@@ -34,6 +34,7 @@ from nextcloud_mcp_server.search.context import (
|
|||||||
get_chunk_bbox_and_page_from_qdrant,
|
get_chunk_bbox_and_page_from_qdrant,
|
||||||
get_chunk_with_context,
|
get_chunk_with_context,
|
||||||
)
|
)
|
||||||
|
from nextcloud_mcp_server.utils.validation import is_valid_nextcloud_doc_id
|
||||||
from nextcloud_mcp_server.vector.oauth_sync import (
|
from nextcloud_mcp_server.vector.oauth_sync import (
|
||||||
NotProvisionedError,
|
NotProvisionedError,
|
||||||
get_user_client_basic_auth,
|
get_user_client_basic_auth,
|
||||||
@@ -502,8 +503,9 @@ async def get_chunk_context(request: Request) -> JSONResponse:
|
|||||||
# otherwise pass through to get_chunk_with_context and bottom out as a
|
# otherwise pass through to get_chunk_with_context and bottom out as a
|
||||||
# 404 from deep inside, not a clear 400. Nextcloud IDs are unsigned
|
# 404 from deep inside, not a clear 400. Nextcloud IDs are unsigned
|
||||||
# ints from MySQL auto_increment; doc_id stays a str downstream
|
# ints from MySQL auto_increment; doc_id stays a str downstream
|
||||||
# (Qdrant payload index is keyword-typed).
|
# (Qdrant payload index is keyword-typed). is_valid_nextcloud_doc_id
|
||||||
if not doc_id.isdigit():
|
# rejects "0", leading zeros, and Unicode digits that pass isdigit().
|
||||||
|
if not is_valid_nextcloud_doc_id(doc_id):
|
||||||
return JSONResponse(
|
return JSONResponse(
|
||||||
{
|
{
|
||||||
"success": False,
|
"success": False,
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ from nextcloud_mcp_server.search.context import (
|
|||||||
get_chunk_bbox_and_page_from_qdrant,
|
get_chunk_bbox_and_page_from_qdrant,
|
||||||
get_chunk_with_context,
|
get_chunk_with_context,
|
||||||
)
|
)
|
||||||
|
from nextcloud_mcp_server.utils.validation import is_valid_nextcloud_doc_id
|
||||||
from nextcloud_mcp_server.vector.oauth_sync import (
|
from nextcloud_mcp_server.vector.oauth_sync import (
|
||||||
NotProvisionedError,
|
NotProvisionedError,
|
||||||
get_user_client_basic_auth,
|
get_user_client_basic_auth,
|
||||||
@@ -564,8 +565,9 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
|
|||||||
# otherwise pass through to get_chunk_with_context and bottom out as a
|
# otherwise pass through to get_chunk_with_context and bottom out as a
|
||||||
# 404 from deep inside, not a clear 400. Nextcloud IDs are unsigned
|
# 404 from deep inside, not a clear 400. Nextcloud IDs are unsigned
|
||||||
# ints from MySQL auto_increment; doc_id stays a str downstream
|
# ints from MySQL auto_increment; doc_id stays a str downstream
|
||||||
# (Qdrant payload index is keyword-typed).
|
# (Qdrant payload index is keyword-typed). is_valid_nextcloud_doc_id
|
||||||
if not doc_id.isdigit():
|
# rejects "0", leading zeros, and Unicode digits that pass isdigit().
|
||||||
|
if not is_valid_nextcloud_doc_id(doc_id):
|
||||||
return JSONResponse(
|
return JSONResponse(
|
||||||
{
|
{
|
||||||
"success": False,
|
"success": False,
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ from qdrant_client.models import FieldCondition, Filter, MatchValue
|
|||||||
|
|
||||||
from nextcloud_mcp_server.client import NextcloudClient
|
from nextcloud_mcp_server.client import NextcloudClient
|
||||||
from nextcloud_mcp_server.config import get_settings
|
from nextcloud_mcp_server.config import get_settings
|
||||||
|
from nextcloud_mcp_server.utils.validation import is_valid_nextcloud_doc_id
|
||||||
from nextcloud_mcp_server.vector.html_processor import html_to_markdown
|
from nextcloud_mcp_server.vector.html_processor import html_to_markdown
|
||||||
from nextcloud_mcp_server.vector.placeholder import get_placeholder_filter
|
from nextcloud_mcp_server.vector.placeholder import get_placeholder_filter
|
||||||
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
from nextcloud_mcp_server.vector.qdrant_client import get_qdrant_client
|
||||||
@@ -577,10 +578,11 @@ async def _fetch_document_text(
|
|||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
if doc_type == "note":
|
if doc_type == "note":
|
||||||
# Note IDs are integers in the Nextcloud API; reject non-numeric
|
# Note IDs are positive ASCII integers (MySQL AUTO_INCREMENT).
|
||||||
# doc_ids explicitly so a malformed payload surfaces in logs
|
# is_valid_nextcloud_doc_id rejects "0", leading zeros, and Unicode
|
||||||
# rather than getting silently swallowed by `except Exception`.
|
# digits that pass str.isdigit(); a malformed payload surfaces in
|
||||||
if not doc_id.isdigit():
|
# logs rather than getting silently swallowed by `except Exception`.
|
||||||
|
if not is_valid_nextcloud_doc_id(doc_id):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Expected numeric note doc_id, got %r — skipping document fetch",
|
"Expected numeric note doc_id, got %r — skipping document fetch",
|
||||||
doc_id,
|
doc_id,
|
||||||
@@ -594,10 +596,11 @@ async def _fetch_document_text(
|
|||||||
content = note.get("content", "")
|
content = note.get("content", "")
|
||||||
return f"{title}\n\n{content}"
|
return f"{title}\n\n{content}"
|
||||||
elif doc_type == "news_item":
|
elif doc_type == "news_item":
|
||||||
# News item IDs are integers in the Nextcloud News API; reject
|
# News item IDs are positive ASCII integers (MySQL AUTO_INCREMENT).
|
||||||
# non-numeric doc_ids explicitly so malformed payloads surface
|
# is_valid_nextcloud_doc_id rejects "0", leading zeros, and Unicode
|
||||||
# rather than getting swallowed by the broad except below.
|
# digits that pass str.isdigit(); malformed payloads surface in
|
||||||
if not doc_id.isdigit():
|
# logs rather than getting swallowed by the broad except below.
|
||||||
|
if not is_valid_nextcloud_doc_id(doc_id):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Expected numeric news_item doc_id, got %r — skipping document fetch",
|
"Expected numeric news_item doc_id, got %r — skipping document fetch",
|
||||||
doc_id,
|
doc_id,
|
||||||
@@ -621,12 +624,13 @@ async def _fetch_document_text(
|
|||||||
content_parts.append(body_markdown)
|
content_parts.append(body_markdown)
|
||||||
return "\n".join(content_parts)
|
return "\n".join(content_parts)
|
||||||
elif doc_type == "deck_card":
|
elif doc_type == "deck_card":
|
||||||
# Deck card IDs are integers in the Nextcloud Deck API; reject
|
# Deck card IDs are positive ASCII integers (MySQL AUTO_INCREMENT).
|
||||||
# non-numeric doc_ids explicitly so malformed payloads surface
|
# is_valid_nextcloud_doc_id rejects "0", leading zeros, and Unicode
|
||||||
# rather than getting swallowed by the broad except below. The
|
# digits that pass str.isdigit(); malformed payloads surface in
|
||||||
# numeric check covers both the metadata-fast-path (line ~600)
|
# logs rather than getting swallowed by the broad except below.
|
||||||
# and the iteration fallback (line ~635).
|
# The numeric check covers both the metadata-fast-path and the
|
||||||
if not doc_id.isdigit():
|
# iteration fallback below.
|
||||||
|
if not is_valid_nextcloud_doc_id(doc_id):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Expected numeric deck_card doc_id, got %r — skipping document fetch",
|
"Expected numeric deck_card doc_id, got %r — skipping document fetch",
|
||||||
doc_id,
|
doc_id,
|
||||||
|
|||||||
@@ -0,0 +1,15 @@
|
|||||||
|
"""Shared validators for primitive types crossing system boundaries."""
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
# Nextcloud object IDs are unsigned ints from MySQL AUTO_INCREMENT, which
|
||||||
|
# starts at 1. Restrict to ASCII positive integers to exclude Unicode digit
|
||||||
|
# classes (e.g. superscripts, Arabic-Indic numerals) that pass str.isdigit()
|
||||||
|
# / str.isdecimal() but would never be valid Nextcloud IDs, and to reject "0"
|
||||||
|
# and leading zeros.
|
||||||
|
_NEXTCLOUD_DOC_ID_RE = re.compile(r"^[1-9][0-9]*$")
|
||||||
|
|
||||||
|
|
||||||
|
def is_valid_nextcloud_doc_id(value: str) -> bool:
|
||||||
|
"""True iff `value` is the str form of a positive ASCII integer (>= 1)."""
|
||||||
|
return bool(_NEXTCLOUD_DOC_ID_RE.fullmatch(value))
|
||||||
@@ -204,12 +204,14 @@ def _group_int_doc_ids(points: list[Any]) -> tuple[dict[str, list[Any]], int]:
|
|||||||
value = payload.get("doc_id")
|
value = payload.get("doc_id")
|
||||||
if value is None or isinstance(value, str):
|
if value is None or isinstance(value, str):
|
||||||
continue
|
continue
|
||||||
if not isinstance(value, int):
|
# Strict type check: bool is a subclass of int in Python, so an
|
||||||
# Producers only ever write int or str; anything else is a
|
# `isinstance(value, int)` guard would let `True`/`False` slip
|
||||||
# producer bug. Stringifying e.g. a float would write "3.0",
|
# through and be stringified to `"True"`/`"False"` — which the
|
||||||
# which producers (str(int)) and the keyword index would
|
# keyword index would never match and the verification side
|
||||||
# never match, and which int() on the verification side
|
# would later reject. Producers only ever write int or str;
|
||||||
# would later reject. Skip and log loudly instead.
|
# anything else (bool, float, etc.) is a producer bug. Skip
|
||||||
|
# and log loudly instead.
|
||||||
|
if type(value) is not int:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Unexpected doc_id type %s on point %s; skipping rewrite",
|
"Unexpected doc_id type %s on point %s; skipping rewrite",
|
||||||
type(value).__name__,
|
type(value).__name__,
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
"""Unit tests for shared boundary validators."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from nextcloud_mcp_server.utils.validation import is_valid_nextcloud_doc_id
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"value",
|
||||||
|
[
|
||||||
|
"1",
|
||||||
|
"42",
|
||||||
|
"1234567890",
|
||||||
|
"9999999999999999999",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_accepts_positive_ascii_integers(value):
|
||||||
|
"""Any positive ASCII integer (no leading zero) is a valid doc_id."""
|
||||||
|
assert is_valid_nextcloud_doc_id(value) is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"value,reason",
|
||||||
|
[
|
||||||
|
("", "empty string"),
|
||||||
|
("0", "MySQL AUTO_INCREMENT starts at 1"),
|
||||||
|
("01", "leading zero"),
|
||||||
|
("00", "leading zeros"),
|
||||||
|
("-1", "negative"),
|
||||||
|
("+1", "explicit sign"),
|
||||||
|
("1.0", "float-like"),
|
||||||
|
(" 1", "leading whitespace"),
|
||||||
|
("1 ", "trailing whitespace"),
|
||||||
|
("1\n", "trailing newline"),
|
||||||
|
("abc", "alphabetic"),
|
||||||
|
("1a", "trailing letter"),
|
||||||
|
("a1", "leading letter"),
|
||||||
|
# Unicode digit classes that pass str.isdigit() but are not ASCII.
|
||||||
|
# `²` (U+00B2) is a superscript and would slip past the old guard.
|
||||||
|
("²", "Unicode superscript-2"),
|
||||||
|
# `٢` (U+0662) Arabic-Indic digit two — passes both isdigit() and
|
||||||
|
# isdecimal(), so only an explicit ASCII regex catches it.
|
||||||
|
("٢", "Arabic-Indic digit two"),
|
||||||
|
# `१` (U+0967) Devanagari digit one — same story.
|
||||||
|
("१", "Devanagari digit one"),
|
||||||
|
# Mixed ASCII + Unicode digits.
|
||||||
|
("1٢", "mixed ASCII + Arabic-Indic"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_rejects_invalid_doc_ids(value, reason):
|
||||||
|
"""Reject empty/zero/leading-zero/non-ASCII/non-digit inputs."""
|
||||||
|
assert is_valid_nextcloud_doc_id(value) is False, f"should reject: {reason}"
|
||||||
@@ -678,6 +678,36 @@ def test_group_int_doc_ids_skips_float_and_warns(caplog):
|
|||||||
assert "99" in msg
|
assert "99" in msg
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_group_int_doc_ids_skips_bool_and_warns(caplog):
|
||||||
|
"""A bool doc_id is not stringified to "True"/"False"; it logs and skips.
|
||||||
|
|
||||||
|
``isinstance(True, int)`` is ``True`` because ``bool`` is a subclass of
|
||||||
|
``int`` in Python, so a naive ``isinstance(value, int)`` guard would let
|
||||||
|
a boolean payload through and write ``str(True)`` → ``"True"`` into
|
||||||
|
Qdrant. Producers never write bools, but the strict ``type(value) is
|
||||||
|
int`` guard ensures any future producer bug surfaces as a WARNING and is
|
||||||
|
not silently stringified.
|
||||||
|
"""
|
||||||
|
bool_point = SimpleNamespace(id=33, payload={"doc_id": True})
|
||||||
|
int_point = SimpleNamespace(id=42, payload={"doc_id": 7})
|
||||||
|
|
||||||
|
with caplog.at_level("WARNING", logger="nextcloud_mcp_server.vector.qdrant_client"):
|
||||||
|
by_value, scanned = _group_int_doc_ids([bool_point, int_point])
|
||||||
|
|
||||||
|
# Only the int point made it into by_value — "True" is *not* a key.
|
||||||
|
assert by_value == {"7": [42]}
|
||||||
|
assert "True" not in by_value
|
||||||
|
assert "False" not in by_value
|
||||||
|
assert scanned == 2
|
||||||
|
|
||||||
|
warnings = [r for r in caplog.records if r.levelname == "WARNING"]
|
||||||
|
assert len(warnings) == 1
|
||||||
|
msg = warnings[0].getMessage()
|
||||||
|
assert "bool" in msg
|
||||||
|
assert "33" in msg
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.unit
|
@pytest.mark.unit
|
||||||
def test_group_int_doc_ids_handles_str_and_missing_silently(caplog):
|
def test_group_int_doc_ids_handles_str_and_missing_silently(caplog):
|
||||||
"""str / missing doc_id payloads are skipped without warning.
|
"""str / missing doc_id payloads are skipped without warning.
|
||||||
|
|||||||
Reference in New Issue
Block a user