feat: dedup shared-file parsing/embedding across users in vector sync
A file shared across many users — directly, or via a group folder shared to a group — was parsed and embedded once per user. Chunk point IDs are user-agnostic (uuid5(tenant_id, doc_id=fileid, chunk_index)), but the per-user freshness gate filtered Qdrant by user_id, so two readers ping-ponged: each overwrote the other's points and each kept seeing "not indexed for me", reprocessing every scan. Production telemetry (note 386945, finding #5) measured identical docs re-processed every few hours at 7-13s each, with PDF parse ~62% of per-doc cost. Layer 1 — tenant-wide dedup: - Thread the scanner's tag-REPORT etag into the file DocumentTask and the chunk payload; index `etag` as a KEYWORD field. - vector/sharing_state.find_indexed_content scrolls tenant-wide (no user_id filter) for a non-placeholder point matching (doc_id, doc_type, etag), gated on embedding_identity in Python so a model switch correctly forces a re-embed. - Scanner skips enqueue and the processor skips fetch/parse/embed when a match exists (cross-worker race-guard before WebDAV read). Dedup is fail-safe: a Qdrant error degrades to "process normally". Layer 2 — observed-access ACL (no admin / GroupFolders API needed): - Each point carries `acl_principals` = the set of user:<uid> whose scanner has observed (hence can read) the file. The per-user tag REPORT is the access oracle; group membership/GroupFolders enumeration is admin-only and unavailable in multi-user modes. - build_ownership_filter ORs MatchAny(acl_principals, ["user:<me>"]) so a deduplicated shared/group-folder point surfaces to every reader; verify-on-read (_verify_files) remains the precise ACL gate. - Deletion/eviction become "release one user": drop the principal and delete the points only when the set empties, so one user untagging a shared file doesn't evict it for the others. Legacy points without the field keep the original per-user delete. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
0919513f21
commit
1c93e7286d
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
@@ -157,26 +158,32 @@ class TestOwnersCacheBehavior:
|
||||
|
||||
|
||||
class TestBuildOwnershipFilter:
|
||||
@staticmethod
|
||||
def _by_key(flt: Filter) -> dict[str, Any]:
|
||||
assert flt.should is not None
|
||||
return {cond.key: cond for cond in flt.should}
|
||||
|
||||
def test_defaults_to_self_only_when_owners_omitted(self) -> None:
|
||||
flt = build_ownership_filter("alice")
|
||||
|
||||
# Self-only: just the user_id branch. Self is NOT duplicated into an
|
||||
# owner_id branch (the user_id branch already covers self-owned content).
|
||||
assert flt.should is not None
|
||||
assert len(flt.should) == 1
|
||||
(user_branch,) = flt.should
|
||||
assert user_branch.key == "user_id"
|
||||
assert user_branch.match.value == "alice"
|
||||
# Self-only: the user_id branch plus the observed-access acl_principals
|
||||
# branch (so a deduplicated shared file the user has claimed is still
|
||||
# findable). No owner_id branch — self is covered by user_id.
|
||||
branches = self._by_key(flt)
|
||||
assert set(branches) == {"user_id", "acl_principals"}
|
||||
assert branches["user_id"].match.value == "alice"
|
||||
assert branches["acl_principals"].match.any == ["user:alice"]
|
||||
|
||||
def test_expands_owner_branch_with_accessible_owners(self) -> None:
|
||||
flt = build_ownership_filter("alice", ["alice", "bob", "carol"])
|
||||
|
||||
owner_branch, user_branch = flt.should
|
||||
branches = self._by_key(flt)
|
||||
assert set(branches) == {"owner_id", "user_id", "acl_principals"}
|
||||
# Owner branch holds only the OTHER owners — self ("alice") is excluded
|
||||
# because the user_id branch already matches self-owned content.
|
||||
assert set(owner_branch.match.any) == {"bob", "carol"}
|
||||
assert user_branch.key == "user_id"
|
||||
assert user_branch.match.value == "alice"
|
||||
assert set(branches["owner_id"].match.any) == {"bob", "carol"}
|
||||
assert branches["user_id"].match.value == "alice"
|
||||
assert branches["acl_principals"].match.any == ["user:alice"]
|
||||
|
||||
def test_explicit_empty_list_omits_owner_branch_keeps_legacy(self) -> None:
|
||||
# Edge case: caller passed an explicit empty list. The owner_id branch
|
||||
@@ -185,11 +192,9 @@ class TestBuildOwnershipFilter:
|
||||
# user still finds their own content from before the migration.
|
||||
flt = build_ownership_filter("alice", [])
|
||||
|
||||
assert flt.should is not None
|
||||
assert len(flt.should) == 1
|
||||
(user_branch,) = flt.should
|
||||
assert user_branch.key == "user_id"
|
||||
assert user_branch.match.value == "alice"
|
||||
branches = self._by_key(flt)
|
||||
assert set(branches) == {"user_id", "acl_principals"}
|
||||
assert branches["user_id"].match.value == "alice"
|
||||
|
||||
|
||||
class TestBuildBaseFilterConditions:
|
||||
|
||||
Reference in New Issue
Block a user