refactor(news): simplify vector sync to fetch all items
Remove the complex starred+unread filtering logic in scan_news_items(). The News app's auto-purge feature (default: 200 items per feed) already limits the total number of items, making explicit filtering unnecessary. Changes: - Replace two API calls (starred + unread) with single all-items call - Remove deduplication logic that merged both lists - Update docstring to explain the simpler approach This reduces code complexity while maintaining the same effective coverage. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -574,11 +574,11 @@ async def scan_news_items(
|
|||||||
scan_id: int,
|
scan_id: int,
|
||||||
) -> int:
|
) -> int:
|
||||||
"""
|
"""
|
||||||
Scan user's News items (starred + unread) and queue changed items.
|
Scan user's News items and queue changed items for indexing.
|
||||||
|
|
||||||
Indexes starred and unread items for semantic search. This provides
|
Indexes all items from the user's feeds. The News app's auto-purge
|
||||||
a balanced approach - important items (starred) and current items
|
feature (default: 200 items per feed) naturally limits the total
|
||||||
(unread) are searchable, while avoiding indexing the entire history.
|
number of items, making explicit filtering unnecessary.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
user_id: User to scan
|
user_id: User to scan
|
||||||
@@ -614,34 +614,19 @@ async def scan_news_items(
|
|||||||
indexed_item_ids = {point.payload["doc_id"] for point in scroll_result[0]}
|
indexed_item_ids = {point.payload["doc_id"] for point in scroll_result[0]}
|
||||||
logger.debug(f"Found {len(indexed_item_ids)} indexed news items in Qdrant")
|
logger.debug(f"Found {len(indexed_item_ids)} indexed news items in Qdrant")
|
||||||
|
|
||||||
# Fetch starred items (type=STARRED)
|
# Fetch all items (News app caps at ~200 per feed via auto-purge)
|
||||||
starred_items = await nc_client.news.get_items(
|
all_items = await nc_client.news.get_items(
|
||||||
batch_size=-1, # Get all
|
|
||||||
type_=NewsItemType.STARRED,
|
|
||||||
get_read=True, # Include read starred items
|
|
||||||
)
|
|
||||||
logger.debug(f"[SCAN-{scan_id}] Found {len(starred_items)} starred news items")
|
|
||||||
|
|
||||||
# Fetch unread items (type=ALL, get_read=False)
|
|
||||||
unread_items = await nc_client.news.get_items(
|
|
||||||
batch_size=-1,
|
batch_size=-1,
|
||||||
type_=NewsItemType.ALL,
|
type_=NewsItemType.ALL,
|
||||||
get_read=False, # Only unread
|
get_read=True,
|
||||||
)
|
)
|
||||||
logger.debug(f"[SCAN-{scan_id}] Found {len(unread_items)} unread news items")
|
logger.debug(f"[SCAN-{scan_id}] Found {len(all_items)} news items")
|
||||||
|
|
||||||
# Combine and deduplicate (an item can be both starred and unread)
|
item_count = len(all_items)
|
||||||
items_by_id: dict[int, dict] = {}
|
|
||||||
for item in starred_items:
|
|
||||||
items_by_id[item["id"]] = item
|
|
||||||
for item in unread_items:
|
|
||||||
items_by_id[item["id"]] = item
|
|
||||||
|
|
||||||
item_count = len(items_by_id)
|
|
||||||
nextcloud_item_ids: set[str] = set()
|
nextcloud_item_ids: set[str] = set()
|
||||||
|
|
||||||
for item_id, item in items_by_id.items():
|
for item in all_items:
|
||||||
doc_id = str(item_id)
|
doc_id = str(item["id"])
|
||||||
nextcloud_item_ids.add(doc_id)
|
nextcloud_item_ids.add(doc_id)
|
||||||
|
|
||||||
# Use lastModified timestamp (microseconds in News API)
|
# Use lastModified timestamp (microseconds in News API)
|
||||||
|
|||||||
Reference in New Issue
Block a user