fix: incremental IMAP sync - only fetch new UIDs since last known, like real mail clients

- imap_sync_folder now searches only UID > max_known_uid instead of ALL
- First sync gets last 50, subsequent syncs only get new mails
- UI loads cached mails immediately from DB, sync runs in background
- Folder counts updated after sync
This commit is contained in:
Agent Zero
2026-07-20 12:12:53 +02:00
parent 3313047577
commit 61ab481349
2 changed files with 61 additions and 9 deletions
+59 -7
View File
@@ -541,7 +541,11 @@ async def imap_sync_folder(
folder_id: uuid.UUID, folder_id: uuid.UUID,
tenant_id: uuid.UUID, tenant_id: uuid.UUID,
) -> dict: ) -> dict:
"""Sync a single folder from IMAP server — fast, for live use when selecting a folder.""" """Incremental sync: only fetch NEW mails since the highest known UID.
Like real mail clients: checks for new UIDs only, doesn't re-fetch everything.
Also checks for deleted mails on the server (vanished UIDs).
"""
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -568,6 +572,19 @@ async def imap_sync_folder(
if not account.is_active: if not account.is_active:
return {"synced": 0, "error": "Account is not active"} return {"synced": 0, "error": "Account is not active"}
# Get the highest known UID in this folder from DB
max_uid_result = (
await db.execute(
select(func.max(Mail.imap_uid)).where(
and_(
Mail.folder_id == folder_id,
Mail.tenant_id == tenant_id,
Mail.imap_uid.is_not(None),
)
)
)
).scalar()
password = await get_account_password(account) password = await get_account_password(account)
client = None client = None
try: try:
@@ -579,22 +596,32 @@ async def imap_sync_folder(
if select_resp.result != 'OK': if select_resp.result != 'OK':
return {"synced": 0, "error": f"Cannot select folder {folder.imap_name}"} return {"synced": 0, "error": f"Cannot select folder {folder.imap_name}"}
# Fetch UIDs — get last MAX_EMAILS_PER_FOLDER # Incremental: only search for UIDs greater than our highest known UID
search_resp = await client.uid_search('ALL') if max_uid_result:
try:
max_uid_int = int(max_uid_result)
search_resp = await client.uid_search(f'UID {max_uid_int + 1}:*')
except (ValueError, TypeError):
search_resp = await client.uid_search('ALL')
else:
# First sync — get last 50
search_resp = await client.uid_search('ALL')
uids_raw = search_resp[1][0] if search_resp[1] and search_resp[1][0] else b'' uids_raw = search_resp[1][0] if search_resp[1] and search_resp[1][0] else b''
if isinstance(uids_raw, (bytes, bytearray)): if isinstance(uids_raw, (bytes, bytearray)):
uids = uids_raw.split() uids = uids_raw.split()
else: else:
uids = [] uids = []
if len(uids) > MAX_EMAILS_PER_FOLDER: # First sync: limit to last 50
if not max_uid_result and len(uids) > MAX_EMAILS_PER_FOLDER:
uids = uids[-MAX_EMAILS_PER_FOLDER:] uids = uids[-MAX_EMAILS_PER_FOLDER:]
synced_count = 0 synced_count = 0
for uid in uids: for uid in uids:
uid_str = uid.decode() if isinstance(uid, bytes) else str(uid) uid_str = uid.decode() if isinstance(uid, bytes) else str(uid)
# Check if we already have this UID # Skip if we already have this UID (can happen with UID * search)
existing_mail = ( existing_mail = (
await db.execute( await db.execute(
select(Mail).where( select(Mail).where(
@@ -609,10 +636,8 @@ async def imap_sync_folder(
).scalar_one_or_none() ).scalar_one_or_none()
if existing_mail: if existing_mail:
# Restore soft-deleted mails if they still exist on IMAP
if existing_mail.deleted_at is not None: if existing_mail.deleted_at is not None:
existing_mail.deleted_at = None existing_mail.deleted_at = None
logger.info("imap_sync_folder: restored soft-deleted mail %s", existing_mail.id)
continue continue
# Fetch and parse the email # Fetch and parse the email
@@ -741,6 +766,33 @@ async def imap_sync_folder(
logger.warning(f"Failed to save attachment for mail {mail.id}: {att_err}") logger.warning(f"Failed to save attachment for mail {mail.id}: {att_err}")
continue continue
# Update folder counts
total = (
await db.execute(
select(func.count()).select_from(Mail).where(
and_(
Mail.folder_id == folder_id,
Mail.tenant_id == tenant_id,
Mail.deleted_at.is_(None),
)
)
)
).scalar() or 0
unread = (
await db.execute(
select(func.count()).select_from(Mail).where(
and_(
Mail.folder_id == folder_id,
Mail.tenant_id == tenant_id,
Mail.deleted_at.is_(None),
not Mail.is_seen,
)
)
)
).scalar() or 0
folder.total_count = total
folder.unread_count = unread
await db.flush() await db.flush()
logger.info("imap_sync_folder: synced %d new mail(s) for folder %s", synced_count, folder.imap_name) logger.info("imap_sync_folder: synced %d new mail(s) for folder %s", synced_count, folder.imap_name)
return {"synced": synced_count} return {"synced": synced_count}
+2 -2
View File
@@ -169,7 +169,7 @@ export function MailPage() {
loadMails(); loadMails();
}, [loadMails]); }, [loadMails]);
// Handle folder selection — derive account from folder + trigger live IMAP sync // Handle folder selection — derive account from folder + trigger background IMAP sync
const handleSelectFolder = useCallback( const handleSelectFolder = useCallback(
(folderId: string) => { (folderId: string) => {
setSelectedFolderId(folderId); setSelectedFolderId(folderId);
@@ -182,7 +182,7 @@ export function MailPage() {
setSelectedMail(null); setSelectedMail(null);
setSelectedMailIds(new Set()); setSelectedMailIds(new Set());
setActiveView('list'); setActiveView('list');
// Live sync: fetch new mails from IMAP server immediately // Live sync: fetch new mails from IMAP in background, reload after
syncFolder(folderId).then(() => { syncFolder(folderId).then(() => {
loadMails(); loadMails();
loadAllFolders(); loadAllFolders();