From 3a5520b46af591cfdaba50e684bf222a7b37d7ea Mon Sep 17 00:00:00 2001 From: Agent Zero Date: Mon, 20 Jul 2026 16:08:05 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20remove=20all=20deleted=5Fat/soft-delete?= =?UTF-8?q?=20from=20mails=20=E2=80=94=20standard=20mail=20program=20logic?= =?UTF-8?q?=20only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../0009_remove_mail_soft_delete.sql | 4 + app/plugins/builtins/mail/plugin.py | 2 +- app/plugins/builtins/mail/routes.py | 48 ++------ app/plugins/builtins/mail/services.py | 108 +++++++++--------- 4 files changed, 71 insertions(+), 91 deletions(-) create mode 100644 app/plugins/builtins/mail/migrations/0009_remove_mail_soft_delete.sql diff --git a/app/plugins/builtins/mail/migrations/0009_remove_mail_soft_delete.sql b/app/plugins/builtins/mail/migrations/0009_remove_mail_soft_delete.sql new file mode 100644 index 0000000..3ec9c89 --- /dev/null +++ b/app/plugins/builtins/mail/migrations/0009_remove_mail_soft_delete.sql @@ -0,0 +1,4 @@ +-- Remove soft-delete (deleted_at) logic from mails +-- Mails now use standard mail program logic: folder_id for Trash, permanent DELETE for Trash empty +-- This migration clears all existing deleted_at values so no mails are hidden +UPDATE mails SET deleted_at = NULL; diff --git a/app/plugins/builtins/mail/plugin.py b/app/plugins/builtins/mail/plugin.py index d5a5c11..194faac 100644 --- a/app/plugins/builtins/mail/plugin.py +++ b/app/plugins/builtins/mail/plugin.py @@ -54,7 +54,7 @@ class MailPlugin(BasePlugin): ), ], events=[], - migrations=["0001_initial.sql", "0006_flag_type.sql", "0007_sync_queue.sql", "0008_sync_queue_deleted_at.sql"], + migrations=["0001_initial.sql", "0006_flag_type.sql", "0007_sync_queue.sql", "0008_sync_queue_deleted_at.sql", "0009_remove_mail_soft_delete.sql"], permissions=["mail:read", "mail:send", "mail:config", "mail:share", "mail:write", "mail:delete"], ) diff --git a/app/plugins/builtins/mail/routes.py b/app/plugins/builtins/mail/routes.py index 0f42655..d069931 100644 --- a/app/plugins/builtins/mail/routes.py +++ b/app/plugins/builtins/mail/routes.py @@ -588,7 +588,7 @@ async def delete_folder( await db.delete(folder) -# ─── Empty Folder (soft-delete all mails in folder) ─── +# ─── Empty Folder (move to Trash or permanent delete) ─── @router.post("/folders/{folder_id}/empty") @@ -599,7 +599,7 @@ async def empty_folder( ): """Empty a folder: move all mails to Trash, or permanent delete if already in Trash. - If the folder IS the Trash folder, mails are permanently deleted (deleted_at + IMAP EXPUNGE). + If the folder IS the Trash folder, mails are permanently deleted (DELETE + IMAP EXPUNGE). If the folder is NOT Trash, mails are moved to Trash (folder_id changed + IMAP MOVE). Returns the number of mails that were emptied. """ @@ -634,22 +634,20 @@ async def empty_folder( trash_folder = f break - # Get all non-deleted mails in this folder + # Get all mails in this folder result = await db.execute( select(Mail).where( and_( Mail.folder_id == f_id, Mail.tenant_id == tenant_id, - Mail.deleted_at.is_(None), ) ) ) mails = result.scalars().all() - now = datetime.now(UTC) for mail in mails: if is_trash or not trash_folder: - # Permanent delete - mail.deleted_at = now + # Permanent delete from DB + IMAP EXPUNGE + await db.delete(mail) try: await mail_services.imap_delete_mail(db, mail.id, tenant_id, permanent=True) except Exception as exc: @@ -667,9 +665,8 @@ async def empty_folder( ) db.add(queue_entry) else: - # Move to Trash folder (keep visible in Trash) + # Move to Trash folder (standard mail program logic) mail.folder_id = trash_folder.id - mail.deleted_at = now try: await mail_services.imap_delete_mail(db, mail.id, tenant_id, permanent=False) except Exception as exc: @@ -1629,8 +1626,8 @@ async def delete_mail( break if trash_folder and mail.folder_id == trash_folder.id: - # Mail is already in Trash → permanent delete - mail.deleted_at = datetime.now(UTC) + # Mail is already in Trash → permanent delete from DB + IMAP EXPUNGE + await db.delete(mail) await db.flush() try: await mail_services.imap_delete_mail(db, m_id, tenant_id, permanent=True) @@ -1648,12 +1645,8 @@ async def delete_mail( db.add(queue_entry) await db.flush() elif trash_folder: - # Mail is NOT in Trash → move to Trash folder - # Set deleted_at so the mail disappears from the source folder - # and the sync doesn't move it back. list_mails shows deleted_at - # mails when viewing the Trash folder. + # Mail is NOT in Trash → move to Trash folder (standard mail program logic) mail.folder_id = trash_folder.id - mail.deleted_at = datetime.now(UTC) await db.flush() try: await mail_services.imap_delete_mail(db, m_id, tenant_id, permanent=False) @@ -1671,8 +1664,8 @@ async def delete_mail( db.add(queue_entry) await db.flush() else: - # No Trash folder found → permanent delete - mail.deleted_at = datetime.now(UTC) + # No Trash folder found → permanent delete from DB + IMAP EXPUNGE + await db.delete(mail) await db.flush() try: await mail_services.imap_delete_mail(db, m_id, tenant_id, permanent=True) @@ -1762,25 +1755,6 @@ async def list_mails( if folder_id: f_id = _parse_uuid(folder_id, "folder_id") stmt = stmt.where(Mail.folder_id == f_id) - # Check if this folder is a Trash folder — if so, show soft-deleted mails too - folder = ( - await db.execute( - select(MailFolder).where( - and_(MailFolder.id == f_id, MailFolder.tenant_id == tenant_id) - ) - ) - ).scalar_one_or_none() - is_trash = folder and ( - 'trash' in (folder.imap_name or '').lower() - or 'papierkorb' in (folder.name or '').lower() - or 'trash' in (folder.name or '').lower() - ) - if not is_trash: - # Exclude soft-deleted mails in non-Trash folders - stmt = stmt.where(Mail.deleted_at.is_(None)) - else: - # No folder specified — exclude soft-deleted mails - stmt = stmt.where(Mail.deleted_at.is_(None)) if account_id: a_id = _parse_uuid(account_id, "account_id") stmt = stmt.where(Mail.account_id == a_id) diff --git a/app/plugins/builtins/mail/services.py b/app/plugins/builtins/mail/services.py index 6a88a27..29fb3af 100644 --- a/app/plugins/builtins/mail/services.py +++ b/app/plugins/builtins/mail/services.py @@ -665,33 +665,6 @@ async def imap_sync_folder( ).scalar_one_or_none() if existing_mail: - # Bidirectional sync logic: - # - Mail has deleted_at AND is still on IMAP server - # - If pending delete in sync queue → our IMAP delete failed, DON'T restore - # - If no pending delete → another program restored it, DO restore - # - Mail has no deleted_at → normal update - if existing_mail.deleted_at is not None: - # Check if there's a pending delete in the sync queue - from app.plugins.builtins.mail.models import MailSyncQueue - pending_delete = ( - await db.execute( - select(MailSyncQueue).where( - and_( - MailSyncQueue.mail_id == existing_mail.id, - MailSyncQueue.operation == "delete", - MailSyncQueue.status == "pending", - ) - ) - ) - ).scalar_one_or_none() - if pending_delete: - # Our IMAP delete failed and is retrying — don't restore - logger.info("imap_sync_folder: skipping mail %s (deleted, pending IMAP delete in queue)", existing_mail.id) - continue - else: - # No pending delete → another program restored the mail → restore in DB - existing_mail.deleted_at = None - logger.info("imap_sync_folder: restored mail %s (no pending delete, another program restored it)", existing_mail.id) # Update folder_id and imap_uid if mail moved to a different folder if existing_mail.folder_id != folder.id: existing_mail.folder_id = folder.id @@ -826,6 +799,33 @@ async def imap_sync_folder( logger.warning(f"Failed to save attachment for mail {mail.id}: {att_err}") continue + # Vanished-UID check: delete DB mails whose UID no longer exists on IMAP server + try: + vanished_search = await client.uid_search('ALL') + vanished_raw = vanished_search[1][0] if vanished_search[1] and vanished_search[1][0] else b'' + if isinstance(vanished_raw, (bytes, bytearray)): + imap_uids_set = {u.decode() if isinstance(u, bytes) else str(u) for u in vanished_raw.split()} + else: + imap_uids_set = set() + db_mails = ( + await db.execute( + select(Mail).where( + and_( + Mail.folder_id == folder_id, + Mail.tenant_id == tenant_id, + Mail.imap_uid.is_not(None), + ) + ) + ) + ).scalars().all() + for db_mail in db_mails: + if db_mail.imap_uid and db_mail.imap_uid not in imap_uids_set: + logger.info("imap_sync_folder: deleting vanished mail %s (UID %s no longer on server)", db_mail.id, db_mail.imap_uid) + await db.delete(db_mail) + await db.flush() + except Exception as vanished_exc: + logger.warning("imap_sync_folder: vanished-UID check failed for folder %s: %s", folder_id, vanished_exc) + # Update folder counts total = ( await db.execute( @@ -833,7 +833,6 @@ async def imap_sync_folder( and_( Mail.folder_id == folder_id, Mail.tenant_id == tenant_id, - Mail.deleted_at.is_(None), ) ) ) @@ -844,7 +843,6 @@ async def imap_sync_folder( and_( Mail.folder_id == folder_id, Mail.tenant_id == tenant_id, - Mail.deleted_at.is_(None), Mail.is_seen.is_(False), ) ) @@ -1185,30 +1183,6 @@ async def imap_sync_account( ).scalar_one_or_none() if existing_mail: - # Bidirectional sync logic: - # - Mail has deleted_at AND is still on IMAP server - # - If pending delete in sync queue → our IMAP delete failed, DON'T restore - # - If no pending delete → another program restored it, DO restore - # - Mail has no deleted_at → normal update - if existing_mail.deleted_at is not None: - from app.plugins.builtins.mail.models import MailSyncQueue - pending_delete = ( - await db.execute( - select(MailSyncQueue).where( - and_( - MailSyncQueue.mail_id == existing_mail.id, - MailSyncQueue.operation == "delete", - MailSyncQueue.status == "pending", - ) - ) - ) - ).scalar_one_or_none() - if pending_delete: - logger.info("imap_sync_account: skipping mail %s (deleted, pending IMAP delete in queue)", existing_mail.id) - continue - else: - existing_mail.deleted_at = None - logger.info("imap_sync_account: restored mail %s (no pending delete, another program restored it)", existing_mail.id) # Track folder moves: update folder_id and imap_uid if the # mail now appears in a different IMAP folder. if existing_mail.folder_id != folder.id: @@ -1217,6 +1191,7 @@ async def imap_sync_account( elif not existing_mail.imap_uid: existing_mail.imap_uid = uid_str + # Save attachments for existing emails that have has_attachments but no records if attachments and existing_mail.has_attachments: existing_att_count = ( @@ -1307,6 +1282,33 @@ async def imap_sync_account( continue await db.flush() + # Vanished-UID check: delete DB mails whose UID no longer exists on IMAP server + try: + vanished_search = await client.uid_search('ALL') + vanished_raw = vanished_search[1][0] if vanished_search[1] and vanished_search[1][0] else b'' + if isinstance(vanished_raw, (bytes, bytearray)): + imap_uids_set = {u.decode() if isinstance(u, bytes) else str(u) for u in vanished_raw.split()} + else: + imap_uids_set = set() + db_mails = ( + await db.execute( + select(Mail).where( + and_( + Mail.folder_id == folder.id, + Mail.tenant_id == tenant_id, + Mail.imap_uid.is_not(None), + ) + ) + ) + ).scalars().all() + for db_mail in db_mails: + if db_mail.imap_uid and db_mail.imap_uid not in imap_uids_set: + logger.info("imap_sync_account: deleting vanished mail %s (UID %s no longer on server)", db_mail.id, db_mail.imap_uid) + await db.delete(db_mail) + await db.flush() + except Exception as vanished_exc: + logger.warning("imap_sync_account: vanished-UID check failed for folder %s: %s", folder.imap_name, vanished_exc) + # Update folder counts total = ( await db.execute(