fix: remove all deleted_at/soft-delete from mails — standard mail program logic only

app/plugins/builtins/mail/migrations/0009_remove_mail_soft_delete.sql

@@ -0,0 +1,4 @@
+-- Remove soft-delete (deleted_at) logic from mails
+-- Mails now use standard mail program logic: folder_id for Trash, permanent DELETE for Trash empty
+-- This migration clears all existing deleted_at values so no mails are hidden
+UPDATE mails SET deleted_at = NULL;

app/plugins/builtins/mail/plugin.py

@@ -54,7 +54,7 @@ class MailPlugin(BasePlugin):
             ),
         ],
         events=[],
-        migrations=["0001_initial.sql", "0006_flag_type.sql", "0007_sync_queue.sql", "0008_sync_queue_deleted_at.sql"],
+        migrations=["0001_initial.sql", "0006_flag_type.sql", "0007_sync_queue.sql", "0008_sync_queue_deleted_at.sql", "0009_remove_mail_soft_delete.sql"],
         permissions=["mail:read", "mail:send", "mail:config", "mail:share", "mail:write", "mail:delete"],
     )
 

app/plugins/builtins/mail/routes.py

@@ -588,7 +588,7 @@ async def delete_folder(
     await db.delete(folder)
 
 
-# ─── Empty Folder (soft-delete all mails in folder) ───
+# ─── Empty Folder (move to Trash or permanent delete) ───
 
 
 @router.post("/folders/{folder_id}/empty")
@@ -599,7 +599,7 @@ async def empty_folder(
 ):
     """Empty a folder: move all mails to Trash, or permanent delete if already in Trash.
 
-    If the folder IS the Trash folder, mails are permanently deleted (deleted_at + IMAP EXPUNGE).
+    If the folder IS the Trash folder, mails are permanently deleted (DELETE + IMAP EXPUNGE).
     If the folder is NOT Trash, mails are moved to Trash (folder_id changed + IMAP MOVE).
     Returns the number of mails that were emptied.
     """
@@ -634,22 +634,20 @@ async def empty_folder(
                 trash_folder = f
                 break
 
-    # Get all non-deleted mails in this folder
+    # Get all mails in this folder
     result = await db.execute(
         select(Mail).where(
             and_(
                 Mail.folder_id == f_id,
                 Mail.tenant_id == tenant_id,
-                Mail.deleted_at.is_(None),
             )
         )
     )
     mails = result.scalars().all()
-    now = datetime.now(UTC)
     for mail in mails:
         if is_trash or not trash_folder:
-            # Permanent delete
+            # Permanent delete from DB + IMAP EXPUNGE
-            mail.deleted_at = now
+            await db.delete(mail)
             try:
                 await mail_services.imap_delete_mail(db, mail.id, tenant_id, permanent=True)
             except Exception as exc:
@@ -667,9 +665,8 @@ async def empty_folder(
                 )
                 db.add(queue_entry)
         else:
-            # Move to Trash folder (keep visible in Trash)
+            # Move to Trash folder (standard mail program logic)
             mail.folder_id = trash_folder.id
-            mail.deleted_at = now
             try:
                 await mail_services.imap_delete_mail(db, mail.id, tenant_id, permanent=False)
             except Exception as exc:
@@ -1629,8 +1626,8 @@ async def delete_mail(
             break
 
     if trash_folder and mail.folder_id == trash_folder.id:
-        # Mail is already in Trash → permanent delete
+        # Mail is already in Trash → permanent delete from DB + IMAP EXPUNGE
-        mail.deleted_at = datetime.now(UTC)
+        await db.delete(mail)
         await db.flush()
         try:
             await mail_services.imap_delete_mail(db, m_id, tenant_id, permanent=True)
@@ -1648,12 +1645,8 @@ async def delete_mail(
             db.add(queue_entry)
             await db.flush()
     elif trash_folder:
-        # Mail is NOT in Trash → move to Trash folder
+        # Mail is NOT in Trash → move to Trash folder (standard mail program logic)
-        # Set deleted_at so the mail disappears from the source folder
-        # and the sync doesn't move it back. list_mails shows deleted_at
-        # mails when viewing the Trash folder.
         mail.folder_id = trash_folder.id
-        mail.deleted_at = datetime.now(UTC)
         await db.flush()
         try:
             await mail_services.imap_delete_mail(db, m_id, tenant_id, permanent=False)
@@ -1671,8 +1664,8 @@ async def delete_mail(
             db.add(queue_entry)
             await db.flush()
     else:
-        # No Trash folder found → permanent delete
+        # No Trash folder found → permanent delete from DB + IMAP EXPUNGE
-        mail.deleted_at = datetime.now(UTC)
+        await db.delete(mail)
         await db.flush()
         try:
             await mail_services.imap_delete_mail(db, m_id, tenant_id, permanent=True)
@@ -1762,25 +1755,6 @@ async def list_mails(
     if folder_id:
         f_id = _parse_uuid(folder_id, "folder_id")
         stmt = stmt.where(Mail.folder_id == f_id)
-        # Check if this folder is a Trash folder — if so, show soft-deleted mails too
-        folder = (
-            await db.execute(
-                select(MailFolder).where(
-                    and_(MailFolder.id == f_id, MailFolder.tenant_id == tenant_id)
-                )
-            )
-        ).scalar_one_or_none()
-        is_trash = folder and (
-            'trash' in (folder.imap_name or '').lower()
-            or 'papierkorb' in (folder.name or '').lower()
-            or 'trash' in (folder.name or '').lower()
-        )
-        if not is_trash:
-            # Exclude soft-deleted mails in non-Trash folders
-            stmt = stmt.where(Mail.deleted_at.is_(None))
-    else:
-        # No folder specified — exclude soft-deleted mails
-        stmt = stmt.where(Mail.deleted_at.is_(None))
     if account_id:
         a_id = _parse_uuid(account_id, "account_id")
         stmt = stmt.where(Mail.account_id == a_id)

app/plugins/builtins/mail/services.py

@@ -665,33 +665,6 @@ async def imap_sync_folder(
                         ).scalar_one_or_none()
 
             if existing_mail:
-                # Bidirectional sync logic:
-                # - Mail has deleted_at AND is still on IMAP server
-                #   - If pending delete in sync queue → our IMAP delete failed, DON'T restore
-                #   - If no pending delete → another program restored it, DO restore
-                # - Mail has no deleted_at → normal update
-                if existing_mail.deleted_at is not None:
-                    # Check if there's a pending delete in the sync queue
-                    from app.plugins.builtins.mail.models import MailSyncQueue
-                    pending_delete = (
-                        await db.execute(
-                            select(MailSyncQueue).where(
-                                and_(
-                                    MailSyncQueue.mail_id == existing_mail.id,
-                                    MailSyncQueue.operation == "delete",
-                                    MailSyncQueue.status == "pending",
-                                )
-                            )
-                        )
-                    ).scalar_one_or_none()
-                    if pending_delete:
-                        # Our IMAP delete failed and is retrying — don't restore
-                        logger.info("imap_sync_folder: skipping mail %s (deleted, pending IMAP delete in queue)", existing_mail.id)
-                        continue
-                    else:
-                        # No pending delete → another program restored the mail → restore in DB
-                        existing_mail.deleted_at = None
-                        logger.info("imap_sync_folder: restored mail %s (no pending delete, another program restored it)", existing_mail.id)
                 # Update folder_id and imap_uid if mail moved to a different folder
                 if existing_mail.folder_id != folder.id:
                     existing_mail.folder_id = folder.id
@@ -826,6 +799,33 @@ async def imap_sync_folder(
                     logger.warning(f"Failed to save attachment for mail {mail.id}: {att_err}")
                     continue
 
+        # Vanished-UID check: delete DB mails whose UID no longer exists on IMAP server
+        try:
+            vanished_search = await client.uid_search('ALL')
+            vanished_raw = vanished_search[1][0] if vanished_search[1] and vanished_search[1][0] else b''
+            if isinstance(vanished_raw, (bytes, bytearray)):
+                imap_uids_set = {u.decode() if isinstance(u, bytes) else str(u) for u in vanished_raw.split()}
+            else:
+                imap_uids_set = set()
+            db_mails = (
+                await db.execute(
+                    select(Mail).where(
+                        and_(
+                            Mail.folder_id == folder_id,
+                            Mail.tenant_id == tenant_id,
+                            Mail.imap_uid.is_not(None),
+                        )
+                    )
+                )
+            ).scalars().all()
+            for db_mail in db_mails:
+                if db_mail.imap_uid and db_mail.imap_uid not in imap_uids_set:
+                    logger.info("imap_sync_folder: deleting vanished mail %s (UID %s no longer on server)", db_mail.id, db_mail.imap_uid)
+                    await db.delete(db_mail)
+            await db.flush()
+        except Exception as vanished_exc:
+            logger.warning("imap_sync_folder: vanished-UID check failed for folder %s: %s", folder_id, vanished_exc)
+
         # Update folder counts
         total = (
             await db.execute(
@@ -833,7 +833,6 @@ async def imap_sync_folder(
                     and_(
                         Mail.folder_id == folder_id,
                         Mail.tenant_id == tenant_id,
-                        Mail.deleted_at.is_(None),
                     )
                 )
             )
@@ -844,7 +843,6 @@ async def imap_sync_folder(
                     and_(
                         Mail.folder_id == folder_id,
                         Mail.tenant_id == tenant_id,
-                        Mail.deleted_at.is_(None),
                         Mail.is_seen.is_(False),
                     )
                 )
@@ -1185,30 +1183,6 @@ async def imap_sync_account(
                         ).scalar_one_or_none()
 
                     if existing_mail:
-                        # Bidirectional sync logic:
-                        # - Mail has deleted_at AND is still on IMAP server
-                        #   - If pending delete in sync queue → our IMAP delete failed, DON'T restore
-                        #   - If no pending delete → another program restored it, DO restore
-                        # - Mail has no deleted_at → normal update
-                        if existing_mail.deleted_at is not None:
-                            from app.plugins.builtins.mail.models import MailSyncQueue
-                            pending_delete = (
-                                await db.execute(
-                                    select(MailSyncQueue).where(
-                                        and_(
-                                            MailSyncQueue.mail_id == existing_mail.id,
-                                            MailSyncQueue.operation == "delete",
-                                            MailSyncQueue.status == "pending",
-                                        )
-                                    )
-                                )
-                            ).scalar_one_or_none()
-                            if pending_delete:
-                                logger.info("imap_sync_account: skipping mail %s (deleted, pending IMAP delete in queue)", existing_mail.id)
-                                continue
-                            else:
-                                existing_mail.deleted_at = None
-                                logger.info("imap_sync_account: restored mail %s (no pending delete, another program restored it)", existing_mail.id)
                         # Track folder moves: update folder_id and imap_uid if the
                         # mail now appears in a different IMAP folder.
                         if existing_mail.folder_id != folder.id:
@@ -1217,6 +1191,7 @@ async def imap_sync_account(
                         elif not existing_mail.imap_uid:
                             existing_mail.imap_uid = uid_str
 
+
                         # Save attachments for existing emails that have has_attachments but no records
                         if attachments and existing_mail.has_attachments:
                             existing_att_count = (
@@ -1307,6 +1282,33 @@ async def imap_sync_account(
                             continue
                     await db.flush()
 
+                # Vanished-UID check: delete DB mails whose UID no longer exists on IMAP server
+                try:
+                    vanished_search = await client.uid_search('ALL')
+                    vanished_raw = vanished_search[1][0] if vanished_search[1] and vanished_search[1][0] else b''
+                    if isinstance(vanished_raw, (bytes, bytearray)):
+                        imap_uids_set = {u.decode() if isinstance(u, bytes) else str(u) for u in vanished_raw.split()}
+                    else:
+                        imap_uids_set = set()
+                    db_mails = (
+                        await db.execute(
+                            select(Mail).where(
+                                and_(
+                                    Mail.folder_id == folder.id,
+                                    Mail.tenant_id == tenant_id,
+                                    Mail.imap_uid.is_not(None),
+                                )
+                            )
+                        )
+                    ).scalars().all()
+                    for db_mail in db_mails:
+                        if db_mail.imap_uid and db_mail.imap_uid not in imap_uids_set:
+                            logger.info("imap_sync_account: deleting vanished mail %s (UID %s no longer on server)", db_mail.id, db_mail.imap_uid)
+                            await db.delete(db_mail)
+                    await db.flush()
+                except Exception as vanished_exc:
+                    logger.warning("imap_sync_account: vanished-UID check failed for folder %s: %s", folder.imap_name, vanished_exc)
+
                 # Update folder counts
                 total = (
                     await db.execute(