fix: remove all deleted_at/soft-delete from mails — standard mail program logic only
This commit is contained in:
Agent Zero
@@ -0,0 +1,4 @@
|
||||
-- Remove soft-delete (deleted_at) logic from mails
|
||||
-- Mails now use standard mail program logic: folder_id for Trash, permanent DELETE for Trash empty
|
||||
-- This migration clears all existing deleted_at values so no mails are hidden
|
||||
UPDATE mails SET deleted_at = NULL;
|
||||
@@ -54,7 +54,7 @@ class MailPlugin(BasePlugin):
|
||||
),
|
||||
],
|
||||
events=[],
|
||||
migrations=["0001_initial.sql", "0006_flag_type.sql", "0007_sync_queue.sql", "0008_sync_queue_deleted_at.sql"],
|
||||
migrations=["0001_initial.sql", "0006_flag_type.sql", "0007_sync_queue.sql", "0008_sync_queue_deleted_at.sql", "0009_remove_mail_soft_delete.sql"],
|
||||
permissions=["mail:read", "mail:send", "mail:config", "mail:share", "mail:write", "mail:delete"],
|
||||
)
|
||||
|
||||
|
||||
@@ -588,7 +588,7 @@ async def delete_folder(
|
||||
await db.delete(folder)
|
||||
|
||||
|
||||
# ─── Empty Folder (soft-delete all mails in folder) ───
|
||||
# ─── Empty Folder (move to Trash or permanent delete) ───
|
||||
|
||||
|
||||
@router.post("/folders/{folder_id}/empty")
|
||||
@@ -599,7 +599,7 @@ async def empty_folder(
|
||||
):
|
||||
"""Empty a folder: move all mails to Trash, or permanent delete if already in Trash.
|
||||
|
||||
If the folder IS the Trash folder, mails are permanently deleted (deleted_at + IMAP EXPUNGE).
|
||||
If the folder IS the Trash folder, mails are permanently deleted (DELETE + IMAP EXPUNGE).
|
||||
If the folder is NOT Trash, mails are moved to Trash (folder_id changed + IMAP MOVE).
|
||||
Returns the number of mails that were emptied.
|
||||
"""
|
||||
@@ -634,22 +634,20 @@ async def empty_folder(
|
||||
trash_folder = f
|
||||
break
|
||||
|
||||
# Get all non-deleted mails in this folder
|
||||
# Get all mails in this folder
|
||||
result = await db.execute(
|
||||
select(Mail).where(
|
||||
and_(
|
||||
Mail.folder_id == f_id,
|
||||
Mail.tenant_id == tenant_id,
|
||||
Mail.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
)
|
||||
mails = result.scalars().all()
|
||||
now = datetime.now(UTC)
|
||||
for mail in mails:
|
||||
if is_trash or not trash_folder:
|
||||
# Permanent delete
|
||||
mail.deleted_at = now
|
||||
# Permanent delete from DB + IMAP EXPUNGE
|
||||
await db.delete(mail)
|
||||
try:
|
||||
await mail_services.imap_delete_mail(db, mail.id, tenant_id, permanent=True)
|
||||
except Exception as exc:
|
||||
@@ -667,9 +665,8 @@ async def empty_folder(
|
||||
)
|
||||
db.add(queue_entry)
|
||||
else:
|
||||
# Move to Trash folder (keep visible in Trash)
|
||||
# Move to Trash folder (standard mail program logic)
|
||||
mail.folder_id = trash_folder.id
|
||||
mail.deleted_at = now
|
||||
try:
|
||||
await mail_services.imap_delete_mail(db, mail.id, tenant_id, permanent=False)
|
||||
except Exception as exc:
|
||||
@@ -1629,8 +1626,8 @@ async def delete_mail(
|
||||
break
|
||||
|
||||
if trash_folder and mail.folder_id == trash_folder.id:
|
||||
# Mail is already in Trash → permanent delete
|
||||
mail.deleted_at = datetime.now(UTC)
|
||||
# Mail is already in Trash → permanent delete from DB + IMAP EXPUNGE
|
||||
await db.delete(mail)
|
||||
await db.flush()
|
||||
try:
|
||||
await mail_services.imap_delete_mail(db, m_id, tenant_id, permanent=True)
|
||||
@@ -1648,12 +1645,8 @@ async def delete_mail(
|
||||
db.add(queue_entry)
|
||||
await db.flush()
|
||||
elif trash_folder:
|
||||
# Mail is NOT in Trash → move to Trash folder
|
||||
# Set deleted_at so the mail disappears from the source folder
|
||||
# and the sync doesn't move it back. list_mails shows deleted_at
|
||||
# mails when viewing the Trash folder.
|
||||
# Mail is NOT in Trash → move to Trash folder (standard mail program logic)
|
||||
mail.folder_id = trash_folder.id
|
||||
mail.deleted_at = datetime.now(UTC)
|
||||
await db.flush()
|
||||
try:
|
||||
await mail_services.imap_delete_mail(db, m_id, tenant_id, permanent=False)
|
||||
@@ -1671,8 +1664,8 @@ async def delete_mail(
|
||||
db.add(queue_entry)
|
||||
await db.flush()
|
||||
else:
|
||||
# No Trash folder found → permanent delete
|
||||
mail.deleted_at = datetime.now(UTC)
|
||||
# No Trash folder found → permanent delete from DB + IMAP EXPUNGE
|
||||
await db.delete(mail)
|
||||
await db.flush()
|
||||
try:
|
||||
await mail_services.imap_delete_mail(db, m_id, tenant_id, permanent=True)
|
||||
@@ -1762,25 +1755,6 @@ async def list_mails(
|
||||
if folder_id:
|
||||
f_id = _parse_uuid(folder_id, "folder_id")
|
||||
stmt = stmt.where(Mail.folder_id == f_id)
|
||||
# Check if this folder is a Trash folder — if so, show soft-deleted mails too
|
||||
folder = (
|
||||
await db.execute(
|
||||
select(MailFolder).where(
|
||||
and_(MailFolder.id == f_id, MailFolder.tenant_id == tenant_id)
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
is_trash = folder and (
|
||||
'trash' in (folder.imap_name or '').lower()
|
||||
or 'papierkorb' in (folder.name or '').lower()
|
||||
or 'trash' in (folder.name or '').lower()
|
||||
)
|
||||
if not is_trash:
|
||||
# Exclude soft-deleted mails in non-Trash folders
|
||||
stmt = stmt.where(Mail.deleted_at.is_(None))
|
||||
else:
|
||||
# No folder specified — exclude soft-deleted mails
|
||||
stmt = stmt.where(Mail.deleted_at.is_(None))
|
||||
if account_id:
|
||||
a_id = _parse_uuid(account_id, "account_id")
|
||||
stmt = stmt.where(Mail.account_id == a_id)
|
||||
|
||||
@@ -665,33 +665,6 @@ async def imap_sync_folder(
|
||||
).scalar_one_or_none()
|
||||
|
||||
if existing_mail:
|
||||
# Bidirectional sync logic:
|
||||
# - Mail has deleted_at AND is still on IMAP server
|
||||
# - If pending delete in sync queue → our IMAP delete failed, DON'T restore
|
||||
# - If no pending delete → another program restored it, DO restore
|
||||
# - Mail has no deleted_at → normal update
|
||||
if existing_mail.deleted_at is not None:
|
||||
# Check if there's a pending delete in the sync queue
|
||||
from app.plugins.builtins.mail.models import MailSyncQueue
|
||||
pending_delete = (
|
||||
await db.execute(
|
||||
select(MailSyncQueue).where(
|
||||
and_(
|
||||
MailSyncQueue.mail_id == existing_mail.id,
|
||||
MailSyncQueue.operation == "delete",
|
||||
MailSyncQueue.status == "pending",
|
||||
)
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if pending_delete:
|
||||
# Our IMAP delete failed and is retrying — don't restore
|
||||
logger.info("imap_sync_folder: skipping mail %s (deleted, pending IMAP delete in queue)", existing_mail.id)
|
||||
continue
|
||||
else:
|
||||
# No pending delete → another program restored the mail → restore in DB
|
||||
existing_mail.deleted_at = None
|
||||
logger.info("imap_sync_folder: restored mail %s (no pending delete, another program restored it)", existing_mail.id)
|
||||
# Update folder_id and imap_uid if mail moved to a different folder
|
||||
if existing_mail.folder_id != folder.id:
|
||||
existing_mail.folder_id = folder.id
|
||||
@@ -826,6 +799,33 @@ async def imap_sync_folder(
|
||||
logger.warning(f"Failed to save attachment for mail {mail.id}: {att_err}")
|
||||
continue
|
||||
|
||||
# Vanished-UID check: delete DB mails whose UID no longer exists on IMAP server
|
||||
try:
|
||||
vanished_search = await client.uid_search('ALL')
|
||||
vanished_raw = vanished_search[1][0] if vanished_search[1] and vanished_search[1][0] else b''
|
||||
if isinstance(vanished_raw, (bytes, bytearray)):
|
||||
imap_uids_set = {u.decode() if isinstance(u, bytes) else str(u) for u in vanished_raw.split()}
|
||||
else:
|
||||
imap_uids_set = set()
|
||||
db_mails = (
|
||||
await db.execute(
|
||||
select(Mail).where(
|
||||
and_(
|
||||
Mail.folder_id == folder_id,
|
||||
Mail.tenant_id == tenant_id,
|
||||
Mail.imap_uid.is_not(None),
|
||||
)
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
for db_mail in db_mails:
|
||||
if db_mail.imap_uid and db_mail.imap_uid not in imap_uids_set:
|
||||
logger.info("imap_sync_folder: deleting vanished mail %s (UID %s no longer on server)", db_mail.id, db_mail.imap_uid)
|
||||
await db.delete(db_mail)
|
||||
await db.flush()
|
||||
except Exception as vanished_exc:
|
||||
logger.warning("imap_sync_folder: vanished-UID check failed for folder %s: %s", folder_id, vanished_exc)
|
||||
|
||||
# Update folder counts
|
||||
total = (
|
||||
await db.execute(
|
||||
@@ -833,7 +833,6 @@ async def imap_sync_folder(
|
||||
and_(
|
||||
Mail.folder_id == folder_id,
|
||||
Mail.tenant_id == tenant_id,
|
||||
Mail.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -844,7 +843,6 @@ async def imap_sync_folder(
|
||||
and_(
|
||||
Mail.folder_id == folder_id,
|
||||
Mail.tenant_id == tenant_id,
|
||||
Mail.deleted_at.is_(None),
|
||||
Mail.is_seen.is_(False),
|
||||
)
|
||||
)
|
||||
@@ -1185,30 +1183,6 @@ async def imap_sync_account(
|
||||
).scalar_one_or_none()
|
||||
|
||||
if existing_mail:
|
||||
# Bidirectional sync logic:
|
||||
# - Mail has deleted_at AND is still on IMAP server
|
||||
# - If pending delete in sync queue → our IMAP delete failed, DON'T restore
|
||||
# - If no pending delete → another program restored it, DO restore
|
||||
# - Mail has no deleted_at → normal update
|
||||
if existing_mail.deleted_at is not None:
|
||||
from app.plugins.builtins.mail.models import MailSyncQueue
|
||||
pending_delete = (
|
||||
await db.execute(
|
||||
select(MailSyncQueue).where(
|
||||
and_(
|
||||
MailSyncQueue.mail_id == existing_mail.id,
|
||||
MailSyncQueue.operation == "delete",
|
||||
MailSyncQueue.status == "pending",
|
||||
)
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if pending_delete:
|
||||
logger.info("imap_sync_account: skipping mail %s (deleted, pending IMAP delete in queue)", existing_mail.id)
|
||||
continue
|
||||
else:
|
||||
existing_mail.deleted_at = None
|
||||
logger.info("imap_sync_account: restored mail %s (no pending delete, another program restored it)", existing_mail.id)
|
||||
# Track folder moves: update folder_id and imap_uid if the
|
||||
# mail now appears in a different IMAP folder.
|
||||
if existing_mail.folder_id != folder.id:
|
||||
@@ -1217,6 +1191,7 @@ async def imap_sync_account(
|
||||
elif not existing_mail.imap_uid:
|
||||
existing_mail.imap_uid = uid_str
|
||||
|
||||
|
||||
# Save attachments for existing emails that have has_attachments but no records
|
||||
if attachments and existing_mail.has_attachments:
|
||||
existing_att_count = (
|
||||
@@ -1307,6 +1282,33 @@ async def imap_sync_account(
|
||||
continue
|
||||
await db.flush()
|
||||
|
||||
# Vanished-UID check: delete DB mails whose UID no longer exists on IMAP server
|
||||
try:
|
||||
vanished_search = await client.uid_search('ALL')
|
||||
vanished_raw = vanished_search[1][0] if vanished_search[1] and vanished_search[1][0] else b''
|
||||
if isinstance(vanished_raw, (bytes, bytearray)):
|
||||
imap_uids_set = {u.decode() if isinstance(u, bytes) else str(u) for u in vanished_raw.split()}
|
||||
else:
|
||||
imap_uids_set = set()
|
||||
db_mails = (
|
||||
await db.execute(
|
||||
select(Mail).where(
|
||||
and_(
|
||||
Mail.folder_id == folder.id,
|
||||
Mail.tenant_id == tenant_id,
|
||||
Mail.imap_uid.is_not(None),
|
||||
)
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
for db_mail in db_mails:
|
||||
if db_mail.imap_uid and db_mail.imap_uid not in imap_uids_set:
|
||||
logger.info("imap_sync_account: deleting vanished mail %s (UID %s no longer on server)", db_mail.id, db_mail.imap_uid)
|
||||
await db.delete(db_mail)
|
||||
await db.flush()
|
||||
except Exception as vanished_exc:
|
||||
logger.warning("imap_sync_account: vanished-UID check failed for folder %s: %s", folder.imap_name, vanished_exc)
|
||||
|
||||
# Update folder counts
|
||||
total = (
|
||||
await db.execute(
|
||||
|
||||
Reference in New Issue
Block a user