fix: mail UI German labels, multi-folder sync, account management, resizable panel fix
This commit is contained in:
Agent Zero
@@ -217,6 +217,134 @@ def account_to_response(account: MailAccount) -> dict:
|
||||
# ─── IMAP Sync Service (F-MAIL-01) ───
|
||||
|
||||
|
||||
# German display names for standard IMAP folders
|
||||
IMAP_FOLDER_NAME_MAP = {
|
||||
"INBOX": "Posteingang",
|
||||
"Sent": "Gesendet",
|
||||
"Sent Items": "Gesendet",
|
||||
"Sent Mail": "Gesendet",
|
||||
"Drafts": "Entwürfe",
|
||||
"Draft": "Entwürfe",
|
||||
"Spam": "Spam",
|
||||
"Junk": "Spam",
|
||||
"Junk Email": "Spam",
|
||||
"Junk E-mail": "Spam",
|
||||
"Trash": "Papierkorb",
|
||||
"Deleted": "Papierkorb",
|
||||
"Deleted Items": "Papierkorb",
|
||||
}
|
||||
|
||||
# Standard IMAP folders that are always considered "standard"
|
||||
STANDARD_IMAP_FOLDERS = {
|
||||
"INBOX", "Sent", "Sent Items", "Sent Mail",
|
||||
"Drafts", "Draft",
|
||||
"Spam", "Junk", "Junk Email", "Junk E-mail",
|
||||
"Trash", "Deleted", "Deleted Items",
|
||||
}
|
||||
|
||||
MAX_EMAILS_PER_FOLDER = 50
|
||||
|
||||
|
||||
def _get_german_folder_name(imap_name: str) -> str:
|
||||
"""Return the German display name for a standard IMAP folder, or the original name."""
|
||||
if imap_name in IMAP_FOLDER_NAME_MAP:
|
||||
return IMAP_FOLDER_NAME_MAP[imap_name]
|
||||
return imap_name
|
||||
|
||||
|
||||
def _parse_imap_list_response(response) -> list[tuple[str, str]]:
|
||||
"""Parse IMAP LIST response into list of (flags, folder_name) tuples."""
|
||||
folders: list[tuple[str, str]] = []
|
||||
lines = response.lines if hasattr(response, 'lines') else response
|
||||
for line in lines:
|
||||
if isinstance(line, (bytes, bytearray)):
|
||||
text = line.decode('utf-8', errors='replace')
|
||||
elif isinstance(line, str):
|
||||
text = line
|
||||
else:
|
||||
continue
|
||||
# IMAP LIST response format: * LIST (\HasChildren) "/" "INBOX"
|
||||
# or: * LIST (\HasNoChildren) "/" "Sent"
|
||||
if 'LIST' not in text:
|
||||
continue
|
||||
# Extract the folder name — it's the last quoted segment
|
||||
# Split by the delimiter (usually " or ')
|
||||
parts = text.split('"')
|
||||
if len(parts) >= 4:
|
||||
# The delimiter is in parts[1], the folder name in parts[3]
|
||||
delimiter = parts[1]
|
||||
folder_name = parts[3]
|
||||
flags = parts[0] if parts[0] else ''
|
||||
folders.append((flags, folder_name))
|
||||
elif len(parts) >= 2:
|
||||
# Fallback: try to extract the last quoted string
|
||||
folder_name = parts[-2] if len(parts) >= 2 else ''
|
||||
if folder_name:
|
||||
folders.append(('', folder_name))
|
||||
return folders
|
||||
|
||||
|
||||
def _build_folder_hierarchy(
|
||||
imap_folders: list[tuple[str, str]],
|
||||
account_id: uuid.UUID,
|
||||
tenant_id: uuid.UUID,
|
||||
existing_folders: dict[str, MailFolder],
|
||||
) -> list[MailFolder]:
|
||||
"""Create or update MailFolder records from IMAP LIST response.
|
||||
Returns the list of folders to add/update.
|
||||
"""
|
||||
result: list[MailFolder] = []
|
||||
# Track delimiter for hierarchy parsing
|
||||
delimiter = '/'
|
||||
|
||||
for flags, imap_name in imap_folders:
|
||||
if not imap_name:
|
||||
continue
|
||||
|
||||
# Determine German display name
|
||||
display_name = _get_german_folder_name(imap_name)
|
||||
is_standard = imap_name in STANDARD_IMAP_FOLDERS
|
||||
|
||||
# Parse parent from hierarchy (e.g. "INBOX/Subfolder")
|
||||
parent_imap_name = None
|
||||
local_name = imap_name
|
||||
if delimiter in imap_name:
|
||||
parts = imap_name.split(delimiter)
|
||||
local_name = parts[-1]
|
||||
parent_imap_name = delimiter.join(parts[:-1])
|
||||
|
||||
# Check if folder already exists
|
||||
if imap_name in existing_folders:
|
||||
folder = existing_folders[imap_name]
|
||||
folder.name = display_name
|
||||
folder.is_standard = is_standard
|
||||
result.append(folder)
|
||||
else:
|
||||
folder = MailFolder(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
name=display_name,
|
||||
imap_name=imap_name,
|
||||
is_standard=is_standard,
|
||||
)
|
||||
result.append(folder)
|
||||
|
||||
# Set parent_id after all folders are created (second pass)
|
||||
|
||||
# Second pass: set parent_id based on IMAP hierarchy
|
||||
folder_by_imap_name = {f.imap_name: f for f in result}
|
||||
for folder in result:
|
||||
if delimiter in folder.imap_name:
|
||||
parts = folder.imap_name.split(delimiter)
|
||||
parent_imap = delimiter.join(parts[:-1])
|
||||
if parent_imap in folder_by_imap_name:
|
||||
parent = folder_by_imap_name[parent_imap]
|
||||
if parent.id:
|
||||
folder.parent_id = parent.id
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def imap_sync_account(
|
||||
db: AsyncSession,
|
||||
account_id: uuid.UUID,
|
||||
@@ -224,8 +352,10 @@ async def imap_sync_account(
|
||||
) -> dict:
|
||||
"""Sync mail folders and messages from IMAP server.
|
||||
|
||||
This function is designed to be called as an ARQ background job.
|
||||
In tests it is mocked — real implementation connects via aioimaplib.
|
||||
Syncs ALL folders from the IMAP server (not just INBOX).
|
||||
For each folder, fetches the last MAX_EMAILS_PER_FOLDER emails
|
||||
(sorted by date descending) to avoid timeouts on large mailboxes.
|
||||
Creates/updates mail_folders records with German display names.
|
||||
"""
|
||||
account = (
|
||||
await db.execute(
|
||||
@@ -239,146 +369,208 @@ async def imap_sync_account(
|
||||
|
||||
password = await get_account_password(account)
|
||||
|
||||
# Import aioimaplib here so tests can mock it
|
||||
try:
|
||||
client = aioimaplib.IMAP4_SSL(host=account.imap_host, port=account.imap_port)
|
||||
await client.wait_hello_from_server()
|
||||
await client.login(account.username, password)
|
||||
await client.select("INBOX")
|
||||
|
||||
# Fetch all message UIDs
|
||||
response = await client.uid_search("ALL")
|
||||
uids = response[1][0].split() if response[1] and response[1][0] else []
|
||||
# 1) LIST all folders from IMAP server
|
||||
list_response = await client.list('""', '"*"')
|
||||
imap_folders = _parse_imap_list_response(list_response)
|
||||
|
||||
synced_count = 0
|
||||
for uid in uids:
|
||||
uid_str = uid.decode() if isinstance(uid, bytes) else str(uid)
|
||||
fetch_resp = await client.uid('fetch', uid_str, '(RFC822)')
|
||||
raw_email = None
|
||||
# aioimaplib Response lines: [header_bytes, data_bytearray, closing_bytes, completion_bytes]
|
||||
for line in (fetch_resp.lines if hasattr(fetch_resp, 'lines') else fetch_resp):
|
||||
if isinstance(line, bytearray):
|
||||
raw_email = bytes(line)
|
||||
break
|
||||
if raw_email is None:
|
||||
continue
|
||||
if isinstance(raw_email, str):
|
||||
raw_email = raw_email.encode()
|
||||
# If LIST returned nothing, fall back to standard folders
|
||||
if not imap_folders:
|
||||
imap_folders = [
|
||||
('', 'INBOX'),
|
||||
('', 'Sent'),
|
||||
('', 'Drafts'),
|
||||
('', 'Spam'),
|
||||
('', 'Trash'),
|
||||
]
|
||||
|
||||
msg = message_from_bytes(raw_email)
|
||||
body_text = ""
|
||||
body_html = ""
|
||||
attachments = []
|
||||
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_type()
|
||||
if ct == "text/plain":
|
||||
body_text = part.get_payload(decode=True).decode("utf-8", errors="replace")
|
||||
elif ct == "text/html":
|
||||
body_html = part.get_payload(decode=True).decode("utf-8", errors="replace")
|
||||
elif part.get_filename():
|
||||
attachments.append(
|
||||
{
|
||||
"filename": part.get_filename(),
|
||||
"mime_type": ct,
|
||||
"size": len(part.get_payload(decode=True) or b""),
|
||||
}
|
||||
)
|
||||
else:
|
||||
ct = msg.get_content_type()
|
||||
payload = msg.get_payload(decode=True)
|
||||
if payload:
|
||||
decoded = payload.decode("utf-8", errors="replace")
|
||||
if ct == "text/html":
|
||||
body_html = decoded
|
||||
else:
|
||||
body_text = decoded
|
||||
|
||||
message_id = msg.get("Message-ID", make_msgid())
|
||||
subject = msg.get("Subject", "")
|
||||
from_addr = msg.get("From", "")
|
||||
to_addrs = msg.get("To", "")
|
||||
cc_addrs = msg.get("Cc", "")
|
||||
refs = msg.get("References", "")
|
||||
in_reply_to = msg.get("In-Reply-To")
|
||||
msg.get("Date", "")
|
||||
|
||||
# Compute thread_id from References/In-Reply-To
|
||||
thread_id = _compute_thread_id(message_id, refs, in_reply_to)
|
||||
|
||||
# Get INBOX folder for this account
|
||||
folder = (
|
||||
await db.execute(
|
||||
select(MailFolder).where(
|
||||
and_(
|
||||
MailFolder.account_id == account.id,
|
||||
MailFolder.imap_name == "INBOX",
|
||||
)
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if folder is None:
|
||||
continue
|
||||
|
||||
mail = Mail(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account.id,
|
||||
folder_id=folder.id,
|
||||
message_id=message_id,
|
||||
thread_id=thread_id,
|
||||
in_reply_to=in_reply_to,
|
||||
references_header=refs,
|
||||
subject=subject,
|
||||
from_address=from_addr,
|
||||
to_addresses=to_addrs,
|
||||
cc_addresses=cc_addrs,
|
||||
body_text=body_text,
|
||||
body_html=body_html,
|
||||
body_html_sanitized=sanitize_html(body_html),
|
||||
has_attachments=len(attachments) > 0,
|
||||
size_bytes=len(raw_email),
|
||||
received_at=datetime.now(UTC),
|
||||
)
|
||||
db.add(mail)
|
||||
synced_count += 1
|
||||
|
||||
await db.flush()
|
||||
|
||||
# Update folder counts
|
||||
folder = (
|
||||
# 2) Load existing folders from DB for this account
|
||||
existing_db_folders = (
|
||||
await db.execute(
|
||||
select(MailFolder).where(
|
||||
and_(
|
||||
MailFolder.account_id == account.id,
|
||||
MailFolder.imap_name == "INBOX",
|
||||
)
|
||||
and_(MailFolder.account_id == account.id, MailFolder.tenant_id == tenant_id)
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if folder:
|
||||
total = (
|
||||
await db.execute(
|
||||
select(text("COUNT(*)")).where(
|
||||
and_(Mail.folder_id == folder.id, Mail.tenant_id == tenant_id)
|
||||
).scalars().all()
|
||||
existing_by_imap: dict[str, MailFolder] = {
|
||||
f.imap_name: f for f in existing_db_folders
|
||||
}
|
||||
|
||||
# 3) Create/update folders in DB
|
||||
db_folders = _build_folder_hierarchy(
|
||||
imap_folders, account.id, tenant_id, existing_by_imap
|
||||
)
|
||||
for folder in db_folders:
|
||||
if folder.id is None:
|
||||
db.add(folder)
|
||||
await db.flush()
|
||||
|
||||
# Build a map of imap_name -> folder_id for email sync
|
||||
folder_by_imap = {f.imap_name: f for f in db_folders}
|
||||
|
||||
synced_count = 0
|
||||
|
||||
# 4) Sync emails for each folder (limit to last 50 per folder)
|
||||
for imap_name, folder in folder_by_imap.items():
|
||||
try:
|
||||
# Select the folder on the IMAP server
|
||||
select_resp = await client.select(f'"{imap_name}"')
|
||||
if select_resp.result != 'OK':
|
||||
continue
|
||||
|
||||
# Fetch UIDs and sort by date descending — get last 50
|
||||
search_resp = await client.uid_search('ALL')
|
||||
uids_raw = search_resp[1][0] if search_resp[1] and search_resp[1][0] else b''
|
||||
if isinstance(uids_raw, (bytes, bytearray)):
|
||||
uids = uids_raw.split()
|
||||
else:
|
||||
uids = []
|
||||
|
||||
# Limit to last MAX_EMAILS_PER_FOLDER UIDs
|
||||
# UIDs are monotonically increasing, so the last ones are the newest
|
||||
if len(uids) > MAX_EMAILS_PER_FOLDER:
|
||||
uids = uids[-MAX_EMAILS_PER_FOLDER:]
|
||||
|
||||
for uid in uids:
|
||||
uid_str = uid.decode() if isinstance(uid, bytes) else str(uid)
|
||||
fetch_resp = await client.uid('fetch', uid_str, '(RFC822)')
|
||||
raw_email = None
|
||||
for line in (fetch_resp.lines if hasattr(fetch_resp, 'lines') else fetch_resp):
|
||||
if isinstance(line, bytearray):
|
||||
raw_email = bytes(line)
|
||||
break
|
||||
if raw_email is None:
|
||||
continue
|
||||
if isinstance(raw_email, str):
|
||||
raw_email = raw_email.encode()
|
||||
|
||||
msg = message_from_bytes(raw_email)
|
||||
body_text = ""
|
||||
body_html = ""
|
||||
attachments = []
|
||||
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_type()
|
||||
if ct == "text/plain":
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload:
|
||||
body_text = payload.decode("utf-8", errors="replace")
|
||||
elif ct == "text/html":
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload:
|
||||
body_html = payload.decode("utf-8", errors="replace")
|
||||
elif part.get_filename():
|
||||
attachments.append(
|
||||
{
|
||||
"filename": part.get_filename(),
|
||||
"mime_type": ct,
|
||||
"size": len(part.get_payload(decode=True) or b""),
|
||||
}
|
||||
)
|
||||
else:
|
||||
ct = msg.get_content_type()
|
||||
payload = msg.get_payload(decode=True)
|
||||
if payload:
|
||||
decoded = payload.decode("utf-8", errors="replace")
|
||||
if ct == "text/html":
|
||||
body_html = decoded
|
||||
else:
|
||||
body_text = decoded
|
||||
|
||||
message_id = msg.get("Message-ID", make_msgid())
|
||||
subject = msg.get("Subject", "")
|
||||
from_addr = msg.get("From", "")
|
||||
to_addrs = msg.get("To", "")
|
||||
cc_addrs = msg.get("Cc", "")
|
||||
refs = msg.get("References", "")
|
||||
in_reply_to = msg.get("In-Reply-To")
|
||||
date_str = msg.get("Date", "")
|
||||
|
||||
# Parse date for received_at
|
||||
received_at = datetime.now(UTC)
|
||||
if date_str:
|
||||
try:
|
||||
from email.utils import parsedate_to_datetime
|
||||
parsed = parsedate_to_datetime(date_str)
|
||||
if parsed:
|
||||
received_at = parsed.astimezone(UTC) if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Compute thread_id from References/In-Reply-To
|
||||
thread_id = _compute_thread_id(message_id, refs, in_reply_to)
|
||||
|
||||
# Check if email already exists (by message_id + folder)
|
||||
existing_mail = (
|
||||
await db.execute(
|
||||
select(Mail).where(
|
||||
and_(
|
||||
Mail.account_id == account.id,
|
||||
Mail.message_id == message_id,
|
||||
Mail.tenant_id == tenant_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing_mail:
|
||||
continue
|
||||
|
||||
mail = Mail(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account.id,
|
||||
folder_id=folder.id,
|
||||
message_id=message_id,
|
||||
thread_id=thread_id,
|
||||
in_reply_to=in_reply_to,
|
||||
references_header=refs,
|
||||
subject=subject,
|
||||
from_address=from_addr,
|
||||
to_addresses=to_addrs,
|
||||
cc_addresses=cc_addrs,
|
||||
body_text=body_text,
|
||||
body_html=body_html,
|
||||
body_html_sanitized=sanitize_html(body_html),
|
||||
has_attachments=len(attachments) > 0,
|
||||
size_bytes=len(raw_email),
|
||||
received_at=received_at,
|
||||
)
|
||||
)
|
||||
).scalar()
|
||||
unread = (
|
||||
await db.execute(
|
||||
select(text("COUNT(*)")).where(
|
||||
and_(
|
||||
Mail.folder_id == folder.id,
|
||||
Mail.tenant_id == tenant_id,
|
||||
not Mail.is_seen,
|
||||
db.add(mail)
|
||||
synced_count += 1
|
||||
|
||||
# Update folder counts
|
||||
total = (
|
||||
await db.execute(
|
||||
select(text("COUNT(*)")).where(
|
||||
and_(Mail.folder_id == folder.id, Mail.tenant_id == tenant_id)
|
||||
)
|
||||
)
|
||||
)
|
||||
).scalar()
|
||||
folder.total_count = total
|
||||
folder.unread_count = unread
|
||||
await db.flush()
|
||||
).scalar()
|
||||
unread = (
|
||||
await db.execute(
|
||||
select(text("COUNT(*)")).where(
|
||||
and_(
|
||||
Mail.folder_id == folder.id,
|
||||
Mail.tenant_id == tenant_id,
|
||||
not Mail.is_seen,
|
||||
)
|
||||
)
|
||||
)
|
||||
).scalar()
|
||||
folder.total_count = total or 0
|
||||
folder.unread_count = unread or 0
|
||||
|
||||
await db.flush()
|
||||
|
||||
except Exception:
|
||||
# Skip folders that can't be selected (e.g. no select permission)
|
||||
continue
|
||||
|
||||
await db.flush()
|
||||
await client.logout()
|
||||
return {"synced": synced_count}
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user