diff --git a/app/plugins/builtins/mail/services.py b/app/plugins/builtins/mail/services.py index 57a3506..6b5a577 100644 --- a/app/plugins/builtins/mail/services.py +++ b/app/plugins/builtins/mail/services.py @@ -217,6 +217,134 @@ def account_to_response(account: MailAccount) -> dict: # ─── IMAP Sync Service (F-MAIL-01) ─── +# German display names for standard IMAP folders +IMAP_FOLDER_NAME_MAP = { + "INBOX": "Posteingang", + "Sent": "Gesendet", + "Sent Items": "Gesendet", + "Sent Mail": "Gesendet", + "Drafts": "Entwürfe", + "Draft": "Entwürfe", + "Spam": "Spam", + "Junk": "Spam", + "Junk Email": "Spam", + "Junk E-mail": "Spam", + "Trash": "Papierkorb", + "Deleted": "Papierkorb", + "Deleted Items": "Papierkorb", +} + +# Standard IMAP folders that are always considered "standard" +STANDARD_IMAP_FOLDERS = { + "INBOX", "Sent", "Sent Items", "Sent Mail", + "Drafts", "Draft", + "Spam", "Junk", "Junk Email", "Junk E-mail", + "Trash", "Deleted", "Deleted Items", +} + +MAX_EMAILS_PER_FOLDER = 50 + + +def _get_german_folder_name(imap_name: str) -> str: + """Return the German display name for a standard IMAP folder, or the original name.""" + if imap_name in IMAP_FOLDER_NAME_MAP: + return IMAP_FOLDER_NAME_MAP[imap_name] + return imap_name + + +def _parse_imap_list_response(response) -> list[tuple[str, str]]: + """Parse IMAP LIST response into list of (flags, folder_name) tuples.""" + folders: list[tuple[str, str]] = [] + lines = response.lines if hasattr(response, 'lines') else response + for line in lines: + if isinstance(line, (bytes, bytearray)): + text = line.decode('utf-8', errors='replace') + elif isinstance(line, str): + text = line + else: + continue + # IMAP LIST response format: * LIST (\HasChildren) "/" "INBOX" + # or: * LIST (\HasNoChildren) "/" "Sent" + if 'LIST' not in text: + continue + # Extract the folder name — it's the last quoted segment + # Split by the delimiter (usually " or ') + parts = text.split('"') + if len(parts) >= 4: + # The delimiter is in parts[1], the folder name in parts[3] + delimiter = parts[1] + folder_name = parts[3] + flags = parts[0] if parts[0] else '' + folders.append((flags, folder_name)) + elif len(parts) >= 2: + # Fallback: try to extract the last quoted string + folder_name = parts[-2] if len(parts) >= 2 else '' + if folder_name: + folders.append(('', folder_name)) + return folders + + +def _build_folder_hierarchy( + imap_folders: list[tuple[str, str]], + account_id: uuid.UUID, + tenant_id: uuid.UUID, + existing_folders: dict[str, MailFolder], +) -> list[MailFolder]: + """Create or update MailFolder records from IMAP LIST response. + Returns the list of folders to add/update. + """ + result: list[MailFolder] = [] + # Track delimiter for hierarchy parsing + delimiter = '/' + + for flags, imap_name in imap_folders: + if not imap_name: + continue + + # Determine German display name + display_name = _get_german_folder_name(imap_name) + is_standard = imap_name in STANDARD_IMAP_FOLDERS + + # Parse parent from hierarchy (e.g. "INBOX/Subfolder") + parent_imap_name = None + local_name = imap_name + if delimiter in imap_name: + parts = imap_name.split(delimiter) + local_name = parts[-1] + parent_imap_name = delimiter.join(parts[:-1]) + + # Check if folder already exists + if imap_name in existing_folders: + folder = existing_folders[imap_name] + folder.name = display_name + folder.is_standard = is_standard + result.append(folder) + else: + folder = MailFolder( + tenant_id=tenant_id, + account_id=account_id, + name=display_name, + imap_name=imap_name, + is_standard=is_standard, + ) + result.append(folder) + + # Set parent_id after all folders are created (second pass) + + # Second pass: set parent_id based on IMAP hierarchy + folder_by_imap_name = {f.imap_name: f for f in result} + for folder in result: + if delimiter in folder.imap_name: + parts = folder.imap_name.split(delimiter) + parent_imap = delimiter.join(parts[:-1]) + if parent_imap in folder_by_imap_name: + parent = folder_by_imap_name[parent_imap] + if parent.id: + folder.parent_id = parent.id + + return result + + async def imap_sync_account( db: AsyncSession, account_id: uuid.UUID, @@ -224,8 +352,10 @@ async def imap_sync_account( ) -> dict: """Sync mail folders and messages from IMAP server. - This function is designed to be called as an ARQ background job. - In tests it is mocked — real implementation connects via aioimaplib. + Syncs ALL folders from the IMAP server (not just INBOX). + For each folder, fetches the last MAX_EMAILS_PER_FOLDER emails + (sorted by date descending) to avoid timeouts on large mailboxes. + Creates/updates mail_folders records with German display names. """ account = ( await db.execute( @@ -239,146 +369,208 @@ async def imap_sync_account( password = await get_account_password(account) - # Import aioimaplib here so tests can mock it try: client = aioimaplib.IMAP4_SSL(host=account.imap_host, port=account.imap_port) await client.wait_hello_from_server() await client.login(account.username, password) - await client.select("INBOX") - # Fetch all message UIDs - response = await client.uid_search("ALL") - uids = response[1][0].split() if response[1] and response[1][0] else [] + # 1) LIST all folders from IMAP server + list_response = await client.list('""', '"*"') + imap_folders = _parse_imap_list_response(list_response) - synced_count = 0 - for uid in uids: - uid_str = uid.decode() if isinstance(uid, bytes) else str(uid) - fetch_resp = await client.uid('fetch', uid_str, '(RFC822)') - raw_email = None - # aioimaplib Response lines: [header_bytes, data_bytearray, closing_bytes, completion_bytes] - for line in (fetch_resp.lines if hasattr(fetch_resp, 'lines') else fetch_resp): - if isinstance(line, bytearray): - raw_email = bytes(line) - break - if raw_email is None: - continue - if isinstance(raw_email, str): - raw_email = raw_email.encode() + # If LIST returned nothing, fall back to standard folders + if not imap_folders: + imap_folders = [ + ('', 'INBOX'), + ('', 'Sent'), + ('', 'Drafts'), + ('', 'Spam'), + ('', 'Trash'), + ] - msg = message_from_bytes(raw_email) - body_text = "" - body_html = "" - attachments = [] - - if msg.is_multipart(): - for part in msg.walk(): - ct = part.get_content_type() - if ct == "text/plain": - body_text = part.get_payload(decode=True).decode("utf-8", errors="replace") - elif ct == "text/html": - body_html = part.get_payload(decode=True).decode("utf-8", errors="replace") - elif part.get_filename(): - attachments.append( - { - "filename": part.get_filename(), - "mime_type": ct, - "size": len(part.get_payload(decode=True) or b""), - } - ) - else: - ct = msg.get_content_type() - payload = msg.get_payload(decode=True) - if payload: - decoded = payload.decode("utf-8", errors="replace") - if ct == "text/html": - body_html = decoded - else: - body_text = decoded - - message_id = msg.get("Message-ID", make_msgid()) - subject = msg.get("Subject", "") - from_addr = msg.get("From", "") - to_addrs = msg.get("To", "") - cc_addrs = msg.get("Cc", "") - refs = msg.get("References", "") - in_reply_to = msg.get("In-Reply-To") - msg.get("Date", "") - - # Compute thread_id from References/In-Reply-To - thread_id = _compute_thread_id(message_id, refs, in_reply_to) - - # Get INBOX folder for this account - folder = ( - await db.execute( - select(MailFolder).where( - and_( - MailFolder.account_id == account.id, - MailFolder.imap_name == "INBOX", - ) - ) - ) - ).scalar_one_or_none() - if folder is None: - continue - - mail = Mail( - tenant_id=tenant_id, - account_id=account.id, - folder_id=folder.id, - message_id=message_id, - thread_id=thread_id, - in_reply_to=in_reply_to, - references_header=refs, - subject=subject, - from_address=from_addr, - to_addresses=to_addrs, - cc_addresses=cc_addrs, - body_text=body_text, - body_html=body_html, - body_html_sanitized=sanitize_html(body_html), - has_attachments=len(attachments) > 0, - size_bytes=len(raw_email), - received_at=datetime.now(UTC), - ) - db.add(mail) - synced_count += 1 - - await db.flush() - - # Update folder counts - folder = ( + # 2) Load existing folders from DB for this account + existing_db_folders = ( await db.execute( select(MailFolder).where( - and_( - MailFolder.account_id == account.id, - MailFolder.imap_name == "INBOX", - ) + and_(MailFolder.account_id == account.id, MailFolder.tenant_id == tenant_id) ) ) - ).scalar_one_or_none() - if folder: - total = ( - await db.execute( - select(text("COUNT(*)")).where( - and_(Mail.folder_id == folder.id, Mail.tenant_id == tenant_id) + ).scalars().all() + existing_by_imap: dict[str, MailFolder] = { + f.imap_name: f for f in existing_db_folders + } + + # 3) Create/update folders in DB + db_folders = _build_folder_hierarchy( + imap_folders, account.id, tenant_id, existing_by_imap + ) + for folder in db_folders: + if folder.id is None: + db.add(folder) + await db.flush() + + # Build a map of imap_name -> folder_id for email sync + folder_by_imap = {f.imap_name: f for f in db_folders} + + synced_count = 0 + + # 4) Sync emails for each folder (limit to last 50 per folder) + for imap_name, folder in folder_by_imap.items(): + try: + # Select the folder on the IMAP server + select_resp = await client.select(f'"{imap_name}"') + if select_resp.result != 'OK': + continue + + # Fetch UIDs and sort by date descending — get last 50 + search_resp = await client.uid_search('ALL') + uids_raw = search_resp[1][0] if search_resp[1] and search_resp[1][0] else b'' + if isinstance(uids_raw, (bytes, bytearray)): + uids = uids_raw.split() + else: + uids = [] + + # Limit to last MAX_EMAILS_PER_FOLDER UIDs + # UIDs are monotonically increasing, so the last ones are the newest + if len(uids) > MAX_EMAILS_PER_FOLDER: + uids = uids[-MAX_EMAILS_PER_FOLDER:] + + for uid in uids: + uid_str = uid.decode() if isinstance(uid, bytes) else str(uid) + fetch_resp = await client.uid('fetch', uid_str, '(RFC822)') + raw_email = None + for line in (fetch_resp.lines if hasattr(fetch_resp, 'lines') else fetch_resp): + if isinstance(line, bytearray): + raw_email = bytes(line) + break + if raw_email is None: + continue + if isinstance(raw_email, str): + raw_email = raw_email.encode() + + msg = message_from_bytes(raw_email) + body_text = "" + body_html = "" + attachments = [] + + if msg.is_multipart(): + for part in msg.walk(): + ct = part.get_content_type() + if ct == "text/plain": + payload = part.get_payload(decode=True) + if payload: + body_text = payload.decode("utf-8", errors="replace") + elif ct == "text/html": + payload = part.get_payload(decode=True) + if payload: + body_html = payload.decode("utf-8", errors="replace") + elif part.get_filename(): + attachments.append( + { + "filename": part.get_filename(), + "mime_type": ct, + "size": len(part.get_payload(decode=True) or b""), + } + ) + else: + ct = msg.get_content_type() + payload = msg.get_payload(decode=True) + if payload: + decoded = payload.decode("utf-8", errors="replace") + if ct == "text/html": + body_html = decoded + else: + body_text = decoded + + message_id = msg.get("Message-ID", make_msgid()) + subject = msg.get("Subject", "") + from_addr = msg.get("From", "") + to_addrs = msg.get("To", "") + cc_addrs = msg.get("Cc", "") + refs = msg.get("References", "") + in_reply_to = msg.get("In-Reply-To") + date_str = msg.get("Date", "") + + # Parse date for received_at + received_at = datetime.now(UTC) + if date_str: + try: + from email.utils import parsedate_to_datetime + parsed = parsedate_to_datetime(date_str) + if parsed: + received_at = parsed.astimezone(UTC) if parsed.tzinfo else parsed.replace(tzinfo=UTC) + except Exception: + pass + + # Compute thread_id from References/In-Reply-To + thread_id = _compute_thread_id(message_id, refs, in_reply_to) + + # Check if email already exists (by message_id + folder) + existing_mail = ( + await db.execute( + select(Mail).where( + and_( + Mail.account_id == account.id, + Mail.message_id == message_id, + Mail.tenant_id == tenant_id, + ) + ) + ) + ).scalar_one_or_none() + if existing_mail: + continue + + mail = Mail( + tenant_id=tenant_id, + account_id=account.id, + folder_id=folder.id, + message_id=message_id, + thread_id=thread_id, + in_reply_to=in_reply_to, + references_header=refs, + subject=subject, + from_address=from_addr, + to_addresses=to_addrs, + cc_addresses=cc_addrs, + body_text=body_text, + body_html=body_html, + body_html_sanitized=sanitize_html(body_html), + has_attachments=len(attachments) > 0, + size_bytes=len(raw_email), + received_at=received_at, ) - ) - ).scalar() - unread = ( - await db.execute( - select(text("COUNT(*)")).where( - and_( - Mail.folder_id == folder.id, - Mail.tenant_id == tenant_id, - not Mail.is_seen, + db.add(mail) + synced_count += 1 + + # Update folder counts + total = ( + await db.execute( + select(text("COUNT(*)")).where( + and_(Mail.folder_id == folder.id, Mail.tenant_id == tenant_id) ) ) - ) - ).scalar() - folder.total_count = total - folder.unread_count = unread - await db.flush() + ).scalar() + unread = ( + await db.execute( + select(text("COUNT(*)")).where( + and_( + Mail.folder_id == folder.id, + Mail.tenant_id == tenant_id, + not Mail.is_seen, + ) + ) + ) + ).scalar() + folder.total_count = total or 0 + folder.unread_count = unread or 0 + await db.flush() + + except Exception: + # Skip folders that can't be selected (e.g. no select permission) + continue + + await db.flush() await client.logout() return {"synced": synced_count} except Exception as e: diff --git a/frontend/src/api/mail.ts b/frontend/src/api/mail.ts index 164819c..252a4b1 100644 --- a/frontend/src/api/mail.ts +++ b/frontend/src/api/mail.ts @@ -27,6 +27,7 @@ export interface MailFolder { id: string; account_id: string; name: string; + imap_name?: string; parent_id: string | null; unread_count: number; total_count: number; diff --git a/frontend/src/components/mail/MailFolderTree.tsx b/frontend/src/components/mail/MailFolderTree.tsx index 8ad25c6..0fc2143 100644 --- a/frontend/src/components/mail/MailFolderTree.tsx +++ b/frontend/src/components/mail/MailFolderTree.tsx @@ -11,6 +11,36 @@ import type { MailAccount, MailFolder } from '@/api/mail'; import { Badge } from '@/components/ui/Badge'; import { EmptyState } from '@/components/ui/EmptyState'; +/** + * Map IMAP folder names to German display names. + */ +const FOLDER_NAME_MAP: Record = { + INBOX: 'Posteingang', + Sent: 'Gesendet', + Drafts: 'Entwürfe', + Spam: 'Spam', + Junk: 'Spam', + Trash: 'Papierkorb', + 'Sent Items': 'Gesendet', + 'Sent Mail': 'Gesendet', + 'Draft': 'Entwürfe', + 'Deleted': 'Papierkorb', + 'Deleted Items': 'Papierkorb', + 'Junk Email': 'Spam', + 'Junk E-mail': 'Spam', +}; + +/** + * Get the display name for a folder — uses the German name from the map + * if the imap_name matches a known folder, otherwise uses the stored name. + */ +function getFolderDisplayName(folder: MailFolder): string { + if (folder.imap_name && FOLDER_NAME_MAP[folder.imap_name]) { + return FOLDER_NAME_MAP[folder.imap_name]; + } + return folder.name; +} + export interface MailFolderTreeProps { accounts: MailAccount[]; folders: MailFolder[]; @@ -90,7 +120,7 @@ function FolderNode({ const hasChildren = folder.children && folder.children.length > 0; return ( -
+
{hasChildren ? ( +