Mail opens faster: the page draws at once from a no-network status, the list, folders and unread counts load side by side, unread counts are cached and time-limited per account, IMAP keeps connections warmer and limits folder counting, messages are prefetched on hover

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

mail-imap.js

@@ -39,18 +39,19 @@ function friendly(a, e) {
 }
 
 // ---- connections: one per account, kept for a minute after the last use
+const IDLE = 5 * 60e3; // a connection is kept this long after its last use: opening one (TLS and login) is the slowest part of a request
 const pool = new Map(); // account id -> { client, timer, key }
-const forget = id => { const e = pool.get(id); if (e) { clearTimeout(e.timer); try { e.client.close(); } catch {} pool.delete(id); } };
+const forget = id => { folderCache.delete(id); boxCache.delete(id); const e = pool.get(id); if (e) { clearTimeout(e.timer); try { e.client.close(); } catch {} pool.delete(id); } };
 const keyOf = a => a.address + '\n' + a.pass;
 async function open(a) {
   const e = pool.get(a.id);
-  if (e && e.key === keyOf(a) && e.client.usable) { clearTimeout(e.timer); e.timer = setTimeout(() => forget(a.id), 60e3).unref(); return e.client; }
+  if (e && e.key === keyOf(a) && e.client.usable) { clearTimeout(e.timer); e.timer = setTimeout(() => forget(a.id), IDLE).unref(); return e.client; }
   forget(a.id);
   const c = cfgOf(a);
   const client = new ImapFlow({ host: c.host, port: c.port, secure: c.secure, ...(c.test ? { doSTARTTLS: false, tls: { rejectUnauthorized: false } } : {}), auth: { user: a.address, pass: a.pass }, logger: false, emitLogs: false, socketTimeout: 60e3, greetingTimeout: 20e3, connectionTimeout: 20e3 });
   client.on('error', () => {}); client.on('close', () => { const cur = pool.get(a.id); if (cur && cur.client === client) pool.delete(a.id); });
   try { await client.connect(); } catch (err) { throw fail(err.authenticationFailed ? 401 : 502, friendly(a, err)); }
-  pool.set(a.id, { client, key: keyOf(a), timer: setTimeout(() => forget(a.id), 60e3).unref() });
+  pool.set(a.id, { client, key: keyOf(a), timer: setTimeout(() => forget(a.id), IDLE).unref() });
   return client;
 }
 /** Run `fn(client)`; if the connection had gone stale, reconnect once. */
@@ -92,20 +93,20 @@ async function pathOf(a, client, folder) {
 const addrOf = x => x ? { name: x.name || '', address: x.address || '' } : { name: '', address: '' };
 const hasAttachment = n => !!n && (((n.disposition || '').toLowerCase() === 'attachment') || (n.childNodes || []).some(hasAttachment));
 
+const folderCache = new Map(); // account id -> { t, v }
 async function folders(a) {
-  return withClient(a, async client => {
+  const c = folderCache.get(a.id); if (c && Date.now() - c.t < 30e3) return c.v;
+  const v = await withClient(a, async client => {
     const { list, alias } = await boxes(a, client), byPath = {}; for (const [k, p] of Object.entries(alias)) byPath[p] = k;
     const skip = b => b.flags.has('\\Noselect') || ['\\Flagged', '\\Important'].includes(b.specialUse);
-    const rows = [];
-    for (const b of list) {
-      if (skip(b)) continue;
-      let st = {}; try { st = await client.status(b.path, { messages: true, unseen: true }); } catch {}
-      const k = byPath[b.path];
-      rows.push({ id: k || 'p-' + Buffer.from(b.path).toString('base64url'), name: k ? NAMES[k] : b.path.replace(/^\[.*?\]\//, ''), unread: st.unseen || 0, total: st.messages || 0, well: !!k });
-    }
+    const keep = list.filter(b => !skip(b)), wellFirst = [...keep.filter(b => byPath[b.path]), ...keep.filter(b => !byPath[b.path])];
+    const counts = new Map(); // one STATUS per folder is a round trip each: the usual six first, then at most ten more (the rest show no number)
+    for (const b of wellFirst.slice(0, Object.keys(byPath).length + 10)) { try { counts.set(b.path, await client.status(b.path, { messages: true, unseen: true })); } catch {} }
+    const rows = keep.map(b => { const k = byPath[b.path], st = counts.get(b.path) || {}; return { id: k || 'p-' + Buffer.from(b.path).toString('base64url'), name: k ? NAMES[k] : b.path.replace(/^\[.*?\]\//, ''), unread: st.unseen || 0, total: st.messages || 0, well: !!k }; });
     const order = Object.keys(NAMES);
     return rows.sort((x, y) => (x.well ? order.indexOf(x.id) : 99) - (y.well ? order.indexOf(y.id) : 99) || x.name.localeCompare(y.name));
   });
+  folderCache.set(a.id, { t: Date.now(), v }); return v;
 }
 
 async function list(a, { folder, skip, q }) {
@@ -144,7 +145,8 @@ async function source(a, mid) {
       if (!dl || !dl.content) throw fail(404, 'That message is gone');
       const chunks = []; let n = 0;
       for await (const ch of dl.content) { n += ch.length; if (n > MAX_SOURCE) throw fail(400, 'That message is too large to open here'); chunks.push(ch); }
-      return { parsed: await simpleParser(Buffer.concat(chunks)), path, uid };
+      const meta = await client.fetchOne(String(uid), { flags: true }, { uid: true }).catch(() => null);
+      return { parsed: await simpleParser(Buffer.concat(chunks)), path, uid, flags: meta ? meta.flags : new Set() };
     } finally { lock.release(); }
   });
 }
@@ -152,11 +154,10 @@ const pick = (list, f) => (list || []).map(x => f(x));
 const addrs = v => (v && v.value ? v.value : []).map(x => ({ name: x.name || '', address: x.address || '' }));
 
 async function message(a, mid) {
-  const { parsed: p, path, uid } = await source(a, mid);
+  const { parsed: p, flags } = await source(a, mid);
   let html = p.html || (p.textAsHtml ? p.textAsHtml : textToHtml(p.text || ''));
   const atts = p.attachments || [];
   for (const x of atts) if (x.cid && /^image\//i.test(x.contentType || '') && x.size < 1.5 * 1024 * 1024) html = html.split('cid:' + x.cid).join(`data:${x.contentType};base64,${x.content.toString('base64')}`);
-  const flags = await withClient(a, async client => { const lock = await client.getMailboxLock(path); try { const m = await client.fetchOne(String(uid), { flags: true }, { uid: true }); return m ? m.flags : new Set(); } finally { lock.release(); } });
   return { id: mid, subject: p.subject || '(No subject)', from: addrs(p.from)[0] || { name: '', address: '' }, to: addrs(p.to), cc: addrs(p.cc), date: (p.date || new Date()).toISOString(), read: flags.has('\\Seen'), flagged: flags.has('\\Flagged'),
     html, attachments: atts.map((x, i) => ({ x, i })).filter(({ x }) => !x.related).map(({ x, i }) => ({ id: String(i), name: x.filename || 'attachment', type: x.contentType, size: x.size })) };
 }

mail.js

@@ -149,7 +149,7 @@ const safeName = n => String(n || 'attachment').replace(/[\r\n"\\/]+/g, '_').sli
 const cleanAttachments = list => { let total = 0; const out = (Array.isArray(list) ? list : []).slice(0, 10).map(a => { const data = String(a.data || ''); total += Buffer.from(data, 'base64').length; return { name: safeName(a.name), type: String(a.type || 'application/octet-stream').slice(0, 100), data }; }); if (total > MAX_ATTACH) throw fail(400, 'Attachments can total about 3 MB'); return out; };
 
 let folderCache = new Map(); // account id -> { t, v }
-const dropFolderCache = id => id ? folderCache.delete(id) : folderCache.clear();
+const dropFolderCache = id => { if (id) { folderCache.delete(id); unreadCache.delete(id); } else { folderCache.clear(); unreadCache.clear(); } };
 
 // ---- Outlook.com through Microsoft Graph. Every function takes the account and Graph's own ids.
 const outlook = {
@@ -205,12 +205,8 @@ const providerOf = a => a.type === 'outlook' ? outlook : imap;
 async function withInline(a, html, attachments, mid) {
   // (contentId only exists on file attachments, so it cannot be selected in the list: fetch each small inline picture whole)
   const inl = (attachments || []).filter(x => x.isInline && /^image\//i.test(x.contentType || '') && (x.size || 0) < 1.5 * 1024 * 1024).slice(0, 15);
-  for (const x of inl) {
-    try {
-      const full = await graph(a, `/me/messages/${id(mid)}/attachments/${id(x.id)}`);
-      if (full && full.contentBytes && full.contentId) html = html.split('cid:' + String(full.contentId).replace(/^<|>$/g, '')).join(`data:${full.contentType || x.contentType};base64,${full.contentBytes}`);
-    } catch { /* leave that picture out */ }
-  }
+  const fulls = await Promise.all(inl.map(x => graph(a, `/me/messages/${id(mid)}/attachments/${id(x.id)}`).catch(() => null))); // all at once
+  inl.forEach((x, i) => { const full = fulls[i]; if (full && full.contentBytes && full.contentId) html = html.split('cid:' + String(full.contentId).replace(/^<|>$/g, '')).join(`data:${full.contentType || x.contentType};base64,${full.contentBytes}`); });
   return html;
 }
 
@@ -246,12 +242,17 @@ async function adminHandle(req, res, p) {
   json(res, 404, { error: 'Not found' });
 }
 
-/** Every unread count in one go (an account that cannot be reached is just left out of the total). */
-async function overview() {
+const unreadCache = new Map(); // account id -> { t, n }: counts are shown at once from here and refreshed when older than 30 s
+const withTimeout = (p, ms) => Promise.race([p, new Promise((_, no) => setTimeout(() => no(Object.assign(new Error('That account did not answer in time'), { status: 504 })), ms))]);
+/** The accounts, with unread counts. `light` makes no network calls at all (what the Mail tab needs to draw itself); otherwise counts older than
+ *  30 seconds are refreshed, each account on its own 6-second limit so one slow or unreachable account cannot hold up the others. */
+async function overview(light) {
   const list = accounts();
   const rows = await Promise.all(list.map(async a => {
-    try { return { ...publicAcct(a), unread: await providerOf(a).unread(a) }; }
-    catch (e) { return { ...publicAcct(a), unread: null, error: e.message, expired: e.status === 409 }; }
+    const c = unreadCache.get(a.id);
+    if (light || (c && Date.now() - c.t < 30e3)) return { ...publicAcct(a), unread: c ? c.n : null };
+    try { const n = await withTimeout(providerOf(a).unread(a), 6000); unreadCache.set(a.id, { t: Date.now(), n }); return { ...publicAcct(a), unread: n }; }
+    catch (e) { return { ...publicAcct(a), unread: c ? c.n : null, error: e.message, expired: e.status === 409 }; }
   }));
   const live = accounts(); // (an account whose sign-in Microsoft revoked was dropped meanwhile)
   const shown = rows.filter(r => live.some(x => x.id === r.id) || !r.expired);
@@ -263,7 +264,7 @@ async function handle(req, res) {
   const u = new URL(req.url, 'http://x'), p = u.pathname, q = u.searchParams;
   try {
     if (p === '/api/mail/admin' || p.startsWith('/api/mail/admin/')) return await adminHandle(req, res, p);
-    if (p === '/api/mail/status') return json(res, 200, { ...(await overview()), outlook: outlookOn(), providers: imap.PRESETS });
+    if (p === '/api/mail/status') return json(res, 200, { ...(await overview(q.get('light') === '1')), outlook: outlookOn(), providers: imap.PRESETS });
     if (p === '/api/mail/connect' && req.method === 'GET') { if (!outlookOn()) throw fail(403, 'Outlook.com is not set up on this server (an admin adds the Microsoft app in Settings)'); return json(res, 200, { url: startUrl(req), redirect: redirectUri(req) }); }
     if (p === '/api/mail/folders') {
       const a = findAccount(q.get('acct') || undefined);

public/mail.js

@@ -24,7 +24,7 @@ const mailHasRemote = html => /<img[^>]+src=["']?https?:|url\(\s*["']?https?:/i.
 // ---- unread badge and Home card
 async function mailPoll() {
   if (!(allowed('mail') && cfg.mail)) return;
-  try { const s = await jfetch('/api/mail/status'); mail.st = s; const n = s.connected && s.unread ? s.unread : 0; if (n !== mailUnread) { mailUnread = n; renderNav(); } } catch {}
+  try { const s = await jfetch('/api/mail/status'); mail.st = s; mail.accts = s.accounts || mail.accts; const n = s.connected && s.unread ? s.unread : 0; if (n !== mailUnread) { mailUnread = n; renderNav(); } } catch {}
 }
 async function renderMailCard(el) {
   if (!el) return;
@@ -125,23 +125,33 @@ async function mailAdminCard() {
 async function viewMail(args = []) {
   const v = $('#view');
   if (!cfg.mail) { v.innerHTML = '<div class="wrap"><h1>Mail</h1><p class="empty">Mail is not available to you.</p></div>'; return; }
-  v.innerHTML = '<div class="wrap"><h1>Mail</h1><p class="empty">Loading…</p></div>';
-  let st; try { st = await jfetch('/api/mail/status'); } catch (e) { v.innerHTML = `<div class="wrap"><h1>Mail</h1><p class="err">${esc(e.message)}</p></div>`; return; }
+  // The accounts come from the server without any network call to the mail providers ("light"), so the page can draw at once; the folders, the message
+  // list and the real unread counts then load side by side instead of one after another.
+  let st = mail.st;
+  if (!st) { v.innerHTML = '<div class="wrap"><h1>Mail</h1><p class="empty">Loading…</p></div>'; try { st = await jfetch('/api/mail/status?light=1'); } catch (e) { v.innerHTML = `<div class="wrap"><h1>Mail</h1><p class="err">${esc(e.message)}</p></div>`; return; } }
   if (!location.hash.startsWith('#/mail')) return;
   mail.st = st; mail.accts = st.accounts || [];
-  if (!st.connected) return mailConnectScreen(st);
-  if (typeof st.unread === 'number' && st.unread !== mailUnread) { mailUnread = st.unread; renderNav(); }
+  if (!st.connected) { mail.st = null; return mailConnectScreen(st); }
   const [kind, key] = args;
   const idAcct = x => String(x || '').includes('~') ? String(x).split('~')[0] : '';
   const want = idAcct(kind === 'f' ? key : kind === 'm' ? key : '') || (mail.accts.some(a => a.id === mail.acct) ? mail.acct : mail.accts[0].id);
   if (want !== mail.acct) { mail.acct = want; mail.folders = []; mail.items = []; if (!(kind === 'f' && idAcct(key))) mail.folder = want + '~inbox'; }
   if (kind === 'f' && key) mail.folder = idAcct(key) ? key : mail.acct + '~' + key;
   else if (!String(mail.folder).startsWith(mail.acct + '~')) mail.folder = mail.acct + '~inbox';
-  if (!mail.folders.length) { try { mail.folders = await jfetch('/api/mail/folders?acct=' + encodeURIComponent(mail.acct)); } catch {} }
   mailShell(kind === 'm' && key ? 'read' : 'list');
-  if (!mail.items.length || kind === 'f' || kind === undefined) mailLoadList(true);
-  else mailDrawList();
+  const acct = mail.acct;
+  if (!mail.items.length || kind === 'f' || kind === undefined) mailLoadList(true); else mailDrawList();
   if (kind === 'm' && key) mailOpen(key); else { mail.msg = null; mailDrawReader(); }
+  // folders (with counts) and the real unread numbers arrive when they arrive
+  jfetch('/api/mail/folders?acct=' + encodeURIComponent(acct)).then(f => { if (mail.acct !== acct) return; mail.folders = f; mailRefreshFolders(); }).catch(() => {});
+  mailPoll().then(() => { if (mail.st && location.hash.startsWith('#/mail')) mailRefreshFolders(); });
+}
+/** Redraw the folder pickers (names and unread counts) without touching the list or the open message. */
+function mailRefreshFolders() {
+  const fl = mail.folders; if (!fl.length) return;
+  const f = $('#mfsel'); if (f) f.innerHTML = fl.map(x => `<option value="${esc(x.id)}" ${x.id === mail.folder ? 'selected' : ''}>${esc(x.name)}${x.unread ? ' (' + x.unread + ')' : ''}</option>`).join('');
+  const side = document.querySelector('.mfold'); if (side) { const sel = side.querySelector('select'); side.innerHTML = (sel ? sel.outerHTML : '') + fl.map(x => `<a href="#/mail/f/${encodeURIComponent(x.id)}" class="${x.id === mail.folder ? 'on' : ''}"><span class="grow">${esc(x.name)}</span>${x.unread ? `<i class="nbadge" style="position:static">${x.unread}</i>` : ''}</a>`).join(''); const s2 = side.querySelector('select'); if (s2) s2.onchange = e => { mail.q = ''; location.hash = '#/mail/f/' + encodeURIComponent(e.target.value + '~inbox'); }; }
+  const acc = $('#macct'); if (acc && mail.accts.length) acc.innerHTML = mail.accts.map(a => `<option value="${esc(a.id)}" ${a.id === mail.acct ? 'selected' : ''}>${esc(a.address)}${a.unread ? ' (' + a.unread + ')' : ''}</option>`).join('');
 }
 
 function mailConnectScreen(st) {
@@ -197,11 +207,16 @@ function mailDrawList() {
   if ($('#mmb')) $('#mmb').onclick = () => mailLoadList(false);
 }
 
+const mailCache = new Map(); // message id -> { t, p: promise }: a message you point at or tap is fetched once and kept for a minute
+const mailFetch = id => { const c = mailCache.get(id); if (c && Date.now() - c.t < 60e3) return c.p; const p = jfetch('/api/mail/message?id=' + encodeURIComponent(id)); mailCache.set(id, { t: Date.now(), p }); p.catch(() => mailCache.delete(id)); if (mailCache.size > 30) mailCache.delete(mailCache.keys().next().value); return p; };
+document.addEventListener('pointerover', e => { const a = e.target.closest && e.target.closest('a.mrow'); if (a && a.getAttribute('href').startsWith('#/mail/m/')) mailFetch(decodeURIComponent(a.getAttribute('href').slice(9))).catch(() => {}); }, { passive: true });
+document.addEventListener('touchstart', e => { const a = e.target.closest && e.target.closest('a.mrow'); if (a && a.getAttribute('href').startsWith('#/mail/m/')) mailFetch(decodeURIComponent(a.getAttribute('href').slice(9))).catch(() => {}); }, { passive: true });
 async function mailOpen(id) {
-  const rd = $('#mread'); if (rd) rd.innerHTML = '<p class="empty" style="padding:16px">Opening…</p>';
+  const rd = $('#mread'), it = mail.items.find(x => x.id === id);
+  if (rd) rd.innerHTML = `<div class="mrhead"><a class="btn ghost mback" href="#/mail/f/${encodeURIComponent(mail.folder)}">${icon('back')}${esc(mailFolderName(mail.folder))}</a>${it ? `<h2 class="msubj">${esc(it.subject)}</h2><p class="mfrom"><b>${esc(mailFrom(it.from))}</b> <small>${esc(it.from.address)}</small></p>` : ''}</div><p class="empty" style="padding:16px">Opening…</p>`; // (what the list already knows, at once)
   mail.images = false;
   try {
-    const m = await jfetch('/api/mail/message?id=' + encodeURIComponent(id));
+    const m = await mailFetch(id);
     if (!(location.hash.startsWith('#/mail/m/'))) return;
     mail.msg = m; mailDrawReader(); mailDrawList();
     if (!m.read) { // opening a message marks it read

tools/mail-imap-test.js

@@ -55,6 +55,9 @@ const free = () => 21000 + Math.floor(Math.random() * 20000);
   ok('and is stored encrypted', !fs.readFileSync(path.join(dir, 'users', 'unate', 'mail-accounts.json'), 'utf8').includes(PASS));
   r = await A('/api/mail/status'); ok('status lists the account with its unread count (2 unread in the inbox)', r.json.connected && r.json.accounts.length === 1 && r.json.accounts[0].unread === 2 && r.json.unread === 2, r.json);
 
+  r = await A('/api/mail/status?light=1'); ok('the light status lists the accounts without asking the mail servers (what lets the Mail tab draw at once)', r.json.connected && r.json.accounts.length === 1 && !r.json.accounts[0].error, r.json);
+  const t0 = Date.now(); r = await A('/api/mail/status'); const t1 = Date.now(); r = await A('/api/mail/status'); ok('the full status is answered from a short-lived cache the second time (counts are refreshed at most every 30 s)', Date.now() - t1 <= Math.max(50, t1 - t0), [t1 - t0, Date.now() - t1]);
+
   // ================= reading =================
   r = await A('/api/mail/folders?acct=' + gm.id); const names = r.json.map(f => f.name);
   ok('folders: Inbox, Sent, Archive (Gmail "All Mail"), Junk, Deleted, then your own; the [Gmail] container is hidden', names.join() === 'Inbox,Sent,Archive,Junk,Deleted,Receipts' && r.json[0].unread === 2 && r.json[0].id.startsWith(gm.id + '~'), r.json);