Mail: instant first paint from the last saved accounts, folders and list; Outlook folders in one batched call; gzip for app files and large mail responses

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

mail.js

@@ -36,7 +36,13 @@ function secretStatus() {
 
 const MAX_BODY = 6 * 1024 * 1024, MAX_ATTACH = 3 * 1024 * 1024;
 const readBody = req => new Promise((ok, no) => { const c = []; let n = 0; req.on('data', d => { n += d.length; if (n > MAX_BODY) { no(new Error('That is too large to send (attachments can total about 3 MB)')); req.destroy(); } else c.push(d); }); req.on('end', () => ok(Buffer.concat(c).toString())); req.on('error', no); });
-const json = (res, code, obj) => { res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store' }); res.end(JSON.stringify(obj)); };
+const json = (res, code, obj) => {
+  const body = Buffer.from(JSON.stringify(obj)), h = { 'content-type': 'application/json', 'cache-control': 'no-store' };
+  if (body.length > 2048 && /\bgzip\b/.test((res.req && res.req.headers['accept-encoding']) || '')) { // message bodies are mostly HTML: gzip makes them several times smaller
+    const z = require('zlib').gzipSync(body); res.writeHead(code, { ...h, 'content-encoding': 'gzip', vary: 'Accept-Encoding' }); return res.end(z);
+  }
+  res.writeHead(code, h); res.end(body);
+};
 const fail = (status, message) => Object.assign(new Error(message), { status });
 
 // ---- each person's accounts (encrypted): [{ id, type: 'outlook' | 'gmail' | 'icloud', address, name, ...credentials }]
@@ -155,10 +161,13 @@ const dropFolderCache = id => { if (id) { folderCache.delete(id); unreadCache.de
 const outlook = {
   async folders(a) {
     const c = folderCache.get(a.id); if (c && Date.now() - c.t < 15e3) return c.v;
-    const sel = '?$select=id,displayName,unreadItemCount,totalItemCount';
+    const sel = '?$select=id,displayName,unreadItemCount,totalItemCount', keys = Object.keys(WELL);
-    const wk = await Promise.all(Object.keys(WELL).map(k => graph(a, `/me/mailFolders/${k}${sel}`).then(f => ({ id: k, realId: f.id, name: WELL[k], unread: f.unreadItemCount || 0, total: f.totalItemCount || 0, well: true })).catch(() => null)));
+    // the usual six and the full list in ONE round trip (Graph's $batch), instead of seven
+    const b = await graph(a, '/$batch', { method: 'POST', body: JSON.stringify({ requests: [...keys.map((k, i) => ({ id: String(i), method: 'GET', url: `/me/mailFolders/${k}${sel}` })), { id: 'all', method: 'GET', url: `/me/mailFolders${sel}&$top=100` }] }) });
+    const by = Object.fromEntries((b.responses || []).map(r => [r.id, r]));
+    const wk = keys.map((k, i) => { const r = by[String(i)]; return r && r.status === 200 ? { id: k, realId: r.body.id, name: WELL[k], unread: r.body.unreadItemCount || 0, total: r.body.totalItemCount || 0, well: true } : null; });
     const known = new Set(wk.filter(Boolean).map(f => f.realId));
-    const rest = (await graph(a, `/me/mailFolders${sel}&$top=100`).catch(() => ({ value: [] }))).value || [];
+    const rest = by.all && by.all.status === 200 ? by.all.body.value || [] : [];
     const v = [...wk.filter(Boolean).map(({ realId, ...f }) => f), ...rest.filter(f => !known.has(f.id) && !['Outbox', 'Conversation History', 'Sync Issues', 'RSS Feeds'].includes(f.displayName)).map(f => ({ id: f.id, name: f.displayName, unread: f.unreadItemCount || 0, total: f.totalItemCount || 0 }))];
     folderCache.set(a.id, { t: Date.now(), v }); return v;
   },

public/mail.js

@@ -4,6 +4,19 @@
 let mailUnread = 0;
 const mail = { st: null, accts: [], acct: '', folders: [], folder: 'inbox', items: [], more: false, next: 0, q: '', msg: null, images: false, loading: false };
 
+// The last mail you looked at is kept on this device (subjects, senders, dates; no message bodies) so Mail can show it the instant you open the tab
+// while the fresh copy loads behind it. Removed when you disconnect an account or sign out.
+const mcKey = k => 'mc:' + (ME ? ME.id : '') + ':' + k;
+const mcGet = k => { try { const o = JSON.parse(localStorage.getItem(mcKey(k))); return o && o.t ? o : null; } catch { return null; } };
+function mcPut(k, v) {
+  try {
+    localStorage.setItem(mcKey(k), JSON.stringify({ t: Date.now(), ...v }));
+    const ix = mcKey('index'), list = (JSON.parse(localStorage.getItem(ix) || '[]')).filter(x => x !== k); list.unshift(k);
+    for (const old of list.splice(16)) localStorage.removeItem(mcKey(old));
+    localStorage.setItem(ix, JSON.stringify(list));
+  } catch { /* storage full or blocked: go without */ }
+}
+function mcClear() { try { for (const k of Object.keys(localStorage)) if (k.startsWith('mc:' + (ME ? ME.id : '') + ':')) localStorage.removeItem(k); } catch {} }
 const mailFrom = f => (f && (f.name || f.address)) || '';
 const mailWhen = iso => {
   const d = new Date(iso), now = new Date(); if (isNaN(d)) return '';
@@ -45,7 +58,7 @@ function mailSettings() {
     box.innerHTML = (s.accounts.length ? s.accounts.map(a => `<div class="row"><span class="grow"><b>${esc(MAIL_LABEL[a.type] || a.type)}</b><small>${esc(a.address || 'Connected')}${a.error ? ' · ' + esc(a.error) : ''}</small></span><button class="btn danger" data-dis="${esc(a.id)}" style="min-height:36px">Disconnect</button></div>`).join('') : `<div class="row"><span class="grow"><b>No mail account yet</b><small>${s.expired ? 'Microsoft signed you out. ' : ''}Add Outlook.com, Gmail or iCloud Mail</small></span></div>`)
       + `<div class="row"><span class="grow"></span><button class="btn ghost" id="madd" style="min-height:36px">Add an account</button></div>`;
     $('#madd').onclick = () => mailAddAccount(s);
-    box.querySelectorAll('[data-dis]').forEach(d => d.onclick = () => { const a = s.accounts.find(x => x.id === d.dataset.dis); confirmSheet('Disconnect ' + (a ? a.address : 'this account') + '?', 'NE Hub forgets the sign-in. Your mailbox is not touched.', 'Disconnect', async () => { try { await jfetch('/api/mail/disconnect', { method: 'POST', body: JSON.stringify({ acct: d.dataset.dis }) }); mail.st = null; mail.items = []; mail.folders = []; mail.acct = ''; await mailPoll(); mailSettings(); } catch (e) { toast(e.message); } }); });
+    box.querySelectorAll('[data-dis]').forEach(d => d.onclick = () => { const a = s.accounts.find(x => x.id === d.dataset.dis); confirmSheet('Disconnect ' + (a ? a.address : 'this account') + '?', 'NE Hub forgets the sign-in. Your mailbox is not touched.', 'Disconnect', async () => { try { await jfetch('/api/mail/disconnect', { method: 'POST', body: JSON.stringify({ acct: d.dataset.dis }) }); mail.st = null; mcClear(); mail.items = []; mail.folders = []; mail.acct = ''; await mailPoll(); mailSettings(); } catch (e) { toast(e.message); } }); });
   }).catch(() => { box.innerHTML = '<div class="row"><span class="grow"><small>Could not check</small></span></div>'; });
   else box.innerHTML = '';
   if (cfg.mailAdmin) mailAdminCard();
@@ -128,22 +141,33 @@ async function viewMail(args = []) {
   // The accounts come from the server without any network call to the mail providers ("light"), so the page can draw at once; the folders, the message
   // list and the real unread counts then load side by side instead of one after another.
   let st = mail.st;
-  if (!st) { v.innerHTML = '<div class="wrap"><h1>Mail</h1><p class="empty">Loading…</p></div>'; try { st = await jfetch('/api/mail/status?light=1'); } catch (e) { v.innerHTML = `<div class="wrap"><h1>Mail</h1><p class="err">${esc(e.message)}</p></div>`; return; } }
+  if (!st) {
+    const saved = mcGet('st'); // the accounts as of last time: draw from them now and check with the server behind it
+    if (saved && saved.connected) {
+      st = saved; mail.st = saved;
+      jfetch('/api/mail/status?light=1').then(fresh => { if (!fresh.connected || fresh.accounts.map(a => a.id).join() !== saved.accounts.map(a => a.id).join()) { mail.st = null; mcClear(); if (location.hash.startsWith('#/mail')) route(); } else { mail.st = { ...fresh, accounts: fresh.accounts.map(a => ({ ...a, unread: (mail.st.accounts.find(x => x.id === a.id) || {}).unread ?? a.unread })) }; mcPut('st', mail.st); } }).catch(() => {});
+    } else {
+      v.innerHTML = '<div class="wrap"><h1>Mail</h1><p class="empty">Loading…</p></div>';
+      try { st = await jfetch('/api/mail/status?light=1'); } catch (e) { v.innerHTML = `<div class="wrap"><h1>Mail</h1><p class="err">${esc(e.message)}</p></div>`; return; }
+      if (st.connected) mcPut('st', { connected: true, accounts: st.accounts, outlook: st.outlook, providers: st.providers });
+    }
+  }
   if (!location.hash.startsWith('#/mail')) return;
   mail.st = st; mail.accts = st.accounts || [];
-  if (!st.connected) { mail.st = null; return mailConnectScreen(st); }
+  if (!st.connected) { mail.st = null; mcClear(); return mailConnectScreen(st); }
   const [kind, key] = args;
   const idAcct = x => String(x || '').includes('~') ? String(x).split('~')[0] : '';
   const want = idAcct(kind === 'f' ? key : kind === 'm' ? key : '') || (mail.accts.some(a => a.id === mail.acct) ? mail.acct : mail.accts[0].id);
   if (want !== mail.acct) { mail.acct = want; mail.folders = []; mail.items = []; if (!(kind === 'f' && idAcct(key))) mail.folder = want + '~inbox'; }
   if (kind === 'f' && key) mail.folder = idAcct(key) ? key : mail.acct + '~' + key;
   else if (!String(mail.folder).startsWith(mail.acct + '~')) mail.folder = mail.acct + '~inbox';
+  if (!mail.folders.length) { const f = mcGet('f:' + mail.acct); if (f) mail.folders = f.folders; }
   mailShell(kind === 'm' && key ? 'read' : 'list');
   const acct = mail.acct;
   if (!mail.items.length || kind === 'f' || kind === undefined) mailLoadList(true); else mailDrawList();
   if (kind === 'm' && key) mailOpen(key); else { mail.msg = null; mailDrawReader(); }
   // folders (with counts) and the real unread numbers arrive when they arrive
-  jfetch('/api/mail/folders?acct=' + encodeURIComponent(acct)).then(f => { if (mail.acct !== acct) return; mail.folders = f; mailRefreshFolders(); }).catch(() => {});
+  jfetch('/api/mail/folders?acct=' + encodeURIComponent(acct)).then(f => { if (mail.acct !== acct) return; mail.folders = f; mcPut('f:' + acct, { folders: f }); mailRefreshFolders(); }).catch(() => {});
   mailPoll().then(() => { if (mail.st && location.hash.startsWith('#/mail')) mailRefreshFolders(); });
 }
 /** Redraw the folder pickers (names and unread counts) without touching the list or the open message. */
@@ -188,14 +212,21 @@ function mailShell(mode) {
 }
 
 async function mailLoadList(reset) {
-  if (reset) { mail.items = []; mail.next = 0; mail.more = false; }
+  const folder = mail.folder, first = reset && !mail.q;
-  const rows = $('#mrows'); if (rows && reset) rows.innerHTML = '<p class="empty">Loading…</p>';
+  if (reset) {
+    mail.items = []; mail.next = 0; mail.more = false;
+    const saved = first && mcGet('l:' + folder);
+    if (saved) { mail.items = saved.items; mail.more = saved.more; mail.next = saved.next; mailDrawList(); } // last time's list, right away
+    else { const rows = $('#mrows'); if (rows) rows.innerHTML = '<p class="empty">Loading…</p>'; }
+  }
+  const showing = reset && mail.items.length; // a saved list is on screen: the fresh one replaces it instead of being added to it
   try {
-    const r = await jfetch(`/api/mail/messages?folder=${encodeURIComponent(mail.folder)}&skip=${mail.next}${mail.q ? '&q=' + encodeURIComponent(mail.q) : ''}`);
+    const r = await jfetch(`/api/mail/messages?folder=${encodeURIComponent(folder)}&skip=${showing ? 0 : mail.next}${mail.q ? '&q=' + encodeURIComponent(mail.q) : ''}`);
-    if (!$('#mrows')) return;
+    if (!$('#mrows') || mail.folder !== folder) return;
-    mail.items = mail.items.concat(r.items); mail.more = r.more; mail.next = r.next;
+    mail.items = showing ? r.items : mail.items.concat(r.items); mail.more = r.more; mail.next = r.next;
+    if (first) mcPut('l:' + folder, { items: r.items, more: r.more, next: r.next });
     mailDrawList();
-  } catch (e) { const el = $('#mrows'); if (el) el.innerHTML = `<p class="err" style="padding:12px">${esc(e.message)}</p>`; if (/Connect again/.test(e.message)) { mail.st = null; setTimeout(route, 1200); } }
+  } catch (e) { if (showing) return; const el = $('#mrows'); if (el) el.innerHTML = `<p class="err" style="padding:12px">${esc(e.message)}</p>`; if (/Connect again/.test(e.message)) { mail.st = null; mcClear(); setTimeout(route, 1200); } }
 }
 function mailDrawList() {
   const rows = $('#mrows'); if (!rows) return;

public/settings.js

@@ -170,6 +170,6 @@ function viewSettings() {
       el.querySelector('#sx').onclick = closeSheet;
     } catch (e) { toast(e.message); }
   };
-  $('#hubout').onclick = async () => { await authPost('/api/auth/logout'); location.replace('/login'); };
+  $('#hubout').onclick = async () => { try { mcClear(); for (const k of Object.keys(localStorage)) if (k.startsWith('dc:')) localStorage.removeItem(k); } catch {} await authPost('/api/auth/logout'); location.replace('/login'); };
   const out = $('#out'); if (out) out.onclick = () => { store.del('sf_token'); toast('Signed out'); viewSettings(); };
 }

server.js

@@ -4,6 +4,7 @@ const http = require('http');
 const fs = require('fs');
 const atomicWrite = require('./atomic');
 const crypto = require('crypto');
+const zlib = require('zlib');
 const path = require('path');
 const { Readable } = require('stream');
 
@@ -121,10 +122,17 @@ function serveStatic(req, res) {
       headers.etag = `W/"${data.length}-${crypto.createHash('sha1').update(data).digest('hex').slice(0, 16)}"`;
       if (req.headers['if-none-match'] === headers.etag) { res.writeHead(304, { etag: headers.etag, 'cache-control': 'no-cache' }); return res.end(); }
     }
+    // text files are sent gzipped when the browser accepts it (the app's scripts are several hundred KB), and the compressed copy is kept per version
+    if (/^(text\/|application\/(json|manifest))|svg/.test(headers['content-type']) && data.length > 1024 && /\bgzip\b/.test(req.headers['accept-encoding'] || '')) {
+      const key = file + ':' + data.length + ':' + (headers.etag || ''); let z = gzCache.get(key);
+      if (!z) { z = zlib.gzipSync(data, { level: 6 }); if (gzCache.size > 200) gzCache.clear(); gzCache.set(key, z); }
+      headers['content-encoding'] = 'gzip'; headers.vary = 'Accept-Encoding'; data = z;
+    }
     res.writeHead(200, headers);
     res.end(data);
   });
 }
+const gzCache = new Map();
 
 // Changes whenever the app's files change, so an open page can tell that a newer version is available.
 const BUILD = (() => { try { return crypto.createHash('sha1').update(fs.readdirSync(PUBLIC).filter(f => /\.(js|css|html)$/.test(f)).sort().map(f => fs.readFileSync(path.join(PUBLIC, f))).join('')).digest('hex').slice(0, 12); } catch { return String(Date.now()); } })();

tools/mail-imap-test.js

@@ -56,7 +56,9 @@ const free = () => 21000 + Math.floor(Math.random() * 20000);
   r = await A('/api/mail/status'); ok('status lists the account with its unread count (2 unread in the inbox)', r.json.connected && r.json.accounts.length === 1 && r.json.accounts[0].unread === 2 && r.json.unread === 2, r.json);
 
   r = await A('/api/mail/status?light=1'); ok('the light status lists the accounts without asking the mail servers (what lets the Mail tab draw at once)', r.json.connected && r.json.accounts.length === 1 && !r.json.accounts[0].error, r.json);
-  const t0 = Date.now(); r = await A('/api/mail/status'); const t1 = Date.now(); r = await A('/api/mail/status'); ok('the full status is answered from a short-lived cache the second time (counts are refreshed at most every 30 s)', Date.now() - t1 <= Math.max(50, t1 - t0), [t1 - t0, Date.now() - t1]);
+  { const first = (await A('/api/mail/status')).json.unread; // unread counts are cached for 30 s: change the real mailbox behind the hub's back and the number stays
+    const c = new ImapFlow({ host: '127.0.0.1', port: imapPort, secure: false, doSTARTTLS: false, auth: { user: ME, pass: PASS }, logger: false }); await c.connect(); const lock = await c.getMailboxLock('INBOX');
+    try { await c.messageFlagsAdd('1', ['\\Seen']); const again = (await A('/api/mail/status')).json.unread; ok('the full status serves a cached unread count for up to 30 s (so opening Mail never waits on the mail servers)', again === first, [first, again]); await c.messageFlagsRemove('1', ['\\Seen']); } finally { lock.release(); await c.logout(); } }
 
   // ================= reading =================
   r = await A('/api/mail/folders?acct=' + gm.id); const names = r.json.map(f => f.name);

tools/mail-test.js

@@ -42,6 +42,7 @@ const SECRET = 'mail-test-secret-'.repeat(3);
     const tok = (rq.headers.authorization || '').replace('Bearer ', '');
     if (!ms.goodTokens.has(tok)) return out(rs, 401, { error: { code: 'InvalidAuthenticationToken', message: 'expired' } });
     const g = p.slice(5);
+    if (g === '/$batch' && rq.method === 'POST') { ms.batches = (ms.batches || 0) + 1; const reqs = JSON.parse(b).requests, self = `http://127.0.0.1:${server.address().port}/v1.0`; const responses = []; for (const q of reqs) { const r = await fetch(self + q.url, { method: q.method, headers: { authorization: rq.headers.authorization } }); responses.push({ id: q.id, status: r.status, body: await r.json().catch(() => ({})) }); } return out(rs, 200, { responses }); }
     if (g === '/me') return out(rs, 200, { displayName: 'Nate', mail: 'nate@outlook.com' });
     let m;
     if ((m = /^\/me\/mailFolders\/(\w+)$/.exec(g))) { const f = { inbox: folder('inbox', 'Inbox', 3, 10), drafts: folder('drafts', 'Drafts', 0, 1), sentitems: folder('sentitems', 'Sent Items', 0, 4), archive: folder('archive', 'Archive', 0, 0), junkemail: folder('junkemail', 'Junk Email', 0, 0), deleteditems: folder('deleteditems', 'Deleted Items', 0, 2) }[m[1]]; return f ? out(rs, 200, f) : out(rs, 404, { error: { message: 'no folder' } }); }
@@ -101,7 +102,8 @@ const SECRET = 'mail-test-secret-'.repeat(3);
   r = await call('uann', '/api/mail/messages'); ok("and cannot read it", r.status === 409, r.status);
 
   // ================= reading =================
-  r = await call('unate', '/api/mail/folders'); const names = (r.json || []).map(f => f.name);
+  ms.batches = 0; r = await call('unate', '/api/mail/folders'); const names = (r.json || []).map(f => f.name);
+  ok('the folder list costs one batched call to Microsoft, not seven', ms.batches === 1, ms.batches);
   ok('folders: the usual six first with unread counts, then your own; internal ones hidden', names.slice(0, 6).join() === 'Inbox,Drafts,Sent,Archive,Junk,Deleted' && names.includes('Receipts') && !names.includes('Outbox') && r.json[0].unread === 3 && names.filter(n => n === 'Inbox').length === 1, r.json);
   r = await call('unate', '/api/mail/messages?folder=inbox'); ok('the inbox lists messages, newest first, with who, read state and attachment flag', r.status === 200 && r.json.items.length === 2 && r.json.items[0].from.address === 'ann@example.com' && r.json.items[0].read === false && r.json.items[0].attach === true && r.json.items[1].flagged === true, r.json);
   ok('and says there may be more', r.json.more === true && r.json.next === 2, r.json);