diff --git a/mail.js b/mail.js index 5424615..b4c25c2 100644 --- a/mail.js +++ b/mail.js @@ -36,7 +36,13 @@ function secretStatus() { const MAX_BODY = 6 * 1024 * 1024, MAX_ATTACH = 3 * 1024 * 1024; const readBody = req => new Promise((ok, no) => { const c = []; let n = 0; req.on('data', d => { n += d.length; if (n > MAX_BODY) { no(new Error('That is too large to send (attachments can total about 3 MB)')); req.destroy(); } else c.push(d); }); req.on('end', () => ok(Buffer.concat(c).toString())); req.on('error', no); }); -const json = (res, code, obj) => { res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store' }); res.end(JSON.stringify(obj)); }; +const json = (res, code, obj) => { + const body = Buffer.from(JSON.stringify(obj)), h = { 'content-type': 'application/json', 'cache-control': 'no-store' }; + if (body.length > 2048 && /\bgzip\b/.test((res.req && res.req.headers['accept-encoding']) || '')) { // message bodies are mostly HTML: gzip makes them several times smaller + const z = require('zlib').gzipSync(body); res.writeHead(code, { ...h, 'content-encoding': 'gzip', vary: 'Accept-Encoding' }); return res.end(z); + } + res.writeHead(code, h); res.end(body); +}; const fail = (status, message) => Object.assign(new Error(message), { status }); // ---- each person's accounts (encrypted): [{ id, type: 'outlook' | 'gmail' | 'icloud', address, name, ...credentials }] @@ -155,10 +161,13 @@ const dropFolderCache = id => { if (id) { folderCache.delete(id); unreadCache.de const outlook = { async folders(a) { const c = folderCache.get(a.id); if (c && Date.now() - c.t < 15e3) return c.v; - const sel = '?$select=id,displayName,unreadItemCount,totalItemCount'; - const wk = await Promise.all(Object.keys(WELL).map(k => graph(a, `/me/mailFolders/${k}${sel}`).then(f => ({ id: k, realId: f.id, name: WELL[k], unread: f.unreadItemCount || 0, total: f.totalItemCount || 0, well: true })).catch(() => null))); + const sel = '?$select=id,displayName,unreadItemCount,totalItemCount', keys = Object.keys(WELL); + // the usual six and the full list in ONE round trip (Graph's $batch), instead of seven + const b = await graph(a, '/$batch', { method: 'POST', body: JSON.stringify({ requests: [...keys.map((k, i) => ({ id: String(i), method: 'GET', url: `/me/mailFolders/${k}${sel}` })), { id: 'all', method: 'GET', url: `/me/mailFolders${sel}&$top=100` }] }) }); + const by = Object.fromEntries((b.responses || []).map(r => [r.id, r])); + const wk = keys.map((k, i) => { const r = by[String(i)]; return r && r.status === 200 ? { id: k, realId: r.body.id, name: WELL[k], unread: r.body.unreadItemCount || 0, total: r.body.totalItemCount || 0, well: true } : null; }); const known = new Set(wk.filter(Boolean).map(f => f.realId)); - const rest = (await graph(a, `/me/mailFolders${sel}&$top=100`).catch(() => ({ value: [] }))).value || []; + const rest = by.all && by.all.status === 200 ? by.all.body.value || [] : []; const v = [...wk.filter(Boolean).map(({ realId, ...f }) => f), ...rest.filter(f => !known.has(f.id) && !['Outbox', 'Conversation History', 'Sync Issues', 'RSS Feeds'].includes(f.displayName)).map(f => ({ id: f.id, name: f.displayName, unread: f.unreadItemCount || 0, total: f.totalItemCount || 0 }))]; folderCache.set(a.id, { t: Date.now(), v }); return v; }, diff --git a/public/mail.js b/public/mail.js index 92cc17b..2aa0373 100644 --- a/public/mail.js +++ b/public/mail.js @@ -4,6 +4,19 @@ let mailUnread = 0; const mail = { st: null, accts: [], acct: '', folders: [], folder: 'inbox', items: [], more: false, next: 0, q: '', msg: null, images: false, loading: false }; +// The last mail you looked at is kept on this device (subjects, senders, dates; no message bodies) so Mail can show it the instant you open the tab +// while the fresh copy loads behind it. Removed when you disconnect an account or sign out. +const mcKey = k => 'mc:' + (ME ? ME.id : '') + ':' + k; +const mcGet = k => { try { const o = JSON.parse(localStorage.getItem(mcKey(k))); return o && o.t ? o : null; } catch { return null; } }; +function mcPut(k, v) { + try { + localStorage.setItem(mcKey(k), JSON.stringify({ t: Date.now(), ...v })); + const ix = mcKey('index'), list = (JSON.parse(localStorage.getItem(ix) || '[]')).filter(x => x !== k); list.unshift(k); + for (const old of list.splice(16)) localStorage.removeItem(mcKey(old)); + localStorage.setItem(ix, JSON.stringify(list)); + } catch { /* storage full or blocked: go without */ } +} +function mcClear() { try { for (const k of Object.keys(localStorage)) if (k.startsWith('mc:' + (ME ? ME.id : '') + ':')) localStorage.removeItem(k); } catch {} } const mailFrom = f => (f && (f.name || f.address)) || ''; const mailWhen = iso => { const d = new Date(iso), now = new Date(); if (isNaN(d)) return ''; @@ -45,7 +58,7 @@ function mailSettings() { box.innerHTML = (s.accounts.length ? s.accounts.map(a => `
${esc(MAIL_LABEL[a.type] || a.type)}${esc(a.address || 'Connected')}${a.error ? ' · ' + esc(a.error) : ''}
`).join('') : `
No mail account yet${s.expired ? 'Microsoft signed you out. ' : ''}Add Outlook.com, Gmail or iCloud Mail
`) + `
`; $('#madd').onclick = () => mailAddAccount(s); - box.querySelectorAll('[data-dis]').forEach(d => d.onclick = () => { const a = s.accounts.find(x => x.id === d.dataset.dis); confirmSheet('Disconnect ' + (a ? a.address : 'this account') + '?', 'NE Hub forgets the sign-in. Your mailbox is not touched.', 'Disconnect', async () => { try { await jfetch('/api/mail/disconnect', { method: 'POST', body: JSON.stringify({ acct: d.dataset.dis }) }); mail.st = null; mail.items = []; mail.folders = []; mail.acct = ''; await mailPoll(); mailSettings(); } catch (e) { toast(e.message); } }); }); + box.querySelectorAll('[data-dis]').forEach(d => d.onclick = () => { const a = s.accounts.find(x => x.id === d.dataset.dis); confirmSheet('Disconnect ' + (a ? a.address : 'this account') + '?', 'NE Hub forgets the sign-in. Your mailbox is not touched.', 'Disconnect', async () => { try { await jfetch('/api/mail/disconnect', { method: 'POST', body: JSON.stringify({ acct: d.dataset.dis }) }); mail.st = null; mcClear(); mail.items = []; mail.folders = []; mail.acct = ''; await mailPoll(); mailSettings(); } catch (e) { toast(e.message); } }); }); }).catch(() => { box.innerHTML = '
Could not check
'; }); else box.innerHTML = ''; if (cfg.mailAdmin) mailAdminCard(); @@ -128,22 +141,33 @@ async function viewMail(args = []) { // The accounts come from the server without any network call to the mail providers ("light"), so the page can draw at once; the folders, the message // list and the real unread counts then load side by side instead of one after another. let st = mail.st; - if (!st) { v.innerHTML = '

Mail

Loading…

'; try { st = await jfetch('/api/mail/status?light=1'); } catch (e) { v.innerHTML = `

Mail

${esc(e.message)}

`; return; } } + if (!st) { + const saved = mcGet('st'); // the accounts as of last time: draw from them now and check with the server behind it + if (saved && saved.connected) { + st = saved; mail.st = saved; + jfetch('/api/mail/status?light=1').then(fresh => { if (!fresh.connected || fresh.accounts.map(a => a.id).join() !== saved.accounts.map(a => a.id).join()) { mail.st = null; mcClear(); if (location.hash.startsWith('#/mail')) route(); } else { mail.st = { ...fresh, accounts: fresh.accounts.map(a => ({ ...a, unread: (mail.st.accounts.find(x => x.id === a.id) || {}).unread ?? a.unread })) }; mcPut('st', mail.st); } }).catch(() => {}); + } else { + v.innerHTML = '

Mail

Loading…

'; + try { st = await jfetch('/api/mail/status?light=1'); } catch (e) { v.innerHTML = `

Mail

${esc(e.message)}

`; return; } + if (st.connected) mcPut('st', { connected: true, accounts: st.accounts, outlook: st.outlook, providers: st.providers }); + } + } if (!location.hash.startsWith('#/mail')) return; mail.st = st; mail.accts = st.accounts || []; - if (!st.connected) { mail.st = null; return mailConnectScreen(st); } + if (!st.connected) { mail.st = null; mcClear(); return mailConnectScreen(st); } const [kind, key] = args; const idAcct = x => String(x || '').includes('~') ? String(x).split('~')[0] : ''; const want = idAcct(kind === 'f' ? key : kind === 'm' ? key : '') || (mail.accts.some(a => a.id === mail.acct) ? mail.acct : mail.accts[0].id); if (want !== mail.acct) { mail.acct = want; mail.folders = []; mail.items = []; if (!(kind === 'f' && idAcct(key))) mail.folder = want + '~inbox'; } if (kind === 'f' && key) mail.folder = idAcct(key) ? key : mail.acct + '~' + key; else if (!String(mail.folder).startsWith(mail.acct + '~')) mail.folder = mail.acct + '~inbox'; + if (!mail.folders.length) { const f = mcGet('f:' + mail.acct); if (f) mail.folders = f.folders; } mailShell(kind === 'm' && key ? 'read' : 'list'); const acct = mail.acct; if (!mail.items.length || kind === 'f' || kind === undefined) mailLoadList(true); else mailDrawList(); if (kind === 'm' && key) mailOpen(key); else { mail.msg = null; mailDrawReader(); } // folders (with counts) and the real unread numbers arrive when they arrive - jfetch('/api/mail/folders?acct=' + encodeURIComponent(acct)).then(f => { if (mail.acct !== acct) return; mail.folders = f; mailRefreshFolders(); }).catch(() => {}); + jfetch('/api/mail/folders?acct=' + encodeURIComponent(acct)).then(f => { if (mail.acct !== acct) return; mail.folders = f; mcPut('f:' + acct, { folders: f }); mailRefreshFolders(); }).catch(() => {}); mailPoll().then(() => { if (mail.st && location.hash.startsWith('#/mail')) mailRefreshFolders(); }); } /** Redraw the folder pickers (names and unread counts) without touching the list or the open message. */ @@ -188,14 +212,21 @@ function mailShell(mode) { } async function mailLoadList(reset) { - if (reset) { mail.items = []; mail.next = 0; mail.more = false; } - const rows = $('#mrows'); if (rows && reset) rows.innerHTML = '

Loading…

'; + const folder = mail.folder, first = reset && !mail.q; + if (reset) { + mail.items = []; mail.next = 0; mail.more = false; + const saved = first && mcGet('l:' + folder); + if (saved) { mail.items = saved.items; mail.more = saved.more; mail.next = saved.next; mailDrawList(); } // last time's list, right away + else { const rows = $('#mrows'); if (rows) rows.innerHTML = '

Loading…

'; } + } + const showing = reset && mail.items.length; // a saved list is on screen: the fresh one replaces it instead of being added to it try { - const r = await jfetch(`/api/mail/messages?folder=${encodeURIComponent(mail.folder)}&skip=${mail.next}${mail.q ? '&q=' + encodeURIComponent(mail.q) : ''}`); - if (!$('#mrows')) return; - mail.items = mail.items.concat(r.items); mail.more = r.more; mail.next = r.next; + const r = await jfetch(`/api/mail/messages?folder=${encodeURIComponent(folder)}&skip=${showing ? 0 : mail.next}${mail.q ? '&q=' + encodeURIComponent(mail.q) : ''}`); + if (!$('#mrows') || mail.folder !== folder) return; + mail.items = showing ? r.items : mail.items.concat(r.items); mail.more = r.more; mail.next = r.next; + if (first) mcPut('l:' + folder, { items: r.items, more: r.more, next: r.next }); mailDrawList(); - } catch (e) { const el = $('#mrows'); if (el) el.innerHTML = `

${esc(e.message)}

`; if (/Connect again/.test(e.message)) { mail.st = null; setTimeout(route, 1200); } } + } catch (e) { if (showing) return; const el = $('#mrows'); if (el) el.innerHTML = `

${esc(e.message)}

`; if (/Connect again/.test(e.message)) { mail.st = null; mcClear(); setTimeout(route, 1200); } } } function mailDrawList() { const rows = $('#mrows'); if (!rows) return; diff --git a/public/settings.js b/public/settings.js index f3ad422..22aa398 100644 --- a/public/settings.js +++ b/public/settings.js @@ -170,6 +170,6 @@ function viewSettings() { el.querySelector('#sx').onclick = closeSheet; } catch (e) { toast(e.message); } }; - $('#hubout').onclick = async () => { await authPost('/api/auth/logout'); location.replace('/login'); }; + $('#hubout').onclick = async () => { try { mcClear(); for (const k of Object.keys(localStorage)) if (k.startsWith('dc:')) localStorage.removeItem(k); } catch {} await authPost('/api/auth/logout'); location.replace('/login'); }; const out = $('#out'); if (out) out.onclick = () => { store.del('sf_token'); toast('Signed out'); viewSettings(); }; } diff --git a/server.js b/server.js index 0b8f71b..1f25f18 100644 --- a/server.js +++ b/server.js @@ -4,6 +4,7 @@ const http = require('http'); const fs = require('fs'); const atomicWrite = require('./atomic'); const crypto = require('crypto'); +const zlib = require('zlib'); const path = require('path'); const { Readable } = require('stream'); @@ -121,10 +122,17 @@ function serveStatic(req, res) { headers.etag = `W/"${data.length}-${crypto.createHash('sha1').update(data).digest('hex').slice(0, 16)}"`; if (req.headers['if-none-match'] === headers.etag) { res.writeHead(304, { etag: headers.etag, 'cache-control': 'no-cache' }); return res.end(); } } + // text files are sent gzipped when the browser accepts it (the app's scripts are several hundred KB), and the compressed copy is kept per version + if (/^(text\/|application\/(json|manifest))|svg/.test(headers['content-type']) && data.length > 1024 && /\bgzip\b/.test(req.headers['accept-encoding'] || '')) { + const key = file + ':' + data.length + ':' + (headers.etag || ''); let z = gzCache.get(key); + if (!z) { z = zlib.gzipSync(data, { level: 6 }); if (gzCache.size > 200) gzCache.clear(); gzCache.set(key, z); } + headers['content-encoding'] = 'gzip'; headers.vary = 'Accept-Encoding'; data = z; + } res.writeHead(200, headers); res.end(data); }); } +const gzCache = new Map(); // Changes whenever the app's files change, so an open page can tell that a newer version is available. const BUILD = (() => { try { return crypto.createHash('sha1').update(fs.readdirSync(PUBLIC).filter(f => /\.(js|css|html)$/.test(f)).sort().map(f => fs.readFileSync(path.join(PUBLIC, f))).join('')).digest('hex').slice(0, 12); } catch { return String(Date.now()); } })(); diff --git a/tools/mail-imap-test.js b/tools/mail-imap-test.js index ba16fd4..c48e7e8 100644 --- a/tools/mail-imap-test.js +++ b/tools/mail-imap-test.js @@ -56,7 +56,9 @@ const free = () => 21000 + Math.floor(Math.random() * 20000); r = await A('/api/mail/status'); ok('status lists the account with its unread count (2 unread in the inbox)', r.json.connected && r.json.accounts.length === 1 && r.json.accounts[0].unread === 2 && r.json.unread === 2, r.json); r = await A('/api/mail/status?light=1'); ok('the light status lists the accounts without asking the mail servers (what lets the Mail tab draw at once)', r.json.connected && r.json.accounts.length === 1 && !r.json.accounts[0].error, r.json); - const t0 = Date.now(); r = await A('/api/mail/status'); const t1 = Date.now(); r = await A('/api/mail/status'); ok('the full status is answered from a short-lived cache the second time (counts are refreshed at most every 30 s)', Date.now() - t1 <= Math.max(50, t1 - t0), [t1 - t0, Date.now() - t1]); + { const first = (await A('/api/mail/status')).json.unread; // unread counts are cached for 30 s: change the real mailbox behind the hub's back and the number stays + const c = new ImapFlow({ host: '127.0.0.1', port: imapPort, secure: false, doSTARTTLS: false, auth: { user: ME, pass: PASS }, logger: false }); await c.connect(); const lock = await c.getMailboxLock('INBOX'); + try { await c.messageFlagsAdd('1', ['\\Seen']); const again = (await A('/api/mail/status')).json.unread; ok('the full status serves a cached unread count for up to 30 s (so opening Mail never waits on the mail servers)', again === first, [first, again]); await c.messageFlagsRemove('1', ['\\Seen']); } finally { lock.release(); await c.logout(); } } // ================= reading ================= r = await A('/api/mail/folders?acct=' + gm.id); const names = r.json.map(f => f.name); diff --git a/tools/mail-test.js b/tools/mail-test.js index 5a80bef..d915362 100644 --- a/tools/mail-test.js +++ b/tools/mail-test.js @@ -42,6 +42,7 @@ const SECRET = 'mail-test-secret-'.repeat(3); const tok = (rq.headers.authorization || '').replace('Bearer ', ''); if (!ms.goodTokens.has(tok)) return out(rs, 401, { error: { code: 'InvalidAuthenticationToken', message: 'expired' } }); const g = p.slice(5); + if (g === '/$batch' && rq.method === 'POST') { ms.batches = (ms.batches || 0) + 1; const reqs = JSON.parse(b).requests, self = `http://127.0.0.1:${server.address().port}/v1.0`; const responses = []; for (const q of reqs) { const r = await fetch(self + q.url, { method: q.method, headers: { authorization: rq.headers.authorization } }); responses.push({ id: q.id, status: r.status, body: await r.json().catch(() => ({})) }); } return out(rs, 200, { responses }); } if (g === '/me') return out(rs, 200, { displayName: 'Nate', mail: 'nate@outlook.com' }); let m; if ((m = /^\/me\/mailFolders\/(\w+)$/.exec(g))) { const f = { inbox: folder('inbox', 'Inbox', 3, 10), drafts: folder('drafts', 'Drafts', 0, 1), sentitems: folder('sentitems', 'Sent Items', 0, 4), archive: folder('archive', 'Archive', 0, 0), junkemail: folder('junkemail', 'Junk Email', 0, 0), deleteditems: folder('deleteditems', 'Deleted Items', 0, 2) }[m[1]]; return f ? out(rs, 200, f) : out(rs, 404, { error: { message: 'no folder' } }); } @@ -101,7 +102,8 @@ const SECRET = 'mail-test-secret-'.repeat(3); r = await call('uann', '/api/mail/messages'); ok("and cannot read it", r.status === 409, r.status); // ================= reading ================= - r = await call('unate', '/api/mail/folders'); const names = (r.json || []).map(f => f.name); + ms.batches = 0; r = await call('unate', '/api/mail/folders'); const names = (r.json || []).map(f => f.name); + ok('the folder list costs one batched call to Microsoft, not seven', ms.batches === 1, ms.batches); ok('folders: the usual six first with unread counts, then your own; internal ones hidden', names.slice(0, 6).join() === 'Inbox,Drafts,Sent,Archive,Junk,Deleted' && names.includes('Receipts') && !names.includes('Outbox') && r.json[0].unread === 3 && names.filter(n => n === 'Inbox').length === 1, r.json); r = await call('unate', '/api/mail/messages?folder=inbox'); ok('the inbox lists messages, newest first, with who, read state and attachment flag', r.status === 200 && r.json.items.length === 2 && r.json.items[0].from.address === 'ann@example.com' && r.json.items[0].read === false && r.json.items[0].attach === true && r.json.items[1].flagged === true, r.json); ok('and says there may be more', r.json.more === true && r.json.next === 2, r.json);