diff --git a/mail-imap.js b/mail-imap.js
index 1c95a9c..7778ed0 100644
--- a/mail-imap.js
+++ b/mail-imap.js
@@ -39,18 +39,19 @@ function friendly(a, e) {
}
// ---- connections: one per account, kept for a minute after the last use
+const IDLE = 5 * 60e3; // a connection is kept this long after its last use: opening one (TLS and login) is the slowest part of a request
const pool = new Map(); // account id -> { client, timer, key }
-const forget = id => { const e = pool.get(id); if (e) { clearTimeout(e.timer); try { e.client.close(); } catch {} pool.delete(id); } };
+const forget = id => { folderCache.delete(id); boxCache.delete(id); const e = pool.get(id); if (e) { clearTimeout(e.timer); try { e.client.close(); } catch {} pool.delete(id); } };
const keyOf = a => a.address + '\n' + a.pass;
async function open(a) {
const e = pool.get(a.id);
- if (e && e.key === keyOf(a) && e.client.usable) { clearTimeout(e.timer); e.timer = setTimeout(() => forget(a.id), 60e3).unref(); return e.client; }
+ if (e && e.key === keyOf(a) && e.client.usable) { clearTimeout(e.timer); e.timer = setTimeout(() => forget(a.id), IDLE).unref(); return e.client; }
forget(a.id);
const c = cfgOf(a);
const client = new ImapFlow({ host: c.host, port: c.port, secure: c.secure, ...(c.test ? { doSTARTTLS: false, tls: { rejectUnauthorized: false } } : {}), auth: { user: a.address, pass: a.pass }, logger: false, emitLogs: false, socketTimeout: 60e3, greetingTimeout: 20e3, connectionTimeout: 20e3 });
client.on('error', () => {}); client.on('close', () => { const cur = pool.get(a.id); if (cur && cur.client === client) pool.delete(a.id); });
try { await client.connect(); } catch (err) { throw fail(err.authenticationFailed ? 401 : 502, friendly(a, err)); }
- pool.set(a.id, { client, key: keyOf(a), timer: setTimeout(() => forget(a.id), 60e3).unref() });
+ pool.set(a.id, { client, key: keyOf(a), timer: setTimeout(() => forget(a.id), IDLE).unref() });
return client;
}
/** Run `fn(client)`; if the connection had gone stale, reconnect once. */
@@ -92,20 +93,20 @@ async function pathOf(a, client, folder) {
const addrOf = x => x ? { name: x.name || '', address: x.address || '' } : { name: '', address: '' };
const hasAttachment = n => !!n && (((n.disposition || '').toLowerCase() === 'attachment') || (n.childNodes || []).some(hasAttachment));
+const folderCache = new Map(); // account id -> { t, v }
async function folders(a) {
- return withClient(a, async client => {
+ const c = folderCache.get(a.id); if (c && Date.now() - c.t < 30e3) return c.v;
+ const v = await withClient(a, async client => {
const { list, alias } = await boxes(a, client), byPath = {}; for (const [k, p] of Object.entries(alias)) byPath[p] = k;
const skip = b => b.flags.has('\\Noselect') || ['\\Flagged', '\\Important'].includes(b.specialUse);
- const rows = [];
- for (const b of list) {
- if (skip(b)) continue;
- let st = {}; try { st = await client.status(b.path, { messages: true, unseen: true }); } catch {}
- const k = byPath[b.path];
- rows.push({ id: k || 'p-' + Buffer.from(b.path).toString('base64url'), name: k ? NAMES[k] : b.path.replace(/^\[.*?\]\//, ''), unread: st.unseen || 0, total: st.messages || 0, well: !!k });
- }
+ const keep = list.filter(b => !skip(b)), wellFirst = [...keep.filter(b => byPath[b.path]), ...keep.filter(b => !byPath[b.path])];
+ const counts = new Map(); // one STATUS per folder is a round trip each: the usual six first, then at most ten more (the rest show no number)
+ for (const b of wellFirst.slice(0, Object.keys(byPath).length + 10)) { try { counts.set(b.path, await client.status(b.path, { messages: true, unseen: true })); } catch {} }
+ const rows = keep.map(b => { const k = byPath[b.path], st = counts.get(b.path) || {}; return { id: k || 'p-' + Buffer.from(b.path).toString('base64url'), name: k ? NAMES[k] : b.path.replace(/^\[.*?\]\//, ''), unread: st.unseen || 0, total: st.messages || 0, well: !!k }; });
const order = Object.keys(NAMES);
return rows.sort((x, y) => (x.well ? order.indexOf(x.id) : 99) - (y.well ? order.indexOf(y.id) : 99) || x.name.localeCompare(y.name));
});
+ folderCache.set(a.id, { t: Date.now(), v }); return v;
}
async function list(a, { folder, skip, q }) {
@@ -144,7 +145,8 @@ async function source(a, mid) {
if (!dl || !dl.content) throw fail(404, 'That message is gone');
const chunks = []; let n = 0;
for await (const ch of dl.content) { n += ch.length; if (n > MAX_SOURCE) throw fail(400, 'That message is too large to open here'); chunks.push(ch); }
- return { parsed: await simpleParser(Buffer.concat(chunks)), path, uid };
+ const meta = await client.fetchOne(String(uid), { flags: true }, { uid: true }).catch(() => null);
+ return { parsed: await simpleParser(Buffer.concat(chunks)), path, uid, flags: meta ? meta.flags : new Set() };
} finally { lock.release(); }
});
}
@@ -152,11 +154,10 @@ const pick = (list, f) => (list || []).map(x => f(x));
const addrs = v => (v && v.value ? v.value : []).map(x => ({ name: x.name || '', address: x.address || '' }));
async function message(a, mid) {
- const { parsed: p, path, uid } = await source(a, mid);
+ const { parsed: p, flags } = await source(a, mid);
let html = p.html || (p.textAsHtml ? p.textAsHtml : textToHtml(p.text || ''));
const atts = p.attachments || [];
for (const x of atts) if (x.cid && /^image\//i.test(x.contentType || '') && x.size < 1.5 * 1024 * 1024) html = html.split('cid:' + x.cid).join(`data:${x.contentType};base64,${x.content.toString('base64')}`);
- const flags = await withClient(a, async client => { const lock = await client.getMailboxLock(path); try { const m = await client.fetchOne(String(uid), { flags: true }, { uid: true }); return m ? m.flags : new Set(); } finally { lock.release(); } });
return { id: mid, subject: p.subject || '(No subject)', from: addrs(p.from)[0] || { name: '', address: '' }, to: addrs(p.to), cc: addrs(p.cc), date: (p.date || new Date()).toISOString(), read: flags.has('\\Seen'), flagged: flags.has('\\Flagged'),
html, attachments: atts.map((x, i) => ({ x, i })).filter(({ x }) => !x.related).map(({ x, i }) => ({ id: String(i), name: x.filename || 'attachment', type: x.contentType, size: x.size })) };
}
diff --git a/mail.js b/mail.js
index 002372b..5424615 100644
--- a/mail.js
+++ b/mail.js
@@ -149,7 +149,7 @@ const safeName = n => String(n || 'attachment').replace(/[\r\n"\\/]+/g, '_').sli
const cleanAttachments = list => { let total = 0; const out = (Array.isArray(list) ? list : []).slice(0, 10).map(a => { const data = String(a.data || ''); total += Buffer.from(data, 'base64').length; return { name: safeName(a.name), type: String(a.type || 'application/octet-stream').slice(0, 100), data }; }); if (total > MAX_ATTACH) throw fail(400, 'Attachments can total about 3 MB'); return out; };
let folderCache = new Map(); // account id -> { t, v }
-const dropFolderCache = id => id ? folderCache.delete(id) : folderCache.clear();
+const dropFolderCache = id => { if (id) { folderCache.delete(id); unreadCache.delete(id); } else { folderCache.clear(); unreadCache.clear(); } };
// ---- Outlook.com through Microsoft Graph. Every function takes the account and Graph's own ids.
const outlook = {
@@ -205,12 +205,8 @@ const providerOf = a => a.type === 'outlook' ? outlook : imap;
async function withInline(a, html, attachments, mid) {
// (contentId only exists on file attachments, so it cannot be selected in the list: fetch each small inline picture whole)
const inl = (attachments || []).filter(x => x.isInline && /^image\//i.test(x.contentType || '') && (x.size || 0) < 1.5 * 1024 * 1024).slice(0, 15);
- for (const x of inl) {
- try {
- const full = await graph(a, `/me/messages/${id(mid)}/attachments/${id(x.id)}`);
- if (full && full.contentBytes && full.contentId) html = html.split('cid:' + String(full.contentId).replace(/^<|>$/g, '')).join(`data:${full.contentType || x.contentType};base64,${full.contentBytes}`);
- } catch { /* leave that picture out */ }
- }
+ const fulls = await Promise.all(inl.map(x => graph(a, `/me/messages/${id(mid)}/attachments/${id(x.id)}`).catch(() => null))); // all at once
+ inl.forEach((x, i) => { const full = fulls[i]; if (full && full.contentBytes && full.contentId) html = html.split('cid:' + String(full.contentId).replace(/^<|>$/g, '')).join(`data:${full.contentType || x.contentType};base64,${full.contentBytes}`); });
return html;
}
@@ -246,12 +242,17 @@ async function adminHandle(req, res, p) {
json(res, 404, { error: 'Not found' });
}
-/** Every unread count in one go (an account that cannot be reached is just left out of the total). */
-async function overview() {
+const unreadCache = new Map(); // account id -> { t, n }: counts are shown at once from here and refreshed when older than 30 s
+const withTimeout = (p, ms) => Promise.race([p, new Promise((_, no) => setTimeout(() => no(Object.assign(new Error('That account did not answer in time'), { status: 504 })), ms))]);
+/** The accounts, with unread counts. `light` makes no network calls at all (what the Mail tab needs to draw itself); otherwise counts older than
+ * 30 seconds are refreshed, each account on its own 6-second limit so one slow or unreachable account cannot hold up the others. */
+async function overview(light) {
const list = accounts();
const rows = await Promise.all(list.map(async a => {
- try { return { ...publicAcct(a), unread: await providerOf(a).unread(a) }; }
- catch (e) { return { ...publicAcct(a), unread: null, error: e.message, expired: e.status === 409 }; }
+ const c = unreadCache.get(a.id);
+ if (light || (c && Date.now() - c.t < 30e3)) return { ...publicAcct(a), unread: c ? c.n : null };
+ try { const n = await withTimeout(providerOf(a).unread(a), 6000); unreadCache.set(a.id, { t: Date.now(), n }); return { ...publicAcct(a), unread: n }; }
+ catch (e) { return { ...publicAcct(a), unread: c ? c.n : null, error: e.message, expired: e.status === 409 }; }
}));
const live = accounts(); // (an account whose sign-in Microsoft revoked was dropped meanwhile)
const shown = rows.filter(r => live.some(x => x.id === r.id) || !r.expired);
@@ -263,7 +264,7 @@ async function handle(req, res) {
const u = new URL(req.url, 'http://x'), p = u.pathname, q = u.searchParams;
try {
if (p === '/api/mail/admin' || p.startsWith('/api/mail/admin/')) return await adminHandle(req, res, p);
- if (p === '/api/mail/status') return json(res, 200, { ...(await overview()), outlook: outlookOn(), providers: imap.PRESETS });
+ if (p === '/api/mail/status') return json(res, 200, { ...(await overview(q.get('light') === '1')), outlook: outlookOn(), providers: imap.PRESETS });
if (p === '/api/mail/connect' && req.method === 'GET') { if (!outlookOn()) throw fail(403, 'Outlook.com is not set up on this server (an admin adds the Microsoft app in Settings)'); return json(res, 200, { url: startUrl(req), redirect: redirectUri(req) }); }
if (p === '/api/mail/folders') {
const a = findAccount(q.get('acct') || undefined);
diff --git a/public/mail.js b/public/mail.js
index b998ce4..92cc17b 100644
--- a/public/mail.js
+++ b/public/mail.js
@@ -24,7 +24,7 @@ const mailHasRemote = html => /]+src=["']?https?:|url\(\s*["']?https?:/i.
// ---- unread badge and Home card
async function mailPoll() {
if (!(allowed('mail') && cfg.mail)) return;
- try { const s = await jfetch('/api/mail/status'); mail.st = s; const n = s.connected && s.unread ? s.unread : 0; if (n !== mailUnread) { mailUnread = n; renderNav(); } } catch {}
+ try { const s = await jfetch('/api/mail/status'); mail.st = s; mail.accts = s.accounts || mail.accts; const n = s.connected && s.unread ? s.unread : 0; if (n !== mailUnread) { mailUnread = n; renderNav(); } } catch {}
}
async function renderMailCard(el) {
if (!el) return;
@@ -125,23 +125,33 @@ async function mailAdminCard() {
async function viewMail(args = []) {
const v = $('#view');
if (!cfg.mail) { v.innerHTML = '
Mail is not available to you.
Loading…
${esc(e.message)}
Loading…
${esc(e.message)}
Opening…
'; + const rd = $('#mread'), it = mail.items.find(x => x.id === id); + if (rd) rd.innerHTML = `${esc(mailFrom(it.from))} ${esc(it.from.address)}
` : ''}Opening…
`; // (what the list already knows, at once) mail.images = false; try { - const m = await jfetch('/api/mail/message?id=' + encodeURIComponent(id)); + const m = await mailFetch(id); if (!(location.hash.startsWith('#/mail/m/'))) return; mail.msg = m; mailDrawReader(); mailDrawList(); if (!m.read) { // opening a message marks it read diff --git a/tools/mail-imap-test.js b/tools/mail-imap-test.js index 077a21d..ba16fd4 100644 --- a/tools/mail-imap-test.js +++ b/tools/mail-imap-test.js @@ -55,6 +55,9 @@ const free = () => 21000 + Math.floor(Math.random() * 20000); ok('and is stored encrypted', !fs.readFileSync(path.join(dir, 'users', 'unate', 'mail-accounts.json'), 'utf8').includes(PASS)); r = await A('/api/mail/status'); ok('status lists the account with its unread count (2 unread in the inbox)', r.json.connected && r.json.accounts.length === 1 && r.json.accounts[0].unread === 2 && r.json.unread === 2, r.json); + r = await A('/api/mail/status?light=1'); ok('the light status lists the accounts without asking the mail servers (what lets the Mail tab draw at once)', r.json.connected && r.json.accounts.length === 1 && !r.json.accounts[0].error, r.json); + const t0 = Date.now(); r = await A('/api/mail/status'); const t1 = Date.now(); r = await A('/api/mail/status'); ok('the full status is answered from a short-lived cache the second time (counts are refreshed at most every 30 s)', Date.now() - t1 <= Math.max(50, t1 - t0), [t1 - t0, Date.now() - t1]); + // ================= reading ================= r = await A('/api/mail/folders?acct=' + gm.id); const names = r.json.map(f => f.name); ok('folders: Inbox, Sent, Archive (Gmail "All Mail"), Junk, Deleted, then your own; the [Gmail] container is hidden', names.join() === 'Inbox,Sent,Archive,Junk,Deleted,Receipts' && r.json[0].unread === 2 && r.json[0].id.startsWith(gm.id + '~'), r.json);