Signal: keep messages that arrive glued together, one chat per person, receipts by time, faster reconnect; edit contacts, clear and remove chats; debug list

- read every JSON value in a frame (lines, arrays), binary frames too
- a person seen by number and by private id is one conversation (merged)
- delivery/read receipts match on message time, not on who sent them
- reconnect every 1-5 s; the chat list says when the link was down
- /api/signal/debug: kinds of messages seen lately (no text)
- edit a contact (kept in the app and sent to Signal), clear messages, remove a chat; right-click menu in the list; name for a new number
- tests: 54 checks

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

README.md

@@ -59,7 +59,8 @@ e.g. hub.necloud.us. Settings: `SEAFILE_URL`, `ONLYOFFICE_URL` env vars in docke
   (if you switch it on in Settings -> Signal alerts) a desktop notification when the app is in the background; sound and notifications are saved per device. It talks to your account through the community `signal-cli-rest-api` bridge, which is one more *linked device* of your account
   (like Signal Desktop; not an official Signal product). The `signal` container in `docker-compose.yml` runs it; remove that service and the hub's `SIGNAL_API_URL` line to skip Signal.
   First time: open the Signal tab, then on your phone Signal -> Settings -> Linked devices -> Link new device and scan the code. You can send and receive text, photos,
-  files, voice messages (played, not recorded), replies, reactions, groups and read receipts; there are no calls. Only messages from after linking appear (Signal does
+  files, voice messages (played, not recorded), replies, reactions, groups and read receipts; there are no calls. In a chat, the ... button edits the contact (the name you give is shown in the app and sent to Signal),
+  clears the messages, or removes the chat from the app (it stays on your phone, and a new message starts it again); right-click a chat in the list for the same menu. A diagnostic list of what arrived lately (kinds only, no message text) is at `/api/signal/debug`. Only messages from after linking appear (Signal does
   not copy older ones to a new device) and they are kept in `/data/signal`. The link keys live in the `nehub-signal` volume (a Docker *volume*, not a host folder: the bridge runs as user 1000 and a root-owned host folder gives "permission denied"; if you do want a folder, `chown -R 1000:1000` it first): whoever gets into your server can read your chats,
   so only give the **Signal** tab to people you would trust with your Signal (it is one shared account, admins always have it). Signal drops a linked device that is
   not connected for about a month; the hub stays connected, so this only matters if the stack is switched off.

public/app.js

@@ -2136,6 +2136,7 @@ function sgEvent(ev) {
     const looking = sgMounted() && sg.cur === ev.conv && document.visibilityState === 'visible' && document.hasFocus();
     if (!looking) { toast('Signal · ' + (ev.name || '') + ': ' + (ev.preview || '')); sgAlert(ev); }
   }
+  if (ev.t === 'gone') sgGone(ev.conv, true);
   if (!sgMounted()) return;
   clearTimeout(sgEvT); sgEvT = setTimeout(() => { if (ev.t === 'status') return sgMount(sg.root, sg.dock); sgRefresh(); }, 150);
 }
@@ -2151,7 +2152,7 @@ async function sgMount(root, dock) {
   if (!st.reachable) { root.innerHTML = `<div class="${cls}">${title}<div class="card" style="max-width:none"><p style="margin-top:0">The Signal service is not answering. Check that its container (<b>signal</b>) is running, then reload.</p></div></div>`; return; }
   if (!st.linked) return sgLinkScreen(root, dock);
   root.innerHTML = `<div class="${cls}"><div class="sg ${sg.cur ? 'open' : ''}"><div class="sgl"><div class="sghead">${dock ? '<b class="sgdt">Chats</b>' : '<h1>Signal</h1>'}<button class="btn ghost" id="sgnew">${icon('edit')}New</button></div>
-    <input type="search" id="sgq" placeholder="Search chats" autocomplete="off"><div class="list sglist" id="sglist"></div><p class="sub" style="margin-top:8px">${esc(st.number)}${st.connected ? '' : ' · reconnecting…'}</p></div>
+    <input type="search" id="sgq" placeholder="Search chats" autocomplete="off"><div class="list sglist" id="sglist"></div><p class="sub" style="margin-top:8px">${esc(st.number)}${st.connected ? '' : ' · reconnecting…'}</p>${st.gap ? `<p class="sub">The link to Signal was down at ${new Date(st.gap.from).toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' })} for ${Math.max(1, Math.round((st.gap.to - st.gap.from) / 1000))} s. A message that arrived then may be missing here.</p>` : ''}</div>
     <div class="sgm" id="sgm"></div></div></div>`;
   sgq('#sgnew').onclick = sgNewChat; sgq('#sgq').oninput = sgDrawList;
   await sgRefresh(true);
@@ -2173,6 +2174,7 @@ function sgDrawList() {
   box.innerHTML = rows.map(c => `<a class="row sgrow ${c.id === sg.cur ? 'on' : ''}" data-id="${esc(c.id)}" href="${sg.dock ? '#' : '#/signal/' + encodeURIComponent(c.id)}"><span class="sgav" style="background:${sgColor(c.id)}">${c.type === 'group' ? icon('chat') : esc(sgInitial(c.name || c.id))}</span>
     <span class="grow"><b>${esc(c.name || c.id)}</b><small>${esc(c.preview || '')}</small></span><span class="sgside"><small>${c.ts ? sgTime(c.ts) : ''}</small>${c.unread ? `<i class="nbadge" style="position:static">${c.unread}</i>` : ''}</span></a>`).join('') || `<p class="empty">${q ? 'Nothing matches' : 'No conversations yet. Tap New, or wait for a message.'}</p>`;
   if (sg.dock) box.onclick = e => { const a = e.target.closest('[data-id]'); if (a) { e.preventDefault(); sgOpen(a.dataset.id); } };
+  box.oncontextmenu = e => { const a = e.target.closest('[data-id]'); if (a) { e.preventDefault(); sgChatMenu(a.dataset.id); } };
 }
 
 const sgLink = t => t.replace(/(https?:\/\/[^\s<]+[^\s<.,;:!?)\]'"])/g, u => `<a href="${u}" target="_blank" rel="noopener">${u}</a>`);
@@ -2200,11 +2202,11 @@ async function sgDrawThread(first) {
   if (sg.cur !== id || !sgq('#sgm')) return; sg.names = r.names || {};
   const c = r.conv, old = box.querySelector('.sgmsgs'), stick = !old || old.scrollHeight - old.scrollTop - old.clientHeight < 80;
   if (!box.querySelector('.sgbar')) {
-    box.innerHTML = `<div class="sgbar">${sg.dock ? `<button type="button" class="ibtn sgback" id="sgbk" aria-label="Back to chats">${icon('back')}</button>` : `<a class="ibtn sgback" href="#/signal" aria-label="Back to chats">${icon('back')}</a>`}<span class="sgav" style="background:${sgColor(c.id)}">${c.type === 'group' ? icon('chat') : esc(sgInitial(c.name || c.id))}</span><span class="grow"><b id="sgtitle"></b><small id="sgsub"></small></span></div>
+    box.innerHTML = `<div class="sgbar">${sg.dock ? `<button type="button" class="ibtn sgback" id="sgbk" aria-label="Back to chats">${icon('back')}</button>` : `<a class="ibtn sgback" href="#/signal" aria-label="Back to chats">${icon('back')}</a>`}<span class="sgav" style="background:${sgColor(c.id)}">${c.type === 'group' ? icon('chat') : esc(sgInitial(c.name || c.id))}</span><span class="grow"><b id="sgtitle"></b><small id="sgsub"></small></span><button type="button" class="ibtn" id="sgmore" aria-label="Chat options" title="Chat options">${icon('more')}</button></div>
       <div class="sgmsgs" id="sgmsgs"></div><div id="sgrep" class="sgrep" hidden></div><div id="sgtray" class="sgtray" hidden></div>
       <form class="sgc" id="sgform"><button type="button" class="ibtn" id="sgat" aria-label="Attach a file">${icon('clip')}</button><input type="file" id="sgfile" multiple hidden>
       <textarea id="sgtext" rows="1" placeholder="Message" enterkeyhint="send"></textarea><button class="ibtn sgsend" id="sgsend" aria-label="Send">${icon('send')}</button></form>`;
-    sgWireComposer(); const bk = sgq('#sgbk'); if (bk) bk.onclick = () => sgOpen(null);
+    sgWireComposer(); const bk = sgq('#sgbk'); if (bk) bk.onclick = () => sgOpen(null); sgq('#sgmore').onclick = () => sgChatMenu(sg.cur);
   }
   sgq('#sgtitle').textContent = c.name || c.id; sgq('#sgsub').textContent = c.type === 'group' ? 'Group' : (c.name && c.name !== c.id ? c.id : '');
   const list = r.msgs, el = sgq('#sgmsgs');
@@ -2251,13 +2253,37 @@ function sgWireComposer() {
     btn.disabled = false; ta.focus();
   };
 }
+// ---- editing contacts and removing chats
+function sgChatMenu(id) {
+  const c = sg.convs.find(x => x.id === id); if (!c) return;
+  const acts = [];
+  if (c.type === 'direct') acts.push({ label: 'Edit contact', icon: 'edit', run: () => sgEditContact(c) });
+  acts.push({ label: 'Clear messages', icon: 'trash', run: () => confirmSheet('Clear all messages?', (c.name || c.id) + ': removed from this app only. Your phone keeps them.', 'Clear', async () => { try { await sgPost('/conv/clear', { id }); await sgRefresh(); } catch (e) { toast(e.message); } }) });
+  acts.push({ label: 'Remove chat', icon: 'trash', danger: true, run: () => confirmSheet('Remove this chat?', 'It is removed from this app only. Your phone keeps it, and a new message from them starts it again here.', 'Remove', async () => { try { await sgPost('/conv/delete', { id }); sgGone(id); } catch (e) { toast(e.message); } }) });
+  actionSheet(c.name || c.id, c.type === 'group' ? 'Group' : c.id, acts);
+}
+function sgGone(id, fromServer) {
+  sg.convs = sg.convs.filter(x => x.id !== id);
+  if (sg.cur === id) { sg.cur = null; sgSaveDock(); if (sg.dock && sgMounted()) sgOpen(null); else if (sgOnView()) location.hash = '#/signal'; }
+  if (!fromServer || sgMounted()) sgRefresh();
+}
+function sgEditContact(c, after) {
+  const el = openSheet('Edit contact', c.id || c.number, `<form id="sgcf"><label for="sgcn">Name</label><input type="text" id="sgcn" value="${esc(!c.name || c.name === (c.id || c.number) ? '' : c.name)}" placeholder="Name" autocomplete="off" maxlength="80">
+    <p class="sub" style="margin:8px 0 0">Shown in this app, and sent to Signal as the contact's name. Leave it empty to use the name Signal has.</p><div class="err" id="sgce"></div>
+    <div class="bar"><button class="btn ghost" type="button" id="sgcx" style="flex:1;justify-content:center">Cancel</button><button class="btn" style="flex:1;justify-content:center">Save</button></div></form>`);
+  el.querySelector('#sgcx').onclick = closeSheet; setTimeout(() => el.querySelector('#sgcn').focus(), 60);
+  el.querySelector('#sgcf').onsubmit = async e => { e.preventDefault();
+    try { await sgPost('/contact', { number: c.id || c.number, name: el.querySelector('#sgcn').value }); closeSheet(); toast('Saved'); if (after) after(); await sgRefresh(); }
+    catch (err) { el.querySelector('#sgce').textContent = err.message; } };
+}
 async function sgNewChat() {
   let contacts = []; try { contacts = (await sgApi('/contacts')).contacts; } catch {}
   const el = openSheet('New chat', '', `<form id="sgnf"><label for="sgnn">Phone number (with country code) or a contact</label><input type="text" id="sgnn" inputmode="tel" autocomplete="off" placeholder="+1 555 123 4567 or a name">
-    <div class="err" id="sgne"></div></form><div class="list" id="sgcl" style="background:var(--panel2);border:0;margin-top:10px;max-height:40dvh;overflow:auto"></div>`);
+    <input type="text" id="sgnm" autocomplete="off" placeholder="Name for a new number (optional)" style="margin-top:8px" maxlength="80"><div class="err" id="sgne"></div><div class="bar"><button class="btn" style="width:100%;justify-content:center">Start chat</button></div></form><div class="list" id="sgcl" style="background:var(--panel2);border:0;margin-top:10px;max-height:40dvh;overflow:auto"></div>`);
-  const input = el.querySelector('#sgnn'), draw = () => { const q = input.value.toLowerCase().trim(); el.querySelector('#sgcl').innerHTML = contacts.filter(c => !q || (c.name || '').toLowerCase().includes(q) || c.number.includes(q)).slice(0, 60).map(c => `<button class="row" data-n="${esc(c.number)}"><span class="sgav" style="background:${sgColor(c.number)}">${esc(sgInitial(c.name || c.number))}</span><span class="grow"><b>${esc(c.name || c.number)}</b><small>${esc(c.number)}</small></span></button>`).join('');
+  const input = el.querySelector('#sgnn'), draw = () => { const q = input.value.toLowerCase().trim(); el.querySelector('#sgcl').innerHTML = contacts.filter(c => !q || (c.name || '').toLowerCase().includes(q) || c.number.includes(q)).slice(0, 60).map(c => `<div class="row it"><button type="button" class="rmain" data-n="${esc(c.number)}"><span class="sgav" style="background:${sgColor(c.number)}">${esc(sgInitial(c.name || c.number))}</span><span class="grow"><b>${esc(c.name || c.number)}</b><small>${esc(c.number)}</small></span></button><button type="button" class="ibtn" data-e="${esc(c.number)}" aria-label="Edit ${esc(c.name || c.number)}" title="Edit contact">${icon('edit')}</button></div>`).join('');
-    el.querySelectorAll('[data-n]').forEach(b => b.onclick = () => go(b.dataset.n)); };
+    el.querySelectorAll('[data-n]').forEach(b => b.onclick = () => go(b.dataset.n));
-  const go = async n => { try { const r = await sgPost('/new', { number: n }); closeSheet(); if (sg.dock) { await sgRefresh(); sgOpen(r.conv.id); } else location.hash = '#/signal/' + encodeURIComponent(r.conv.id); } catch (e) { el.querySelector('#sgne').textContent = e.message; } };
+    el.querySelectorAll('[data-e]').forEach(b => b.onclick = () => { const k = contacts.find(x => x.number === b.dataset.e); sgEditContact(k, async () => { try { contacts = (await sgApi('/contacts')).contacts; } catch {} }); }); };
+  const go = async n => { try { const r = await sgPost('/new', { number: n, name: el.querySelector('#sgnm').value }); closeSheet(); if (sg.dock) { await sgRefresh(); sgOpen(r.conv.id); } else location.hash = '#/signal/' + encodeURIComponent(r.conv.id); } catch (e) { el.querySelector('#sgne').textContent = e.message; } };
   input.oninput = draw; draw(); el.querySelector('#sgnf').onsubmit = e => { e.preventDefault(); go(input.value); }; setTimeout(() => input.focus(), 60);
 }
 function sgLinkScreen(root, dock) {

signal.js

@@ -7,6 +7,9 @@ const crypto = require('crypto');
 const { Readable } = require('stream');
 
 const MAX_PER_CONV = 1500;
+const trace = []; // what has been seen lately (kinds of messages and connection changes, never the text), for working out why a message did not show
+const note = (kind, extra) => { trace.push({ at: new Date().toISOString(), kind, ...extra }); if (trace.length > 300) trace.shift(); };
+const ids = () => (state.db.ids = state.db.ids || {}); // a contact can show up by phone number or only by Signal's private id (uuid); this remembers which is whose
 const state = { api: '', dir: '', account: '', db: { account: '', convs: {}, msgs: {} }, ws: null, wsUp: false, backoff: 1000, clients: new Set(), saveT: null,
   contacts: { at: 0, map: new Map() }, groups: { at: 0, list: [] }, started: false };
 
@@ -45,9 +48,9 @@ async function loadContacts(force) {
   try {
     const list = await apiJson('GET', '/v1/contacts/' + encodeURIComponent(state.account));
     const m = new Map();
-    for (const c of list || []) { const n = c.name || c.nickname && c.nickname.name || [c.given_name, c.family_name].filter(Boolean).join(' ') || c.profile_name || (c.profile && c.profile.given_name) || ''; if (c.number) m.set(c.number, n); }
+    for (const c of list || []) { const n = c.name || c.nickname && c.nickname.name || [c.given_name, c.family_name].filter(Boolean).join(' ') || c.profile_name || (c.profile && c.profile.given_name) || ''; if (c.number) m.set(c.number, n); if (c.number && c.uuid && ids()[c.uuid] !== c.number) { ids()[c.uuid] = c.number; mergeConv(c.uuid, c.number); } }
     state.contacts.map = m;
-    for (const c of Object.values(state.db.convs)) if (c.type === 'direct' && m.get(c.id) && c.name !== m.get(c.id)) { c.name = m.get(c.id); }
+    for (const c of Object.values(state.db.convs)) if (c.type === 'direct') { const n = displayName(c.id, c.name !== c.id ? c.name : ''); if (n && c.name !== n) c.name = n; }
     saveSoon();
   } catch (e) { state.contacts.at = Date.now() - 9 * 60e3; /* try again in a minute */ }
 }
@@ -62,7 +65,18 @@ async function loadGroups(force) {
   } catch (e) { state.groups.at = Date.now() - 4 * 60e3; }
 }
 const groupApiId = internal => { const g = state.groups.list.find(x => x.internal_id === internal); return g ? g.id : 'group.' + Buffer.from(internal).toString('base64'); };
-const displayName = (id, hint) => state.contacts.map.get(id) || hint || id;
+const nicks = () => (state.db.nick = state.db.nick || {}); // names you chose for contacts in this app
+const displayName = (id, hint) => nicks()[id] || state.contacts.map.get(id) || hint || id;
+
+function setNick(number, name) {
+  name = name.trim().slice(0, 80);
+  if (name) nicks()[number] = name; else delete nicks()[number];
+  const c = state.db.convs[number]; if (c) { c.name = displayName(number, ''); emit({ t: 'update', conv: number }); }
+  saveSoon(); return name;
+}
+function dropFiles(cid) { // pictures you sent that were kept here
+  for (const m of state.db.msgs[cid] || []) for (const a of m.att || []) if (a.local && /^local-[0-9a-f]+$/.test(a.id)) { try { fs.unlinkSync(path.join(state.dir, 'out', a.id)); } catch { /* already gone */ } }
+}
 
 // ---- keeping the conversations
 function emit(ev) {
@@ -95,6 +109,22 @@ function addMessage(cid, c, m) {
   return true;
 }
 
+// One person, one conversation, whether Signal names them by number or only by private id.
+function peer(number, uuid) {
+  const map = ids();
+  if (number && uuid && map[uuid] !== number) { map[uuid] = number; mergeConv(uuid, number); saveSoon(); }
+  return number || (uuid && map[uuid]) || uuid || '';
+}
+function mergeConv(from, to) {
+  const a = state.db.convs[from]; if (!a || from === to) return;
+  const b = state.db.convs[to] || (state.db.convs[to] = { id: to, type: 'direct', name: a.name, ts: 0, preview: '', unread: 0 });
+  const list = (state.db.msgs[to] = state.db.msgs[to] || []);
+  for (const m of state.db.msgs[from] || []) if (!list.some(x => x.id === m.id || (x.ts === m.ts && x.out === m.out))) list.push(m);
+  list.sort((x, y) => x.ts - y.ts); b.unread = (b.unread || 0) + (a.unread || 0);
+  const last = list[list.length - 1]; if (last) { b.ts = last.ts; b.preview = (last.out ? 'You: ' : '') + previewOf(last); }
+  if (b.name === to && a.name !== from) b.name = a.name;
+  delete state.db.convs[from]; delete state.db.msgs[from]; emit({ t: 'update', conv: to });
+}
 function ingestData(dm, from, fromName, ts, out, dest) {
   const gid = dm.groupInfo && dm.groupInfo.groupId;
   const cid = gid ? 'g:' + gid : (out ? dest : from);
@@ -130,7 +160,6 @@ function ingestReceipt(r, who) {
   if (!status) return;
   let changed = null;
   for (const [cid, list] of Object.entries(state.db.msgs)) {
-    if (cid !== who && !cid.startsWith('g:')) continue;
     for (const m of list) if (m.out && (r.timestamps || []).includes(m.ts) && (m.status !== 'read')) { m.status = status; changed = cid; }
   }
   if (changed) { saveSoon(); emit({ t: 'update', conv: changed }); }
@@ -138,35 +167,65 @@ function ingestReceipt(r, who) {
 
 function ingest(env) {
   if (!env) return;
-  const ts = env.timestamp || Date.now(), who = env.sourceNumber || env.source || env.sourceUuid || '', name = env.sourceName || '';
+  const ts = env.timestamp || Date.now(), who = peer(env.sourceNumber, env.sourceUuid || (/^[0-9a-f-]{36}$/i.test(env.source || '') ? env.source : '')) || env.source || '', name = env.sourceName || '';
   if (env.dataMessage) ingestData(env.dataMessage, who, name, env.dataMessage.timestamp || ts, false);
   else if (env.editMessage) ingestEdit(env.editMessage, who, ts, false);
   else if (env.syncMessage && env.syncMessage.sentMessage) {
     const sm = env.syncMessage.sentMessage;
-    if (sm.editMessage) ingestEdit(sm.editMessage, state.account, ts, true, sm.destinationNumber || sm.destination);
+    if (sm.editMessage) ingestEdit(sm.editMessage, state.account, ts, true, peer(sm.destinationNumber, sm.destinationUuid) || sm.destination);
-    else ingestData(sm, state.account, '', sm.timestamp || ts, true, sm.destinationNumber || sm.destination || sm.destinationUuid);
+    else ingestData(sm, state.account, '', sm.timestamp || ts, true, peer(sm.destinationNumber, sm.destinationUuid) || sm.destination);
   } else if (env.receiptMessage) ingestReceipt(env.receiptMessage, who);
   else if (env.typingMessage) emit({ t: 'typing', conv: env.typingMessage.groupId ? 'g:' + env.typingMessage.groupId : who, stop: env.typingMessage.action === 'STOPPED' });
 }
 
+// A frame normally holds one message, but messages that arrive close together could come glued together (one JSON per line, an array, or
+// back to back), so read every JSON value in it instead of giving up when the whole thing is not one value.
+function jsonValues(text) {
+  try { const v = JSON.parse(text); return Array.isArray(v) ? v : [v]; } catch { /* more than one value, or damaged */ }
+  const out = []; let depth = 0, start = -1, inStr = false, esc = false;
+  for (let i = 0; i < text.length; i++) {
+    const c = text[i];
+    if (inStr) { if (esc) esc = false; else if (c === '\\') esc = true; else if (c === '"') inStr = false; continue; }
+    if (c === '"') inStr = true;
+    else if (c === '{') { if (depth++ === 0) start = i; }
+    else if (c === '}' && depth > 0 && --depth === 0) { try { out.push(JSON.parse(text.slice(start, i + 1))); } catch { /* skip this one */ } }
+  }
+  return out;
+}
 function handleRaw(text) {
-  let m; try { m = JSON.parse(text); } catch { return; }
+  const values = jsonValues(text);
-  const env = m.envelope || (m.params && m.params.envelope) || (m.result && m.result.envelope);
+  if (!values.length) { note('unreadable', { bytes: String(text).length }); return; }
-  try { ingest(env); } catch (e) { console.error('[signal] could not read a message:', e.message); }
+  if (values.length > 1) note('several-in-one-frame', { count: values.length });
+  for (const m of values) {
+    const env = m && (m.envelope || (m.params && m.params.envelope) || (m.result && m.result.envelope));
+    const kinds = env ? Object.keys(env).filter(k => /Message$/.test(k)) : ['no-envelope:' + Object.keys(m || {}).join(',')];
+    note('envelope', { kinds, from: env && (env.sourceNumber || env.source || '').slice(-4), group: !!(env && env.dataMessage && env.dataMessage.groupInfo), text: !!(env && ((env.dataMessage && env.dataMessage.message) || (env.syncMessage && env.syncMessage.sentMessage && env.syncMessage.sentMessage.message))), att: !!(env && env.dataMessage && (env.dataMessage.attachments || []).length) });
+    try { ingest(env); } catch (e) { note('error', { message: e.message }); console.error('[signal] could not read a message:', e.message); }
+  }
 }
 
 // ---- the link to signal-cli
+// signal-cli hands a message only to whoever is listening at that moment, so the hub reconnects quickly and says so when it was away.
 function connect() {
   if (!state.account || state.ws || typeof WebSocket === 'undefined') return;
   const url = state.api.replace(/^http/, 'ws') + '/v1/receive/' + encodeURIComponent(state.account);
   let ws;
-  try { ws = new WebSocket(url); } catch (e) { return setTimeout(connect, state.backoff = Math.min(state.backoff * 2, 30000)); }
+  try { ws = new WebSocket(url); } catch (e) { return retrySoon(); }
   state.ws = ws;
-  ws.onopen = () => { state.wsUp = true; state.backoff = 1000; emit({ t: 'status' }); };
+  ws.onopen = () => {
-  ws.onmessage = e => handleRaw(typeof e.data === 'string' ? e.data : String(e.data));
+    state.wsUp = true; state.backoff = 1000; note('connected');
-  const gone = () => { if (state.ws !== ws) return; state.ws = null; state.wsUp = false; emit({ t: 'status' }); const t = setTimeout(connect, state.backoff = Math.min(state.backoff * 2, 30000)); t.unref && t.unref(); };
+    if (state.downSince) { const from = state.downSince; state.downSince = 0; if (Date.now() - from > 2500) state.gap = { from, to: Date.now() }; console.log(`[signal] connected again after ${Math.round((Date.now() - from) / 1000)} s`); } else console.log('[signal] connected');
+    emit({ t: 'status' });
+  };
+  const text = d => typeof d === 'string' ? handleRaw(d) : d && typeof d.text === 'function' ? d.text().then(handleRaw) : d instanceof ArrayBuffer ? handleRaw(Buffer.from(d).toString('utf8')) : handleRaw(String(d)); // (a binary frame is read as text too)
+  ws.onmessage = e => text(e.data);
+  const gone = ev => {
+    if (state.ws !== ws) return; state.ws = null; state.wsUp = false; if (!state.downSince) state.downSince = Date.now();
+    note('disconnected', { code: ev && ev.code, reason: ev && ev.reason }); console.log('[signal] connection lost' + (ev && ev.code ? ' (code ' + ev.code + ')' : '')); emit({ t: 'status' }); retrySoon();
+  };
   ws.onclose = gone; ws.onerror = gone;
 }
+function retrySoon() { const t = setTimeout(connect, state.backoff); state.backoff = Math.min(state.backoff + 1000, 5000); t.unref && t.unref(); } // 1 s, 2 s ... never longer than 5 s
 async function refreshAccount() {
   try {
     const list = await apiJson('GET', '/v1/accounts');
@@ -218,7 +277,8 @@ async function send(cid, text, atts, quoteTs) {
 async function handle(req, res) {
   const u = new URL(req.url, 'http://x'), p = u.pathname.replace(/^\/api\/signal/, '') || '/';
   try {
-    if (p === '/status') return json(res, 200, { linked: !!state.account, number: state.account, connected: state.wsUp, reachable: state.reachable !== false, unread: totalUnread() });
+    if (p === '/status') return json(res, 200, { linked: !!state.account, number: state.account, connected: state.wsUp, reachable: state.reachable !== false, unread: totalUnread(), gap: state.gap || null });
+    if (p === '/debug') return json(res, 200, { account: state.account ? '…' + state.account.slice(-4) : '', connected: state.wsUp, gap: state.gap || null, convs: Object.keys(state.db.convs).length, trace }); // what arrived lately: kinds only, no message text
     if (p === '/qr') { // a new link code each time; scanned in Signal -> Settings -> Linked devices
       const r = await api('GET', '/v1/qrcodelink?device_name=' + encodeURIComponent('NE Hub'), null, 60000);
       if (!r.ok) return json(res, 502, { error: 'Could not get a link code from the Signal service' });
@@ -232,7 +292,12 @@ async function handle(req, res) {
     }
     if (!state.account) return json(res, 409, { error: 'Signal is not linked yet' });
     if (p === '/convs') return json(res, 200, { convs: Object.values(state.db.convs).sort((a, b) => b.ts - a.ts), unread: totalUnread() });
-    if (p === '/contacts') { await loadContacts(false); return json(res, 200, { contacts: [...state.contacts.map].map(([number, name]) => ({ number, name })).sort((a, b) => (a.name || a.number).localeCompare(b.name || b.number)) }); }
+    if (p === '/contacts') {
+      await loadContacts(false);
+      const all = new Map(state.contacts.map); for (const n of Object.keys(nicks())) if (!all.has(n)) all.set(n, '');
+      for (const c of Object.values(state.db.convs)) if (c.type === 'direct' && /^\+/.test(c.id) && !all.has(c.id)) all.set(c.id, '');
+      return json(res, 200, { contacts: [...all].map(([number, name]) => ({ number, name: nicks()[number] || name, mine: !!nicks()[number] })).sort((a, b) => (a.name || a.number).localeCompare(b.name || b.number)) });
+    }
     let m;
     if ((m = /^\/conv\/(.+)$/.exec(p)) && req.method === 'GET') {
       const id = decodeURIComponent(m[1]), c = state.db.convs[id]; if (!c) return json(res, 404, { error: 'Unknown conversation' });
@@ -243,8 +308,25 @@ async function handle(req, res) {
     if (p === '/new' && req.method === 'POST') {
       const b = JSON.parse(await readBody(req, 10000)), n = String(b.number || '').replace(/[\s()-]/g, '');
       if (!/^\+\d{6,15}$/.test(n)) return json(res, 400, { error: 'Enter a phone number with its country code, like +15551234567' });
+      if (String(b.name || '').trim()) setNick(n, String(b.name));
       const c = conv(n, 'direct', ''); saveSoon(); return json(res, 200, { conv: c });
     }
+    if (p === '/contact' && req.method === 'POST') { // rename a contact (an empty name goes back to the name Signal has)
+      const b = JSON.parse(await readBody(req, 10000)), n = String(b.number || '').replace(/[\s()-]/g, '');
+      if (!/^\+\d{6,15}$/.test(n) && !/^[0-9a-f-]{36}$/i.test(n)) return json(res, 400, { error: 'That is not a phone number' });
+      const name = setNick(n, String(b.name || ''));
+      let bridge = false;
+      if (name && /^\+/.test(n)) { try { await apiJson('PUT', '/v1/contacts/' + encodeURIComponent(state.account), { recipient: n, name }); bridge = true; } catch { /* kept in this app anyway */ } }
+      return json(res, 200, { ok: true, name: displayName(n, ''), bridge });
+    }
+    if (p === '/conv/delete' && req.method === 'POST') { // remove a chat from this app (it stays on your phone; a new message starts it again)
+      const b = JSON.parse(await readBody(req, 10000)); if (!state.db.convs[b.id]) return json(res, 404, { error: 'Unknown conversation' });
+      dropFiles(b.id); delete state.db.convs[b.id]; delete state.db.msgs[b.id]; saveSoon(); emit({ t: 'gone', conv: b.id }); return json(res, 200, { ok: true });
+    }
+    if (p === '/conv/clear' && req.method === 'POST') { // empty a chat but keep it in the list
+      const b = JSON.parse(await readBody(req, 10000)), c = state.db.convs[b.id]; if (!c) return json(res, 404, { error: 'Unknown conversation' });
+      dropFiles(b.id); state.db.msgs[b.id] = []; c.preview = ''; c.unread = 0; saveSoon(); emit({ t: 'update', conv: b.id }); return json(res, 200, { ok: true });
+    }
     if (p === '/read' && req.method === 'POST') {
       const b = JSON.parse(await readBody(req, 10000)), c = state.db.convs[b.id]; if (!c) return json(res, 404, { error: 'Unknown conversation' });
       c.unread = 0; saveSoon(); emit({ t: 'read', conv: c.id });

tools/signal-test.js

@@ -13,15 +13,17 @@ const SECRET = 'signal-test-secret-'.repeat(2), ME = '+15550001';
 
 (async () => {
   // ---- fake signal-cli-rest-api
-  const fake = { accounts: [], sends: [], receipts: [], reactions: [], sockets: new Set(), contacts: [{ number: '+15551234567', name: 'Sam Rivera' }], groups: [{ id: 'group.' + Buffer.from('GID123==').toString('base64'), internal_id: 'GID123==', name: 'Family' }] };
+  const fake = { accounts: [], puts: [], sends: [], receipts: [], reactions: [], sockets: new Set(), contacts: [{ number: '+15551234567', name: 'Sam Rivera' }], groups: [{ id: 'group.' + Buffer.from('GID123==').toString('base64'), internal_id: 'GID123==', name: 'Family' }] };
   const frame = text => { const p = Buffer.from(text), n = p.length; const h = n < 126 ? Buffer.from([0x81, n]) : Buffer.from([0x81, 126, n >> 8, n & 255]); return Buffer.concat([h, p]); };
-  const push = o => { for (const s of fake.sockets) s.write(frame(JSON.stringify(o))); };
+  const pushRaw = text => { for (const s of fake.sockets) s.write(frame(text)); };
+  const push = (o, binary) => { for (const s of fake.sockets) { const f = frame(JSON.stringify(o)); if (binary) f[0] = 0x82; s.write(f); } };
   const png = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==', 'base64');
   const srv = http.createServer((rq, rs) => {
     const body = []; rq.on('data', d => body.push(d)); rq.on('end', () => {
       const b = Buffer.concat(body).toString(), u = new URL(rq.url, 'http://x'), j = (c, o) => { rs.writeHead(c, { 'content-type': 'application/json' }); rs.end(JSON.stringify(o)); };
       if (u.pathname === '/v1/accounts') return j(200, fake.accounts);
       if (u.pathname === '/v1/qrcodelink') { rs.writeHead(200, { 'content-type': 'image/png' }); return rs.end(png); }
+      if (u.pathname.startsWith('/v1/contacts/') && rq.method === 'PUT') { fake.puts.push(JSON.parse(b)); return j(204, {}); }
       if (u.pathname.startsWith('/v1/contacts/')) return j(200, fake.contacts);
       if (u.pathname.startsWith('/v1/groups/')) return j(200, fake.groups);
       if (u.pathname === '/v2/send') { fake.sends.push(JSON.parse(b)); return j(201, { timestamp: String(1700000000000 + fake.sends.length * 1000) }); }
@@ -87,6 +89,31 @@ const SECRET = 'signal-test-secret-'.repeat(2), ME = '+15550001';
   r = await call('/api/signal/attachment/att-1'); ok('and it can be fetched through the hub', r.status === 200 && r.headers.get('content-type') === 'image/png' && (await r.arrayBuffer()).byteLength === png.length, r.status);
   r = await call('/api/signal/attachment/..%2F..%2Fetc%2Fpasswd'); ok('attachment names cannot wander off', r.status === 400 || r.status === 404, r.status);
 
+  push({ account: ME, envelope: { sourceNumber: '+15551234567', sourceName: 'Sam', timestamp: T0 + 3500, dataMessage: { timestamp: T0 + 3500, message: 'sent as a binary frame' } } }, true);
+  th = await until(async () => { const t = await getJ('/api/signal/conv/' + encodeURIComponent('+15551234567')); return t.msgs.some(m => m.text === 'sent as a binary frame') ? t : false; });
+  ok('a message that comes in a binary frame is read too', !!th);
+  const e1 = { account: ME, envelope: { sourceNumber: '+15551234567', sourceName: 'Sam', timestamp: T0 + 3510, dataMessage: { timestamp: T0 + 3510, message: 'glued one' } } };
+  const e2 = { account: ME, envelope: { sourceNumber: '+15551234567', sourceName: 'Sam', timestamp: T0 + 3520, dataMessage: { timestamp: T0 + 3520, message: 'glued two with {braces} and "quotes"' } } };
+  pushRaw(JSON.stringify(e1) + '\n' + JSON.stringify(e2));
+  pushRaw(JSON.stringify([{ ...e1, envelope: { ...e1.envelope, timestamp: T0 + 3530, dataMessage: { timestamp: T0 + 3530, message: 'in an array' } } }]));
+  th = await until(async () => { const t = await getJ('/api/signal/conv/' + encodeURIComponent('+15551234567')); return ['glued one', 'glued two with {braces} and "quotes"', 'in an array'].every(x => t.msgs.some(m => m.text === x)) ? t : false; });
+  ok('messages that arrive glued together in one frame (or in a list) are all kept', !!th);
+  const dbg = await getJ('/api/signal/debug');
+  ok('there is a log of what arrived (kinds only), to find out why a message did not show', dbg.trace.some(x => x.kind === 'envelope' && x.kinds.includes('dataMessage')) && !JSON.stringify(dbg).includes('Hi there'), dbg.trace.slice(-2));
+  r = await call('/api/signal/debug', {}, 'ufriend1'); ok('and only people with the Signal tab can see it', r.status === 403, r.status);
+
+  // the same person by private id only (no number), and a conversation that had been started under that id
+  const UU = '0d1f2a3b-4c5d-6e7f-8091-a2b3c4d5e6f7';
+  push({ account: ME, envelope: { sourceUuid: UU, source: UU, sourceName: 'Lee', timestamp: T0 + 3600, dataMessage: { timestamp: T0 + 3600, message: 'from a private id' } } });
+  await until(async () => (await getJ('/api/signal/convs')).convs.some(c => c.id === UU));
+  push({ account: ME, envelope: { sourceNumber: '+15558889999', sourceUuid: UU, sourceName: 'Lee', timestamp: T0 + 3700, dataMessage: { timestamp: T0 + 3700, message: 'now with the number' } } });
+  const merged = await until(async () => { const c = (await getJ('/api/signal/convs')).convs; return !c.some(x => x.id === UU) && c.some(x => x.id === '+15558889999') ? c : false; });
+  const lee = merged && (await getJ('/api/signal/conv/' + encodeURIComponent('+15558889999'))).msgs.map(m => m.text).join('|');
+  ok('a person seen first by private id and then by number is one conversation, with both messages', lee === 'from a private id|now with the number', lee);
+  push({ account: ME, envelope: { sourceUuid: UU, source: UU, timestamp: T0 + 3800, dataMessage: { timestamp: T0 + 3800, message: 'later, id only' } } });
+  const lee2 = await until(async () => { const t = await getJ('/api/signal/conv/' + encodeURIComponent('+15558889999')); return t.msgs.some(m => m.text === 'later, id only') ? t : false; });
+  ok('and later messages that carry only the id land in the same place', !!lee2);
+
   // reactions, receipts, edits, deletes
   push({ account: ME, envelope: { sourceNumber: '+15551234567', timestamp: T0 + 4000, dataMessage: { timestamp: T0 + 4000, reaction: { emoji: '👍', targetAuthorNumber: ME, targetSentTimestamp: T0 + 2000 } } } });
   push({ account: ME, envelope: { sourceNumber: '+15551234567', timestamp: T0 + 5000, editMessage: { targetSentTimestamp: T0, dataMessage: { message: 'Hi there (edited)' } } } });
@@ -98,6 +125,7 @@ const SECRET = 'signal-test-secret-'.repeat(2), ME = '+15550001';
   r = await post('/api/signal/send', { to: '+15551234567', text: 'On my way' }); let sent = (await r.json()).msg;
   ok('sending to a person', r.status === 200 && fake.sends.length === 1 && fake.sends[0].number === ME && fake.sends[0].recipients[0] === '+15551234567' && fake.sends[0].message === 'On my way', fake.sends);
   ok('your sent message appears at once', sent && sent.out && sent.status === 'sent');
+  push({ account: ME, envelope: { sourceUuid: '11111111-2222-3333-4444-555555555555', timestamp: T0 + 6900, receiptMessage: { isDelivery: true, timestamps: [sent.ts] } } }); // (a receipt from someone Signal names only by id)
   push({ account: ME, envelope: { sourceNumber: '+15551234567', timestamp: T0 + 7000, receiptMessage: { isDelivery: true, timestamps: [sent.ts] } } });
   push({ account: ME, envelope: { sourceNumber: '+15551234567', timestamp: T0 + 8000, receiptMessage: { isRead: true, timestamps: [sent.ts] } } });
   th = await until(async () => { const t = await getJ('/api/signal/conv/' + encodeURIComponent('+15551234567')); const m = t.msgs.find(x => x.ts === sent.ts); return m && m.status === 'read' ? t : false; });
@@ -120,6 +148,30 @@ const SECRET = 'signal-test-secret-'.repeat(2), ME = '+15550001';
   ok('starting a chat with a new number', r.status === 200 && nc.id === '+15550009999', nc);
   r = await post('/api/signal/new', { number: 'abc' }); ok('a bad number is refused', r.status === 400);
 
+  // editing contacts and removing chats
+  r = await post('/api/signal/contact', { number: '+15551234567', name: '  Sammy  ' });
+  convs = (await getJ('/api/signal/convs')).convs;
+  ok('a contact can be renamed, here and in Signal', r.status === 200 && convs.find(c => c.id === '+15551234567').name === 'Sammy' && fake.puts[0].recipient === '+15551234567' && fake.puts[0].name === 'Sammy', [convs.map(c => c.name), fake.puts]);
+  push({ account: ME, envelope: { sourceNumber: '+15551234567', sourceName: 'Sam', timestamp: T0 + 9500, dataMessage: { timestamp: T0 + 9500, message: 'name should stay' } } });
+  await until(async () => (await getJ('/api/signal/conv/' + encodeURIComponent('+15551234567'))).msgs.some(m => m.text === 'name should stay'));
+  ok('your name for them is kept when they write again', (await getJ('/api/signal/convs')).convs.find(c => c.id === '+15551234567').name === 'Sammy');
+  const cl = (await getJ('/api/signal/contacts')).contacts; ok('the contact list shows your name', cl.find(c => c.number === '+15551234567').name === 'Sammy' && cl.find(c => c.number === '+15551234567').mine === true, cl);
+  r = await post('/api/signal/contact', { number: '+15551234567', name: '' });
+  ok('an empty name goes back to the name Signal has', (await getJ('/api/signal/convs')).convs.find(c => c.id === '+15551234567').name === 'Sam Rivera');
+  r = await post('/api/signal/contact', { number: 'nope', name: 'x' }); ok('a bad number is refused', r.status === 400, r.status);
+  r = await post('/api/signal/new', { number: '+15550007777', name: 'Pat' }); ok('a new chat can be started with a name', (await r.json()).conv.name === 'Pat');
+  r = await post('/api/signal/send', { to: '+15550007777', text: 'pic', attachments: [{ name: 'p.png', type: 'image/png', data: png.toString('base64') }] });
+  const outDir = path.join(dataDir, 'signal', 'out'), filesBefore = fs.readdirSync(outDir).length;
+  r = await post('/api/signal/conv/clear', { id: 'g:GID123==' });
+  ok('clearing a chat empties it but keeps it', r.status === 200 && (await getJ('/api/signal/conv/' + encodeURIComponent('g:GID123=='))).msgs.length === 0 && (await getJ('/api/signal/convs')).convs.some(c => c.id === 'g:GID123=='));
+  r = await post('/api/signal/conv/delete', { id: '+15550007777' });
+  ok('removing a chat takes it out of the list', r.status === 200 && !(await getJ('/api/signal/convs')).convs.some(c => c.id === '+15550007777'));
+  ok('and deletes the pictures kept for it', fs.readdirSync(outDir).length === filesBefore - 1, [filesBefore, fs.readdirSync(outDir).length]);
+  ok('the live feed says so', events.some(e => e.t === 'gone' && e.conv === '+15550007777'), events.slice(-3));
+  r = await post('/api/signal/conv/delete', { id: 'nobody' }); ok('removing something unknown is refused', r.status === 404);
+  push({ account: ME, envelope: { sourceNumber: '+15550007777', sourceName: 'Pat', timestamp: T0 + 9800, dataMessage: { timestamp: T0 + 9800, message: 'back again' } } });
+  ok('a new message from a removed chat starts it again', !!(await until(async () => (await getJ('/api/signal/convs')).convs.some(c => c.id === '+15550007777'))));
+
   // restart: history stays; the socket coming back
   ac.abort(); child.kill(); await wait(800); const before = JSON.parse(fs.readFileSync(path.join(dataDir, 'signal', 'store.json'), 'utf8'));
   ok('conversations are saved on disk', Object.keys(before.convs).length >= 3, Object.keys(before.convs));
@@ -128,7 +180,7 @@ const SECRET = 'signal-test-secret-'.repeat(2), ME = '+15550001';
   ok('after a restart the history is still there', Array.isArray(convs) && convs.some(c => c.id === '+15551234567'), convs);
   await until(async () => (await getJ('/api/signal/status')).connected);
   for (const s of fake.sockets) s.destroy(); // the service drops the connection
-  const back = await until(async () => { const s = await getJ('/api/signal/status'); return s.connected && fake.sockets.size > 0; }, 12000);
+  const back = await until(async () => { const s = await getJ('/api/signal/status'); return s.connected && fake.sockets.size > 0; }, 6000);
   ok('it reconnects by itself if the Signal service drops the link', !!back);
   push({ account: ME, envelope: { sourceNumber: '+15551234567', timestamp: T0 + 9000, dataMessage: { timestamp: T0 + 9000, message: 'Still there?' } } });
   ok('and messages keep arriving', !!(await until(async () => (await getJ('/api/signal/conv/' + encodeURIComponent('+15551234567'))).msgs.some(m => m.text === 'Still there?'))));