Siteforge/server.js

import express from 'express'
import { hasVisitorConsent } from './server/helpers.js'
import { isValidAdminSession } from './server/auth.js'
import { BACKUP_INTERVAL_MS } from './server/config.js'
import { state } from './server/state.js'
import {
  loadHitStatsFromDisk,
  loadVisitorStatsFromDisk,
  loadContactSubmissionsFromDisk,
  loadReplyTemplatesFromDisk,
  loadReplyHistoryFromDisk,
  loadQuestionsFromDisk,
  loadDraftQuestionsFromDisk,
  loadStudyUsersFromDisk,
  loadStudyCommunityFromDisk,
  loadStudyRemindersFromDisk,
  loadStudyCommentsFromDisk,
  loadStudyCertificatesFromDisk,
  loadEpisodeScriptsFromDisk,
  migrateStudyNotesIfNeeded,
  loadDownloadCountsFromDisk,
  loadQrCodesFromDisk,
  loadEpisodePlaysFromDisk,
  loadAnalyticsEventsFromDisk,
  loadEmailSettingsFromDisk,
  loadAuditLogFromDisk,
  loadWebhooksFromDisk,
  loadStudySessionsFromDisk,
  queueStudySessionsWrite,
  loadNewsletterIssuesFromDisk,
  createBackupSnapshot,
  refreshContentCaches,
  queueHitStatsWrite,
} from './server/data.js'
import { logResendEmailAlignmentWarnings, sendStudyReminderEmail, sendStudyReengagementEmail, sendStudyWeeklyDigestEmail } from './server/email.js'
import {
  detectBot,
  sanitizeUserAgent,
  shouldCountHit,
  recordHit,
  scheduleStudyReminders,
  scheduleReengagementEmails,
  scheduleWeeklyDigestEmails,
  backfillCompletedStudies,
} from './server/study-helpers.js'
import { recordVisitor } from './server/routes/analytics.js'

// Route registrars
import { register as registerAdminAuth } from './server/routes/admin-auth.js'
import { register as registerAdminContent, runScheduledSeriesTransition } from './server/routes/admin-content.js'
import { register as registerAdminAssets } from './server/routes/admin-assets.js'
import { register as registerAdminBackup } from './server/routes/admin-backup.js'
import { register as registerAdminDashboard } from './server/routes/admin-dashboard.js'
import { register as registerStudyAuth } from './server/routes/study-auth.js'
import { register as registerStudyData } from './server/routes/study-data.js'
import { register as registerStudyAccount } from './server/routes/study-account.js'
import { register as registerStudyComments } from './server/routes/study-comments.js'
import { register as registerStudyCertificate } from './server/routes/study-certificate.js'
import { register as registerEpisodeScripts } from './server/routes/episode-scripts.js'
import { register as registerContact } from './server/routes/contact.js'
import { register as registerInboundEmail } from './server/routes/inbound-email.js'
import { register as registerQuestions } from './server/routes/questions.js'
import { register as registerAnalytics } from './server/routes/analytics.js'
import { register as registerDownloads } from './server/routes/downloads.js'
import { register as registerEpisodes } from './server/routes/episodes.js'
import { register as registerNuggets } from './server/routes/nuggets.js'
import { register as registerNuggetsHostUpload } from './server/routes/nuggets-host-upload.js'
import { register as registerNuggetsWorkflow } from './server/routes/nuggets-workflow.js'
import { register as registerPodcastWorkflow } from './server/routes/podcast-workflow.js'
import { register as registerSeasonCheck } from './server/season-check.js'
import { warmFeedSnapshot } from './server/feed-release.js'
import { preferIpv4ForSeafile } from './server/seafile-dns.js'
import { startNuggetsSeafileWatcher } from './server/nuggets-seafile-watcher.js'
import { register as registerFeed } from './server/routes/feed.js'
import { register as registerQrCodes } from './server/routes/qr-codes.js'
import { register as registerBiblePassage } from './server/routes/bible-passage.js'
import { register as registerNewsletter } from './server/routes/newsletter.js'
import { register as registerPublic } from './server/routes/public.js'

preferIpv4ForSeafile()

const app = express()
app.use(express.json({ limit: '10mb' }))
const trustProxyHops = Number(process.env.TRUST_PROXY_HOPS ?? 1)
app.set('trust proxy', Number.isFinite(trustProxyHops) && trustProxyHops >= 0 ? trustProxyHops : 1)

app.use((_req, res, next) => {
  res.setHeader('X-Content-Type-Options', 'nosniff')
  res.setHeader('X-Frame-Options', 'SAMEORIGIN')
  res.setHeader('Referrer-Policy', 'strict-origin-when-cross-origin')
  res.setHeader('Permissions-Policy', 'camera=(), microphone=(), geolocation=(), payment=()')
  if (process.env.NODE_ENV === 'production') {
    res.setHeader('Strict-Transport-Security', 'max-age=31536000')
  }
  res.setHeader(
    'Content-Security-Policy',
    [
      "default-src 'self'",
      "script-src 'self' 'unsafe-inline' https://www.truthforlife.org https://ajax.googleapis.com https://storage.ko-fi.com https://static.cloudflareinsights.com",
      "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com https://www.truthforlife.org",
      "font-src 'self' https://fonts.gstatic.com",
      "img-src 'self' data: https:",
      "media-src 'self' https:",
      "frame-src https:",
      "connect-src 'self' https:",
    ].join('; '),
  )
  next()
})

// Register API routes
registerAdminAuth(app)
registerAdminContent(app)
registerAdminAssets(app)
registerAdminBackup(app)
registerAdminDashboard(app)
registerStudyAuth(app)
registerStudyData(app)
registerStudyAccount(app)
registerStudyComments(app)
registerStudyCertificate(app)
registerEpisodeScripts(app)
registerContact(app)
registerInboundEmail(app)
registerQuestions(app)
registerAnalytics(app)
registerDownloads(app)
registerEpisodes(app)
registerNuggets(app)
registerNuggetsHostUpload(app)
registerNuggetsWorkflow(app)
registerPodcastWorkflow(app)
registerSeasonCheck(app)
registerFeed(app)
registerQrCodes(app)
registerBiblePassage(app)
registerNewsletter(app)

// Hit-counting middleware (must come before public routes)
app.use((req, res, next) => {
  if (shouldCountHit(req)) {
    const ua = sanitizeUserAgent(req.get('user-agent'))
    const botDetection = detectBot(ua)
    recordHit(req.path, botDetection.isBot, botDetection.reason)
    queueHitStatsWrite()
    if (hasVisitorConsent(req) && !botDetection.isBot && !isValidAdminSession(req)) {
      try {
        recordVisitor(req, res)
      } catch (err) {
        console.error('[visitor-stats] failed to record visitor:', err)
      }
    }
  }
  next()
})

// Public routes (robots, sitemap, static, SPA fallback)
registerPublic(app)

// JSON error responses for API routes instead of Express's default HTML page.
app.use((err, req, res, _next) => {
  if (res.headersSent) return
  const status = Number.isInteger(err?.status) && err.status >= 400 && err.status < 600 ? err.status : 500
  if (status >= 500) console.error('[error]', req.method, req.path, err)
  if (req.path.startsWith('/api/')) {
    res.status(status).json({ message: status >= 500 ? 'Internal server error.' : String(err?.message ?? 'Request failed.') })
    return
  }
  res.status(status).type('text/plain').send(status >= 500 ? 'Internal server error.' : 'Bad request.')
})

const PORT = Number(process.env.PORT ?? 4173)
Promise.all([
  loadHitStatsFromDisk(),
  loadVisitorStatsFromDisk(),
  loadContactSubmissionsFromDisk(),
  loadReplyTemplatesFromDisk(),
  loadReplyHistoryFromDisk(),
  loadQuestionsFromDisk(),
  loadDraftQuestionsFromDisk(),
  loadStudyUsersFromDisk(),
  loadStudyCommunityFromDisk(),
  loadStudyRemindersFromDisk(),
  loadStudyCommentsFromDisk(),
  loadStudyCertificatesFromDisk(),
  loadEpisodeScriptsFromDisk(),
  migrateStudyNotesIfNeeded(),
  loadDownloadCountsFromDisk(),
  loadQrCodesFromDisk(),
  loadEpisodePlaysFromDisk(),
  loadAnalyticsEventsFromDisk(),
  loadEmailSettingsFromDisk(),
  loadAuditLogFromDisk(),
  loadWebhooksFromDisk(),
  loadStudySessionsFromDisk(),
  loadNewsletterIssuesFromDisk(),
  refreshContentCaches(),
])
  .catch(err => {
    console.error('[stats] failed to load persisted stats:', err)
  })
  .then(() => backfillCompletedStudies())
  .then(updated => { if (updated > 0) console.log(`[study] marked ${updated} finished studies as completed`) })
  .catch(err => {
    console.error('[study] completed-study backfill failed:', err)
  })
  .finally(() => {
    createBackupSnapshot('startup').catch(() => {})
    setInterval(() => {
      createBackupSnapshot('scheduled').catch(() => {})
    }, BACKUP_INTERVAL_MS)

    startNuggetsSeafileWatcher()
    warmFeedSnapshot()

    // Purge expired study sessions every hour
    setInterval(() => {
      const now = Date.now()
      let pruned = false
      for (const [token, session] of state.studySessions) {
        if (session.expiresAt <= now) { state.studySessions.delete(token); pruned = true }
      }
      if (pruned) queueStudySessionsWrite()
    }, 60 * 60 * 1000)

    // Send study reminder emails for newly released lessons every hour
    setInterval(() => {
      scheduleStudyReminders(sendStudyReminderEmail).catch(err => {
        console.error('[study-reminders] failed to schedule reminders:', err)
      })
    }, 60 * 60 * 1000)
    scheduleStudyReminders(sendStudyReminderEmail).catch(err => {
      console.error('[study-reminders] failed to schedule reminders:', err)
    })

    // Send re-engagement emails to inactive study users every hour
    setInterval(() => {
      scheduleReengagementEmails(sendStudyReengagementEmail).catch(err => {
        console.error('[study-reengagement] failed to schedule emails:', err)
      })
    }, 60 * 60 * 1000)
    scheduleReengagementEmails(sendStudyReengagementEmail).catch(err => {
      console.error('[study-reengagement] failed to schedule emails:', err)
    })

    // Send weekly lesson digest emails every hour (only actually fires every 7 days per user)
    setInterval(() => {
      scheduleWeeklyDigestEmails(sendStudyWeeklyDigestEmail).catch(err => {
        console.error('[study-digest] failed to schedule weekly digest:', err)
      })
    }, 60 * 60 * 1000)
    scheduleWeeklyDigestEmails(sendStudyWeeklyDigestEmail).catch(err => {
      console.error('[study-digest] failed to schedule weekly digest:', err)
    })

    // Check for a scheduled series transition every minute
    setInterval(() => {
      runScheduledSeriesTransition().catch(err => {
        console.error('[series-scheduler] error:', err)
      })
    }, 60 * 1000)
    runScheduledSeriesTransition().catch(err => {
      console.error('[series-scheduler] error:', err)
    })

    const server = app.listen(PORT, () => {
      logResendEmailAlignmentWarnings()
      console.log(`Siteforge listening on http://localhost:${PORT}`)
    })

    // On every deploy the running container gets SIGTERM'd and replaced.
    // All the queueXWrite() helpers in data.js debounce disk writes onto
    // per-resource promise chains — without draining those here, a redeploy
    // can kill the process mid-write and silently lose the last few
    // analytics events (or any other pending write) of that container's life.
    function collectPendingWrites() {
      return [
        state.hitStatsWritePromise,
        state.visitorStatsWritePromise,
        state.contactSubmissionsWritePromise,
        state.questionsWritePromise,
        state.draftQuestionsWritePromise,
        state.replyTemplatesWritePromise,
        state.replyHistoryWritePromise,
        state.studyUsersWritePromise,
        state.studyCommunityWritePromise,
        state.studyRemindersWritePromise,
        state.studyCommentsWritePromise,
        state.studyCertificatesWritePromise,
        state.episodeScriptsWritePromise,
        state.downloadCountsWritePromise,
        state.episodePlaysWritePromise,
        state.analyticsEventsWritePromise,
        state.qrCodesWritePromise,
        state.emailSettingsWritePromise,
        state.auditLogWritePromise,
        state.webhooksWritePromise,
        state.studySessionsWritePromise,
        ...state.studyNotesWriteQueues.values(),
        ...state.studyProgressWriteQueues.values(),
      ]
    }

    let shuttingDown = false
    async function gracefulShutdown(signal) {
      if (shuttingDown) return
      shuttingDown = true
      console.log(`[shutdown] ${signal} received — closing server and flushing pending writes...`)
      server.close()
      const forceExit = setTimeout(() => {
        console.error('[shutdown] timed out waiting for pending writes; exiting anyway')
        process.exit(1)
      }, 10_000)
      try {
        await Promise.all(collectPendingWrites())
        console.log('[shutdown] all pending writes flushed')
      } catch (err) {
        console.error('[shutdown] error while flushing writes:', err)
      } finally {
        clearTimeout(forceExit)
        process.exit(0)
      }
    }

    process.on('SIGTERM', () => gracefulShutdown('SIGTERM'))
    process.on('SIGINT', () => gracefulShutdown('SIGINT'))
  })