Overview: report liveness and a Source whose poll loop stopped

src/app/http.ts

@@ -44,6 +44,8 @@ export interface HttpHost {
   readonly health: Record<string, Health>
   readonly machine: Machine
   readonly masterId: string
+  /** When this process started serving, read by the overview's liveness readout. */
+  readonly startedAt: string
   /** Resolves once startup finishes or setup determines what is still missing. */
   readonly ready: Promise<void>
   /** Resolved deployment; throws if read while waiting for setup (`!configured()`). */
@@ -160,7 +162,7 @@ function routes(runtime: HttpHost): Routes {
   return {
     'GET state': () => stateView(runtime),
     'GET overview': () => overview(views(runtime), runtime.health, policyVersion(store.settings()),
-      executorLoad(store, runtime.executor.activeCount()), runtime.setupProblem ?? runtime.feishuProblem),
+      executorLoad(store, runtime.executor.activeCount()), runtime.setupProblem ?? runtime.feishuProblem, runtime.startedAt),
     'GET search': ({ query }) => search(views(runtime), query.q ?? '', runtime.sourceLinks),
     'GET sources': () => sourcesStatus(views(runtime), runtime.health, setupState(runtime)),
     'GET settings': () => settingsView(store, runtime.sources, runtime.health, setupState(runtime), runtime.machine.stateDir),

src/app/runtime.ts

@@ -74,6 +74,8 @@ export class Runtime extends Service implements SourceHost {
   private readonly board = new HealthBoard()
   private readonly runtimeHealth = this.board.claim('runtime', { state: 'starting', detail: '' })
   readonly masterId: string
+  /** When this process started serving, reported by the overview so a restart is visible rather than inferred. */
+  readonly startedAt = new Date().toISOString()
   private readonly sourceLinks = new TextCache(200_000)
   private readonly stopped = new AbortController()
   private readonly disposers: (() => void)[] = []
@@ -220,7 +222,7 @@ export class Runtime extends Service implements SourceHost {
     const runtime = this
     const { store, sources, prompts, sessions, settings, engine, executor, health, machine, masterId, ready, sourceLinks, setup, passes } = this
     return { context: this.ctx, store, sources, prompts, sessions, settings, engine, executor, health, machine, masterId, ready, sourceLinks,
-      get deployment() { return setup.deployment },
+      startedAt: this.startedAt,      get deployment() { return setup.deployment },
       get setupProblem() { return setup.problem }, get feishuProblem() { return setup.feishuProblem },
       configured: () => setup.configured(), links: () => runtime.links(), ticketEnvironment: ticket => runtime.ticketEnvironment(ticket),
       checkSetup: () => { setup.check() }, requestPass: () => { passes.request() }, models: () => this.ctx.sessionController.modelCatalog() }

src/app/views/overview.ts

@@ -22,6 +22,23 @@ function unjudged(store: Store, since: string): { thread: Thread; decision: Deci
     return [{ thread, decision }]
   })
 }
+/** A Source counts as stalled after this many of its own poll intervals without a fresh health report, but never sooner. */
+const STALL_POLLS = 3, STALL_FLOOR_MS = 180_000
+/** The interval a Source polls at, from its config; a Source without the field polls every 60 seconds. */
+function pollIntervalMs(config: Record<string, unknown> | undefined): number {
+  const seconds = Number(config?.pollSeconds)
+  return Number.isFinite(seconds) && seconds >= 60 ? seconds * 1000 : 60_000
+}
+/**
+ * Milliseconds since a Source last finished a poll round, read from the newest `at` its health entries carry. A Source
+ * reports `at` once per round, so a value that stops moving means its loop stopped: the service answers but polls nothing.
+ */
+const pollTimes = (parts: readonly { at?: string }[]): number[] =>
+  parts.flatMap(part => part.at === undefined ? [] : [Date.parse(part.at)]).filter(Number.isFinite)
+export function pollAgeMs(parts: readonly { at?: string }[]): number | undefined {
+  const times = pollTimes(parts)
+  return times.length ? Date.now() - Math.max(...times) : undefined
+}
 /** Reports error health when any thread has an undismissed `unavailable` eval from the last hour without a newer decision or active eval. */
 export function evalsHealth(store: Store, now = Date.now()): Health {
   const since = new Date(now - 3_600_000).toISOString()
@@ -34,8 +51,10 @@ export function evalsHealth(store: Store, now = Date.now()): Health {
 /**
  * Builds the operator overview: active system problems, tickets awaiting human action, executor load, and 24-hour trigger flow.
  * @param setup - Setup problem to surface first (where setup stopped, or a Source paused by setup).
+ * @param startedAt - When this process started serving, for the liveness readout.
  */
-export function overview(host: ViewHost, health: HealthMap, policyVersionValue: string, load: ExecutorLoad, setup?: SetupProblem) {
+export function overview(host: ViewHost, health: HealthMap, policyVersionValue: string, load: ExecutorLoad, setup?: SetupProblem,
+  startedAt?: string) {
   const { store, sources } = host
   const view = snapshot(host)
   const settings = store.settings()
@@ -45,12 +64,25 @@ export function overview(host: ViewHost, health: HealthMap, policyVersionValue:
   const running = instances.filter(instance => instance.enabled && instance.installed).map(instance => sourceHealth(sources, instance, health))
   // Exclude source-owned and policy-sync health entries so only core runtime errors surface as `runtime-error`.
   const sourceParts = new Set(instances.flatMap(instance => sources.kind(instance.kind)?.health ?? []))
+  const configOf = new Map(instances.map(instance => [instance.id, instance.config]))
+  /**
+   * A Source that answers but stopped polling: its last report is older than a few of its own intervals. Only Sources
+   * that have reported at least one round are judged, so a Source still starting up is never called stalled.
+   */
+  const stalled = running.flatMap(source => {
+    const age = pollAgeMs(source.health)
+    if (age === undefined) return []
+    const interval = pollIntervalMs(configOf.get(source.id))
+    if (age <= Math.max(STALL_POLLS * interval, STALL_FLOOR_MS)) return []
+    return [{ kind: 'stalled' as const, source: { id: source.id, title: source.title }, ageMs: age, intervalMs: interval }]
+  })
   const problems = [
     ...(setup ? [{ kind: 'setup' as const, ...setup }] : []),
     ...(running.length ? [] : [{ kind: 'no-source' as const }]),
     // Only allowed senders can originate work, so empty allowlists mean no request can ever become a ticket.
     ...(anyoneAllowed(settings) ? [] : [{ kind: 'no-allowed' as const }]),
     ...running.flatMap(source => source.health.filter(item => item.state === 'error').map(item => ({ kind: 'source-error' as const, source: { id: source.id, title: source.title, health: item } }))),
+    ...stalled,
     ...(settings.paused ? [{ kind: 'executor-paused' as const }] : []),
     // Unavailable Evals behind the `evals` health entry are listed individually as `eval-unavailable` below.
     ...Object.entries(health).filter(([name, item]) => item.state === 'error' && !sourceParts.has(name) && !name.startsWith('sync:')
@@ -68,8 +100,22 @@ export function overview(host: ViewHost, health: HealthMap, policyVersionValue:
   const stats = Object.values(store.read.triggerStats(since))
   const received = stats.reduce((sum, item) => sum + item.received, 0), filtered = stats.reduce((sum, item) => sum + item.filtered, 0)
   const judgement = store.read.flow(since)
+  /**
+   * Whether the service is alive is one question; whether it is doing anything is another. Restarts are invisible in the
+   * flow counters, and a dead poll loop answers every request, so both are reported explicitly.
+   */
+  const polled = running.flatMap(source => source.health)
+  const newestPoll = pollTimes(polled)
+  const liveness = {
+    startedAt: startedAt ?? null,
+    uptimeMs: startedAt === undefined ? null : Date.now() - Date.parse(startedAt),
+    lastPollAt: newestPoll.length ? new Date(Math.max(...newestPoll)).toISOString() : null,
+    pollAgeMs: pollAgeMs(polled) ?? null,
+    polling: running.length > 0 && newestPoll.length > 0 && !stalled.length,
+  }
   return {
     problems,
+    liveness,
     person: open.filter(ticket => ticketStage(ticket).stage === 'person').map(ticket => ticketRow(view, ticket)),
     executor: { ...load, maxWorkers: settings.executor.maxWorkers, idle: open.filter(ticket => ticketStage(ticket).stage === 'idle').length },
     flow: { received, filtered, triggers: received - filtered, judged: judgement.judged, accepted: judgement.accepted,

src/client/locales.ts

@@ -128,6 +128,15 @@ const TEXT = {
   problemReply: ['{source} 回复发送失败（已重试 {attempts} 次）', '{source} reply failing (retried {attempts} times)'],
   problemRuntime: ['运行异常：{name}', '{name} failing'],
   problemEvalUnavailable: ['评估未完成：{thread}', 'Eval unavailable: {thread}'],
+  problemStalled: ['{source} 最后一次轮询在 {age}，此后收不到新触发', '{source} last polled {age}: the service answers, but no Trigger arrives'],
+  problemStalledHint: ['轮询循环已经停了。先看日志有没有报错；没有报错时通常是进程虽在、事件循环被卡住或被挂起。',
+    'The poll loop stopped. Check the log for errors first; without one the process is usually alive with its event loop blocked or suspended.'],
+  livenessTitle: ['存活', 'Liveness'],
+  livenessUptime: ['已运行', 'Uptime'],
+  livenessLastPoll: ['最近轮询', 'Last poll'],
+  livenessPolling: ['在轮询', 'Polling'],
+  livenessNotPolling: ['不在轮询', 'Not polling'],
+  livenessNever: ['尚无', 'Never'],
   sourceErrorHint: ['来源会在每次轮询时自动重试。持续出现时检查网络和该来源的登录状态。', 'The Source retries on every poll. If this persists, check the network and the Source\'s sign-in.'],
   details: ['详情 ›', 'Details ›'],
   lastDay: ['过去 24 小时', 'Last 24 hours'],

src/client/overview.tsx

@@ -5,7 +5,7 @@ import { Button, Icon } from './controls.tsx'
 import { useAction, usePanel, usePanelData } from './provider.tsx'
 import { setupPage, settingsRoute, type Route } from './route.ts'
 import { TicketLine } from './rows.tsx'
-import { healthText, localized, reasonLabel, valueLabel, readableMarkers } from './text.ts'
+import { duration, healthText, localized, reasonLabel, valueLabel, readableMarkers } from './text.ts'
 import { CheckSetupButton, ConfirmButton, Hint, LoadState, Section, SetupMessage } from './widgets.tsx'
 
 function ProblemLine({ problem, onChanged }: { problem: Problem; onChanged: () => Promise<void> }) {
@@ -52,6 +52,8 @@ function ProblemLine({ problem, onChanged }: { problem: Problem; onChanged: () =
         <Button variant="outline" size="sm" icon={<Icon name="dismiss" size={14} />} disabled={post.busy}
           onClick={() => { void post.run(() => call('POST decisions/:id/dismiss', { id: problem.decisionId })) }}>{t('dismiss')}</Button>
       </>)
+    case 'stalled': return row(t('problemStalled', { source: localized(t, problem.source.title),
+      age: duration(problem.ageMs, t('lang')) }), undefined, { page: 'source' }, t('problemStalledHint'))
     case 'outgoing': return row(t('problemReply', { source: valueLabel(t, 'source', problem.outgoing.channel.source),
       attempts: problem.outgoing.attempts }),
       raw(problem.outgoing.error), undefined)
@@ -81,6 +83,15 @@ export function OverviewPage() {
       <ul className="dt-list dt-cards">{data.problems.map((problem, index) => <ProblemLine key={index} problem={problem}
         onChanged={reload} />)}</ul>
     </Section>}
+    <Section title={t('livenessTitle')}>
+      {/* A stopped poll loop answers every request, so the readout sits on the overview rather than behind a diagnostics page. */}
+      <div className="dt-stats">
+        {stat(t('livenessUptime'), duration(data.liveness.uptimeMs, t('lang')) || t('livenessNever'))}
+        {stat(t('livenessLastPoll'), data.liveness.lastPollAt
+          ? duration(data.liveness.pollAgeMs, t('lang')) : t('livenessNever'))}
+        {stat(t('livenessPolling'), data.liveness.polling ? t('livenessPolling') : t('livenessNotPolling'))}
+      </div>
+    </Section>
     <Section title={t('lastDay')} action={more({ page: 'trigger', query: { filter: 'recent' } })}>
       {/* Received and evaluated counts operate on Triggers rather than Threads or Decisions, so they do not link to filtered views. */}
       <div className="dt-stats">

src/client/text.ts

@@ -57,6 +57,15 @@ export function when(value: string | undefined, lang: string): string {
   const year = sameYear ? {} : { year: 'numeric' } as const
   return date.toLocaleString(timeLocale(lang), { ...year, month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' })
 }
+/** A span of milliseconds as a short localized phrase, for readouts such as uptime and time since the last poll. */
+export function duration(ms: number | null | undefined, lang: string): string {
+  if (ms === null || ms === undefined || !Number.isFinite(ms) || ms < 0) return ''
+  const relative = new Intl.RelativeTimeFormat(timeLocale(lang), { numeric: 'auto' })
+  if (ms < 60_000) return relative.format(-Math.round(ms / 1000), 'second')
+  if (ms < 3_600_000) return relative.format(-Math.round(ms / 60_000), 'minute')
+  if (ms < 86_400_000) return relative.format(-Math.round(ms / 3_600_000), 'hour')
+  return relative.format(-Math.round(ms / 86_400_000), 'day')
+}
 export function fullTime(value: string | undefined, lang: string): string {
   const date = new Date(value ?? '')
   return Number.isNaN(date.getTime()) ? '' : date.toLocaleString(timeLocale(lang))