diff --git a/src/app/http.ts b/src/app/http.ts index e055d87..9a37217 100644 --- a/src/app/http.ts +++ b/src/app/http.ts @@ -44,6 +44,8 @@ export interface HttpHost { readonly health: Record readonly machine: Machine readonly masterId: string + /** When this process started serving, read by the overview's liveness readout. */ + readonly startedAt: string /** Resolves once startup finishes or setup determines what is still missing. */ readonly ready: Promise /** Resolved deployment; throws if read while waiting for setup (`!configured()`). */ @@ -160,7 +162,7 @@ function routes(runtime: HttpHost): Routes { return { 'GET state': () => stateView(runtime), 'GET overview': () => overview(views(runtime), runtime.health, policyVersion(store.settings()), - executorLoad(store, runtime.executor.activeCount()), runtime.setupProblem ?? runtime.feishuProblem), + executorLoad(store, runtime.executor.activeCount()), runtime.setupProblem ?? runtime.feishuProblem, runtime.startedAt), 'GET search': ({ query }) => search(views(runtime), query.q ?? '', runtime.sourceLinks), 'GET sources': () => sourcesStatus(views(runtime), runtime.health, setupState(runtime)), 'GET settings': () => settingsView(store, runtime.sources, runtime.health, setupState(runtime), runtime.machine.stateDir), diff --git a/src/app/runtime.ts b/src/app/runtime.ts index d3cf74e..693eb4b 100644 --- a/src/app/runtime.ts +++ b/src/app/runtime.ts @@ -74,6 +74,8 @@ export class Runtime extends Service implements SourceHost { private readonly board = new HealthBoard() private readonly runtimeHealth = this.board.claim('runtime', { state: 'starting', detail: '' }) readonly masterId: string + /** When this process started serving, reported by the overview so a restart is visible rather than inferred. */ + readonly startedAt = new Date().toISOString() private readonly sourceLinks = new TextCache(200_000) private readonly stopped = new AbortController() private readonly disposers: (() => void)[] = [] @@ -220,7 +222,7 @@ export class Runtime extends Service implements SourceHost { const runtime = this const { store, sources, prompts, sessions, settings, engine, executor, health, machine, masterId, ready, sourceLinks, setup, passes } = this return { context: this.ctx, store, sources, prompts, sessions, settings, engine, executor, health, machine, masterId, ready, sourceLinks, - get deployment() { return setup.deployment }, + startedAt: this.startedAt, get deployment() { return setup.deployment }, get setupProblem() { return setup.problem }, get feishuProblem() { return setup.feishuProblem }, configured: () => setup.configured(), links: () => runtime.links(), ticketEnvironment: ticket => runtime.ticketEnvironment(ticket), checkSetup: () => { setup.check() }, requestPass: () => { passes.request() }, models: () => this.ctx.sessionController.modelCatalog() } diff --git a/src/app/views/overview.ts b/src/app/views/overview.ts index 66f0a77..ac78bc3 100644 --- a/src/app/views/overview.ts +++ b/src/app/views/overview.ts @@ -22,6 +22,23 @@ function unjudged(store: Store, since: string): { thread: Thread; decision: Deci return [{ thread, decision }] }) } +/** A Source counts as stalled after this many of its own poll intervals without a fresh health report, but never sooner. */ +const STALL_POLLS = 3, STALL_FLOOR_MS = 180_000 +/** The interval a Source polls at, from its config; a Source without the field polls every 60 seconds. */ +function pollIntervalMs(config: Record | undefined): number { + const seconds = Number(config?.pollSeconds) + return Number.isFinite(seconds) && seconds >= 60 ? seconds * 1000 : 60_000 +} +/** + * Milliseconds since a Source last finished a poll round, read from the newest `at` its health entries carry. A Source + * reports `at` once per round, so a value that stops moving means its loop stopped: the service answers but polls nothing. + */ +const pollTimes = (parts: readonly { at?: string }[]): number[] => + parts.flatMap(part => part.at === undefined ? [] : [Date.parse(part.at)]).filter(Number.isFinite) +export function pollAgeMs(parts: readonly { at?: string }[]): number | undefined { + const times = pollTimes(parts) + return times.length ? Date.now() - Math.max(...times) : undefined +} /** Reports error health when any thread has an undismissed `unavailable` eval from the last hour without a newer decision or active eval. */ export function evalsHealth(store: Store, now = Date.now()): Health { const since = new Date(now - 3_600_000).toISOString() @@ -34,8 +51,10 @@ export function evalsHealth(store: Store, now = Date.now()): Health { /** * Builds the operator overview: active system problems, tickets awaiting human action, executor load, and 24-hour trigger flow. * @param setup - Setup problem to surface first (where setup stopped, or a Source paused by setup). + * @param startedAt - When this process started serving, for the liveness readout. */ -export function overview(host: ViewHost, health: HealthMap, policyVersionValue: string, load: ExecutorLoad, setup?: SetupProblem) { +export function overview(host: ViewHost, health: HealthMap, policyVersionValue: string, load: ExecutorLoad, setup?: SetupProblem, + startedAt?: string) { const { store, sources } = host const view = snapshot(host) const settings = store.settings() @@ -45,12 +64,25 @@ export function overview(host: ViewHost, health: HealthMap, policyVersionValue: const running = instances.filter(instance => instance.enabled && instance.installed).map(instance => sourceHealth(sources, instance, health)) // Exclude source-owned and policy-sync health entries so only core runtime errors surface as `runtime-error`. const sourceParts = new Set(instances.flatMap(instance => sources.kind(instance.kind)?.health ?? [])) + const configOf = new Map(instances.map(instance => [instance.id, instance.config])) + /** + * A Source that answers but stopped polling: its last report is older than a few of its own intervals. Only Sources + * that have reported at least one round are judged, so a Source still starting up is never called stalled. + */ + const stalled = running.flatMap(source => { + const age = pollAgeMs(source.health) + if (age === undefined) return [] + const interval = pollIntervalMs(configOf.get(source.id)) + if (age <= Math.max(STALL_POLLS * interval, STALL_FLOOR_MS)) return [] + return [{ kind: 'stalled' as const, source: { id: source.id, title: source.title }, ageMs: age, intervalMs: interval }] + }) const problems = [ ...(setup ? [{ kind: 'setup' as const, ...setup }] : []), ...(running.length ? [] : [{ kind: 'no-source' as const }]), // Only allowed senders can originate work, so empty allowlists mean no request can ever become a ticket. ...(anyoneAllowed(settings) ? [] : [{ kind: 'no-allowed' as const }]), ...running.flatMap(source => source.health.filter(item => item.state === 'error').map(item => ({ kind: 'source-error' as const, source: { id: source.id, title: source.title, health: item } }))), + ...stalled, ...(settings.paused ? [{ kind: 'executor-paused' as const }] : []), // Unavailable Evals behind the `evals` health entry are listed individually as `eval-unavailable` below. ...Object.entries(health).filter(([name, item]) => item.state === 'error' && !sourceParts.has(name) && !name.startsWith('sync:') @@ -68,8 +100,22 @@ export function overview(host: ViewHost, health: HealthMap, policyVersionValue: const stats = Object.values(store.read.triggerStats(since)) const received = stats.reduce((sum, item) => sum + item.received, 0), filtered = stats.reduce((sum, item) => sum + item.filtered, 0) const judgement = store.read.flow(since) + /** + * Whether the service is alive is one question; whether it is doing anything is another. Restarts are invisible in the + * flow counters, and a dead poll loop answers every request, so both are reported explicitly. + */ + const polled = running.flatMap(source => source.health) + const newestPoll = pollTimes(polled) + const liveness = { + startedAt: startedAt ?? null, + uptimeMs: startedAt === undefined ? null : Date.now() - Date.parse(startedAt), + lastPollAt: newestPoll.length ? new Date(Math.max(...newestPoll)).toISOString() : null, + pollAgeMs: pollAgeMs(polled) ?? null, + polling: running.length > 0 && newestPoll.length > 0 && !stalled.length, + } return { problems, + liveness, person: open.filter(ticket => ticketStage(ticket).stage === 'person').map(ticket => ticketRow(view, ticket)), executor: { ...load, maxWorkers: settings.executor.maxWorkers, idle: open.filter(ticket => ticketStage(ticket).stage === 'idle').length }, flow: { received, filtered, triggers: received - filtered, judged: judgement.judged, accepted: judgement.accepted, diff --git a/src/client/locales.ts b/src/client/locales.ts index 1aa3c55..8b86a85 100644 --- a/src/client/locales.ts +++ b/src/client/locales.ts @@ -128,6 +128,15 @@ const TEXT = { problemReply: ['{source} 回复发送失败（已重试 {attempts} 次）', '{source} reply failing (retried {attempts} times)'], problemRuntime: ['运行异常：{name}', '{name} failing'], problemEvalUnavailable: ['评估未完成：{thread}', 'Eval unavailable: {thread}'], + problemStalled: ['{source} 最后一次轮询在 {age}，此后收不到新触发', '{source} last polled {age}: the service answers, but no Trigger arrives'], + problemStalledHint: ['轮询循环已经停了。先看日志有没有报错；没有报错时通常是进程虽在、事件循环被卡住或被挂起。', + 'The poll loop stopped. Check the log for errors first; without one the process is usually alive with its event loop blocked or suspended.'], + livenessTitle: ['存活', 'Liveness'], + livenessUptime: ['已运行', 'Uptime'], + livenessLastPoll: ['最近轮询', 'Last poll'], + livenessPolling: ['在轮询', 'Polling'], + livenessNotPolling: ['不在轮询', 'Not polling'], + livenessNever: ['尚无', 'Never'], sourceErrorHint: ['来源会在每次轮询时自动重试。持续出现时检查网络和该来源的登录状态。', 'The Source retries on every poll. If this persists, check the network and the Source\'s sign-in.'], details: ['详情 ›', 'Details ›'], lastDay: ['过去 24 小时', 'Last 24 hours'], diff --git a/src/client/overview.tsx b/src/client/overview.tsx index 721a437..772b93d 100644 --- a/src/client/overview.tsx +++ b/src/client/overview.tsx @@ -5,7 +5,7 @@ import { Button, Icon } from './controls.tsx' import { useAction, usePanel, usePanelData } from './provider.tsx' import { setupPage, settingsRoute, type Route } from './route.ts' import { TicketLine } from './rows.tsx' -import { healthText, localized, reasonLabel, valueLabel, readableMarkers } from './text.ts' +import { duration, healthText, localized, reasonLabel, valueLabel, readableMarkers } from './text.ts' import { CheckSetupButton, ConfirmButton, Hint, LoadState, Section, SetupMessage } from './widgets.tsx' function ProblemLine({ problem, onChanged }: { problem: Problem; onChanged: () => Promise }) { @@ -52,6 +52,8 @@ function ProblemLine({ problem, onChanged }: { problem: Problem; onChanged: () = ) + case 'stalled': return row(t('problemStalled', { source: localized(t, problem.source.title), + age: duration(problem.ageMs, t('lang')) }), undefined, { page: 'source' }, t('problemStalledHint')) case 'outgoing': return row(t('problemReply', { source: valueLabel(t, 'source', problem.outgoing.channel.source), attempts: problem.outgoing.attempts }), raw(problem.outgoing.error), undefined) @@ -81,6 +83,15 @@ export function OverviewPage() {
    {data.problems.map((problem, index) => )}
} +
+ {/* A stopped poll loop answers every request, so the readout sits on the overview rather than behind a diagnostics page. */} +
+ {stat(t('livenessUptime'), duration(data.liveness.uptimeMs, t('lang')) || t('livenessNever'))} + {stat(t('livenessLastPoll'), data.liveness.lastPollAt + ? duration(data.liveness.pollAgeMs, t('lang')) : t('livenessNever'))} + {stat(t('livenessPolling'), data.liveness.polling ? t('livenessPolling') : t('livenessNotPolling'))} +
+
{/* Received and evaluated counts operate on Triggers rather than Threads or Decisions, so they do not link to filtered views. */}
diff --git a/src/client/text.ts b/src/client/text.ts index 5736922..17812d7 100644 --- a/src/client/text.ts +++ b/src/client/text.ts @@ -57,6 +57,15 @@ export function when(value: string | undefined, lang: string): string { const year = sameYear ? {} : { year: 'numeric' } as const return date.toLocaleString(timeLocale(lang), { ...year, month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' }) } +/** A span of milliseconds as a short localized phrase, for readouts such as uptime and time since the last poll. */ +export function duration(ms: number | null | undefined, lang: string): string { + if (ms === null || ms === undefined || !Number.isFinite(ms) || ms < 0) return '' + const relative = new Intl.RelativeTimeFormat(timeLocale(lang), { numeric: 'auto' }) + if (ms < 60_000) return relative.format(-Math.round(ms / 1000), 'second') + if (ms < 3_600_000) return relative.format(-Math.round(ms / 60_000), 'minute') + if (ms < 86_400_000) return relative.format(-Math.round(ms / 3_600_000), 'hour') + return relative.format(-Math.round(ms / 86_400_000), 'day') +} export function fullTime(value: string | undefined, lang: string): string { const date = new Date(value ?? '') return Number.isNaN(date.getTime()) ? '' : date.toLocaleString(timeLocale(lang)) diff --git a/tests/client.spec.tsx b/tests/client.spec.tsx index 2241799..5d6dc58 100644 --- a/tests/client.spec.tsx +++ b/tests/client.spec.tsx @@ -42,7 +42,8 @@ const openMaster = async (container: HTMLElement) => { } const serve = () => vi.stubGlobal('fetch', vi.fn(async (url: string) => { const path = new URL(url, 'http://127.0.0.1').pathname.replace('/orchestrator/api/', '') - if (path === 'overview') return Response.json({ problems: [], person: [], executor: { running: 0, queued: 0, maxWorkers: 3, idle: 0 }, flow: { triggers: 0, filtered: 0, judged: 0, accepted: 0, evaluating: 0 }, policyVersion: 'v' }) + if (path === 'overview') return Response.json({ problems: [], liveness: { startedAt: null, uptimeMs: null, lastPollAt: null, pollAgeMs: null, polling: false }, person: [], + executor: { running: 0, queued: 0, maxWorkers: 3, idle: 0 }, flow: { triggers: 0, filtered: 0, judged: 0, accepted: 0, evaluating: 0 }, policyVersion: 'v' }) if (path === 'state') return Response.json({ masterId: 'master', settings: SettingsSchema.parse({}), revisions: { policy: 0, eval: 0, executor: 0 }, sources: [], policy_version: 'v', health: {}, outgoing: [], load: { running: 0, queued: 0 }, setup: { machine: { stateDir: '/srv/state', worktreeRoot: '/srv/state/worktrees', blockedCommands: [], promptMaxChars: 100000, admissionContextMaxPages: 10, admissionContextTimeoutMs: 15000 }, problem: null, feishuProblem: null, accounts: null } }) return Response.json({ error: 'unavailable' }, { status: 503 }) diff --git a/tests/fixtures/panel-server.ts b/tests/fixtures/panel-server.ts index 3d30883..f2e1136 100644 --- a/tests/fixtures/panel-server.ts +++ b/tests/fixtures/panel-server.ts @@ -20,7 +20,7 @@ import { githubChannel } from '../../src/sources/github-trigger.ts' import { feishuKind } from '../../src/sources/feishu.ts' import { githubKind } from '../../src/sources/github.ts' import type { Health } from '../../src/sources/host.ts' -import type { SourceKind } from '../../src/sources/kind.ts' +import type { Localized, SourceKind } from '../../src/sources/kind.ts' import { testDeployment } from './deployment.ts' export const MEMBER = 'ou_alice' @@ -110,6 +110,8 @@ export interface PanelServer { opened: string[] dropped: DroppedReply[] failing: Extract['outgoing'][] + /** Sources whose poll loop the fixture reports as stopped, with how long ago their last round was. */ + stalled: { id: string; title: Localized; ageMs: number; intervalMs: number }[] } export let server: PanelServer @@ -195,8 +197,13 @@ const overviewView = (): ApiResponse<'GET overview'> => { ...Object.entries(server.health).filter(([name, item]) => item.state === 'error' && !KINDS.some(kind => kind.health.includes(name)) && !name.startsWith('sync:') && name !== 'evals') .map(([name, item]) => ({ kind: 'runtime-error' as const, name, detail: item.detail })), + ...server.stalled.map(item => ({ kind: 'stalled' as const, source: { id: item.id, title: item.title }, ageMs: item.ageMs, + intervalMs: item.intervalMs })), ...Object.values(server.settings.policy.sources).some(part => part.allowed.length) ? [] : [{ kind: 'no-allowed' as const }], ...server.failing.map(outgoing => ({ kind: 'outgoing' as const, outgoing }))], + // The fixture panel renders the readout, so a poll round a few seconds old stands for a running Source. + liveness: { startedAt: '2026-09-22T00:00:00Z', uptimeMs: 3_600_000, lastPollAt: new Date(Date.now() - 4_000).toISOString(), + pollAgeMs: 4_000, polling: true }, person: server.person, executor: { running: 2, queued: 0, maxWorkers: server.settings.executor.maxWorkers, idle: 1 }, flow: { received: 12, filtered: 2, triggers: 10, judged: 5, accepted: 1, evaluating: 0, dropped: server.dropped.filter(item => item.dropped.at >= dayAgo()).length }, @@ -249,7 +256,7 @@ function initial(): PanelServer { en: 'Feishu message' } } const defaults = { revision: 0, previousRevision: null, savedAt: '', source: 'defaults' } return { - health: {}, down: false, pending: 0, opened: [], requests: [], misuse: [], queued: new Set(), dropped: [], failing: [], + health: {}, down: false, pending: 0, opened: [], requests: [], misuse: [], queued: new Set(), dropped: [], failing: [], stalled: [], setup: { machine: MACHINE, problem: null, feishuProblem: null, accounts: { repository: 'acme/widgets', agent: 'agent-bot', feishu: { appId: 'cli_bot', brand: 'feishu', botId: 'ou_bot', name: 'Widgets Bot' } } }, diff --git a/tests/panel-overview.spec.tsx b/tests/panel-overview.spec.tsx index 1ac9f61..ac5475f 100644 --- a/tests/panel-overview.spec.tsx +++ b/tests/panel-overview.spec.tsx @@ -2,6 +2,7 @@ // The overview. import { expect, it } from 'vitest' import { en, zh } from '../src/client/locales.ts' +import { duration } from '../src/client/text.ts' import { usePanel, mount, remount, all, find, tab, click, text } from './fixtures/panel.tsx' import { server, droppedReply, threadSummary } from './fixtures/panel-server.ts' @@ -10,7 +11,8 @@ it('leads the overview with the pending Tickets, then problems, the last day and await mount() expect(tab(zh['tab.overview']).getAttribute('aria-selected')).toBe('true') expect(all('[role="tab"]').map(tab => tab.textContent)).toEqual(['概览', '来源', '触发', '评估', '执行', '设置']) - expect(all('.dt-section-title').map(title => title.textContent)).toEqual(['待处理工单2', '系统问题2', '过去 24 小时详情 ›', '执行器详情 ›']) + expect(all('.dt-section-title').map(title => title.textContent)).toEqual(['待处理工单2', '系统问题2', '存活', '过去 24 小时详情 ›', + '执行器详情 ›']) expect(all('.dt-cards .dt-ticket-stage').map(item => item.textContent)).toEqual(['待回答', '待处理失败']) expect(all('.dt-cards .dt-att-what').map(item => item.textContent)).toEqual(['飞书 来源异常', '评估未完成：这个文案好奇怪']) // Actions sit below the title and Threads, apart from the stage, each with its icon. @@ -18,7 +20,13 @@ it('leads the overview with the pending Tickets, then problems, the last day and && [...row.querySelectorAll('.dt-ticket-actions button, .dt-ticket-actions a')].every(action => action.querySelector('svg')))).toBe(true) expect(text()).toContain('来源会在每次轮询时自动重试') expect(text()).toContain('需要确认按钮颜色') - expect(all('.dt-stat').map(stat => stat.textContent)).toEqual(['到达12', '被过滤2', '已评估5', '接单或续做1', '执行中2 / 3', '排队0', '等待新触发1']) + // Durations are formatted by the interface language, so the readout is asserted by label, and only the polling word verbatim. + expect(all('.dt-stat-label').map(label => label.textContent)).toEqual([zh.livenessUptime, zh.livenessLastPoll, zh.livenessPolling, + zh.flowReceivedLabel, zh.flowFilteredLabel, zh.flowJudgedLabel, zh.flowAcceptedLabel, zh.executingLabel, zh.queuedLabel, + zh.idleLabel]) + expect(all('.dt-stat-value').map(value => value.textContent).slice(3)).toEqual(['12', '2', '5', '1', '2 / 3', '0', '1']) + expect(all('.dt-stat-value').map(value => value.textContent).slice(0, 3)).toEqual([expect.stringContaining('小时'), expect + .stringContaining('秒'), zh.livenessPolling]) expect(all('.dt-chip').map(chip => chip.textContent)[0]).toBe('GitHub#42CI 失败1') expect(document.querySelector('.dt-chip-problem')?.textContent).toBe('CI 失败1') expect(document.querySelector('a.dt-chip-link')?.href).toBe('https://github.com/acme/widgets/pull/42') @@ -126,10 +134,10 @@ it('words a refused retry in the interface language by its code', async () => { it('shows the overview in English', async () => { await mount(en) expect(all('.dt-section-title').map(title => title.textContent)).toEqual([`${en.pendingTickets}2`, `${en.problems}2`, - `${en.lastDay}${en.details}`, `${en['term.executor']}${en.details}`]) + en.livenessTitle, `${en.lastDay}${en.details}`, `${en['term.executor']}${en.details}`]) expect(all('.dt-cards .dt-ticket-stage').map(item => item.textContent)).toEqual([en['stage.answer'], en['stage.failed']]) - expect(all('.dt-stat-label').map(label => label.textContent)).toEqual([en.flowReceivedLabel, en.flowFilteredLabel, en.flowJudgedLabel, - en.flowAcceptedLabel, en.executingLabel, en.queuedLabel, en.idleLabel]) + expect(all('.dt-stat-label').map(label => label.textContent)).toEqual([en.livenessUptime, en.livenessLastPoll, en.livenessPolling, + en.flowReceivedLabel, en.flowFilteredLabel, en.flowJudgedLabel, en.flowAcceptedLabel, en.executingLabel, en.queuedLabel, en.idleLabel]) }) it('lists the unavailable Evals one by one, without the health entry that counts them as a failure of dsh-superagent', async () => { @@ -157,3 +165,12 @@ it('says when no Source is on, an Eval is past its deadline or a message keeps f await click('评估超时未决定：#9 迟迟未决') expect(server.requests.some(request => request.path.startsWith('threads/github:9'))).toBe(true) }) + +it('says when a Source stopped polling, with how long its last round was and the way to it', async () => { + server.stalled = [{ id: 'github', title: { zh: 'GitHub', en: 'GitHub' }, ageMs: 26 * 60_000, intervalMs: 60_000 }] + await mount() + const line = all('.dt-cards .dt-att-what').map(item => item.textContent).find(title => title?.includes('GitHub 最后一次轮询在')) + // The same duration helper the panel uses, so the assertion does not depend on the locale's spacing rules. + expect(line).toBe(zh.problemStalled.replace('{source}', 'GitHub').replace('{age}', duration(26 * 60_000, 'zh'))) + expect(text()).toContain(zh.problemStalledHint) +}) diff --git a/tests/views.spec.ts b/tests/views.spec.ts index 8056e9e..5665e89 100644 --- a/tests/views.spec.ts +++ b/tests/views.spec.ts @@ -11,7 +11,7 @@ import { snapshot, threadSummary, ticketSummary, triggerView } from '../src/app/ import { basisFacts, ticketDetail } from '../src/app/views/tickets.ts' import { threadDetail } from '../src/app/views/threads.ts' import { droppedReplies, sourcesStatus } from '../src/app/views/sources.ts' -import { overview } from '../src/app/views/overview.ts' +import { overview, pollAgeMs } from '../src/app/views/overview.ts' import { markdown, plain } from '../src/app/views/text.ts' import { policyHistory } from '../src/app/settings-history.ts' import { KV } from '../src/kv.ts' @@ -319,3 +319,39 @@ it('reads each ticket a page\'s follow-ups reached once, and nothing for a messa expect(followupOwners(store.triggers.get('f3')!, records)).toEqual([]) expect([records.tickets.get, records.threads.get].map(read => read.mock.calls.length)).toEqual([0, 0]) }) + +it('reads the age of a poll round from the newest report a Source carries, and nothing before it reports one', () => { + const now = Date.now() + expect(pollAgeMs([{}, {}])).toBeUndefined() + expect(pollAgeMs([{ at: new Date(now - 300_000).toISOString() }, { at: new Date(now - 60_000).toISOString() }])) + .toBeGreaterThanOrEqual(59_000) + expect(pollAgeMs([{ at: 'not a time' }])).toBeUndefined() +}) + +it('lists a Source whose poll loop stopped past a few of its own intervals, and never one still starting or off', () => { + const host = views() + const long = new Date(Date.now() - 20 * 60_000).toISOString() + // github polls every 60 seconds by default, so 20 minutes is well past the floor: the loop stopped. + expect(problems(host, { github: { state: 'ready', detail: '', at: long } }).filter(item => item.kind === 'stalled')) + .toEqual([{ kind: 'stalled', source: { id: 'github', title: githubKind.title }, ageMs: expect.any(Number), intervalMs: 60_000 }]) + // A Source that never reported a round is starting, not stalled; the grid an operator reads says so separately. + expect(problems(host, { github: { state: 'starting', detail: '' } }).filter(item => item.kind === 'stalled')).toEqual([]) + // A fresh round, or one off Source that is switched off, is not stalled either. + const fresh = new Date(Date.now() - 5_000).toISOString() + expect(problems(host, { github: { state: 'ready', detail: '', at: fresh } }).filter(item => item.kind === 'stalled')).toEqual([]) + store.saveSettings({ ...store.settings(), sources: { ...store.settings().sources, github: { enabled: false, config: {} } } }) + expect(problems(host, { github: { state: 'ready', detail: '', at: long } }).filter(item => item.kind === 'stalled')).toEqual([]) +}) + +it('reports whether the service is alive and polling apart from whether any Trigger arrived', () => { + const host = views() + const startedAt = new Date(Date.now() - 7_200_000).toISOString() + const polled = new Date(Date.now() - 30_000).toISOString() + const view = overview(host, { github: { state: 'ready', detail: '', at: polled } }, 'v1', load, undefined, startedAt) + expect(view.liveness).toMatchObject({ startedAt, lastPollAt: polled, polling: true }) + expect(view.liveness.uptimeMs).toBeGreaterThanOrEqual(7_190_000) + // The process can answer for hours with nothing running: no Source enables no polling, whatever the uptime says. + store.saveSettings({ ...store.settings(), sources: { github: { enabled: false, config: {} }, feishu: { enabled: false, config: {} } } }) + expect(overview(host, {}, 'v1', load, undefined, startedAt).liveness).toMatchObject({ polling: false, lastPollAt: null, pollAgeMs: null }) + expect(overview(host, {}, 'v1', load).liveness).toMatchObject({ startedAt: null, uptimeMs: null, polling: false }) +})