Overview: report liveness and a Source whose poll loop stopped
This commit is contained in:
+3
-1
@@ -44,6 +44,8 @@ export interface HttpHost {
|
||||
readonly health: Record<string, Health>
|
||||
readonly machine: Machine
|
||||
readonly masterId: string
|
||||
/** When this process started serving, read by the overview's liveness readout. */
|
||||
readonly startedAt: string
|
||||
/** Resolves once startup finishes or setup determines what is still missing. */
|
||||
readonly ready: Promise<void>
|
||||
/** Resolved deployment; throws if read while waiting for setup (`!configured()`). */
|
||||
@@ -160,7 +162,7 @@ function routes(runtime: HttpHost): Routes {
|
||||
return {
|
||||
'GET state': () => stateView(runtime),
|
||||
'GET overview': () => overview(views(runtime), runtime.health, policyVersion(store.settings()),
|
||||
executorLoad(store, runtime.executor.activeCount()), runtime.setupProblem ?? runtime.feishuProblem),
|
||||
executorLoad(store, runtime.executor.activeCount()), runtime.setupProblem ?? runtime.feishuProblem, runtime.startedAt),
|
||||
'GET search': ({ query }) => search(views(runtime), query.q ?? '', runtime.sourceLinks),
|
||||
'GET sources': () => sourcesStatus(views(runtime), runtime.health, setupState(runtime)),
|
||||
'GET settings': () => settingsView(store, runtime.sources, runtime.health, setupState(runtime), runtime.machine.stateDir),
|
||||
|
||||
+3
-1
@@ -74,6 +74,8 @@ export class Runtime extends Service implements SourceHost {
|
||||
private readonly board = new HealthBoard()
|
||||
private readonly runtimeHealth = this.board.claim('runtime', { state: 'starting', detail: '' })
|
||||
readonly masterId: string
|
||||
/** When this process started serving, reported by the overview so a restart is visible rather than inferred. */
|
||||
readonly startedAt = new Date().toISOString()
|
||||
private readonly sourceLinks = new TextCache(200_000)
|
||||
private readonly stopped = new AbortController()
|
||||
private readonly disposers: (() => void)[] = []
|
||||
@@ -220,7 +222,7 @@ export class Runtime extends Service implements SourceHost {
|
||||
const runtime = this
|
||||
const { store, sources, prompts, sessions, settings, engine, executor, health, machine, masterId, ready, sourceLinks, setup, passes } = this
|
||||
return { context: this.ctx, store, sources, prompts, sessions, settings, engine, executor, health, machine, masterId, ready, sourceLinks,
|
||||
get deployment() { return setup.deployment },
|
||||
startedAt: this.startedAt, get deployment() { return setup.deployment },
|
||||
get setupProblem() { return setup.problem }, get feishuProblem() { return setup.feishuProblem },
|
||||
configured: () => setup.configured(), links: () => runtime.links(), ticketEnvironment: ticket => runtime.ticketEnvironment(ticket),
|
||||
checkSetup: () => { setup.check() }, requestPass: () => { passes.request() }, models: () => this.ctx.sessionController.modelCatalog() }
|
||||
|
||||
@@ -22,6 +22,23 @@ function unjudged(store: Store, since: string): { thread: Thread; decision: Deci
|
||||
return [{ thread, decision }]
|
||||
})
|
||||
}
|
||||
/** A Source counts as stalled after this many of its own poll intervals without a fresh health report, but never sooner. */
|
||||
const STALL_POLLS = 3, STALL_FLOOR_MS = 180_000
|
||||
/** The interval a Source polls at, from its config; a Source without the field polls every 60 seconds. */
|
||||
function pollIntervalMs(config: Record<string, unknown> | undefined): number {
|
||||
const seconds = Number(config?.pollSeconds)
|
||||
return Number.isFinite(seconds) && seconds >= 60 ? seconds * 1000 : 60_000
|
||||
}
|
||||
/**
|
||||
* Milliseconds since a Source last finished a poll round, read from the newest `at` its health entries carry. A Source
|
||||
* reports `at` once per round, so a value that stops moving means its loop stopped: the service answers but polls nothing.
|
||||
*/
|
||||
const pollTimes = (parts: readonly { at?: string }[]): number[] =>
|
||||
parts.flatMap(part => part.at === undefined ? [] : [Date.parse(part.at)]).filter(Number.isFinite)
|
||||
export function pollAgeMs(parts: readonly { at?: string }[]): number | undefined {
|
||||
const times = pollTimes(parts)
|
||||
return times.length ? Date.now() - Math.max(...times) : undefined
|
||||
}
|
||||
/** Reports error health when any thread has an undismissed `unavailable` eval from the last hour without a newer decision or active eval. */
|
||||
export function evalsHealth(store: Store, now = Date.now()): Health {
|
||||
const since = new Date(now - 3_600_000).toISOString()
|
||||
@@ -34,8 +51,10 @@ export function evalsHealth(store: Store, now = Date.now()): Health {
|
||||
/**
|
||||
* Builds the operator overview: active system problems, tickets awaiting human action, executor load, and 24-hour trigger flow.
|
||||
* @param setup - Setup problem to surface first (where setup stopped, or a Source paused by setup).
|
||||
* @param startedAt - When this process started serving, for the liveness readout.
|
||||
*/
|
||||
export function overview(host: ViewHost, health: HealthMap, policyVersionValue: string, load: ExecutorLoad, setup?: SetupProblem) {
|
||||
export function overview(host: ViewHost, health: HealthMap, policyVersionValue: string, load: ExecutorLoad, setup?: SetupProblem,
|
||||
startedAt?: string) {
|
||||
const { store, sources } = host
|
||||
const view = snapshot(host)
|
||||
const settings = store.settings()
|
||||
@@ -45,12 +64,25 @@ export function overview(host: ViewHost, health: HealthMap, policyVersionValue:
|
||||
const running = instances.filter(instance => instance.enabled && instance.installed).map(instance => sourceHealth(sources, instance, health))
|
||||
// Exclude source-owned and policy-sync health entries so only core runtime errors surface as `runtime-error`.
|
||||
const sourceParts = new Set(instances.flatMap(instance => sources.kind(instance.kind)?.health ?? []))
|
||||
const configOf = new Map(instances.map(instance => [instance.id, instance.config]))
|
||||
/**
|
||||
* A Source that answers but stopped polling: its last report is older than a few of its own intervals. Only Sources
|
||||
* that have reported at least one round are judged, so a Source still starting up is never called stalled.
|
||||
*/
|
||||
const stalled = running.flatMap(source => {
|
||||
const age = pollAgeMs(source.health)
|
||||
if (age === undefined) return []
|
||||
const interval = pollIntervalMs(configOf.get(source.id))
|
||||
if (age <= Math.max(STALL_POLLS * interval, STALL_FLOOR_MS)) return []
|
||||
return [{ kind: 'stalled' as const, source: { id: source.id, title: source.title }, ageMs: age, intervalMs: interval }]
|
||||
})
|
||||
const problems = [
|
||||
...(setup ? [{ kind: 'setup' as const, ...setup }] : []),
|
||||
...(running.length ? [] : [{ kind: 'no-source' as const }]),
|
||||
// Only allowed senders can originate work, so empty allowlists mean no request can ever become a ticket.
|
||||
...(anyoneAllowed(settings) ? [] : [{ kind: 'no-allowed' as const }]),
|
||||
...running.flatMap(source => source.health.filter(item => item.state === 'error').map(item => ({ kind: 'source-error' as const, source: { id: source.id, title: source.title, health: item } }))),
|
||||
...stalled,
|
||||
...(settings.paused ? [{ kind: 'executor-paused' as const }] : []),
|
||||
// Unavailable Evals behind the `evals` health entry are listed individually as `eval-unavailable` below.
|
||||
...Object.entries(health).filter(([name, item]) => item.state === 'error' && !sourceParts.has(name) && !name.startsWith('sync:')
|
||||
@@ -68,8 +100,22 @@ export function overview(host: ViewHost, health: HealthMap, policyVersionValue:
|
||||
const stats = Object.values(store.read.triggerStats(since))
|
||||
const received = stats.reduce((sum, item) => sum + item.received, 0), filtered = stats.reduce((sum, item) => sum + item.filtered, 0)
|
||||
const judgement = store.read.flow(since)
|
||||
/**
|
||||
* Whether the service is alive is one question; whether it is doing anything is another. Restarts are invisible in the
|
||||
* flow counters, and a dead poll loop answers every request, so both are reported explicitly.
|
||||
*/
|
||||
const polled = running.flatMap(source => source.health)
|
||||
const newestPoll = pollTimes(polled)
|
||||
const liveness = {
|
||||
startedAt: startedAt ?? null,
|
||||
uptimeMs: startedAt === undefined ? null : Date.now() - Date.parse(startedAt),
|
||||
lastPollAt: newestPoll.length ? new Date(Math.max(...newestPoll)).toISOString() : null,
|
||||
pollAgeMs: pollAgeMs(polled) ?? null,
|
||||
polling: running.length > 0 && newestPoll.length > 0 && !stalled.length,
|
||||
}
|
||||
return {
|
||||
problems,
|
||||
liveness,
|
||||
person: open.filter(ticket => ticketStage(ticket).stage === 'person').map(ticket => ticketRow(view, ticket)),
|
||||
executor: { ...load, maxWorkers: settings.executor.maxWorkers, idle: open.filter(ticket => ticketStage(ticket).stage === 'idle').length },
|
||||
flow: { received, filtered, triggers: received - filtered, judged: judgement.judged, accepted: judgement.accepted,
|
||||
|
||||
@@ -128,6 +128,15 @@ const TEXT = {
|
||||
problemReply: ['{source} 回复发送失败(已重试 {attempts} 次)', '{source} reply failing (retried {attempts} times)'],
|
||||
problemRuntime: ['运行异常:{name}', '{name} failing'],
|
||||
problemEvalUnavailable: ['评估未完成:{thread}', 'Eval unavailable: {thread}'],
|
||||
problemStalled: ['{source} 最后一次轮询在 {age},此后收不到新触发', '{source} last polled {age}: the service answers, but no Trigger arrives'],
|
||||
problemStalledHint: ['轮询循环已经停了。先看日志有没有报错;没有报错时通常是进程虽在、事件循环被卡住或被挂起。',
|
||||
'The poll loop stopped. Check the log for errors first; without one the process is usually alive with its event loop blocked or suspended.'],
|
||||
livenessTitle: ['存活', 'Liveness'],
|
||||
livenessUptime: ['已运行', 'Uptime'],
|
||||
livenessLastPoll: ['最近轮询', 'Last poll'],
|
||||
livenessPolling: ['在轮询', 'Polling'],
|
||||
livenessNotPolling: ['不在轮询', 'Not polling'],
|
||||
livenessNever: ['尚无', 'Never'],
|
||||
sourceErrorHint: ['来源会在每次轮询时自动重试。持续出现时检查网络和该来源的登录状态。', 'The Source retries on every poll. If this persists, check the network and the Source\'s sign-in.'],
|
||||
details: ['详情 ›', 'Details ›'],
|
||||
lastDay: ['过去 24 小时', 'Last 24 hours'],
|
||||
|
||||
+12
-1
@@ -5,7 +5,7 @@ import { Button, Icon } from './controls.tsx'
|
||||
import { useAction, usePanel, usePanelData } from './provider.tsx'
|
||||
import { setupPage, settingsRoute, type Route } from './route.ts'
|
||||
import { TicketLine } from './rows.tsx'
|
||||
import { healthText, localized, reasonLabel, valueLabel, readableMarkers } from './text.ts'
|
||||
import { duration, healthText, localized, reasonLabel, valueLabel, readableMarkers } from './text.ts'
|
||||
import { CheckSetupButton, ConfirmButton, Hint, LoadState, Section, SetupMessage } from './widgets.tsx'
|
||||
|
||||
function ProblemLine({ problem, onChanged }: { problem: Problem; onChanged: () => Promise<void> }) {
|
||||
@@ -52,6 +52,8 @@ function ProblemLine({ problem, onChanged }: { problem: Problem; onChanged: () =
|
||||
<Button variant="outline" size="sm" icon={<Icon name="dismiss" size={14} />} disabled={post.busy}
|
||||
onClick={() => { void post.run(() => call('POST decisions/:id/dismiss', { id: problem.decisionId })) }}>{t('dismiss')}</Button>
|
||||
</>)
|
||||
case 'stalled': return row(t('problemStalled', { source: localized(t, problem.source.title),
|
||||
age: duration(problem.ageMs, t('lang')) }), undefined, { page: 'source' }, t('problemStalledHint'))
|
||||
case 'outgoing': return row(t('problemReply', { source: valueLabel(t, 'source', problem.outgoing.channel.source),
|
||||
attempts: problem.outgoing.attempts }),
|
||||
raw(problem.outgoing.error), undefined)
|
||||
@@ -81,6 +83,15 @@ export function OverviewPage() {
|
||||
<ul className="dt-list dt-cards">{data.problems.map((problem, index) => <ProblemLine key={index} problem={problem}
|
||||
onChanged={reload} />)}</ul>
|
||||
</Section>}
|
||||
<Section title={t('livenessTitle')}>
|
||||
{/* A stopped poll loop answers every request, so the readout sits on the overview rather than behind a diagnostics page. */}
|
||||
<div className="dt-stats">
|
||||
{stat(t('livenessUptime'), duration(data.liveness.uptimeMs, t('lang')) || t('livenessNever'))}
|
||||
{stat(t('livenessLastPoll'), data.liveness.lastPollAt
|
||||
? duration(data.liveness.pollAgeMs, t('lang')) : t('livenessNever'))}
|
||||
{stat(t('livenessPolling'), data.liveness.polling ? t('livenessPolling') : t('livenessNotPolling'))}
|
||||
</div>
|
||||
</Section>
|
||||
<Section title={t('lastDay')} action={more({ page: 'trigger', query: { filter: 'recent' } })}>
|
||||
{/* Received and evaluated counts operate on Triggers rather than Threads or Decisions, so they do not link to filtered views. */}
|
||||
<div className="dt-stats">
|
||||
|
||||
@@ -57,6 +57,15 @@ export function when(value: string | undefined, lang: string): string {
|
||||
const year = sameYear ? {} : { year: 'numeric' } as const
|
||||
return date.toLocaleString(timeLocale(lang), { ...year, month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' })
|
||||
}
|
||||
/** A span of milliseconds as a short localized phrase, for readouts such as uptime and time since the last poll. */
|
||||
export function duration(ms: number | null | undefined, lang: string): string {
|
||||
if (ms === null || ms === undefined || !Number.isFinite(ms) || ms < 0) return ''
|
||||
const relative = new Intl.RelativeTimeFormat(timeLocale(lang), { numeric: 'auto' })
|
||||
if (ms < 60_000) return relative.format(-Math.round(ms / 1000), 'second')
|
||||
if (ms < 3_600_000) return relative.format(-Math.round(ms / 60_000), 'minute')
|
||||
if (ms < 86_400_000) return relative.format(-Math.round(ms / 3_600_000), 'hour')
|
||||
return relative.format(-Math.round(ms / 86_400_000), 'day')
|
||||
}
|
||||
export function fullTime(value: string | undefined, lang: string): string {
|
||||
const date = new Date(value ?? '')
|
||||
return Number.isNaN(date.getTime()) ? '' : date.toLocaleString(timeLocale(lang))
|
||||
|
||||
@@ -42,7 +42,8 @@ const openMaster = async (container: HTMLElement) => {
|
||||
}
|
||||
const serve = () => vi.stubGlobal('fetch', vi.fn(async (url: string) => {
|
||||
const path = new URL(url, 'http://127.0.0.1').pathname.replace('/orchestrator/api/', '')
|
||||
if (path === 'overview') return Response.json({ problems: [], person: [], executor: { running: 0, queued: 0, maxWorkers: 3, idle: 0 }, flow: { triggers: 0, filtered: 0, judged: 0, accepted: 0, evaluating: 0 }, policyVersion: 'v' })
|
||||
if (path === 'overview') return Response.json({ problems: [], liveness: { startedAt: null, uptimeMs: null, lastPollAt: null, pollAgeMs: null, polling: false }, person: [],
|
||||
executor: { running: 0, queued: 0, maxWorkers: 3, idle: 0 }, flow: { triggers: 0, filtered: 0, judged: 0, accepted: 0, evaluating: 0 }, policyVersion: 'v' })
|
||||
if (path === 'state') return Response.json({ masterId: 'master', settings: SettingsSchema.parse({}), revisions: { policy: 0, eval: 0, executor: 0 }, sources: [], policy_version: 'v', health: {}, outgoing: [], load: { running: 0, queued: 0 },
|
||||
setup: { machine: { stateDir: '/srv/state', worktreeRoot: '/srv/state/worktrees', blockedCommands: [], promptMaxChars: 100000, admissionContextMaxPages: 10, admissionContextTimeoutMs: 15000 }, problem: null, feishuProblem: null, accounts: null } })
|
||||
return Response.json({ error: 'unavailable' }, { status: 503 })
|
||||
|
||||
Vendored
+9
-2
@@ -20,7 +20,7 @@ import { githubChannel } from '../../src/sources/github-trigger.ts'
|
||||
import { feishuKind } from '../../src/sources/feishu.ts'
|
||||
import { githubKind } from '../../src/sources/github.ts'
|
||||
import type { Health } from '../../src/sources/host.ts'
|
||||
import type { SourceKind } from '../../src/sources/kind.ts'
|
||||
import type { Localized, SourceKind } from '../../src/sources/kind.ts'
|
||||
import { testDeployment } from './deployment.ts'
|
||||
|
||||
export const MEMBER = 'ou_alice'
|
||||
@@ -110,6 +110,8 @@ export interface PanelServer {
|
||||
opened: string[]
|
||||
dropped: DroppedReply[]
|
||||
failing: Extract<Problem, { kind: 'outgoing' }>['outgoing'][]
|
||||
/** Sources whose poll loop the fixture reports as stopped, with how long ago their last round was. */
|
||||
stalled: { id: string; title: Localized; ageMs: number; intervalMs: number }[]
|
||||
}
|
||||
export let server: PanelServer
|
||||
|
||||
@@ -195,8 +197,13 @@ const overviewView = (): ApiResponse<'GET overview'> => {
|
||||
...Object.entries(server.health).filter(([name, item]) => item.state === 'error' && !KINDS.some(kind => kind.health.includes(name))
|
||||
&& !name.startsWith('sync:') && name !== 'evals')
|
||||
.map(([name, item]) => ({ kind: 'runtime-error' as const, name, detail: item.detail })),
|
||||
...server.stalled.map(item => ({ kind: 'stalled' as const, source: { id: item.id, title: item.title }, ageMs: item.ageMs,
|
||||
intervalMs: item.intervalMs })),
|
||||
...Object.values(server.settings.policy.sources).some(part => part.allowed.length) ? [] : [{ kind: 'no-allowed' as const }],
|
||||
...server.failing.map(outgoing => ({ kind: 'outgoing' as const, outgoing }))],
|
||||
// The fixture panel renders the readout, so a poll round a few seconds old stands for a running Source.
|
||||
liveness: { startedAt: '2026-09-22T00:00:00Z', uptimeMs: 3_600_000, lastPollAt: new Date(Date.now() - 4_000).toISOString(),
|
||||
pollAgeMs: 4_000, polling: true },
|
||||
person: server.person, executor: { running: 2, queued: 0, maxWorkers: server.settings.executor.maxWorkers, idle: 1 },
|
||||
flow: { received: 12, filtered: 2, triggers: 10, judged: 5, accepted: 1, evaluating: 0,
|
||||
dropped: server.dropped.filter(item => item.dropped.at >= dayAgo()).length },
|
||||
@@ -249,7 +256,7 @@ function initial(): PanelServer {
|
||||
en: 'Feishu message' } }
|
||||
const defaults = { revision: 0, previousRevision: null, savedAt: '', source: 'defaults' }
|
||||
return {
|
||||
health: {}, down: false, pending: 0, opened: [], requests: [], misuse: [], queued: new Set(), dropped: [], failing: [],
|
||||
health: {}, down: false, pending: 0, opened: [], requests: [], misuse: [], queued: new Set(), dropped: [], failing: [], stalled: [],
|
||||
setup: { machine: MACHINE, problem: null, feishuProblem: null,
|
||||
accounts: { repository: 'acme/widgets', agent: 'agent-bot', feishu: { appId: 'cli_bot', brand: 'feishu', botId: 'ou_bot',
|
||||
name: 'Widgets Bot' } } },
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// The overview.
|
||||
import { expect, it } from 'vitest'
|
||||
import { en, zh } from '../src/client/locales.ts'
|
||||
import { duration } from '../src/client/text.ts'
|
||||
import { usePanel, mount, remount, all, find, tab, click, text } from './fixtures/panel.tsx'
|
||||
import { server, droppedReply, threadSummary } from './fixtures/panel-server.ts'
|
||||
|
||||
@@ -10,7 +11,8 @@ it('leads the overview with the pending Tickets, then problems, the last day and
|
||||
await mount()
|
||||
expect(tab(zh['tab.overview']).getAttribute('aria-selected')).toBe('true')
|
||||
expect(all('[role="tab"]').map(tab => tab.textContent)).toEqual(['概览', '来源', '触发', '评估', '执行', '设置'])
|
||||
expect(all('.dt-section-title').map(title => title.textContent)).toEqual(['待处理工单2', '系统问题2', '过去 24 小时详情 ›', '执行器详情 ›'])
|
||||
expect(all('.dt-section-title').map(title => title.textContent)).toEqual(['待处理工单2', '系统问题2', '存活', '过去 24 小时详情 ›',
|
||||
'执行器详情 ›'])
|
||||
expect(all('.dt-cards .dt-ticket-stage').map(item => item.textContent)).toEqual(['待回答', '待处理失败'])
|
||||
expect(all('.dt-cards .dt-att-what').map(item => item.textContent)).toEqual(['飞书 来源异常', '评估未完成:这个文案好奇怪'])
|
||||
// Actions sit below the title and Threads, apart from the stage, each with its icon.
|
||||
@@ -18,7 +20,13 @@ it('leads the overview with the pending Tickets, then problems, the last day and
|
||||
&& [...row.querySelectorAll('.dt-ticket-actions button, .dt-ticket-actions a')].every(action => action.querySelector('svg')))).toBe(true)
|
||||
expect(text()).toContain('来源会在每次轮询时自动重试')
|
||||
expect(text()).toContain('需要确认按钮颜色')
|
||||
expect(all('.dt-stat').map(stat => stat.textContent)).toEqual(['到达12', '被过滤2', '已评估5', '接单或续做1', '执行中2 / 3', '排队0', '等待新触发1'])
|
||||
// Durations are formatted by the interface language, so the readout is asserted by label, and only the polling word verbatim.
|
||||
expect(all('.dt-stat-label').map(label => label.textContent)).toEqual([zh.livenessUptime, zh.livenessLastPoll, zh.livenessPolling,
|
||||
zh.flowReceivedLabel, zh.flowFilteredLabel, zh.flowJudgedLabel, zh.flowAcceptedLabel, zh.executingLabel, zh.queuedLabel,
|
||||
zh.idleLabel])
|
||||
expect(all('.dt-stat-value').map(value => value.textContent).slice(3)).toEqual(['12', '2', '5', '1', '2 / 3', '0', '1'])
|
||||
expect(all('.dt-stat-value').map(value => value.textContent).slice(0, 3)).toEqual([expect.stringContaining('小时'), expect
|
||||
.stringContaining('秒'), zh.livenessPolling])
|
||||
expect(all('.dt-chip').map(chip => chip.textContent)[0]).toBe('GitHub#42CI 失败1')
|
||||
expect(document.querySelector('.dt-chip-problem')?.textContent).toBe('CI 失败1')
|
||||
expect(document.querySelector<HTMLAnchorElement>('a.dt-chip-link')?.href).toBe('https://github.com/acme/widgets/pull/42')
|
||||
@@ -126,10 +134,10 @@ it('words a refused retry in the interface language by its code', async () => {
|
||||
it('shows the overview in English', async () => {
|
||||
await mount(en)
|
||||
expect(all('.dt-section-title').map(title => title.textContent)).toEqual([`${en.pendingTickets}2`, `${en.problems}2`,
|
||||
`${en.lastDay}${en.details}`, `${en['term.executor']}${en.details}`])
|
||||
en.livenessTitle, `${en.lastDay}${en.details}`, `${en['term.executor']}${en.details}`])
|
||||
expect(all('.dt-cards .dt-ticket-stage').map(item => item.textContent)).toEqual([en['stage.answer'], en['stage.failed']])
|
||||
expect(all('.dt-stat-label').map(label => label.textContent)).toEqual([en.flowReceivedLabel, en.flowFilteredLabel, en.flowJudgedLabel,
|
||||
en.flowAcceptedLabel, en.executingLabel, en.queuedLabel, en.idleLabel])
|
||||
expect(all('.dt-stat-label').map(label => label.textContent)).toEqual([en.livenessUptime, en.livenessLastPoll, en.livenessPolling,
|
||||
en.flowReceivedLabel, en.flowFilteredLabel, en.flowJudgedLabel, en.flowAcceptedLabel, en.executingLabel, en.queuedLabel, en.idleLabel])
|
||||
})
|
||||
|
||||
it('lists the unavailable Evals one by one, without the health entry that counts them as a failure of dsh-superagent', async () => {
|
||||
@@ -157,3 +165,12 @@ it('says when no Source is on, an Eval is past its deadline or a message keeps f
|
||||
await click('评估超时未决定:#9 迟迟未决')
|
||||
expect(server.requests.some(request => request.path.startsWith('threads/github:9'))).toBe(true)
|
||||
})
|
||||
|
||||
it('says when a Source stopped polling, with how long its last round was and the way to it', async () => {
|
||||
server.stalled = [{ id: 'github', title: { zh: 'GitHub', en: 'GitHub' }, ageMs: 26 * 60_000, intervalMs: 60_000 }]
|
||||
await mount()
|
||||
const line = all('.dt-cards .dt-att-what').map(item => item.textContent).find(title => title?.includes('GitHub 最后一次轮询在'))
|
||||
// The same duration helper the panel uses, so the assertion does not depend on the locale's spacing rules.
|
||||
expect(line).toBe(zh.problemStalled.replace('{source}', 'GitHub').replace('{age}', duration(26 * 60_000, 'zh')))
|
||||
expect(text()).toContain(zh.problemStalledHint)
|
||||
})
|
||||
|
||||
+37
-1
@@ -11,7 +11,7 @@ import { snapshot, threadSummary, ticketSummary, triggerView } from '../src/app/
|
||||
import { basisFacts, ticketDetail } from '../src/app/views/tickets.ts'
|
||||
import { threadDetail } from '../src/app/views/threads.ts'
|
||||
import { droppedReplies, sourcesStatus } from '../src/app/views/sources.ts'
|
||||
import { overview } from '../src/app/views/overview.ts'
|
||||
import { overview, pollAgeMs } from '../src/app/views/overview.ts'
|
||||
import { markdown, plain } from '../src/app/views/text.ts'
|
||||
import { policyHistory } from '../src/app/settings-history.ts'
|
||||
import { KV } from '../src/kv.ts'
|
||||
@@ -319,3 +319,39 @@ it('reads each ticket a page\'s follow-ups reached once, and nothing for a messa
|
||||
expect(followupOwners(store.triggers.get('f3')!, records)).toEqual([])
|
||||
expect([records.tickets.get, records.threads.get].map(read => read.mock.calls.length)).toEqual([0, 0])
|
||||
})
|
||||
|
||||
it('reads the age of a poll round from the newest report a Source carries, and nothing before it reports one', () => {
|
||||
const now = Date.now()
|
||||
expect(pollAgeMs([{}, {}])).toBeUndefined()
|
||||
expect(pollAgeMs([{ at: new Date(now - 300_000).toISOString() }, { at: new Date(now - 60_000).toISOString() }]))
|
||||
.toBeGreaterThanOrEqual(59_000)
|
||||
expect(pollAgeMs([{ at: 'not a time' }])).toBeUndefined()
|
||||
})
|
||||
|
||||
it('lists a Source whose poll loop stopped past a few of its own intervals, and never one still starting or off', () => {
|
||||
const host = views()
|
||||
const long = new Date(Date.now() - 20 * 60_000).toISOString()
|
||||
// github polls every 60 seconds by default, so 20 minutes is well past the floor: the loop stopped.
|
||||
expect(problems(host, { github: { state: 'ready', detail: '', at: long } }).filter(item => item.kind === 'stalled'))
|
||||
.toEqual([{ kind: 'stalled', source: { id: 'github', title: githubKind.title }, ageMs: expect.any(Number), intervalMs: 60_000 }])
|
||||
// A Source that never reported a round is starting, not stalled; the grid an operator reads says so separately.
|
||||
expect(problems(host, { github: { state: 'starting', detail: '' } }).filter(item => item.kind === 'stalled')).toEqual([])
|
||||
// A fresh round, or one off Source that is switched off, is not stalled either.
|
||||
const fresh = new Date(Date.now() - 5_000).toISOString()
|
||||
expect(problems(host, { github: { state: 'ready', detail: '', at: fresh } }).filter(item => item.kind === 'stalled')).toEqual([])
|
||||
store.saveSettings({ ...store.settings(), sources: { ...store.settings().sources, github: { enabled: false, config: {} } } })
|
||||
expect(problems(host, { github: { state: 'ready', detail: '', at: long } }).filter(item => item.kind === 'stalled')).toEqual([])
|
||||
})
|
||||
|
||||
it('reports whether the service is alive and polling apart from whether any Trigger arrived', () => {
|
||||
const host = views()
|
||||
const startedAt = new Date(Date.now() - 7_200_000).toISOString()
|
||||
const polled = new Date(Date.now() - 30_000).toISOString()
|
||||
const view = overview(host, { github: { state: 'ready', detail: '', at: polled } }, 'v1', load, undefined, startedAt)
|
||||
expect(view.liveness).toMatchObject({ startedAt, lastPollAt: polled, polling: true })
|
||||
expect(view.liveness.uptimeMs).toBeGreaterThanOrEqual(7_190_000)
|
||||
// The process can answer for hours with nothing running: no Source enables no polling, whatever the uptime says.
|
||||
store.saveSettings({ ...store.settings(), sources: { github: { enabled: false, config: {} }, feishu: { enabled: false, config: {} } } })
|
||||
expect(overview(host, {}, 'v1', load, undefined, startedAt).liveness).toMatchObject({ polling: false, lastPollAt: null, pollAgeMs: null })
|
||||
expect(overview(host, {}, 'v1', load).liveness).toMatchObject({ startedAt: null, uptimeMs: null, polling: false })
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user