Add per-node health checks with status indicators and scheduled probes.

This commit is contained in:
2026-07-19 22:40:21 +02:00
parent 5307ba1a7b
commit aac9e82766
20 changed files with 1674 additions and 65 deletions
+88 -1
View File
@@ -454,11 +454,98 @@
.node-list-heading-main {
display: flex;
flex-wrap: wrap;
align-items: baseline;
align-items: center;
gap: 0.5rem 1rem;
min-width: 0;
}
.node-health-dot {
display: inline-block;
width: 0.55rem;
height: 0.55rem;
border-radius: 50%;
flex-shrink: 0;
vertical-align: middle;
box-shadow: inset 0 0 0 1px color-mix(in srgb, currentColor 25%, transparent);
}
.node-health-dot-ok {
background: var(--ctp-green);
color: var(--ctp-green);
}
.node-health-dot-error {
background: var(--ctp-red);
color: var(--ctp-red);
}
.node-health-dot-unknown {
background: var(--ctp-overlay0);
color: var(--ctp-overlay0);
}
.node-healthcheck-tab {
display: flex;
flex-direction: column;
gap: 1rem;
}
.node-health-interval {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 0.5rem 0.75rem;
margin: 0.55rem 0 0.35rem;
}
.node-health-interval-label {
display: inline-flex;
flex-wrap: wrap;
align-items: center;
gap: 0.4rem 0.5rem;
font-size: 0.85rem;
color: var(--ctp-subtext0);
}
.node-health-interval-input {
width: 5.5rem;
padding: 0.3rem 0.45rem;
border: 1px solid var(--ctp-surface1);
border-radius: 6px;
background: var(--ctp-surface0);
color: var(--ctp-text);
font: inherit;
}
.node-health-interval-unit {
padding: 0.3rem 0.45rem;
border: 1px solid var(--ctp-surface1);
border-radius: 6px;
background: var(--ctp-surface0);
color: var(--ctp-text);
font: inherit;
}
.node-health-interval-save {
padding: 0.3rem 0.7rem;
border: 1px solid var(--ctp-surface1);
border-radius: 6px;
background: var(--ctp-surface0);
color: var(--ctp-text);
font: inherit;
font-size: 0.85rem;
cursor: pointer;
}
.node-health-interval-save:hover:not(:disabled) {
border-color: var(--ctp-overlay0);
}
.node-health-interval-save:disabled {
opacity: 0.65;
cursor: not-allowed;
}
.node-edit {
flex-shrink: 0;
margin-left: auto;
+97 -45
View File
@@ -29,7 +29,7 @@ import {
patchAction,
patchUser,
reauth,
testNodeSSH,
runNodeHealthCheck,
type AccessMode,
type Action,
type ActionEnvVar,
@@ -76,6 +76,7 @@ import { LoginPage } from './LoginPage'
import { NodeDetailPanel } from './NodeDetailPanel'
import {
countAuthFailuresToday,
countNodeHealthErrorsBySection,
countNodesBySection,
type NodeSectionCounts,
} from './sidebarCounts'
@@ -2407,6 +2408,26 @@ function resourceTabsFor(section: ResourceSectionId): ReadonlyArray<{
]
}
function healthDotClass(healthOk: boolean | null | undefined): string {
if (healthOk === true) {
return 'node-health-dot node-health-dot-ok'
}
if (healthOk === false) {
return 'node-health-dot node-health-dot-error'
}
return 'node-health-dot node-health-dot-unknown'
}
function healthDotLabel(healthOk: boolean | null | undefined): string {
if (healthOk === true) {
return 'Healthy'
}
if (healthOk === false) {
return 'Unhealthy'
}
return 'Health unknown'
}
function ResourceOverviewTab({
section,
canRead,
@@ -2421,10 +2442,7 @@ function ResourceOverviewTab({
const [nodes, setNodes] = useState<ReadonlyArray<Node>>([])
const [isLoading, setIsLoading] = useState(true)
const [loadError, setLoadError] = useState<string | null>(null)
const [testingNodeId, setTestingNodeId] = useState<string | null>(null)
const [testResults, setTestResults] = useState(
{} as Record<string, { ok: boolean; message: string }>,
)
const [checkingNodeId, setCheckingNodeId] = useState<string | null>(null)
const [me, setMe] = useState<MeResponse | null>(null)
const [pendingDeleteNodeId, setPendingDeleteNodeId] = useState<string | null>(
null,
@@ -2491,29 +2509,29 @@ function ResourceOverviewTab({
}
}, [section, canRead])
async function handleTestSSH(nodeId: string) {
setTestingNodeId(nodeId)
setTestResults((previous) => {
const next = { ...previous }
delete next[nodeId]
return next
})
async function handleHealthCheck(nodeId: string) {
setCheckingNodeId(nodeId)
try {
const result = await testNodeSSH(nodeId)
setTestResults((previous) => ({
...previous,
[nodeId]: { ok: result.ok, message: result.message },
}))
const response = await runNodeHealthCheck(nodeId)
setNodes((previous) =>
previous.map((node) => (node.id === nodeId ? response.node : node)),
)
} catch (err) {
setTestResults((previous) => ({
...previous,
[nodeId]: {
ok: false,
message: err instanceof Error ? err.message : 'SSH test failed',
},
}))
setNodes((previous) =>
previous.map((node) => {
if (node.id !== nodeId) {
return node
}
return {
...node,
health_ok: false,
health_message:
err instanceof Error ? err.message : 'Health check failed',
}
}),
)
} finally {
setTestingNodeId(null)
setCheckingNodeId(null)
}
}
@@ -2571,11 +2589,16 @@ function ResourceOverviewTab({
) : null}
<ul className="node-list">
{nodes.map((node) => {
const testResult = testResults[node.id]
return (
<li key={node.id} className="node-list-item">
<div className="node-list-heading">
<div className="node-list-heading-main">
<span
className={healthDotClass(node.health_ok)}
title={healthDotLabel(node.health_ok)}
aria-label={healthDotLabel(node.health_ok)}
role="img"
/>
<strong>{node.name}</strong>
<code className="node-id">{node.id}</code>
</div>
@@ -2584,7 +2607,13 @@ function ResourceOverviewTab({
className="node-edit"
onClick={() => {
navigateTo(
pathFor(section, 'overview', 'overview', node.id, 'actions'),
pathFor(
section,
'overview',
'overview',
node.id,
'healthcheck',
),
)
}}
>
@@ -2595,20 +2624,37 @@ function ResourceOverviewTab({
{node.host_ip} · {node.username} · group {node.group_name} ·{' '}
{node.key_algo}
</p>
{node.health_message ? (
<p
className={
node.health_ok
? 'node-test-result node-test-result-ok'
: node.health_ok === false
? 'node-test-result node-test-result-fail'
: 'node-test-result'
}
role="status"
>
{node.health_message}
{node.health_last_checked_at
? ` · ${new Date(node.health_last_checked_at).toLocaleString()}`
: ''}
</p>
) : null}
{canExec || canDelete ? (
<div className="node-actions">
{canExec ? (
<button
type="button"
className="node-test-ssh"
disabled={testingNodeId === node.id}
disabled={checkingNodeId === node.id}
onClick={() => {
void handleTestSSH(node.id)
void handleHealthCheck(node.id)
}}
>
{testingNodeId === node.id
? 'Testing SSH…'
: 'Test SSH connection'}
{checkingNodeId === node.id
? 'Checking health…'
: 'Health check'}
</button>
) : null}
{canDelete ? (
@@ -2622,19 +2668,6 @@ function ResourceOverviewTab({
Delete
</button>
) : null}
{testResult ? (
<p
className={
testResult.ok
? 'node-test-result node-test-result-ok'
: 'node-test-result node-test-result-fail'
}
role="status"
>
{testResult.ok ? 'Succeeded: ' : 'Failed: '}
{testResult.message}
</p>
) : null}
</div>
) : null}
</li>
@@ -3211,6 +3244,7 @@ function LogsPanel() {
{ id: 'update', label: 'update' },
{ id: 'delete', label: 'delete' },
{ id: 'test_ssh', label: 'test_ssh' },
{ id: 'health_check', label: 'health_check' },
]
const authOutcomeFilters: ReadonlyArray<{
@@ -3812,6 +3846,8 @@ function App() {
const [canReadLogs, setCanReadLogs] = useState(false)
const [nodeSectionCounts, setNodeSectionCounts] =
useState<NodeSectionCounts | null>(null)
const [nodeHealthErrorCounts, setNodeHealthErrorCounts] =
useState<NodeSectionCounts | null>(null)
const [authFailuresToday, setAuthFailuresToday] = useState<number | null>(
null,
)
@@ -3893,10 +3929,14 @@ function App() {
})
if (!isCancelled) {
setNodeSectionCounts(countNodesBySection(nodesResponse.nodes))
setNodeHealthErrorCounts(
countNodeHealthErrorsBySection(nodesResponse.nodes),
)
}
} catch {
if (!isCancelled) {
setNodeSectionCounts(null)
setNodeHealthErrorCounts(null)
}
}
@@ -4099,6 +4139,10 @@ function App() {
isResourceSectionId(item.id) && nodeSectionCounts !== null
? nodeSectionCounts[item.id]
: null
const healthErrorCount =
isResourceSectionId(item.id) && nodeHealthErrorCounts !== null
? nodeHealthErrorCounts[item.id]
: null
return (
<button
key={item.id}
@@ -4112,6 +4156,14 @@ function App() {
onClick={() => goToSection(item.id)}
>
<span className="nav-item-label">{item.label}</span>
{healthErrorCount !== null && healthErrorCount > 0 ? (
<span
className="nav-count-badge nav-count-badge-alert"
aria-label={`${healthErrorCount} unhealthy`}
>
{healthErrorCount}
</span>
) : null}
{sectionCount !== null ? (
<span className="nav-count-badge" aria-hidden="true">
{sectionCount}
+272 -1
View File
@@ -22,10 +22,12 @@ import {
fetchNode,
patchActionGroup,
patchActionGroupItem,
patchNode,
reorderActionGroupItems,
resolveNodeCommands,
runActionGroup,
runActionGroupItem,
runNodeHealthCheck,
} from './api/client'
import {
collectRelativeCommandNames,
@@ -45,6 +47,7 @@ const WEEKDAY_LABELS = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'] as cons
const NODE_DETAIL_TABS: ReadonlyArray<{ id: NodeDetailTabId; label: string }> = [
{ id: 'overview', label: 'Overview' },
{ id: 'healthcheck', label: 'Healthcheck' },
{ id: 'actions', label: 'Actions' },
{ id: 'logs', label: 'Logs' },
]
@@ -325,6 +328,8 @@ export function NodeDetailPanel({
const canDelete = Boolean(me?.permissions?.includes('actions.delete'))
const canRun = Boolean(me?.permissions?.includes('actions.execute'))
const canReadLogs = Boolean(me?.permissions?.includes('logs.read'))
const canExecNodes = Boolean(me?.permissions?.includes('nodes.exec'))
const canUpdateNodes = Boolean(me?.permissions?.includes('nodes.update'))
return (
<div className="config-panel node-detail-panel">
@@ -367,6 +372,14 @@ export function NodeDetailPanel({
{activeTab === 'overview' ? (
<NodeDetailOverviewTab node={node} />
) : null}
{activeTab === 'healthcheck' ? (
<NodeHealthcheckTab
node={node}
canExec={canExecNodes}
canUpdate={canUpdateNodes}
onNodeUpdated={setNode}
/>
) : null}
{activeTab === 'actions' ? (
<NodeActionsTab
nodeId={node.id}
@@ -385,12 +398,63 @@ export function NodeDetailPanel({
)
}
function healthDotClass(healthOk: boolean | null | undefined): string {
if (healthOk === true) {
return 'node-health-dot node-health-dot-ok'
}
if (healthOk === false) {
return 'node-health-dot node-health-dot-error'
}
return 'node-health-dot node-health-dot-unknown'
}
function healthDotLabel(healthOk: boolean | null | undefined): string {
if (healthOk === true) {
return 'Healthy'
}
if (healthOk === false) {
return 'Unhealthy'
}
return 'Health unknown'
}
function intervalToFormValues(intervalSeconds: number): {
value: number
unit: 'seconds' | 'minutes'
} {
if (intervalSeconds > 0 && intervalSeconds % 60 === 0) {
return { value: intervalSeconds / 60, unit: 'minutes' }
}
return { value: intervalSeconds, unit: 'seconds' }
}
function formValuesToIntervalSeconds(
value: number,
unit: 'seconds' | 'minutes',
): number {
if (!Number.isFinite(value) || value < 0) {
return 0
}
const rounded = Math.floor(value)
if (unit === 'minutes') {
return rounded * 60
}
return rounded
}
function NodeDetailOverviewTab({ node }: { node: Node }) {
return (
<dl className="node-detail-overview">
<div>
<dt>Name</dt>
<dd>{node.name}</dd>
<dd>
<span
className={healthDotClass(node.health_ok)}
title={healthDotLabel(node.health_ok)}
aria-hidden="true"
/>{' '}
{node.name}
</dd>
</div>
<div>
<dt>Host IP</dt>
@@ -416,6 +480,213 @@ function NodeDetailOverviewTab({ node }: { node: Node }) {
)
}
function NodeHealthcheckTab({
node,
canExec,
canUpdate,
onNodeUpdated,
}: {
node: Node
canExec: boolean
canUpdate: boolean
onNodeUpdated: (node: Node) => void
}) {
const [intervalDraft, setIntervalDraft] = useState(() =>
intervalToFormValues(node.health_check_interval_seconds ?? 0),
)
const [isSavingInterval, setIsSavingInterval] = useState(false)
const [intervalError, setIntervalError] = useState<string | null>(null)
const [isChecking, setIsChecking] = useState(false)
const [checkError, setCheckError] = useState<string | null>(null)
useEffect(() => {
setIntervalDraft(
intervalToFormValues(node.health_check_interval_seconds ?? 0),
)
setIntervalError(null)
}, [node.id, node.health_check_interval_seconds])
async function handleSaveInterval() {
const seconds = formValuesToIntervalSeconds(
intervalDraft.value,
intervalDraft.unit,
)
setIsSavingInterval(true)
setIntervalError(null)
try {
const response = await patchNode(node.id, {
health_check_interval_seconds: seconds,
})
onNodeUpdated(response.node)
setIntervalDraft(
intervalToFormValues(response.node.health_check_interval_seconds ?? 0),
)
} catch (err) {
setIntervalError(
err instanceof Error ? err.message : 'Failed to save interval',
)
} finally {
setIsSavingInterval(false)
}
}
async function handleHealthCheck() {
setIsChecking(true)
setCheckError(null)
try {
const response = await runNodeHealthCheck(node.id)
onNodeUpdated(response.node)
} catch (err) {
setCheckError(
err instanceof Error ? err.message : 'Health check failed',
)
} finally {
setIsChecking(false)
}
}
return (
<div className="node-healthcheck-tab">
<dl className="node-detail-overview">
<div>
<dt>Status</dt>
<dd>
<span
className={healthDotClass(node.health_ok)}
title={healthDotLabel(node.health_ok)}
aria-hidden="true"
/>{' '}
{healthDotLabel(node.health_ok)}
</dd>
</div>
<div>
<dt>Last check</dt>
<dd>
{node.health_last_checked_at
? new Date(node.health_last_checked_at).toLocaleString()
: 'Never'}
</dd>
</div>
<div>
<dt>Result</dt>
<dd>
{node.health_message
? node.health_message
: 'Not checked yet'}
</dd>
</div>
<div>
<dt>Ping</dt>
<dd>
{node.health_ping_ok === true
? 'OK'
: node.health_ping_ok === false
? 'Failed'
: '—'}
</dd>
</div>
<div>
<dt>SSH</dt>
<dd>
{node.health_ssh_ok === true
? 'OK'
: node.health_ssh_ok === false
? 'Failed'
: '—'}
</dd>
</div>
</dl>
{canUpdate ? (
<div className="node-health-interval">
<label className="node-health-interval-label">
Health check every
<input
type="number"
min={0}
step={1}
className="node-health-interval-input"
value={intervalDraft.value}
disabled={isSavingInterval}
onChange={(event) => {
const nextValue = Number(event.target.value)
setIntervalDraft((previous) => ({
...previous,
value: Number.isFinite(nextValue) ? nextValue : 0,
}))
setIntervalError(null)
}}
/>
<select
className="node-health-interval-unit"
value={intervalDraft.unit}
disabled={isSavingInterval}
onChange={(event) => {
setIntervalDraft((previous) => ({
...previous,
unit: event.target.value as 'seconds' | 'minutes',
}))
setIntervalError(null)
}}
>
<option value="seconds">seconds</option>
<option value="minutes">minutes</option>
</select>
</label>
<button
type="button"
className="node-health-interval-save"
disabled={isSavingInterval}
onClick={() => {
void handleSaveInterval()
}}
>
{isSavingInterval ? 'Saving…' : 'Save interval'}
</button>
{intervalError ? (
<p className="config-error" role="alert">
{intervalError}
</p>
) : null}
<p className="config-hint">Use 0 to disable scheduled checks.</p>
</div>
) : (
<p className="config-hint">
Interval:{' '}
{(node.health_check_interval_seconds ?? 0) === 0
? 'Disabled'
: `Every ${node.health_check_interval_seconds} seconds`}
. You need <code>nodes.update</code> to change this.
</p>
)}
{canExec ? (
<div className="node-actions">
<button
type="button"
className="node-test-ssh"
disabled={isChecking}
onClick={() => {
void handleHealthCheck()
}}
>
{isChecking ? 'Checking health…' : 'Health check'}
</button>
{checkError ? (
<p className="config-error" role="alert">
{checkError}
</p>
) : null}
</div>
) : (
<p className="config-hint">
You need <code>nodes.exec</code> to run a health check.
</p>
)}
</div>
)
}
function NodeActionsTab({
nodeId,
nodeUsername,
+57
View File
@@ -705,6 +705,63 @@ describe('nodes API', () => {
}),
)
})
it('posts to the health-check endpoint', async () => {
const { runNodeHealthCheck } = await import('./client')
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
node: {
id: '11111111-2222-4333-8444-555555555555',
health_ok: true,
health_message: 'ping ok (icmp); ssh ok',
},
}),
})
const result = await runNodeHealthCheck(
'11111111-2222-4333-8444-555555555555',
fetchMock,
)
expect(result.node.health_ok).toBe(true)
expect(fetchMock).toHaveBeenCalledWith(
'http://localhost:8080/api/v1/nodes/11111111-2222-4333-8444-555555555555/health-check',
expect.objectContaining({
method: 'POST',
credentials: 'include',
}),
)
})
it('patches node health interval', async () => {
const { patchNode } = await import('./client')
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
node: {
id: '11111111-2222-4333-8444-555555555555',
health_check_interval_seconds: 300,
},
}),
})
const result = await patchNode(
'11111111-2222-4333-8444-555555555555',
{ health_check_interval_seconds: 300 },
fetchMock,
)
expect(result.node.health_check_interval_seconds).toBe(300)
expect(fetchMock).toHaveBeenCalledWith(
'http://localhost:8080/api/v1/nodes/11111111-2222-4333-8444-555555555555',
expect.objectContaining({
method: 'PATCH',
credentials: 'include',
body: JSON.stringify({ health_check_interval_seconds: 300 }),
}),
)
})
})
describe('actions API', () => {
+45 -1
View File
@@ -658,6 +658,12 @@ export type Node = {
public_key: string
key_algo: string
created_at: string
health_check_interval_seconds?: number
health_last_checked_at?: string | null
health_ping_ok?: boolean | null
health_ssh_ok?: boolean | null
health_ok?: boolean | null
health_message?: string
}
export type NodesResponse = {
@@ -774,6 +780,44 @@ export async function testNodeSSH(
return (await response.json()) as NodeSSHTestResponse
}
export type PatchNodeRequest = {
health_check_interval_seconds: number
}
export async function patchNode(
id: string,
payload: PatchNodeRequest,
fetchImpl: typeof fetch = fetch,
): Promise<NodeResponse> {
const response = await apiFetch(
`/api/v1/nodes/${encodeURIComponent(id)}`,
{
method: 'PATCH',
body: JSON.stringify(payload),
},
fetchImpl,
)
if (!response.ok) {
throw new Error(await readErrorMessage(response))
}
return (await response.json()) as NodeResponse
}
export async function runNodeHealthCheck(
id: string,
fetchImpl: typeof fetch = fetch,
): Promise<NodeResponse> {
const response = await apiFetch(
`/api/v1/nodes/${encodeURIComponent(id)}/health-check`,
{ method: 'POST' },
fetchImpl,
)
if (!response.ok) {
throw new Error(await readErrorMessage(response))
}
return (await response.json()) as NodeResponse
}
export type ResolveCommandsResponse = {
paths: Record<string, string>
missing: ReadonlyArray<string>
@@ -799,7 +843,7 @@ export async function resolveNodeCommands(
return (await response.json()) as ResolveCommandsResponse
}
export type NodeAuditAction = 'create' | 'delete' | 'test_ssh' | 'update'
export type NodeAuditAction = 'create' | 'delete' | 'test_ssh' | 'health_check' | 'update'
export type NodeAuditEvent = {
id: string
+1
View File
@@ -59,6 +59,7 @@ describe('parseLocation', () => {
'/configuration/advanced',
'/containers/node-1',
'/containers/node-1/actions',
'/containers/node-1/healthcheck',
'/vms/node-2/logs',
]) {
const location = parseLocation(path)
+2 -1
View File
@@ -22,7 +22,7 @@ export type ConfigTabId =
export type ResourceTabId = 'overview' | 'security' | 'add'
export type NodeDetailTabId = 'overview' | 'actions' | 'logs'
export type NodeDetailTabId = 'overview' | 'healthcheck' | 'actions' | 'logs'
export type ResourceSectionId = 'containers' | 'vms' | 'docker'
@@ -54,6 +54,7 @@ const RESOURCE_TAB_IDS: ReadonlySet<string> = new Set([
const NODE_DETAIL_TAB_IDS: ReadonlySet<string> = new Set([
'overview',
'healthcheck',
'actions',
'logs',
])
+36
View File
@@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest'
import type { AuthAuditEvent, Node } from './api/client'
import {
countAuthFailuresToday,
countNodeHealthErrorsBySection,
countNodesBySection,
} from './sidebarCounts'
@@ -37,6 +38,41 @@ describe('countNodesBySection', () => {
})
})
describe('countNodeHealthErrorsBySection', () => {
it('counts only nodes with health_ok false', () => {
expect(
countNodeHealthErrorsBySection([
{ kind: 'container', health_ok: false },
{ kind: 'container', health_ok: true },
{ kind: 'container', health_ok: null },
{ kind: 'container' },
{ kind: 'docker', health_ok: false },
{ kind: 'vm', health_ok: false },
{ kind: 'vm', health_ok: false },
{ kind: 'vm', health_ok: true },
]),
).toEqual({
containers: 1,
docker: 1,
vms: 2,
})
})
it('returns zeros when no unhealthy nodes', () => {
expect(
countNodeHealthErrorsBySection([
{ kind: 'container', health_ok: true },
{ kind: 'docker' },
{ kind: 'vm', health_ok: null },
]),
).toEqual({
containers: 0,
docker: 0,
vms: 0,
})
})
})
describe('countAuthFailuresToday', () => {
const now = new Date(2026, 6, 18, 15, 30, 0)
+17
View File
@@ -24,6 +24,23 @@ export function countNodesBySection(
return counts
}
export function countNodeHealthErrorsBySection(
nodes: ReadonlyArray<Pick<Node, 'kind' | 'health_ok'>>,
): NodeSectionCounts {
const counts: NodeSectionCounts = {
containers: 0,
docker: 0,
vms: 0,
}
for (const node of nodes) {
if (node.health_ok !== false) {
continue
}
counts[KIND_TO_SECTION[node.kind]] += 1
}
return counts
}
export function countAuthFailuresToday(
events: ReadonlyArray<Pick<AuthAuditEvent, 'at' | 'action' | 'outcome'>>,
now: Date = new Date(),