From aac9e827665c2b10e632329cb53aa431dc13546d Mon Sep 17 00:00:00 2001 From: squid Date: Sun, 19 Jul 2026 22:40:21 +0200 Subject: [PATCH] Add per-node health checks with status indicators and scheduled probes. --- service/cmd/server/main.go | 3 + service/internal/api/authz.go | 17 ++ service/internal/api/nodes_handlers.go | 159 ++++++++++ .../api/nodes_health_handlers_test.go | 211 +++++++++++++ service/internal/api/router.go | 5 + service/internal/api/setup_auth_handlers.go | 28 +- service/internal/auth/ping.go | 89 ++++++ service/internal/auth/ping_test.go | 96 ++++++ service/internal/health/checker.go | 286 ++++++++++++++++++ service/internal/health/checker_test.go | 164 ++++++++++ service/internal/settings/types.go | 17 +- webui/src/App.css | 89 +++++- webui/src/App.tsx | 142 ++++++--- webui/src/NodeDetailPanel.tsx | 273 ++++++++++++++++- webui/src/api/client.test.ts | 57 ++++ webui/src/api/client.ts | 46 ++- webui/src/navigation.test.ts | 1 + webui/src/navigation.ts | 3 +- webui/src/sidebarCounts.test.ts | 36 +++ webui/src/sidebarCounts.ts | 17 ++ 20 files changed, 1674 insertions(+), 65 deletions(-) create mode 100644 service/internal/api/nodes_health_handlers_test.go create mode 100644 service/internal/auth/ping.go create mode 100644 service/internal/auth/ping_test.go create mode 100644 service/internal/health/checker.go create mode 100644 service/internal/health/checker_test.go diff --git a/service/cmd/server/main.go b/service/cmd/server/main.go index 40a02b1..0ca3e58 100644 --- a/service/cmd/server/main.go +++ b/service/cmd/server/main.go @@ -57,6 +57,9 @@ func main() { if app.Executor != nil { app.Executor.StopScheduler() } + if app.HealthChecker != nil { + app.HealthChecker.StopScheduler() + } shutdownContext, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() diff --git a/service/internal/api/authz.go b/service/internal/api/authz.go index 5aab6b0..83b7279 100644 --- a/service/internal/api/authz.go +++ b/service/internal/api/authz.go @@ -10,6 +10,7 @@ import ( var errUsersManageRequired = errors.New("permission users.manage required") var errNodesReadRequired = errors.New("permission nodes.read required") var errNodesExecRequired = errors.New("permission nodes.exec required") +var errNodesUpdateRequired = errors.New("permission nodes.update required") var errNodesDeleteRequired = errors.New("permission nodes.delete required") var errActionsCreateRequired = errors.New("permission actions.create required") var errActionsUpdateRequired = errors.New("permission actions.update required") @@ -135,6 +136,22 @@ func (app *App) authorizeNodesExec(request *http.Request) error { return nil } +func (app *App) authorizeNodesUpdate(request *http.Request) error { + user, ok := UserFromContext(request.Context()) + if !ok { + return errors.New("authentication required") + } + + payload, err := loadSettingsOrDefault(app.ConfigDir) + if err != nil { + return err + } + if !userHasPermission(user, payload.Groups, "nodes.update") { + return errNodesUpdateRequired + } + return nil +} + func (app *App) authorizeNodesDelete(request *http.Request) error { user, ok := UserFromContext(request.Context()) if !ok { diff --git a/service/internal/api/nodes_handlers.go b/service/internal/api/nodes_handlers.go index d22163b..600aa00 100644 --- a/service/internal/api/nodes_handlers.go +++ b/service/internal/api/nodes_handlers.go @@ -3,6 +3,7 @@ package api import ( "encoding/json" "errors" + "fmt" "net" "net/http" "strings" @@ -231,6 +232,164 @@ func (app *App) nodesTestSSHHandler(writer http.ResponseWriter, request *http.Re }) } +type patchNodeRequest struct { + HealthCheckIntervalSeconds *int `json:"health_check_interval_seconds"` +} + +func (app *App) nodesPatchHandler(writer http.ResponseWriter, request *http.Request) { + if err := app.authorizeNodesUpdate(request); err != nil { + writeJSON(writer, http.StatusForbidden, apiErrorResponse{Error: err.Error()}) + return + } + + user, ok := UserFromContext(request.Context()) + if !ok { + writeJSON(writer, http.StatusUnauthorized, apiErrorResponse{Error: "authentication required"}) + return + } + + nodeID := strings.TrimSpace(request.PathValue("id")) + if nodeID == "" { + writeJSON(writer, http.StatusBadRequest, apiErrorResponse{Error: "missing node id"}) + return + } + + var payload patchNodeRequest + if err := json.NewDecoder(request.Body).Decode(&payload); err != nil { + writeJSON(writer, http.StatusBadRequest, apiErrorResponse{Error: "invalid JSON body"}) + return + } + if payload.HealthCheckIntervalSeconds == nil { + writeJSON(writer, http.StatusBadRequest, apiErrorResponse{ + Error: "health_check_interval_seconds is required", + }) + return + } + if *payload.HealthCheckIntervalSeconds < 0 { + writeJSON(writer, http.StatusBadRequest, apiErrorResponse{ + Error: "health_check_interval_seconds must be >= 0", + }) + return + } + + settingsPayload, err := loadSettingsOrDefault(app.ConfigDir) + if err != nil { + writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()}) + return + } + + store, err := settings.LoadNodesOrEmpty(app.ConfigDir) + if err != nil { + writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()}) + return + } + + nodeIndex := -1 + var node settings.Node + for index, candidate := range store.Nodes { + if candidate.ID != nodeID { + continue + } + node = candidate + nodeIndex = index + break + } + if nodeIndex < 0 { + writeJSON(writer, http.StatusNotFound, apiErrorResponse{Error: "node not found"}) + return + } + if !userCanAccessNode(user, settingsPayload.Groups, node) { + writeJSON(writer, http.StatusForbidden, apiErrorResponse{Error: "permission denied"}) + return + } + + node.HealthCheckIntervalSeconds = *payload.HealthCheckIntervalSeconds + store.Nodes[nodeIndex] = node + if err := settings.SaveNodes(app.ConfigDir, store); err != nil { + writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()}) + return + } + + _ = appendNodeAudit( + app.ConfigDir, + settings.NodeAuditActionUpdate, + user.Username, + node, + fmt.Sprintf("health_check_interval_seconds=%d", node.HealthCheckIntervalSeconds), + ) + writeJSON(writer, http.StatusOK, nodeResponse{Node: node}) +} + +func (app *App) nodesHealthCheckHandler(writer http.ResponseWriter, request *http.Request) { + if err := app.authorizeNodesExec(request); err != nil { + writeJSON(writer, http.StatusForbidden, apiErrorResponse{Error: err.Error()}) + return + } + if err := app.requireKey(); err != nil { + writeJSON(writer, http.StatusServiceUnavailable, apiErrorResponse{Error: err.Error()}) + return + } + if app.HealthChecker == nil { + writeJSON(writer, http.StatusServiceUnavailable, apiErrorResponse{Error: "health checker unavailable"}) + return + } + + user, ok := UserFromContext(request.Context()) + if !ok { + writeJSON(writer, http.StatusUnauthorized, apiErrorResponse{Error: "authentication required"}) + return + } + + nodeID := strings.TrimSpace(request.PathValue("id")) + if nodeID == "" { + writeJSON(writer, http.StatusBadRequest, apiErrorResponse{Error: "missing node id"}) + return + } + + settingsPayload, err := loadSettingsOrDefault(app.ConfigDir) + if err != nil { + writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()}) + return + } + + store, err := settings.LoadNodesOrEmpty(app.ConfigDir) + if err != nil { + writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()}) + return + } + + var node settings.Node + found := false + for _, candidate := range store.Nodes { + if candidate.ID != nodeID { + continue + } + node = candidate + found = true + break + } + if !found { + writeJSON(writer, http.StatusNotFound, apiErrorResponse{Error: "node not found"}) + return + } + if !userCanAccessNode(user, settingsPayload.Groups, node) { + writeJSON(writer, http.StatusForbidden, apiErrorResponse{Error: "permission denied"}) + return + } + + updated, err := app.HealthChecker.CheckNodeByID(nodeID, user.Username) + if err != nil { + if strings.Contains(err.Error(), "already running") { + writeJSON(writer, http.StatusConflict, apiErrorResponse{Error: err.Error()}) + return + } + writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()}) + return + } + + writeJSON(writer, http.StatusOK, nodeResponse{Node: updated}) +} + func (app *App) nodesCreateHandler(writer http.ResponseWriter, request *http.Request) { if err := app.authorizeNodesExec(request); err != nil { writeJSON(writer, http.StatusForbidden, apiErrorResponse{Error: err.Error()}) diff --git a/service/internal/api/nodes_health_handlers_test.go b/service/internal/api/nodes_health_handlers_test.go new file mode 100644 index 0000000..0589ece --- /dev/null +++ b/service/internal/api/nodes_health_handlers_test.go @@ -0,0 +1,211 @@ +package api + +import ( + "bytes" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "codeberg.org/SquidSE/ClusterCanvas/service/internal/auth" +) + +func TestNodesHealthCheckPersistsStatus(t *testing.T) { + configDir := t.TempDir() + cookie := seedCompletedSetup(t, configDir) + router, app := NewRouterWithApp(configDir) + + app.HealthChecker.PingFn = func(hostIP string) auth.PingResult { + return auth.PingResult{OK: true, Method: auth.PingMethodICMP} + } + app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error { + return nil + } + + createBody := []byte(`{ + "id":"bbbbbbbb-bbbb-4ccc-8ddd-eeeeeeeeeeee", + "kind":"container", + "name":"healthy-node", + "host_ip":"10.20.30.40", + "group_name":"Administrators", + "generate":{"algorithm":"ed25519"} + }`) + createRequest := withSession( + httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)), + cookie, + ) + createRecorder := httptest.NewRecorder() + router.ServeHTTP(createRecorder, createRequest) + if createRecorder.Code != http.StatusCreated { + t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + + healthRequest := withSession( + httptest.NewRequest( + http.MethodPost, + "/api/v1/nodes/bbbbbbbb-bbbb-4ccc-8ddd-eeeeeeeeeeee/health-check", + nil, + ), + cookie, + ) + healthRecorder := httptest.NewRecorder() + router.ServeHTTP(healthRecorder, healthRequest) + if healthRecorder.Code != http.StatusOK { + t.Fatalf("health status = %d body=%s", healthRecorder.Code, healthRecorder.Body.String()) + } + + var response nodeResponse + if err := json.Unmarshal(healthRecorder.Body.Bytes(), &response); err != nil { + t.Fatalf("decode: %v", err) + } + if response.Node.HealthOK == nil || !*response.Node.HealthOK { + t.Fatalf("health_ok = %#v", response.Node.HealthOK) + } + if response.Node.HealthMessage == "" { + t.Fatal("expected health message") + } +} + +func TestNodesHealthCheckRecordsFailure(t *testing.T) { + configDir := t.TempDir() + cookie := seedCompletedSetup(t, configDir) + router, app := NewRouterWithApp(configDir) + + app.HealthChecker.PingFn = func(hostIP string) auth.PingResult { + return auth.PingResult{ + OK: false, + Method: auth.PingMethodTCP, + Error: errors.New("unreachable"), + } + } + app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error { + return errors.New("ssh failed") + } + + createBody := []byte(`{ + "id":"cccccccc-bbbb-4ccc-8ddd-eeeeeeeeeeee", + "kind":"vm", + "name":"unhealthy-node", + "host_ip":"10.20.30.41", + "group_name":"Administrators", + "generate":{"algorithm":"ed25519"} + }`) + createRequest := withSession( + httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)), + cookie, + ) + createRecorder := httptest.NewRecorder() + router.ServeHTTP(createRecorder, createRequest) + if createRecorder.Code != http.StatusCreated { + t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + + healthRequest := withSession( + httptest.NewRequest( + http.MethodPost, + "/api/v1/nodes/cccccccc-bbbb-4ccc-8ddd-eeeeeeeeeeee/health-check", + nil, + ), + cookie, + ) + healthRecorder := httptest.NewRecorder() + router.ServeHTTP(healthRecorder, healthRequest) + if healthRecorder.Code != http.StatusOK { + t.Fatalf("health status = %d body=%s", healthRecorder.Code, healthRecorder.Body.String()) + } + + var response nodeResponse + if err := json.Unmarshal(healthRecorder.Body.Bytes(), &response); err != nil { + t.Fatalf("decode: %v", err) + } + if response.Node.HealthOK == nil || *response.Node.HealthOK { + t.Fatalf("expected failure, got %#v", response.Node.HealthOK) + } +} + +func TestNodesPatchHealthInterval(t *testing.T) { + configDir := t.TempDir() + cookie := seedCompletedSetup(t, configDir) + router := NewRouter(configDir) + + createBody := []byte(`{ + "id":"dddddddd-bbbb-4ccc-8ddd-eeeeeeeeeeee", + "kind":"docker", + "name":"interval-node", + "host_ip":"10.20.30.42", + "group_name":"Administrators", + "generate":{"algorithm":"ed25519"} + }`) + createRequest := withSession( + httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)), + cookie, + ) + createRecorder := httptest.NewRecorder() + router.ServeHTTP(createRecorder, createRequest) + if createRecorder.Code != http.StatusCreated { + t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + + patchBody := []byte(`{"health_check_interval_seconds":120}`) + patchRequest := withSession( + httptest.NewRequest( + http.MethodPatch, + "/api/v1/nodes/dddddddd-bbbb-4ccc-8ddd-eeeeeeeeeeee", + bytes.NewReader(patchBody), + ), + cookie, + ) + patchRecorder := httptest.NewRecorder() + router.ServeHTTP(patchRecorder, patchRequest) + if patchRecorder.Code != http.StatusOK { + t.Fatalf("patch status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String()) + } + + var response nodeResponse + if err := json.Unmarshal(patchRecorder.Body.Bytes(), &response); err != nil { + t.Fatalf("decode: %v", err) + } + if response.Node.HealthCheckIntervalSeconds != 120 { + t.Fatalf("interval = %d", response.Node.HealthCheckIntervalSeconds) + } +} + +func TestNodesPatchHealthIntervalRejectsNegative(t *testing.T) { + configDir := t.TempDir() + cookie := seedCompletedSetup(t, configDir) + router := NewRouter(configDir) + + createBody := []byte(`{ + "id":"eeeeeeee-bbbb-4ccc-8ddd-eeeeeeeeeeee", + "kind":"container", + "name":"bad-interval", + "host_ip":"10.20.30.43", + "group_name":"Administrators", + "generate":{"algorithm":"ed25519"} + }`) + createRequest := withSession( + httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)), + cookie, + ) + createRecorder := httptest.NewRecorder() + router.ServeHTTP(createRecorder, createRequest) + if createRecorder.Code != http.StatusCreated { + t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + + patchBody := []byte(`{"health_check_interval_seconds":-5}`) + patchRequest := withSession( + httptest.NewRequest( + http.MethodPatch, + "/api/v1/nodes/eeeeeeee-bbbb-4ccc-8ddd-eeeeeeeeeeee", + bytes.NewReader(patchBody), + ), + cookie, + ) + patchRecorder := httptest.NewRecorder() + router.ServeHTTP(patchRecorder, patchRequest) + if patchRecorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String()) + } +} diff --git a/service/internal/api/router.go b/service/internal/api/router.go index 350452a..c11d9a8 100644 --- a/service/internal/api/router.go +++ b/service/internal/api/router.go @@ -22,6 +22,9 @@ func NewRouterWithApp(configDir string) (http.Handler, *App) { if app.Executor != nil { app.Executor.StartScheduler() } + if app.HealthChecker != nil { + app.HealthChecker.StartScheduler() + } mux := http.NewServeMux() mux.HandleFunc("GET /health", HealthHandler) @@ -62,8 +65,10 @@ func NewRouterWithApp(configDir string) (http.Handler, *App) { mux.HandleFunc("GET /api/v1/nodes", app.nodesListHandler) mux.HandleFunc("POST /api/v1/nodes", app.nodesCreateHandler) mux.HandleFunc("GET /api/v1/nodes/{id}", app.nodesGetHandler) + mux.HandleFunc("PATCH /api/v1/nodes/{id}", app.nodesPatchHandler) mux.HandleFunc("DELETE /api/v1/nodes/{id}", app.nodesDeleteHandler) mux.HandleFunc("POST /api/v1/nodes/{id}/test-ssh", app.nodesTestSSHHandler) + mux.HandleFunc("POST /api/v1/nodes/{id}/health-check", app.nodesHealthCheckHandler) mux.HandleFunc("POST /api/v1/nodes/{id}/resolve-commands", app.nodesResolveCommandsHandler) mux.HandleFunc("GET /api/v1/nodes/{id}/action-groups", app.actionGroupsListHandler) mux.HandleFunc("POST /api/v1/nodes/{id}/action-groups", app.actionGroupsCreateHandler) diff --git a/service/internal/api/setup_auth_handlers.go b/service/internal/api/setup_auth_handlers.go index 4b0f7bb..12dd6b3 100644 --- a/service/internal/api/setup_auth_handlers.go +++ b/service/internal/api/setup_auth_handlers.go @@ -11,6 +11,7 @@ import ( "time" "codeberg.org/SquidSE/ClusterCanvas/service/internal/auth" + "codeberg.org/SquidSE/ClusterCanvas/service/internal/health" "codeberg.org/SquidSE/ClusterCanvas/service/internal/runner" "codeberg.org/SquidSE/ClusterCanvas/service/internal/settings" ) @@ -77,10 +78,11 @@ type meResponse struct { // App holds shared API dependencies. type App struct { - ConfigDir string - Key []byte - Sessions *auth.SessionManager - Executor *runner.Executor + ConfigDir string + Key []byte + Sessions *auth.SessionManager + Executor *runner.Executor + HealthChecker *health.Checker pendingTOTPMu sync.Mutex pendingTOTP map[string]string // username -> secret (setup only) @@ -91,18 +93,20 @@ func newApp(configDir string) (*App, error) { if err != nil { // Key is required for setup complete / auth; status still works without it. app := &App{ - ConfigDir: configDir, - pendingTOTP: map[string]string{}, - Executor: runner.NewExecutor(configDir, nil), + ConfigDir: configDir, + pendingTOTP: map[string]string{}, + Executor: runner.NewExecutor(configDir, nil), + HealthChecker: health.NewChecker(configDir, nil), } return app, nil } app := &App{ - ConfigDir: configDir, - Key: key, - Sessions: auth.NewSessionManager(configDir, key), - pendingTOTP: map[string]string{}, - Executor: runner.NewExecutor(configDir, key), + ConfigDir: configDir, + Key: key, + Sessions: auth.NewSessionManager(configDir, key), + pendingTOTP: map[string]string{}, + Executor: runner.NewExecutor(configDir, key), + HealthChecker: health.NewChecker(configDir, key), } return app, nil } diff --git a/service/internal/auth/ping.go b/service/internal/auth/ping.go new file mode 100644 index 0000000..d9d10d3 --- /dev/null +++ b/service/internal/auth/ping.go @@ -0,0 +1,89 @@ +package auth + +import ( + "fmt" + "net" + "os/exec" + "strings" + "time" +) + +const ( + PingMethodICMP = "icmp" + PingMethodTCP = "tcp" + + defaultPingCount = "1" + defaultPingWaitSec = "2" + defaultTCPTimeout = 3 * time.Second +) + +// PingResult is the outcome of a reachability probe. +type PingResult struct { + OK bool + Method string + Error error +} + +type pingCommandFunc func(hostIP string) error +type tcpDialFunc func(address string, timeout time.Duration) error + +var ( + runICMPPing = defaultRunICMPPing + dialTCP = defaultDialTCP +) + +// PingHost probes host reachability: ICMP first, then TCP :22 when ICMP is +// unavailable or the host does not respond to ICMP. +func PingHost(hostIP string) PingResult { + hostIP = strings.TrimSpace(hostIP) + if hostIP == "" { + return PingResult{OK: false, Error: fmt.Errorf("host IP is required")} + } + + icmpErr := runICMPPing(hostIP) + if icmpErr == nil { + return PingResult{OK: true, Method: PingMethodICMP} + } + + address := net.JoinHostPort(hostIP, defaultSSHPort) + tcpErr := dialTCP(address, defaultTCPTimeout) + if tcpErr == nil { + return PingResult{OK: true, Method: PingMethodTCP} + } + + return PingResult{ + OK: false, + Method: PingMethodTCP, + Error: fmt.Errorf( + "icmp: %v; tcp %s: %w", + icmpErr, + address, + tcpErr, + ), + } +} + +func defaultRunICMPPing(hostIP string) error { + pingPath, err := exec.LookPath("ping") + if err != nil { + return fmt.Errorf("ping unavailable: %w", err) + } + + cmd := exec.Command(pingPath, "-c", defaultPingCount, "-W", defaultPingWaitSec, hostIP) + output, runErr := cmd.CombinedOutput() + if runErr == nil { + return nil + } + + combined := strings.TrimSpace(string(output) + " " + runErr.Error()) + return fmt.Errorf("%s", strings.TrimSpace(combined)) +} + +func defaultDialTCP(address string, timeout time.Duration) error { + conn, err := net.DialTimeout("tcp", address, timeout) + if err != nil { + return err + } + _ = conn.Close() + return nil +} diff --git a/service/internal/auth/ping_test.go b/service/internal/auth/ping_test.go new file mode 100644 index 0000000..6a8b999 --- /dev/null +++ b/service/internal/auth/ping_test.go @@ -0,0 +1,96 @@ +package auth + +import ( + "errors" + "testing" + "time" +) + +func TestPingHostUsesICMPWhenAvailable(t *testing.T) { + originalPing := runICMPPing + originalDial := dialTCP + t.Cleanup(func() { + runICMPPing = originalPing + dialTCP = originalDial + }) + + runICMPPing = func(hostIP string) error { + if hostIP != "10.0.0.1" { + t.Fatalf("host = %q", hostIP) + } + return nil + } + dialTCP = func(address string, timeout time.Duration) error { + t.Fatal("tcp should not be used when icmp succeeds") + return nil + } + + result := PingHost("10.0.0.1") + if !result.OK { + t.Fatalf("expected ok, got %#v", result) + } + if result.Method != PingMethodICMP { + t.Fatalf("method = %q", result.Method) + } +} + +func TestPingHostFallsBackToTCPWhenICMPFails(t *testing.T) { + originalPing := runICMPPing + originalDial := dialTCP + t.Cleanup(func() { + runICMPPing = originalPing + dialTCP = originalDial + }) + + runICMPPing = func(hostIP string) error { + return errors.New("ping unavailable: executable file not found") + } + dialTCP = func(address string, timeout time.Duration) error { + if address != "10.0.0.2:22" { + t.Fatalf("address = %q", address) + } + return nil + } + + result := PingHost("10.0.0.2") + if !result.OK { + t.Fatalf("expected ok, got %#v", result) + } + if result.Method != PingMethodTCP { + t.Fatalf("method = %q", result.Method) + } +} + +func TestPingHostFailsWhenBothFail(t *testing.T) { + originalPing := runICMPPing + originalDial := dialTCP + t.Cleanup(func() { + runICMPPing = originalPing + dialTCP = originalDial + }) + + runICMPPing = func(hostIP string) error { + return errors.New("100% packet loss") + } + dialTCP = func(address string, timeout time.Duration) error { + return errors.New("connection refused") + } + + result := PingHost("10.0.0.3") + if result.OK { + t.Fatal("expected failure") + } + if result.Error == nil { + t.Fatal("expected error") + } + if result.Method != PingMethodTCP { + t.Fatalf("method = %q", result.Method) + } +} + +func TestPingHostRequiresHostIP(t *testing.T) { + result := PingHost(" ") + if result.OK || result.Error == nil { + t.Fatalf("expected failure, got %#v", result) + } +} diff --git a/service/internal/health/checker.go b/service/internal/health/checker.go new file mode 100644 index 0000000..f5be113 --- /dev/null +++ b/service/internal/health/checker.go @@ -0,0 +1,286 @@ +package health + +import ( + "fmt" + "sync" + "time" + + "codeberg.org/SquidSE/ClusterCanvas/service/internal/auth" + "codeberg.org/SquidSE/ClusterCanvas/service/internal/settings" +) + +const schedulerTickInterval = 15 * time.Second + +// Result is the outcome of a node health check. +type Result struct { + PingOK bool + SSHOK bool + OK bool + Message string + CheckedAt time.Time +} + +// Checker runs and schedules per-node health checks. +type Checker struct { + ConfigDir string + Key []byte + + mu sync.Mutex + running map[string]struct{} + stopCh chan struct{} + stopped chan struct{} + tickerOn bool + + // Optional overrides for tests. + PingFn func(hostIP string) auth.PingResult + SSHFn func(hostIP string, username string, privateKeyPEM string, passphrase string) error + NowFn func() time.Time +} + +// NewChecker creates a health checker for the given config directory and key. +func NewChecker(configDir string, key []byte) *Checker { + return &Checker{ + ConfigDir: configDir, + Key: key, + running: map[string]struct{}{}, + stopCh: make(chan struct{}), + stopped: make(chan struct{}), + } +} + +// StartScheduler begins the background health-check ticker. +func (checker *Checker) StartScheduler() { + checker.mu.Lock() + if checker.tickerOn { + checker.mu.Unlock() + return + } + checker.tickerOn = true + checker.mu.Unlock() + + go func() { + defer close(checker.stopped) + ticker := time.NewTicker(schedulerTickInterval) + defer ticker.Stop() + for { + select { + case <-checker.stopCh: + return + case <-ticker.C: + checker.tickDueChecks() + } + } + }() +} + +// StopScheduler stops the background ticker and waits for it to exit. +func (checker *Checker) StopScheduler() { + checker.mu.Lock() + if !checker.tickerOn { + checker.mu.Unlock() + return + } + checker.tickerOn = false + checker.mu.Unlock() + close(checker.stopCh) + <-checker.stopped +} + +func (checker *Checker) tryLockNode(nodeID string) bool { + checker.mu.Lock() + defer checker.mu.Unlock() + if _, ok := checker.running[nodeID]; ok { + return false + } + checker.running[nodeID] = struct{}{} + return true +} + +func (checker *Checker) unlockNode(nodeID string) { + checker.mu.Lock() + defer checker.mu.Unlock() + delete(checker.running, nodeID) +} + +func (checker *Checker) now() time.Time { + if checker.NowFn != nil { + return checker.NowFn() + } + return time.Now().UTC() +} + +func (checker *Checker) tickDueChecks() { + if len(checker.Key) == 0 { + return + } + store, err := settings.LoadNodesOrEmpty(checker.ConfigDir) + if err != nil { + return + } + now := checker.now() + for _, node := range store.Nodes { + if !IsHealthCheckDue(node, now) { + continue + } + nodeID := node.ID + go func() { + _, _ = checker.CheckNodeByID(nodeID, "scheduler") + }() + } +} + +// IsHealthCheckDue reports whether a node should be probed now. +func IsHealthCheckDue(node settings.Node, now time.Time) bool { + if node.HealthCheckIntervalSeconds <= 0 { + return false + } + if node.HealthLastCheckedAt == nil { + return true + } + elapsed := now.Sub(node.HealthLastCheckedAt.UTC()) + return elapsed >= time.Duration(node.HealthCheckIntervalSeconds)*time.Second +} + +// CheckNodeByID loads credentials, runs ping + SSH, and persists the result. +func (checker *Checker) CheckNodeByID(nodeID string, actor string) (settings.Node, error) { + if !checker.tryLockNode(nodeID) { + store, err := settings.LoadNodesOrEmpty(checker.ConfigDir) + if err != nil { + return settings.Node{}, err + } + for _, node := range store.Nodes { + if node.ID == nodeID { + return node, fmt.Errorf("health check already running for node") + } + } + return settings.Node{}, fmt.Errorf("node not found") + } + defer checker.unlockNode(nodeID) + + if len(checker.Key) == 0 { + return settings.Node{}, fmt.Errorf("%s is not set", settings.ConfigKeyEnvVar) + } + + store, err := settings.LoadNodesOrEmpty(checker.ConfigDir) + if err != nil { + return settings.Node{}, err + } + + nodeIndex := -1 + var node settings.Node + for index, candidate := range store.Nodes { + if candidate.ID != nodeID { + continue + } + node = candidate + nodeIndex = index + break + } + if nodeIndex < 0 { + return settings.Node{}, fmt.Errorf("node not found") + } + + keyStore, err := settings.LoadNodeKeysOrEmpty(checker.ConfigDir, checker.Key) + if err != nil { + return settings.Node{}, err + } + + var keyEntry settings.NodeKeyEntry + keyFound := false + for _, entry := range keyStore.Keys { + if entry.NodeID == node.ID { + keyEntry = entry + keyFound = true + break + } + } + if !keyFound { + return settings.Node{}, fmt.Errorf("private key not found for node") + } + + result := checker.runChecks(node, keyEntry) + updated := applyHealthResult(node, result) + store.Nodes[nodeIndex] = updated + if err := settings.SaveNodes(checker.ConfigDir, store); err != nil { + return settings.Node{}, err + } + + _ = appendHealthAudit(checker.ConfigDir, actor, updated, result.Message) + return updated, nil +} + +func (checker *Checker) runChecks(node settings.Node, keyEntry settings.NodeKeyEntry) Result { + checkedAt := checker.now() + pingFn := checker.PingFn + if pingFn == nil { + pingFn = auth.PingHost + } + sshFn := checker.SSHFn + if sshFn == nil { + sshFn = auth.TestSSHConnection + } + + pingResult := pingFn(node.HostIP) + sshErr := sshFn(node.HostIP, node.Username, keyEntry.PrivateKey, keyEntry.Passphrase) + sshOK := sshErr == nil + + message := buildHealthMessage(pingResult, sshOK, sshErr) + return Result{ + PingOK: pingResult.OK, + SSHOK: sshOK, + OK: pingResult.OK && sshOK, + Message: message, + CheckedAt: checkedAt, + } +} + +func buildHealthMessage(pingResult auth.PingResult, sshOK bool, sshErr error) string { + var parts []string + if pingResult.OK { + parts = append(parts, fmt.Sprintf("ping ok (%s)", pingResult.Method)) + } else if pingResult.Error != nil { + parts = append(parts, fmt.Sprintf("ping failed: %v", pingResult.Error)) + } else { + parts = append(parts, "ping failed") + } + + if sshOK { + parts = append(parts, "ssh ok") + } else if sshErr != nil { + parts = append(parts, fmt.Sprintf("ssh failed: %v", sshErr)) + } else { + parts = append(parts, "ssh failed") + } + + return fmt.Sprintf("%s; %s", parts[0], parts[1]) +} + +func applyHealthResult(node settings.Node, result Result) settings.Node { + checkedAt := result.CheckedAt + pingOK := result.PingOK + sshOK := result.SSHOK + healthOK := result.OK + node.HealthLastCheckedAt = &checkedAt + node.HealthPingOK = &pingOK + node.HealthSSHOK = &sshOK + node.HealthOK = &healthOK + node.HealthMessage = result.Message + return node +} + +func appendHealthAudit(configDir string, actor string, node settings.Node, detail string) error { + eventID, err := auth.NewUUID() + if err != nil { + eventID = fmt.Sprintf("health-%d", time.Now().UnixNano()) + } + return settings.AppendNodeAuditEvent(configDir, settings.NodeAuditEvent{ + ID: eventID, + At: time.Now().UTC(), + Action: settings.NodeAuditActionHealthCheck, + Actor: actor, + NodeID: node.ID, + NodeName: node.Name, + NodeKind: node.Kind, + Detail: detail, + }) +} diff --git a/service/internal/health/checker_test.go b/service/internal/health/checker_test.go new file mode 100644 index 0000000..16655bb --- /dev/null +++ b/service/internal/health/checker_test.go @@ -0,0 +1,164 @@ +package health + +import ( + "errors" + "testing" + "time" + + "codeberg.org/SquidSE/ClusterCanvas/service/internal/auth" + "codeberg.org/SquidSE/ClusterCanvas/service/internal/settings" +) + +func TestIsHealthCheckDue(t *testing.T) { + now := time.Date(2026, 7, 19, 12, 0, 0, 0, time.UTC) + + t.Run("disabled", func(t *testing.T) { + if IsHealthCheckDue(settings.Node{HealthCheckIntervalSeconds: 0}, now) { + t.Fatal("expected not due") + } + }) + + t.Run("never checked", func(t *testing.T) { + if !IsHealthCheckDue(settings.Node{HealthCheckIntervalSeconds: 60}, now) { + t.Fatal("expected due") + } + }) + + t.Run("interval elapsed", func(t *testing.T) { + checked := now.Add(-61 * time.Second) + node := settings.Node{ + HealthCheckIntervalSeconds: 60, + HealthLastCheckedAt: &checked, + } + if !IsHealthCheckDue(node, now) { + t.Fatal("expected due") + } + }) + + t.Run("interval not elapsed", func(t *testing.T) { + checked := now.Add(-30 * time.Second) + node := settings.Node{ + HealthCheckIntervalSeconds: 60, + HealthLastCheckedAt: &checked, + } + if IsHealthCheckDue(node, now) { + t.Fatal("expected not due") + } + }) +} + +func TestCheckNodeByIDPersistsResult(t *testing.T) { + dir := t.TempDir() + keyBytes := make([]byte, 32) + for index := range keyBytes { + keyBytes[index] = byte(index + 1) + } + + nodeID := "11111111-2222-4333-8444-555555555555" + store := settings.NodeStore{ + Nodes: []settings.Node{{ + ID: nodeID, + Kind: settings.NodeKindContainer, + Name: "ct-1", + HostIP: "10.9.9.9", + Username: "clustercanvas", + GroupName: "Administrators", + KeyAlgo: "ed25519", + CreatedAt: time.Now().UTC(), + }}, + } + if err := settings.SaveNodes(dir, store); err != nil { + t.Fatalf("SaveNodes: %v", err) + } + if err := settings.SaveNodeKeys(dir, settings.NodeKeyStore{ + Keys: []settings.NodeKeyEntry{{ + NodeID: nodeID, + PrivateKey: "dummy-key", + Algorithm: "ed25519", + }}, + }, keyBytes); err != nil { + t.Fatalf("SaveNodeKeys: %v", err) + } + + checker := NewChecker(dir, keyBytes) + checker.NowFn = func() time.Time { + return time.Date(2026, 7, 19, 15, 0, 0, 0, time.UTC) + } + checker.PingFn = func(hostIP string) auth.PingResult { + return auth.PingResult{OK: true, Method: auth.PingMethodTCP} + } + checker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error { + return nil + } + + updated, err := checker.CheckNodeByID(nodeID, "tester") + if err != nil { + t.Fatalf("CheckNodeByID: %v", err) + } + if updated.HealthOK == nil || !*updated.HealthOK { + t.Fatalf("health_ok = %#v", updated.HealthOK) + } + if updated.HealthPingOK == nil || !*updated.HealthPingOK { + t.Fatal("expected ping ok") + } + if updated.HealthSSHOK == nil || !*updated.HealthSSHOK { + t.Fatal("expected ssh ok") + } + if updated.HealthMessage == "" { + t.Fatal("expected message") + } + + loaded, err := settings.LoadNodes(dir) + if err != nil { + t.Fatalf("LoadNodes: %v", err) + } + if loaded.Nodes[0].HealthOK == nil || !*loaded.Nodes[0].HealthOK { + t.Fatal("persisted health_ok missing") + } +} + +func TestCheckNodeByIDRecordsFailure(t *testing.T) { + dir := t.TempDir() + keyBytes := make([]byte, 32) + for index := range keyBytes { + keyBytes[index] = byte(index + 3) + } + + nodeID := "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + if err := settings.SaveNodes(dir, settings.NodeStore{ + Nodes: []settings.Node{{ + ID: nodeID, + Kind: settings.NodeKindVM, + Name: "vm-1", + HostIP: "10.8.8.8", + Username: "clustercanvas", + }}, + }); err != nil { + t.Fatalf("SaveNodes: %v", err) + } + if err := settings.SaveNodeKeys(dir, settings.NodeKeyStore{ + Keys: []settings.NodeKeyEntry{{ + NodeID: nodeID, + PrivateKey: "dummy-key", + Algorithm: "ed25519", + }}, + }, keyBytes); err != nil { + t.Fatalf("SaveNodeKeys: %v", err) + } + + checker := NewChecker(dir, keyBytes) + checker.PingFn = func(hostIP string) auth.PingResult { + return auth.PingResult{OK: false, Method: auth.PingMethodTCP, Error: errors.New("unreachable")} + } + checker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error { + return errors.New("ssh dial failed") + } + + updated, err := checker.CheckNodeByID(nodeID, "tester") + if err != nil { + t.Fatalf("CheckNodeByID: %v", err) + } + if updated.HealthOK == nil || *updated.HealthOK { + t.Fatalf("expected health failure, got %#v", updated.HealthOK) + } +} diff --git a/service/internal/settings/types.go b/service/internal/settings/types.go index 6565362..8b0e696 100644 --- a/service/internal/settings/types.go +++ b/service/internal/settings/types.go @@ -105,6 +105,14 @@ type Node struct { PublicKey string `json:"public_key"` KeyAlgo string `json:"key_algo"` CreatedAt time.Time `json:"created_at"` + + // Health check configuration and last result (persisted). + HealthCheckIntervalSeconds int `json:"health_check_interval_seconds"` + HealthLastCheckedAt *time.Time `json:"health_last_checked_at,omitempty"` + HealthPingOK *bool `json:"health_ping_ok,omitempty"` + HealthSSHOK *bool `json:"health_ssh_ok,omitempty"` + HealthOK *bool `json:"health_ok,omitempty"` + HealthMessage string `json:"health_message,omitempty"` } // NodeStore is the plain JSON payload in nodes.json. @@ -284,10 +292,11 @@ type NodeKeyStore struct { type NodeAuditAction string const ( - NodeAuditActionCreate NodeAuditAction = "create" - NodeAuditActionDelete NodeAuditAction = "delete" - NodeAuditActionTestSSH NodeAuditAction = "test_ssh" - NodeAuditActionUpdate NodeAuditAction = "update" + NodeAuditActionCreate NodeAuditAction = "create" + NodeAuditActionDelete NodeAuditAction = "delete" + NodeAuditActionTestSSH NodeAuditAction = "test_ssh" + NodeAuditActionHealthCheck NodeAuditAction = "health_check" + NodeAuditActionUpdate NodeAuditAction = "update" ) // NodeAuditEvent is one append-only record of a node mutation. diff --git a/webui/src/App.css b/webui/src/App.css index 6ceabc6..9a112fb 100644 --- a/webui/src/App.css +++ b/webui/src/App.css @@ -454,11 +454,98 @@ .node-list-heading-main { display: flex; flex-wrap: wrap; - align-items: baseline; + align-items: center; gap: 0.5rem 1rem; min-width: 0; } +.node-health-dot { + display: inline-block; + width: 0.55rem; + height: 0.55rem; + border-radius: 50%; + flex-shrink: 0; + vertical-align: middle; + box-shadow: inset 0 0 0 1px color-mix(in srgb, currentColor 25%, transparent); +} + +.node-health-dot-ok { + background: var(--ctp-green); + color: var(--ctp-green); +} + +.node-health-dot-error { + background: var(--ctp-red); + color: var(--ctp-red); +} + +.node-health-dot-unknown { + background: var(--ctp-overlay0); + color: var(--ctp-overlay0); +} + +.node-healthcheck-tab { + display: flex; + flex-direction: column; + gap: 1rem; +} + +.node-health-interval { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0.5rem 0.75rem; + margin: 0.55rem 0 0.35rem; +} + +.node-health-interval-label { + display: inline-flex; + flex-wrap: wrap; + align-items: center; + gap: 0.4rem 0.5rem; + font-size: 0.85rem; + color: var(--ctp-subtext0); +} + +.node-health-interval-input { + width: 5.5rem; + padding: 0.3rem 0.45rem; + border: 1px solid var(--ctp-surface1); + border-radius: 6px; + background: var(--ctp-surface0); + color: var(--ctp-text); + font: inherit; +} + +.node-health-interval-unit { + padding: 0.3rem 0.45rem; + border: 1px solid var(--ctp-surface1); + border-radius: 6px; + background: var(--ctp-surface0); + color: var(--ctp-text); + font: inherit; +} + +.node-health-interval-save { + padding: 0.3rem 0.7rem; + border: 1px solid var(--ctp-surface1); + border-radius: 6px; + background: var(--ctp-surface0); + color: var(--ctp-text); + font: inherit; + font-size: 0.85rem; + cursor: pointer; +} + +.node-health-interval-save:hover:not(:disabled) { + border-color: var(--ctp-overlay0); +} + +.node-health-interval-save:disabled { + opacity: 0.65; + cursor: not-allowed; +} + .node-edit { flex-shrink: 0; margin-left: auto; diff --git a/webui/src/App.tsx b/webui/src/App.tsx index e09a173..a460778 100644 --- a/webui/src/App.tsx +++ b/webui/src/App.tsx @@ -29,7 +29,7 @@ import { patchAction, patchUser, reauth, - testNodeSSH, + runNodeHealthCheck, type AccessMode, type Action, type ActionEnvVar, @@ -76,6 +76,7 @@ import { LoginPage } from './LoginPage' import { NodeDetailPanel } from './NodeDetailPanel' import { countAuthFailuresToday, + countNodeHealthErrorsBySection, countNodesBySection, type NodeSectionCounts, } from './sidebarCounts' @@ -2407,6 +2408,26 @@ function resourceTabsFor(section: ResourceSectionId): ReadonlyArray<{ ] } +function healthDotClass(healthOk: boolean | null | undefined): string { + if (healthOk === true) { + return 'node-health-dot node-health-dot-ok' + } + if (healthOk === false) { + return 'node-health-dot node-health-dot-error' + } + return 'node-health-dot node-health-dot-unknown' +} + +function healthDotLabel(healthOk: boolean | null | undefined): string { + if (healthOk === true) { + return 'Healthy' + } + if (healthOk === false) { + return 'Unhealthy' + } + return 'Health unknown' +} + function ResourceOverviewTab({ section, canRead, @@ -2421,10 +2442,7 @@ function ResourceOverviewTab({ const [nodes, setNodes] = useState>([]) const [isLoading, setIsLoading] = useState(true) const [loadError, setLoadError] = useState(null) - const [testingNodeId, setTestingNodeId] = useState(null) - const [testResults, setTestResults] = useState( - {} as Record, - ) + const [checkingNodeId, setCheckingNodeId] = useState(null) const [me, setMe] = useState(null) const [pendingDeleteNodeId, setPendingDeleteNodeId] = useState( null, @@ -2491,29 +2509,29 @@ function ResourceOverviewTab({ } }, [section, canRead]) - async function handleTestSSH(nodeId: string) { - setTestingNodeId(nodeId) - setTestResults((previous) => { - const next = { ...previous } - delete next[nodeId] - return next - }) + async function handleHealthCheck(nodeId: string) { + setCheckingNodeId(nodeId) try { - const result = await testNodeSSH(nodeId) - setTestResults((previous) => ({ - ...previous, - [nodeId]: { ok: result.ok, message: result.message }, - })) + const response = await runNodeHealthCheck(nodeId) + setNodes((previous) => + previous.map((node) => (node.id === nodeId ? response.node : node)), + ) } catch (err) { - setTestResults((previous) => ({ - ...previous, - [nodeId]: { - ok: false, - message: err instanceof Error ? err.message : 'SSH test failed', - }, - })) + setNodes((previous) => + previous.map((node) => { + if (node.id !== nodeId) { + return node + } + return { + ...node, + health_ok: false, + health_message: + err instanceof Error ? err.message : 'Health check failed', + } + }), + ) } finally { - setTestingNodeId(null) + setCheckingNodeId(null) } } @@ -2571,11 +2589,16 @@ function ResourceOverviewTab({ ) : null}
    {nodes.map((node) => { - const testResult = testResults[node.id] return (
  • + {node.name} {node.id}
    @@ -2584,7 +2607,13 @@ function ResourceOverviewTab({ className="node-edit" onClick={() => { navigateTo( - pathFor(section, 'overview', 'overview', node.id, 'actions'), + pathFor( + section, + 'overview', + 'overview', + node.id, + 'healthcheck', + ), ) }} > @@ -2595,20 +2624,37 @@ function ResourceOverviewTab({ {node.host_ip} · {node.username} · group {node.group_name} ·{' '} {node.key_algo}

    + {node.health_message ? ( +

    + {node.health_message} + {node.health_last_checked_at + ? ` · ${new Date(node.health_last_checked_at).toLocaleString()}` + : ''} +

    + ) : null} {canExec || canDelete ? (
    {canExec ? ( ) : null} {canDelete ? ( @@ -2622,19 +2668,6 @@ function ResourceOverviewTab({ Delete ) : null} - {testResult ? ( -

    - {testResult.ok ? 'Succeeded: ' : 'Failed: '} - {testResult.message} -

    - ) : null}
    ) : null}
    • @@ -3211,6 +3244,7 @@ function LogsPanel() { { id: 'update', label: 'update' }, { id: 'delete', label: 'delete' }, { id: 'test_ssh', label: 'test_ssh' }, + { id: 'health_check', label: 'health_check' }, ] const authOutcomeFilters: ReadonlyArray<{ @@ -3812,6 +3846,8 @@ function App() { const [canReadLogs, setCanReadLogs] = useState(false) const [nodeSectionCounts, setNodeSectionCounts] = useState(null) + const [nodeHealthErrorCounts, setNodeHealthErrorCounts] = + useState(null) const [authFailuresToday, setAuthFailuresToday] = useState( null, ) @@ -3893,10 +3929,14 @@ function App() { }) if (!isCancelled) { setNodeSectionCounts(countNodesBySection(nodesResponse.nodes)) + setNodeHealthErrorCounts( + countNodeHealthErrorsBySection(nodesResponse.nodes), + ) } } catch { if (!isCancelled) { setNodeSectionCounts(null) + setNodeHealthErrorCounts(null) } } @@ -4099,6 +4139,10 @@ function App() { isResourceSectionId(item.id) && nodeSectionCounts !== null ? nodeSectionCounts[item.id] : null + const healthErrorCount = + isResourceSectionId(item.id) && nodeHealthErrorCounts !== null + ? nodeHealthErrorCounts[item.id] + : null return ( + {intervalError ? ( +

    + {intervalError} +

    + ) : null} +

    Use 0 to disable scheduled checks.

    + + ) : ( +

    + Interval:{' '} + {(node.health_check_interval_seconds ?? 0) === 0 + ? 'Disabled' + : `Every ${node.health_check_interval_seconds} seconds`} + . You need nodes.update to change this. +

    + )} + + {canExec ? ( +
    + + {checkError ? ( +

    + {checkError} +

    + ) : null} +
    + ) : ( +

    + You need nodes.exec to run a health check. +

    + )} + + ) +} + function NodeActionsTab({ nodeId, nodeUsername, diff --git a/webui/src/api/client.test.ts b/webui/src/api/client.test.ts index c8dc0c5..4304953 100644 --- a/webui/src/api/client.test.ts +++ b/webui/src/api/client.test.ts @@ -705,6 +705,63 @@ describe('nodes API', () => { }), ) }) + + it('posts to the health-check endpoint', async () => { + const { runNodeHealthCheck } = await import('./client') + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ + node: { + id: '11111111-2222-4333-8444-555555555555', + health_ok: true, + health_message: 'ping ok (icmp); ssh ok', + }, + }), + }) + + const result = await runNodeHealthCheck( + '11111111-2222-4333-8444-555555555555', + fetchMock, + ) + + expect(result.node.health_ok).toBe(true) + expect(fetchMock).toHaveBeenCalledWith( + 'http://localhost:8080/api/v1/nodes/11111111-2222-4333-8444-555555555555/health-check', + expect.objectContaining({ + method: 'POST', + credentials: 'include', + }), + ) + }) + + it('patches node health interval', async () => { + const { patchNode } = await import('./client') + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ + node: { + id: '11111111-2222-4333-8444-555555555555', + health_check_interval_seconds: 300, + }, + }), + }) + + const result = await patchNode( + '11111111-2222-4333-8444-555555555555', + { health_check_interval_seconds: 300 }, + fetchMock, + ) + + expect(result.node.health_check_interval_seconds).toBe(300) + expect(fetchMock).toHaveBeenCalledWith( + 'http://localhost:8080/api/v1/nodes/11111111-2222-4333-8444-555555555555', + expect.objectContaining({ + method: 'PATCH', + credentials: 'include', + body: JSON.stringify({ health_check_interval_seconds: 300 }), + }), + ) + }) }) describe('actions API', () => { diff --git a/webui/src/api/client.ts b/webui/src/api/client.ts index 25c05b8..31fac37 100644 --- a/webui/src/api/client.ts +++ b/webui/src/api/client.ts @@ -658,6 +658,12 @@ export type Node = { public_key: string key_algo: string created_at: string + health_check_interval_seconds?: number + health_last_checked_at?: string | null + health_ping_ok?: boolean | null + health_ssh_ok?: boolean | null + health_ok?: boolean | null + health_message?: string } export type NodesResponse = { @@ -774,6 +780,44 @@ export async function testNodeSSH( return (await response.json()) as NodeSSHTestResponse } +export type PatchNodeRequest = { + health_check_interval_seconds: number +} + +export async function patchNode( + id: string, + payload: PatchNodeRequest, + fetchImpl: typeof fetch = fetch, +): Promise { + const response = await apiFetch( + `/api/v1/nodes/${encodeURIComponent(id)}`, + { + method: 'PATCH', + body: JSON.stringify(payload), + }, + fetchImpl, + ) + if (!response.ok) { + throw new Error(await readErrorMessage(response)) + } + return (await response.json()) as NodeResponse +} + +export async function runNodeHealthCheck( + id: string, + fetchImpl: typeof fetch = fetch, +): Promise { + const response = await apiFetch( + `/api/v1/nodes/${encodeURIComponent(id)}/health-check`, + { method: 'POST' }, + fetchImpl, + ) + if (!response.ok) { + throw new Error(await readErrorMessage(response)) + } + return (await response.json()) as NodeResponse +} + export type ResolveCommandsResponse = { paths: Record missing: ReadonlyArray @@ -799,7 +843,7 @@ export async function resolveNodeCommands( return (await response.json()) as ResolveCommandsResponse } -export type NodeAuditAction = 'create' | 'delete' | 'test_ssh' | 'update' +export type NodeAuditAction = 'create' | 'delete' | 'test_ssh' | 'health_check' | 'update' export type NodeAuditEvent = { id: string diff --git a/webui/src/navigation.test.ts b/webui/src/navigation.test.ts index a7edffd..81e57c7 100644 --- a/webui/src/navigation.test.ts +++ b/webui/src/navigation.test.ts @@ -59,6 +59,7 @@ describe('parseLocation', () => { '/configuration/advanced', '/containers/node-1', '/containers/node-1/actions', + '/containers/node-1/healthcheck', '/vms/node-2/logs', ]) { const location = parseLocation(path) diff --git a/webui/src/navigation.ts b/webui/src/navigation.ts index 3fd4ce0..989379a 100644 --- a/webui/src/navigation.ts +++ b/webui/src/navigation.ts @@ -22,7 +22,7 @@ export type ConfigTabId = export type ResourceTabId = 'overview' | 'security' | 'add' -export type NodeDetailTabId = 'overview' | 'actions' | 'logs' +export type NodeDetailTabId = 'overview' | 'healthcheck' | 'actions' | 'logs' export type ResourceSectionId = 'containers' | 'vms' | 'docker' @@ -54,6 +54,7 @@ const RESOURCE_TAB_IDS: ReadonlySet = new Set([ const NODE_DETAIL_TAB_IDS: ReadonlySet = new Set([ 'overview', + 'healthcheck', 'actions', 'logs', ]) diff --git a/webui/src/sidebarCounts.test.ts b/webui/src/sidebarCounts.test.ts index 542ff0b..54e408e 100644 --- a/webui/src/sidebarCounts.test.ts +++ b/webui/src/sidebarCounts.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest' import type { AuthAuditEvent, Node } from './api/client' import { countAuthFailuresToday, + countNodeHealthErrorsBySection, countNodesBySection, } from './sidebarCounts' @@ -37,6 +38,41 @@ describe('countNodesBySection', () => { }) }) +describe('countNodeHealthErrorsBySection', () => { + it('counts only nodes with health_ok false', () => { + expect( + countNodeHealthErrorsBySection([ + { kind: 'container', health_ok: false }, + { kind: 'container', health_ok: true }, + { kind: 'container', health_ok: null }, + { kind: 'container' }, + { kind: 'docker', health_ok: false }, + { kind: 'vm', health_ok: false }, + { kind: 'vm', health_ok: false }, + { kind: 'vm', health_ok: true }, + ]), + ).toEqual({ + containers: 1, + docker: 1, + vms: 2, + }) + }) + + it('returns zeros when no unhealthy nodes', () => { + expect( + countNodeHealthErrorsBySection([ + { kind: 'container', health_ok: true }, + { kind: 'docker' }, + { kind: 'vm', health_ok: null }, + ]), + ).toEqual({ + containers: 0, + docker: 0, + vms: 0, + }) + }) +}) + describe('countAuthFailuresToday', () => { const now = new Date(2026, 6, 18, 15, 30, 0) diff --git a/webui/src/sidebarCounts.ts b/webui/src/sidebarCounts.ts index d348959..c3a9ae6 100644 --- a/webui/src/sidebarCounts.ts +++ b/webui/src/sidebarCounts.ts @@ -24,6 +24,23 @@ export function countNodesBySection( return counts } +export function countNodeHealthErrorsBySection( + nodes: ReadonlyArray>, +): NodeSectionCounts { + const counts: NodeSectionCounts = { + containers: 0, + docker: 0, + vms: 0, + } + for (const node of nodes) { + if (node.health_ok !== false) { + continue + } + counts[KIND_TO_SECTION[node.kind]] += 1 + } + return counts +} + export function countAuthFailuresToday( events: ReadonlyArray>, now: Date = new Date(),