Add per-node health checks with status indicators and scheduled probes.

service/internal/api/authz.go

@@ -10,6 +10,7 @@ import (
 var errUsersManageRequired = errors.New("permission users.manage required")
 var errNodesReadRequired = errors.New("permission nodes.read required")
 var errNodesExecRequired = errors.New("permission nodes.exec required")
+var errNodesUpdateRequired = errors.New("permission nodes.update required")
 var errNodesDeleteRequired = errors.New("permission nodes.delete required")
 var errActionsCreateRequired = errors.New("permission actions.create required")
 var errActionsUpdateRequired = errors.New("permission actions.update required")
@@ -135,6 +136,22 @@ func (app *App) authorizeNodesExec(request *http.Request) error {
 	return nil
 }
 
+func (app *App) authorizeNodesUpdate(request *http.Request) error {
+	user, ok := UserFromContext(request.Context())
+	if !ok {
+		return errors.New("authentication required")
+	}
+
+	payload, err := loadSettingsOrDefault(app.ConfigDir)
+	if err != nil {
+		return err
+	}
+	if !userHasPermission(user, payload.Groups, "nodes.update") {
+		return errNodesUpdateRequired
+	}
+	return nil
+}
+
 func (app *App) authorizeNodesDelete(request *http.Request) error {
 	user, ok := UserFromContext(request.Context())
 	if !ok {

service/internal/api/nodes_handlers.go

@@ -3,6 +3,7 @@ package api
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net"
 	"net/http"
 	"strings"
@@ -231,6 +232,164 @@ func (app *App) nodesTestSSHHandler(writer http.ResponseWriter, request *http.Re
 	})
 }
 
+type patchNodeRequest struct {
+	HealthCheckIntervalSeconds *int `json:"health_check_interval_seconds"`
+}
+
+func (app *App) nodesPatchHandler(writer http.ResponseWriter, request *http.Request) {
+	if err := app.authorizeNodesUpdate(request); err != nil {
+		writeJSON(writer, http.StatusForbidden, apiErrorResponse{Error: err.Error()})
+		return
+	}
+
+	user, ok := UserFromContext(request.Context())
+	if !ok {
+		writeJSON(writer, http.StatusUnauthorized, apiErrorResponse{Error: "authentication required"})
+		return
+	}
+
+	nodeID := strings.TrimSpace(request.PathValue("id"))
+	if nodeID == "" {
+		writeJSON(writer, http.StatusBadRequest, apiErrorResponse{Error: "missing node id"})
+		return
+	}
+
+	var payload patchNodeRequest
+	if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
+		writeJSON(writer, http.StatusBadRequest, apiErrorResponse{Error: "invalid JSON body"})
+		return
+	}
+	if payload.HealthCheckIntervalSeconds == nil {
+		writeJSON(writer, http.StatusBadRequest, apiErrorResponse{
+			Error: "health_check_interval_seconds is required",
+		})
+		return
+	}
+	if *payload.HealthCheckIntervalSeconds < 0 {
+		writeJSON(writer, http.StatusBadRequest, apiErrorResponse{
+			Error: "health_check_interval_seconds must be >= 0",
+		})
+		return
+	}
+
+	settingsPayload, err := loadSettingsOrDefault(app.ConfigDir)
+	if err != nil {
+		writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()})
+		return
+	}
+
+	store, err := settings.LoadNodesOrEmpty(app.ConfigDir)
+	if err != nil {
+		writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()})
+		return
+	}
+
+	nodeIndex := -1
+	var node settings.Node
+	for index, candidate := range store.Nodes {
+		if candidate.ID != nodeID {
+			continue
+		}
+		node = candidate
+		nodeIndex = index
+		break
+	}
+	if nodeIndex < 0 {
+		writeJSON(writer, http.StatusNotFound, apiErrorResponse{Error: "node not found"})
+		return
+	}
+	if !userCanAccessNode(user, settingsPayload.Groups, node) {
+		writeJSON(writer, http.StatusForbidden, apiErrorResponse{Error: "permission denied"})
+		return
+	}
+
+	node.HealthCheckIntervalSeconds = *payload.HealthCheckIntervalSeconds
+	store.Nodes[nodeIndex] = node
+	if err := settings.SaveNodes(app.ConfigDir, store); err != nil {
+		writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()})
+		return
+	}
+
+	_ = appendNodeAudit(
+		app.ConfigDir,
+		settings.NodeAuditActionUpdate,
+		user.Username,
+		node,
+		fmt.Sprintf("health_check_interval_seconds=%d", node.HealthCheckIntervalSeconds),
+	)
+	writeJSON(writer, http.StatusOK, nodeResponse{Node: node})
+}
+
+func (app *App) nodesHealthCheckHandler(writer http.ResponseWriter, request *http.Request) {
+	if err := app.authorizeNodesExec(request); err != nil {
+		writeJSON(writer, http.StatusForbidden, apiErrorResponse{Error: err.Error()})
+		return
+	}
+	if err := app.requireKey(); err != nil {
+		writeJSON(writer, http.StatusServiceUnavailable, apiErrorResponse{Error: err.Error()})
+		return
+	}
+	if app.HealthChecker == nil {
+		writeJSON(writer, http.StatusServiceUnavailable, apiErrorResponse{Error: "health checker unavailable"})
+		return
+	}
+
+	user, ok := UserFromContext(request.Context())
+	if !ok {
+		writeJSON(writer, http.StatusUnauthorized, apiErrorResponse{Error: "authentication required"})
+		return
+	}
+
+	nodeID := strings.TrimSpace(request.PathValue("id"))
+	if nodeID == "" {
+		writeJSON(writer, http.StatusBadRequest, apiErrorResponse{Error: "missing node id"})
+		return
+	}
+
+	settingsPayload, err := loadSettingsOrDefault(app.ConfigDir)
+	if err != nil {
+		writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()})
+		return
+	}
+
+	store, err := settings.LoadNodesOrEmpty(app.ConfigDir)
+	if err != nil {
+		writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()})
+		return
+	}
+
+	var node settings.Node
+	found := false
+	for _, candidate := range store.Nodes {
+		if candidate.ID != nodeID {
+			continue
+		}
+		node = candidate
+		found = true
+		break
+	}
+	if !found {
+		writeJSON(writer, http.StatusNotFound, apiErrorResponse{Error: "node not found"})
+		return
+	}
+	if !userCanAccessNode(user, settingsPayload.Groups, node) {
+		writeJSON(writer, http.StatusForbidden, apiErrorResponse{Error: "permission denied"})
+		return
+	}
+
+	updated, err := app.HealthChecker.CheckNodeByID(nodeID, user.Username)
+	if err != nil {
+		if strings.Contains(err.Error(), "already running") {
+			writeJSON(writer, http.StatusConflict, apiErrorResponse{Error: err.Error()})
+			return
+		}
+		writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()})
+		return
+	}
+
+	writeJSON(writer, http.StatusOK, nodeResponse{Node: updated})
+}
+
 func (app *App) nodesCreateHandler(writer http.ResponseWriter, request *http.Request) {
 	if err := app.authorizeNodesExec(request); err != nil {
 		writeJSON(writer, http.StatusForbidden, apiErrorResponse{Error: err.Error()})

service/internal/api/nodes_health_handlers_test.go

@@ -0,0 +1,211 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"codeberg.org/SquidSE/ClusterCanvas/service/internal/auth"
+)
+
+func TestNodesHealthCheckPersistsStatus(t *testing.T) {
+	configDir := t.TempDir()
+	cookie := seedCompletedSetup(t, configDir)
+	router, app := NewRouterWithApp(configDir)
+
+	app.HealthChecker.PingFn = func(hostIP string) auth.PingResult {
+		return auth.PingResult{OK: true, Method: auth.PingMethodICMP}
+	}
+	app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error {
+		return nil
+	}
+
+	createBody := []byte(`{
+		"id":"bbbbbbbb-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+		"kind":"container",
+		"name":"healthy-node",
+		"host_ip":"10.20.30.40",
+		"group_name":"Administrators",
+		"generate":{"algorithm":"ed25519"}
+	}`)
+	createRequest := withSession(
+		httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
+		cookie,
+	)
+	createRecorder := httptest.NewRecorder()
+	router.ServeHTTP(createRecorder, createRequest)
+	if createRecorder.Code != http.StatusCreated {
+		t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
+	}
+
+	healthRequest := withSession(
+		httptest.NewRequest(
+			http.MethodPost,
+			"/api/v1/nodes/bbbbbbbb-bbbb-4ccc-8ddd-eeeeeeeeeeee/health-check",
+			nil,
+		),
+		cookie,
+	)
+	healthRecorder := httptest.NewRecorder()
+	router.ServeHTTP(healthRecorder, healthRequest)
+	if healthRecorder.Code != http.StatusOK {
+		t.Fatalf("health status = %d body=%s", healthRecorder.Code, healthRecorder.Body.String())
+	}
+
+	var response nodeResponse
+	if err := json.Unmarshal(healthRecorder.Body.Bytes(), &response); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if response.Node.HealthOK == nil || !*response.Node.HealthOK {
+		t.Fatalf("health_ok = %#v", response.Node.HealthOK)
+	}
+	if response.Node.HealthMessage == "" {
+		t.Fatal("expected health message")
+	}
+}
+
+func TestNodesHealthCheckRecordsFailure(t *testing.T) {
+	configDir := t.TempDir()
+	cookie := seedCompletedSetup(t, configDir)
+	router, app := NewRouterWithApp(configDir)
+
+	app.HealthChecker.PingFn = func(hostIP string) auth.PingResult {
+		return auth.PingResult{
+			OK:     false,
+			Method: auth.PingMethodTCP,
+			Error:  errors.New("unreachable"),
+		}
+	}
+	app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error {
+		return errors.New("ssh failed")
+	}
+
+	createBody := []byte(`{
+		"id":"cccccccc-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+		"kind":"vm",
+		"name":"unhealthy-node",
+		"host_ip":"10.20.30.41",
+		"group_name":"Administrators",
+		"generate":{"algorithm":"ed25519"}
+	}`)
+	createRequest := withSession(
+		httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
+		cookie,
+	)
+	createRecorder := httptest.NewRecorder()
+	router.ServeHTTP(createRecorder, createRequest)
+	if createRecorder.Code != http.StatusCreated {
+		t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
+	}
+
+	healthRequest := withSession(
+		httptest.NewRequest(
+			http.MethodPost,
+			"/api/v1/nodes/cccccccc-bbbb-4ccc-8ddd-eeeeeeeeeeee/health-check",
+			nil,
+		),
+		cookie,
+	)
+	healthRecorder := httptest.NewRecorder()
+	router.ServeHTTP(healthRecorder, healthRequest)
+	if healthRecorder.Code != http.StatusOK {
+		t.Fatalf("health status = %d body=%s", healthRecorder.Code, healthRecorder.Body.String())
+	}
+
+	var response nodeResponse
+	if err := json.Unmarshal(healthRecorder.Body.Bytes(), &response); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if response.Node.HealthOK == nil || *response.Node.HealthOK {
+		t.Fatalf("expected failure, got %#v", response.Node.HealthOK)
+	}
+}
+
+func TestNodesPatchHealthInterval(t *testing.T) {
+	configDir := t.TempDir()
+	cookie := seedCompletedSetup(t, configDir)
+	router := NewRouter(configDir)
+
+	createBody := []byte(`{
+		"id":"dddddddd-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+		"kind":"docker",
+		"name":"interval-node",
+		"host_ip":"10.20.30.42",
+		"group_name":"Administrators",
+		"generate":{"algorithm":"ed25519"}
+	}`)
+	createRequest := withSession(
+		httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
+		cookie,
+	)
+	createRecorder := httptest.NewRecorder()
+	router.ServeHTTP(createRecorder, createRequest)
+	if createRecorder.Code != http.StatusCreated {
+		t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
+	}
+
+	patchBody := []byte(`{"health_check_interval_seconds":120}`)
+	patchRequest := withSession(
+		httptest.NewRequest(
+			http.MethodPatch,
+			"/api/v1/nodes/dddddddd-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+			bytes.NewReader(patchBody),
+		),
+		cookie,
+	)
+	patchRecorder := httptest.NewRecorder()
+	router.ServeHTTP(patchRecorder, patchRequest)
+	if patchRecorder.Code != http.StatusOK {
+		t.Fatalf("patch status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String())
+	}
+
+	var response nodeResponse
+	if err := json.Unmarshal(patchRecorder.Body.Bytes(), &response); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if response.Node.HealthCheckIntervalSeconds != 120 {
+		t.Fatalf("interval = %d", response.Node.HealthCheckIntervalSeconds)
+	}
+}
+
+func TestNodesPatchHealthIntervalRejectsNegative(t *testing.T) {
+	configDir := t.TempDir()
+	cookie := seedCompletedSetup(t, configDir)
+	router := NewRouter(configDir)
+
+	createBody := []byte(`{
+		"id":"eeeeeeee-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+		"kind":"container",
+		"name":"bad-interval",
+		"host_ip":"10.20.30.43",
+		"group_name":"Administrators",
+		"generate":{"algorithm":"ed25519"}
+	}`)
+	createRequest := withSession(
+		httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
+		cookie,
+	)
+	createRecorder := httptest.NewRecorder()
+	router.ServeHTTP(createRecorder, createRequest)
+	if createRecorder.Code != http.StatusCreated {
+		t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
+	}
+
+	patchBody := []byte(`{"health_check_interval_seconds":-5}`)
+	patchRequest := withSession(
+		httptest.NewRequest(
+			http.MethodPatch,
+			"/api/v1/nodes/eeeeeeee-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+			bytes.NewReader(patchBody),
+		),
+		cookie,
+	)
+	patchRecorder := httptest.NewRecorder()
+	router.ServeHTTP(patchRecorder, patchRequest)
+	if patchRecorder.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String())
+	}
+}

service/internal/api/router.go

@@ -22,6 +22,9 @@ func NewRouterWithApp(configDir string) (http.Handler, *App) {
 	if app.Executor != nil {
 		app.Executor.StartScheduler()
 	}
+	if app.HealthChecker != nil {
+		app.HealthChecker.StartScheduler()
+	}
 
 	mux := http.NewServeMux()
 	mux.HandleFunc("GET /health", HealthHandler)
@@ -62,8 +65,10 @@ func NewRouterWithApp(configDir string) (http.Handler, *App) {
 	mux.HandleFunc("GET /api/v1/nodes", app.nodesListHandler)
 	mux.HandleFunc("POST /api/v1/nodes", app.nodesCreateHandler)
 	mux.HandleFunc("GET /api/v1/nodes/{id}", app.nodesGetHandler)
+	mux.HandleFunc("PATCH /api/v1/nodes/{id}", app.nodesPatchHandler)
 	mux.HandleFunc("DELETE /api/v1/nodes/{id}", app.nodesDeleteHandler)
 	mux.HandleFunc("POST /api/v1/nodes/{id}/test-ssh", app.nodesTestSSHHandler)
+	mux.HandleFunc("POST /api/v1/nodes/{id}/health-check", app.nodesHealthCheckHandler)
 	mux.HandleFunc("POST /api/v1/nodes/{id}/resolve-commands", app.nodesResolveCommandsHandler)
 	mux.HandleFunc("GET /api/v1/nodes/{id}/action-groups", app.actionGroupsListHandler)
 	mux.HandleFunc("POST /api/v1/nodes/{id}/action-groups", app.actionGroupsCreateHandler)

service/internal/api/setup_auth_handlers.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"codeberg.org/SquidSE/ClusterCanvas/service/internal/auth"
+	"codeberg.org/SquidSE/ClusterCanvas/service/internal/health"
 	"codeberg.org/SquidSE/ClusterCanvas/service/internal/runner"
 	"codeberg.org/SquidSE/ClusterCanvas/service/internal/settings"
 )
@@ -77,10 +78,11 @@ type meResponse struct {
 
 // App holds shared API dependencies.
 type App struct {
-	ConfigDir string
-	Key       []byte
-	Sessions  *auth.SessionManager
-	Executor  *runner.Executor
+	ConfigDir     string
+	Key           []byte
+	Sessions      *auth.SessionManager
+	Executor      *runner.Executor
+	HealthChecker *health.Checker
 
 	pendingTOTPMu sync.Mutex
 	pendingTOTP   map[string]string // username -> secret (setup only)
@@ -91,18 +93,20 @@ func newApp(configDir string) (*App, error) {
 	if err != nil {
 		// Key is required for setup complete / auth; status still works without it.
 		app := &App{
-			ConfigDir:   configDir,
-			pendingTOTP: map[string]string{},
-			Executor:    runner.NewExecutor(configDir, nil),
+			ConfigDir:     configDir,
+			pendingTOTP:   map[string]string{},
+			Executor:      runner.NewExecutor(configDir, nil),
+			HealthChecker: health.NewChecker(configDir, nil),
 		}
 		return app, nil
 	}
 	app := &App{
-		ConfigDir:   configDir,
-		Key:         key,
-		Sessions:    auth.NewSessionManager(configDir, key),
-		pendingTOTP: map[string]string{},
-		Executor:    runner.NewExecutor(configDir, key),
+		ConfigDir:     configDir,
+		Key:           key,
+		Sessions:      auth.NewSessionManager(configDir, key),
+		pendingTOTP:   map[string]string{},
+		Executor:      runner.NewExecutor(configDir, key),
+		HealthChecker: health.NewChecker(configDir, key),
 	}
 	return app, nil
 }