Add per-node health checks with status indicators and scheduled probes.

service/cmd/server/main.go

@@ -57,6 +57,9 @@ func main() {
 	if app.Executor != nil {
 		app.Executor.StopScheduler()
 	}
+	if app.HealthChecker != nil {
+		app.HealthChecker.StopScheduler()
+	}
 
 	shutdownContext, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()

service/internal/api/authz.go

@@ -10,6 +10,7 @@ import (
 var errUsersManageRequired = errors.New("permission users.manage required")
 var errNodesReadRequired = errors.New("permission nodes.read required")
 var errNodesExecRequired = errors.New("permission nodes.exec required")
+var errNodesUpdateRequired = errors.New("permission nodes.update required")
 var errNodesDeleteRequired = errors.New("permission nodes.delete required")
 var errActionsCreateRequired = errors.New("permission actions.create required")
 var errActionsUpdateRequired = errors.New("permission actions.update required")
@@ -135,6 +136,22 @@ func (app *App) authorizeNodesExec(request *http.Request) error {
 	return nil
 }
 
+func (app *App) authorizeNodesUpdate(request *http.Request) error {
+	user, ok := UserFromContext(request.Context())
+	if !ok {
+		return errors.New("authentication required")
+	}
+
+	payload, err := loadSettingsOrDefault(app.ConfigDir)
+	if err != nil {
+		return err
+	}
+	if !userHasPermission(user, payload.Groups, "nodes.update") {
+		return errNodesUpdateRequired
+	}
+	return nil
+}
+
 func (app *App) authorizeNodesDelete(request *http.Request) error {
 	user, ok := UserFromContext(request.Context())
 	if !ok {

service/internal/api/nodes_handlers.go

@@ -3,6 +3,7 @@ package api
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net"
 	"net/http"
 	"strings"
@@ -231,6 +232,164 @@ func (app *App) nodesTestSSHHandler(writer http.ResponseWriter, request *http.Re
 	})
 }
 
+type patchNodeRequest struct {
+	HealthCheckIntervalSeconds *int `json:"health_check_interval_seconds"`
+}
+
+func (app *App) nodesPatchHandler(writer http.ResponseWriter, request *http.Request) {
+	if err := app.authorizeNodesUpdate(request); err != nil {
+		writeJSON(writer, http.StatusForbidden, apiErrorResponse{Error: err.Error()})
+		return
+	}
+
+	user, ok := UserFromContext(request.Context())
+	if !ok {
+		writeJSON(writer, http.StatusUnauthorized, apiErrorResponse{Error: "authentication required"})
+		return
+	}
+
+	nodeID := strings.TrimSpace(request.PathValue("id"))
+	if nodeID == "" {
+		writeJSON(writer, http.StatusBadRequest, apiErrorResponse{Error: "missing node id"})
+		return
+	}
+
+	var payload patchNodeRequest
+	if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
+		writeJSON(writer, http.StatusBadRequest, apiErrorResponse{Error: "invalid JSON body"})
+		return
+	}
+	if payload.HealthCheckIntervalSeconds == nil {
+		writeJSON(writer, http.StatusBadRequest, apiErrorResponse{
+			Error: "health_check_interval_seconds is required",
+		})
+		return
+	}
+	if *payload.HealthCheckIntervalSeconds < 0 {
+		writeJSON(writer, http.StatusBadRequest, apiErrorResponse{
+			Error: "health_check_interval_seconds must be >= 0",
+		})
+		return
+	}
+
+	settingsPayload, err := loadSettingsOrDefault(app.ConfigDir)
+	if err != nil {
+		writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()})
+		return
+	}
+
+	store, err := settings.LoadNodesOrEmpty(app.ConfigDir)
+	if err != nil {
+		writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()})
+		return
+	}
+
+	nodeIndex := -1
+	var node settings.Node
+	for index, candidate := range store.Nodes {
+		if candidate.ID != nodeID {
+			continue
+		}
+		node = candidate
+		nodeIndex = index
+		break
+	}
+	if nodeIndex < 0 {
+		writeJSON(writer, http.StatusNotFound, apiErrorResponse{Error: "node not found"})
+		return
+	}
+	if !userCanAccessNode(user, settingsPayload.Groups, node) {
+		writeJSON(writer, http.StatusForbidden, apiErrorResponse{Error: "permission denied"})
+		return
+	}
+
+	node.HealthCheckIntervalSeconds = *payload.HealthCheckIntervalSeconds
+	store.Nodes[nodeIndex] = node
+	if err := settings.SaveNodes(app.ConfigDir, store); err != nil {
+		writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()})
+		return
+	}
+
+	_ = appendNodeAudit(
+		app.ConfigDir,
+		settings.NodeAuditActionUpdate,
+		user.Username,
+		node,
+		fmt.Sprintf("health_check_interval_seconds=%d", node.HealthCheckIntervalSeconds),
+	)
+	writeJSON(writer, http.StatusOK, nodeResponse{Node: node})
+}
+
+func (app *App) nodesHealthCheckHandler(writer http.ResponseWriter, request *http.Request) {
+	if err := app.authorizeNodesExec(request); err != nil {
+		writeJSON(writer, http.StatusForbidden, apiErrorResponse{Error: err.Error()})
+		return
+	}
+	if err := app.requireKey(); err != nil {
+		writeJSON(writer, http.StatusServiceUnavailable, apiErrorResponse{Error: err.Error()})
+		return
+	}
+	if app.HealthChecker == nil {
+		writeJSON(writer, http.StatusServiceUnavailable, apiErrorResponse{Error: "health checker unavailable"})
+		return
+	}
+
+	user, ok := UserFromContext(request.Context())
+	if !ok {
+		writeJSON(writer, http.StatusUnauthorized, apiErrorResponse{Error: "authentication required"})
+		return
+	}
+
+	nodeID := strings.TrimSpace(request.PathValue("id"))
+	if nodeID == "" {
+		writeJSON(writer, http.StatusBadRequest, apiErrorResponse{Error: "missing node id"})
+		return
+	}
+
+	settingsPayload, err := loadSettingsOrDefault(app.ConfigDir)
+	if err != nil {
+		writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()})
+		return
+	}
+
+	store, err := settings.LoadNodesOrEmpty(app.ConfigDir)
+	if err != nil {
+		writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()})
+		return
+	}
+
+	var node settings.Node
+	found := false
+	for _, candidate := range store.Nodes {
+		if candidate.ID != nodeID {
+			continue
+		}
+		node = candidate
+		found = true
+		break
+	}
+	if !found {
+		writeJSON(writer, http.StatusNotFound, apiErrorResponse{Error: "node not found"})
+		return
+	}
+	if !userCanAccessNode(user, settingsPayload.Groups, node) {
+		writeJSON(writer, http.StatusForbidden, apiErrorResponse{Error: "permission denied"})
+		return
+	}
+
+	updated, err := app.HealthChecker.CheckNodeByID(nodeID, user.Username)
+	if err != nil {
+		if strings.Contains(err.Error(), "already running") {
+			writeJSON(writer, http.StatusConflict, apiErrorResponse{Error: err.Error()})
+			return
+		}
+		writeJSON(writer, http.StatusInternalServerError, apiErrorResponse{Error: err.Error()})
+		return
+	}
+
+	writeJSON(writer, http.StatusOK, nodeResponse{Node: updated})
+}
+
 func (app *App) nodesCreateHandler(writer http.ResponseWriter, request *http.Request) {
 	if err := app.authorizeNodesExec(request); err != nil {
 		writeJSON(writer, http.StatusForbidden, apiErrorResponse{Error: err.Error()})

service/internal/api/nodes_health_handlers_test.go

@@ -0,0 +1,211 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"codeberg.org/SquidSE/ClusterCanvas/service/internal/auth"
+)
+
+func TestNodesHealthCheckPersistsStatus(t *testing.T) {
+	configDir := t.TempDir()
+	cookie := seedCompletedSetup(t, configDir)
+	router, app := NewRouterWithApp(configDir)
+
+	app.HealthChecker.PingFn = func(hostIP string) auth.PingResult {
+		return auth.PingResult{OK: true, Method: auth.PingMethodICMP}
+	}
+	app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error {
+		return nil
+	}
+
+	createBody := []byte(`{
+		"id":"bbbbbbbb-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+		"kind":"container",
+		"name":"healthy-node",
+		"host_ip":"10.20.30.40",
+		"group_name":"Administrators",
+		"generate":{"algorithm":"ed25519"}
+	}`)
+	createRequest := withSession(
+		httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
+		cookie,
+	)
+	createRecorder := httptest.NewRecorder()
+	router.ServeHTTP(createRecorder, createRequest)
+	if createRecorder.Code != http.StatusCreated {
+		t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
+	}
+
+	healthRequest := withSession(
+		httptest.NewRequest(
+			http.MethodPost,
+			"/api/v1/nodes/bbbbbbbb-bbbb-4ccc-8ddd-eeeeeeeeeeee/health-check",
+			nil,
+		),
+		cookie,
+	)
+	healthRecorder := httptest.NewRecorder()
+	router.ServeHTTP(healthRecorder, healthRequest)
+	if healthRecorder.Code != http.StatusOK {
+		t.Fatalf("health status = %d body=%s", healthRecorder.Code, healthRecorder.Body.String())
+	}
+
+	var response nodeResponse
+	if err := json.Unmarshal(healthRecorder.Body.Bytes(), &response); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if response.Node.HealthOK == nil || !*response.Node.HealthOK {
+		t.Fatalf("health_ok = %#v", response.Node.HealthOK)
+	}
+	if response.Node.HealthMessage == "" {
+		t.Fatal("expected health message")
+	}
+}
+
+func TestNodesHealthCheckRecordsFailure(t *testing.T) {
+	configDir := t.TempDir()
+	cookie := seedCompletedSetup(t, configDir)
+	router, app := NewRouterWithApp(configDir)
+
+	app.HealthChecker.PingFn = func(hostIP string) auth.PingResult {
+		return auth.PingResult{
+			OK:     false,
+			Method: auth.PingMethodTCP,
+			Error:  errors.New("unreachable"),
+		}
+	}
+	app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error {
+		return errors.New("ssh failed")
+	}
+
+	createBody := []byte(`{
+		"id":"cccccccc-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+		"kind":"vm",
+		"name":"unhealthy-node",
+		"host_ip":"10.20.30.41",
+		"group_name":"Administrators",
+		"generate":{"algorithm":"ed25519"}
+	}`)
+	createRequest := withSession(
+		httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
+		cookie,
+	)
+	createRecorder := httptest.NewRecorder()
+	router.ServeHTTP(createRecorder, createRequest)
+	if createRecorder.Code != http.StatusCreated {
+		t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
+	}
+
+	healthRequest := withSession(
+		httptest.NewRequest(
+			http.MethodPost,
+			"/api/v1/nodes/cccccccc-bbbb-4ccc-8ddd-eeeeeeeeeeee/health-check",
+			nil,
+		),
+		cookie,
+	)
+	healthRecorder := httptest.NewRecorder()
+	router.ServeHTTP(healthRecorder, healthRequest)
+	if healthRecorder.Code != http.StatusOK {
+		t.Fatalf("health status = %d body=%s", healthRecorder.Code, healthRecorder.Body.String())
+	}
+
+	var response nodeResponse
+	if err := json.Unmarshal(healthRecorder.Body.Bytes(), &response); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if response.Node.HealthOK == nil || *response.Node.HealthOK {
+		t.Fatalf("expected failure, got %#v", response.Node.HealthOK)
+	}
+}
+
+func TestNodesPatchHealthInterval(t *testing.T) {
+	configDir := t.TempDir()
+	cookie := seedCompletedSetup(t, configDir)
+	router := NewRouter(configDir)
+
+	createBody := []byte(`{
+		"id":"dddddddd-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+		"kind":"docker",
+		"name":"interval-node",
+		"host_ip":"10.20.30.42",
+		"group_name":"Administrators",
+		"generate":{"algorithm":"ed25519"}
+	}`)
+	createRequest := withSession(
+		httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
+		cookie,
+	)
+	createRecorder := httptest.NewRecorder()
+	router.ServeHTTP(createRecorder, createRequest)
+	if createRecorder.Code != http.StatusCreated {
+		t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
+	}
+
+	patchBody := []byte(`{"health_check_interval_seconds":120}`)
+	patchRequest := withSession(
+		httptest.NewRequest(
+			http.MethodPatch,
+			"/api/v1/nodes/dddddddd-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+			bytes.NewReader(patchBody),
+		),
+		cookie,
+	)
+	patchRecorder := httptest.NewRecorder()
+	router.ServeHTTP(patchRecorder, patchRequest)
+	if patchRecorder.Code != http.StatusOK {
+		t.Fatalf("patch status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String())
+	}
+
+	var response nodeResponse
+	if err := json.Unmarshal(patchRecorder.Body.Bytes(), &response); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if response.Node.HealthCheckIntervalSeconds != 120 {
+		t.Fatalf("interval = %d", response.Node.HealthCheckIntervalSeconds)
+	}
+}
+
+func TestNodesPatchHealthIntervalRejectsNegative(t *testing.T) {
+	configDir := t.TempDir()
+	cookie := seedCompletedSetup(t, configDir)
+	router := NewRouter(configDir)
+
+	createBody := []byte(`{
+		"id":"eeeeeeee-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+		"kind":"container",
+		"name":"bad-interval",
+		"host_ip":"10.20.30.43",
+		"group_name":"Administrators",
+		"generate":{"algorithm":"ed25519"}
+	}`)
+	createRequest := withSession(
+		httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
+		cookie,
+	)
+	createRecorder := httptest.NewRecorder()
+	router.ServeHTTP(createRecorder, createRequest)
+	if createRecorder.Code != http.StatusCreated {
+		t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
+	}
+
+	patchBody := []byte(`{"health_check_interval_seconds":-5}`)
+	patchRequest := withSession(
+		httptest.NewRequest(
+			http.MethodPatch,
+			"/api/v1/nodes/eeeeeeee-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+			bytes.NewReader(patchBody),
+		),
+		cookie,
+	)
+	patchRecorder := httptest.NewRecorder()
+	router.ServeHTTP(patchRecorder, patchRequest)
+	if patchRecorder.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String())
+	}
+}

service/internal/api/router.go

@@ -22,6 +22,9 @@ func NewRouterWithApp(configDir string) (http.Handler, *App) {
 	if app.Executor != nil {
 		app.Executor.StartScheduler()
 	}
+	if app.HealthChecker != nil {
+		app.HealthChecker.StartScheduler()
+	}
 
 	mux := http.NewServeMux()
 	mux.HandleFunc("GET /health", HealthHandler)
@@ -62,8 +65,10 @@ func NewRouterWithApp(configDir string) (http.Handler, *App) {
 	mux.HandleFunc("GET /api/v1/nodes", app.nodesListHandler)
 	mux.HandleFunc("POST /api/v1/nodes", app.nodesCreateHandler)
 	mux.HandleFunc("GET /api/v1/nodes/{id}", app.nodesGetHandler)
+	mux.HandleFunc("PATCH /api/v1/nodes/{id}", app.nodesPatchHandler)
 	mux.HandleFunc("DELETE /api/v1/nodes/{id}", app.nodesDeleteHandler)
 	mux.HandleFunc("POST /api/v1/nodes/{id}/test-ssh", app.nodesTestSSHHandler)
+	mux.HandleFunc("POST /api/v1/nodes/{id}/health-check", app.nodesHealthCheckHandler)
 	mux.HandleFunc("POST /api/v1/nodes/{id}/resolve-commands", app.nodesResolveCommandsHandler)
 	mux.HandleFunc("GET /api/v1/nodes/{id}/action-groups", app.actionGroupsListHandler)
 	mux.HandleFunc("POST /api/v1/nodes/{id}/action-groups", app.actionGroupsCreateHandler)

service/internal/api/setup_auth_handlers.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"codeberg.org/SquidSE/ClusterCanvas/service/internal/auth"
+	"codeberg.org/SquidSE/ClusterCanvas/service/internal/health"
 	"codeberg.org/SquidSE/ClusterCanvas/service/internal/runner"
 	"codeberg.org/SquidSE/ClusterCanvas/service/internal/settings"
 )
@@ -77,10 +78,11 @@ type meResponse struct {
 
 // App holds shared API dependencies.
 type App struct {
-	ConfigDir string
-	Key       []byte
-	Sessions  *auth.SessionManager
-	Executor  *runner.Executor
+	ConfigDir     string
+	Key           []byte
+	Sessions      *auth.SessionManager
+	Executor      *runner.Executor
+	HealthChecker *health.Checker
 
 	pendingTOTPMu sync.Mutex
 	pendingTOTP   map[string]string // username -> secret (setup only)
@@ -91,18 +93,20 @@ func newApp(configDir string) (*App, error) {
 	if err != nil {
 		// Key is required for setup complete / auth; status still works without it.
 		app := &App{
-			ConfigDir:   configDir,
-			pendingTOTP: map[string]string{},
-			Executor:    runner.NewExecutor(configDir, nil),
+			ConfigDir:     configDir,
+			pendingTOTP:   map[string]string{},
+			Executor:      runner.NewExecutor(configDir, nil),
+			HealthChecker: health.NewChecker(configDir, nil),
 		}
 		return app, nil
 	}
 	app := &App{
-		ConfigDir:   configDir,
-		Key:         key,
-		Sessions:    auth.NewSessionManager(configDir, key),
-		pendingTOTP: map[string]string{},
-		Executor:    runner.NewExecutor(configDir, key),
+		ConfigDir:     configDir,
+		Key:           key,
+		Sessions:      auth.NewSessionManager(configDir, key),
+		pendingTOTP:   map[string]string{},
+		Executor:      runner.NewExecutor(configDir, key),
+		HealthChecker: health.NewChecker(configDir, key),
 	}
 	return app, nil
 }

service/internal/auth/ping.go

@@ -0,0 +1,89 @@
+package auth
+
+import (
+	"fmt"
+	"net"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+const (
+	PingMethodICMP = "icmp"
+	PingMethodTCP  = "tcp"
+
+	defaultPingCount   = "1"
+	defaultPingWaitSec = "2"
+	defaultTCPTimeout  = 3 * time.Second
+)
+
+// PingResult is the outcome of a reachability probe.
+type PingResult struct {
+	OK     bool
+	Method string
+	Error  error
+}
+
+type pingCommandFunc func(hostIP string) error
+type tcpDialFunc func(address string, timeout time.Duration) error
+
+var (
+	runICMPPing = defaultRunICMPPing
+	dialTCP     = defaultDialTCP
+)
+
+// PingHost probes host reachability: ICMP first, then TCP :22 when ICMP is
+// unavailable or the host does not respond to ICMP.
+func PingHost(hostIP string) PingResult {
+	hostIP = strings.TrimSpace(hostIP)
+	if hostIP == "" {
+		return PingResult{OK: false, Error: fmt.Errorf("host IP is required")}
+	}
+
+	icmpErr := runICMPPing(hostIP)
+	if icmpErr == nil {
+		return PingResult{OK: true, Method: PingMethodICMP}
+	}
+
+	address := net.JoinHostPort(hostIP, defaultSSHPort)
+	tcpErr := dialTCP(address, defaultTCPTimeout)
+	if tcpErr == nil {
+		return PingResult{OK: true, Method: PingMethodTCP}
+	}
+
+	return PingResult{
+		OK:     false,
+		Method: PingMethodTCP,
+		Error: fmt.Errorf(
+			"icmp: %v; tcp %s: %w",
+			icmpErr,
+			address,
+			tcpErr,
+		),
+	}
+}
+
+func defaultRunICMPPing(hostIP string) error {
+	pingPath, err := exec.LookPath("ping")
+	if err != nil {
+		return fmt.Errorf("ping unavailable: %w", err)
+	}
+
+	cmd := exec.Command(pingPath, "-c", defaultPingCount, "-W", defaultPingWaitSec, hostIP)
+	output, runErr := cmd.CombinedOutput()
+	if runErr == nil {
+		return nil
+	}
+
+	combined := strings.TrimSpace(string(output) + " " + runErr.Error())
+	return fmt.Errorf("%s", strings.TrimSpace(combined))
+}
+
+func defaultDialTCP(address string, timeout time.Duration) error {
+	conn, err := net.DialTimeout("tcp", address, timeout)
+	if err != nil {
+		return err
+	}
+	_ = conn.Close()
+	return nil
+}

service/internal/auth/ping_test.go

@@ -0,0 +1,96 @@
+package auth
+
+import (
+	"errors"
+	"testing"
+	"time"
+)
+
+func TestPingHostUsesICMPWhenAvailable(t *testing.T) {
+	originalPing := runICMPPing
+	originalDial := dialTCP
+	t.Cleanup(func() {
+		runICMPPing = originalPing
+		dialTCP = originalDial
+	})
+
+	runICMPPing = func(hostIP string) error {
+		if hostIP != "10.0.0.1" {
+			t.Fatalf("host = %q", hostIP)
+		}
+		return nil
+	}
+	dialTCP = func(address string, timeout time.Duration) error {
+		t.Fatal("tcp should not be used when icmp succeeds")
+		return nil
+	}
+
+	result := PingHost("10.0.0.1")
+	if !result.OK {
+		t.Fatalf("expected ok, got %#v", result)
+	}
+	if result.Method != PingMethodICMP {
+		t.Fatalf("method = %q", result.Method)
+	}
+}
+
+func TestPingHostFallsBackToTCPWhenICMPFails(t *testing.T) {
+	originalPing := runICMPPing
+	originalDial := dialTCP
+	t.Cleanup(func() {
+		runICMPPing = originalPing
+		dialTCP = originalDial
+	})
+
+	runICMPPing = func(hostIP string) error {
+		return errors.New("ping unavailable: executable file not found")
+	}
+	dialTCP = func(address string, timeout time.Duration) error {
+		if address != "10.0.0.2:22" {
+			t.Fatalf("address = %q", address)
+		}
+		return nil
+	}
+
+	result := PingHost("10.0.0.2")
+	if !result.OK {
+		t.Fatalf("expected ok, got %#v", result)
+	}
+	if result.Method != PingMethodTCP {
+		t.Fatalf("method = %q", result.Method)
+	}
+}
+
+func TestPingHostFailsWhenBothFail(t *testing.T) {
+	originalPing := runICMPPing
+	originalDial := dialTCP
+	t.Cleanup(func() {
+		runICMPPing = originalPing
+		dialTCP = originalDial
+	})
+
+	runICMPPing = func(hostIP string) error {
+		return errors.New("100% packet loss")
+	}
+	dialTCP = func(address string, timeout time.Duration) error {
+		return errors.New("connection refused")
+	}
+
+	result := PingHost("10.0.0.3")
+	if result.OK {
+		t.Fatal("expected failure")
+	}
+	if result.Error == nil {
+		t.Fatal("expected error")
+	}
+	if result.Method != PingMethodTCP {
+		t.Fatalf("method = %q", result.Method)
+	}
+}
+
+func TestPingHostRequiresHostIP(t *testing.T) {
+	result := PingHost("  ")
+	if result.OK || result.Error == nil {
+		t.Fatalf("expected failure, got %#v", result)
+	}
+}

service/internal/health/checker.go

@@ -0,0 +1,286 @@
+package health
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"codeberg.org/SquidSE/ClusterCanvas/service/internal/auth"
+	"codeberg.org/SquidSE/ClusterCanvas/service/internal/settings"
+)
+
+const schedulerTickInterval = 15 * time.Second
+
+// Result is the outcome of a node health check.
+type Result struct {
+	PingOK   bool
+	SSHOK    bool
+	OK       bool
+	Message  string
+	CheckedAt time.Time
+}
+
+// Checker runs and schedules per-node health checks.
+type Checker struct {
+	ConfigDir string
+	Key       []byte
+
+	mu       sync.Mutex
+	running  map[string]struct{}
+	stopCh   chan struct{}
+	stopped  chan struct{}
+	tickerOn bool
+
+	// Optional overrides for tests.
+	PingFn func(hostIP string) auth.PingResult
+	SSHFn  func(hostIP string, username string, privateKeyPEM string, passphrase string) error
+	NowFn  func() time.Time
+}
+
+// NewChecker creates a health checker for the given config directory and key.
+func NewChecker(configDir string, key []byte) *Checker {
+	return &Checker{
+		ConfigDir: configDir,
+		Key:       key,
+		running:   map[string]struct{}{},
+		stopCh:    make(chan struct{}),
+		stopped:   make(chan struct{}),
+	}
+}
+
+// StartScheduler begins the background health-check ticker.
+func (checker *Checker) StartScheduler() {
+	checker.mu.Lock()
+	if checker.tickerOn {
+		checker.mu.Unlock()
+		return
+	}
+	checker.tickerOn = true
+	checker.mu.Unlock()
+
+	go func() {
+		defer close(checker.stopped)
+		ticker := time.NewTicker(schedulerTickInterval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-checker.stopCh:
+				return
+			case <-ticker.C:
+				checker.tickDueChecks()
+			}
+		}
+	}()
+}
+
+// StopScheduler stops the background ticker and waits for it to exit.
+func (checker *Checker) StopScheduler() {
+	checker.mu.Lock()
+	if !checker.tickerOn {
+		checker.mu.Unlock()
+		return
+	}
+	checker.tickerOn = false
+	checker.mu.Unlock()
+	close(checker.stopCh)
+	<-checker.stopped
+}
+
+func (checker *Checker) tryLockNode(nodeID string) bool {
+	checker.mu.Lock()
+	defer checker.mu.Unlock()
+	if _, ok := checker.running[nodeID]; ok {
+		return false
+	}
+	checker.running[nodeID] = struct{}{}
+	return true
+}
+
+func (checker *Checker) unlockNode(nodeID string) {
+	checker.mu.Lock()
+	defer checker.mu.Unlock()
+	delete(checker.running, nodeID)
+}
+
+func (checker *Checker) now() time.Time {
+	if checker.NowFn != nil {
+		return checker.NowFn()
+	}
+	return time.Now().UTC()
+}
+
+func (checker *Checker) tickDueChecks() {
+	if len(checker.Key) == 0 {
+		return
+	}
+	store, err := settings.LoadNodesOrEmpty(checker.ConfigDir)
+	if err != nil {
+		return
+	}
+	now := checker.now()
+	for _, node := range store.Nodes {
+		if !IsHealthCheckDue(node, now) {
+			continue
+		}
+		nodeID := node.ID
+		go func() {
+			_, _ = checker.CheckNodeByID(nodeID, "scheduler")
+		}()
+	}
+}
+
+// IsHealthCheckDue reports whether a node should be probed now.
+func IsHealthCheckDue(node settings.Node, now time.Time) bool {
+	if node.HealthCheckIntervalSeconds <= 0 {
+		return false
+	}
+	if node.HealthLastCheckedAt == nil {
+		return true
+	}
+	elapsed := now.Sub(node.HealthLastCheckedAt.UTC())
+	return elapsed >= time.Duration(node.HealthCheckIntervalSeconds)*time.Second
+}
+
+// CheckNodeByID loads credentials, runs ping + SSH, and persists the result.
+func (checker *Checker) CheckNodeByID(nodeID string, actor string) (settings.Node, error) {
+	if !checker.tryLockNode(nodeID) {
+		store, err := settings.LoadNodesOrEmpty(checker.ConfigDir)
+		if err != nil {
+			return settings.Node{}, err
+		}
+		for _, node := range store.Nodes {
+			if node.ID == nodeID {
+				return node, fmt.Errorf("health check already running for node")
+			}
+		}
+		return settings.Node{}, fmt.Errorf("node not found")
+	}
+	defer checker.unlockNode(nodeID)
+
+	if len(checker.Key) == 0 {
+		return settings.Node{}, fmt.Errorf("%s is not set", settings.ConfigKeyEnvVar)
+	}
+
+	store, err := settings.LoadNodesOrEmpty(checker.ConfigDir)
+	if err != nil {
+		return settings.Node{}, err
+	}
+
+	nodeIndex := -1
+	var node settings.Node
+	for index, candidate := range store.Nodes {
+		if candidate.ID != nodeID {
+			continue
+		}
+		node = candidate
+		nodeIndex = index
+		break
+	}
+	if nodeIndex < 0 {
+		return settings.Node{}, fmt.Errorf("node not found")
+	}
+
+	keyStore, err := settings.LoadNodeKeysOrEmpty(checker.ConfigDir, checker.Key)
+	if err != nil {
+		return settings.Node{}, err
+	}
+
+	var keyEntry settings.NodeKeyEntry
+	keyFound := false
+	for _, entry := range keyStore.Keys {
+		if entry.NodeID == node.ID {
+			keyEntry = entry
+			keyFound = true
+			break
+		}
+	}
+	if !keyFound {
+		return settings.Node{}, fmt.Errorf("private key not found for node")
+	}
+
+	result := checker.runChecks(node, keyEntry)
+	updated := applyHealthResult(node, result)
+	store.Nodes[nodeIndex] = updated
+	if err := settings.SaveNodes(checker.ConfigDir, store); err != nil {
+		return settings.Node{}, err
+	}
+
+	_ = appendHealthAudit(checker.ConfigDir, actor, updated, result.Message)
+	return updated, nil
+}
+
+func (checker *Checker) runChecks(node settings.Node, keyEntry settings.NodeKeyEntry) Result {
+	checkedAt := checker.now()
+	pingFn := checker.PingFn
+	if pingFn == nil {
+		pingFn = auth.PingHost
+	}
+	sshFn := checker.SSHFn
+	if sshFn == nil {
+		sshFn = auth.TestSSHConnection
+	}
+
+	pingResult := pingFn(node.HostIP)
+	sshErr := sshFn(node.HostIP, node.Username, keyEntry.PrivateKey, keyEntry.Passphrase)
+	sshOK := sshErr == nil
+
+	message := buildHealthMessage(pingResult, sshOK, sshErr)
+	return Result{
+		PingOK:    pingResult.OK,
+		SSHOK:     sshOK,
+		OK:        pingResult.OK && sshOK,
+		Message:   message,
+		CheckedAt: checkedAt,
+	}
+}
+
+func buildHealthMessage(pingResult auth.PingResult, sshOK bool, sshErr error) string {
+	var parts []string
+	if pingResult.OK {
+		parts = append(parts, fmt.Sprintf("ping ok (%s)", pingResult.Method))
+	} else if pingResult.Error != nil {
+		parts = append(parts, fmt.Sprintf("ping failed: %v", pingResult.Error))
+	} else {
+		parts = append(parts, "ping failed")
+	}
+
+	if sshOK {
+		parts = append(parts, "ssh ok")
+	} else if sshErr != nil {
+		parts = append(parts, fmt.Sprintf("ssh failed: %v", sshErr))
+	} else {
+		parts = append(parts, "ssh failed")
+	}
+
+	return fmt.Sprintf("%s; %s", parts[0], parts[1])
+}
+
+func applyHealthResult(node settings.Node, result Result) settings.Node {
+	checkedAt := result.CheckedAt
+	pingOK := result.PingOK
+	sshOK := result.SSHOK
+	healthOK := result.OK
+	node.HealthLastCheckedAt = &checkedAt
+	node.HealthPingOK = &pingOK
+	node.HealthSSHOK = &sshOK
+	node.HealthOK = &healthOK
+	node.HealthMessage = result.Message
+	return node
+}
+
+func appendHealthAudit(configDir string, actor string, node settings.Node, detail string) error {
+	eventID, err := auth.NewUUID()
+	if err != nil {
+		eventID = fmt.Sprintf("health-%d", time.Now().UnixNano())
+	}
+	return settings.AppendNodeAuditEvent(configDir, settings.NodeAuditEvent{
+		ID:       eventID,
+		At:       time.Now().UTC(),
+		Action:   settings.NodeAuditActionHealthCheck,
+		Actor:    actor,
+		NodeID:   node.ID,
+		NodeName: node.Name,
+		NodeKind: node.Kind,
+		Detail:   detail,
+	})
+}

service/internal/health/checker_test.go

@@ -0,0 +1,164 @@
+package health
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"codeberg.org/SquidSE/ClusterCanvas/service/internal/auth"
+	"codeberg.org/SquidSE/ClusterCanvas/service/internal/settings"
+)
+
+func TestIsHealthCheckDue(t *testing.T) {
+	now := time.Date(2026, 7, 19, 12, 0, 0, 0, time.UTC)
+
+	t.Run("disabled", func(t *testing.T) {
+		if IsHealthCheckDue(settings.Node{HealthCheckIntervalSeconds: 0}, now) {
+			t.Fatal("expected not due")
+		}
+	})
+
+	t.Run("never checked", func(t *testing.T) {
+		if !IsHealthCheckDue(settings.Node{HealthCheckIntervalSeconds: 60}, now) {
+			t.Fatal("expected due")
+		}
+	})
+
+	t.Run("interval elapsed", func(t *testing.T) {
+		checked := now.Add(-61 * time.Second)
+		node := settings.Node{
+			HealthCheckIntervalSeconds: 60,
+			HealthLastCheckedAt:        &checked,
+		}
+		if !IsHealthCheckDue(node, now) {
+			t.Fatal("expected due")
+		}
+	})
+
+	t.Run("interval not elapsed", func(t *testing.T) {
+		checked := now.Add(-30 * time.Second)
+		node := settings.Node{
+			HealthCheckIntervalSeconds: 60,
+			HealthLastCheckedAt:        &checked,
+		}
+		if IsHealthCheckDue(node, now) {
+			t.Fatal("expected not due")
+		}
+	})
+}
+
+func TestCheckNodeByIDPersistsResult(t *testing.T) {
+	dir := t.TempDir()
+	keyBytes := make([]byte, 32)
+	for index := range keyBytes {
+		keyBytes[index] = byte(index + 1)
+	}
+
+	nodeID := "11111111-2222-4333-8444-555555555555"
+	store := settings.NodeStore{
+		Nodes: []settings.Node{{
+			ID:        nodeID,
+			Kind:      settings.NodeKindContainer,
+			Name:      "ct-1",
+			HostIP:    "10.9.9.9",
+			Username:  "clustercanvas",
+			GroupName: "Administrators",
+			KeyAlgo:   "ed25519",
+			CreatedAt: time.Now().UTC(),
+		}},
+	}
+	if err := settings.SaveNodes(dir, store); err != nil {
+		t.Fatalf("SaveNodes: %v", err)
+	}
+	if err := settings.SaveNodeKeys(dir, settings.NodeKeyStore{
+		Keys: []settings.NodeKeyEntry{{
+			NodeID:     nodeID,
+			PrivateKey: "dummy-key",
+			Algorithm:  "ed25519",
+		}},
+	}, keyBytes); err != nil {
+		t.Fatalf("SaveNodeKeys: %v", err)
+	}
+
+	checker := NewChecker(dir, keyBytes)
+	checker.NowFn = func() time.Time {
+		return time.Date(2026, 7, 19, 15, 0, 0, 0, time.UTC)
+	}
+	checker.PingFn = func(hostIP string) auth.PingResult {
+		return auth.PingResult{OK: true, Method: auth.PingMethodTCP}
+	}
+	checker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error {
+		return nil
+	}
+
+	updated, err := checker.CheckNodeByID(nodeID, "tester")
+	if err != nil {
+		t.Fatalf("CheckNodeByID: %v", err)
+	}
+	if updated.HealthOK == nil || !*updated.HealthOK {
+		t.Fatalf("health_ok = %#v", updated.HealthOK)
+	}
+	if updated.HealthPingOK == nil || !*updated.HealthPingOK {
+		t.Fatal("expected ping ok")
+	}
+	if updated.HealthSSHOK == nil || !*updated.HealthSSHOK {
+		t.Fatal("expected ssh ok")
+	}
+	if updated.HealthMessage == "" {
+		t.Fatal("expected message")
+	}
+
+	loaded, err := settings.LoadNodes(dir)
+	if err != nil {
+		t.Fatalf("LoadNodes: %v", err)
+	}
+	if loaded.Nodes[0].HealthOK == nil || !*loaded.Nodes[0].HealthOK {
+		t.Fatal("persisted health_ok missing")
+	}
+}
+
+func TestCheckNodeByIDRecordsFailure(t *testing.T) {
+	dir := t.TempDir()
+	keyBytes := make([]byte, 32)
+	for index := range keyBytes {
+		keyBytes[index] = byte(index + 3)
+	}
+
+	nodeID := "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"
+	if err := settings.SaveNodes(dir, settings.NodeStore{
+		Nodes: []settings.Node{{
+			ID:       nodeID,
+			Kind:     settings.NodeKindVM,
+			Name:     "vm-1",
+			HostIP:   "10.8.8.8",
+			Username: "clustercanvas",
+		}},
+	}); err != nil {
+		t.Fatalf("SaveNodes: %v", err)
+	}
+	if err := settings.SaveNodeKeys(dir, settings.NodeKeyStore{
+		Keys: []settings.NodeKeyEntry{{
+			NodeID:     nodeID,
+			PrivateKey: "dummy-key",
+			Algorithm:  "ed25519",
+		}},
+	}, keyBytes); err != nil {
+		t.Fatalf("SaveNodeKeys: %v", err)
+	}
+
+	checker := NewChecker(dir, keyBytes)
+	checker.PingFn = func(hostIP string) auth.PingResult {
+		return auth.PingResult{OK: false, Method: auth.PingMethodTCP, Error: errors.New("unreachable")}
+	}
+	checker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error {
+		return errors.New("ssh dial failed")
+	}
+
+	updated, err := checker.CheckNodeByID(nodeID, "tester")
+	if err != nil {
+		t.Fatalf("CheckNodeByID: %v", err)
+	}
+	if updated.HealthOK == nil || *updated.HealthOK {
+		t.Fatalf("expected health failure, got %#v", updated.HealthOK)
+	}
+}

service/internal/settings/types.go

@@ -105,6 +105,14 @@ type Node struct {
 	PublicKey string    `json:"public_key"`
 	KeyAlgo   string    `json:"key_algo"`
 	CreatedAt time.Time `json:"created_at"`
+
+	// Health check configuration and last result (persisted).
+	HealthCheckIntervalSeconds int        `json:"health_check_interval_seconds"`
+	HealthLastCheckedAt        *time.Time `json:"health_last_checked_at,omitempty"`
+	HealthPingOK               *bool      `json:"health_ping_ok,omitempty"`
+	HealthSSHOK                *bool      `json:"health_ssh_ok,omitempty"`
+	HealthOK                   *bool      `json:"health_ok,omitempty"`
+	HealthMessage              string     `json:"health_message,omitempty"`
 }
 
 // NodeStore is the plain JSON payload in nodes.json.
@@ -284,10 +292,11 @@ type NodeKeyStore struct {
 type NodeAuditAction string
 
 const (
-	NodeAuditActionCreate  NodeAuditAction = "create"
-	NodeAuditActionDelete  NodeAuditAction = "delete"
-	NodeAuditActionTestSSH NodeAuditAction = "test_ssh"
-	NodeAuditActionUpdate  NodeAuditAction = "update"
+	NodeAuditActionCreate      NodeAuditAction = "create"
+	NodeAuditActionDelete      NodeAuditAction = "delete"
+	NodeAuditActionTestSSH     NodeAuditAction = "test_ssh"
+	NodeAuditActionHealthCheck NodeAuditAction = "health_check"
+	NodeAuditActionUpdate      NodeAuditAction = "update"
 )
 
 // NodeAuditEvent is one append-only record of a node mutation.