Add per-node health checks with status indicators and scheduled probes.

2026-07-19 22:40:21 +02:00
parent 5307ba1a7b
commit aac9e82766
20 changed files with 1674 additions and 65 deletions
@@ -0,0 +1,211 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"codeberg.org/SquidSE/ClusterCanvas/service/internal/auth"
+)
+
+func TestNodesHealthCheckPersistsStatus(t *testing.T) {
+	configDir := t.TempDir()
+	cookie := seedCompletedSetup(t, configDir)
+	router, app := NewRouterWithApp(configDir)
+
+	app.HealthChecker.PingFn = func(hostIP string) auth.PingResult {
+		return auth.PingResult{OK: true, Method: auth.PingMethodICMP}
+	}
+	app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error {
+		return nil
+	}
+
+	createBody := []byte(`{
+		"id":"bbbbbbbb-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+		"kind":"container",
+		"name":"healthy-node",
+		"host_ip":"10.20.30.40",
+		"group_name":"Administrators",
+		"generate":{"algorithm":"ed25519"}
+	}`)
+	createRequest := withSession(
+		httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
+		cookie,
+	)
+	createRecorder := httptest.NewRecorder()
+	router.ServeHTTP(createRecorder, createRequest)
+	if createRecorder.Code != http.StatusCreated {
+		t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
+	}
+
+	healthRequest := withSession(
+		httptest.NewRequest(
+			http.MethodPost,
+			"/api/v1/nodes/bbbbbbbb-bbbb-4ccc-8ddd-eeeeeeeeeeee/health-check",
+			nil,
+		),
+		cookie,
+	)
+	healthRecorder := httptest.NewRecorder()
+	router.ServeHTTP(healthRecorder, healthRequest)
+	if healthRecorder.Code != http.StatusOK {
+		t.Fatalf("health status = %d body=%s", healthRecorder.Code, healthRecorder.Body.String())
+	}
+
+	var response nodeResponse
+	if err := json.Unmarshal(healthRecorder.Body.Bytes(), &response); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if response.Node.HealthOK == nil || !*response.Node.HealthOK {
+		t.Fatalf("health_ok = %#v", response.Node.HealthOK)
+	}
+	if response.Node.HealthMessage == "" {
+		t.Fatal("expected health message")
+	}
+}
+
+func TestNodesHealthCheckRecordsFailure(t *testing.T) {
+	configDir := t.TempDir()
+	cookie := seedCompletedSetup(t, configDir)
+	router, app := NewRouterWithApp(configDir)
+
+	app.HealthChecker.PingFn = func(hostIP string) auth.PingResult {
+		return auth.PingResult{
+			OK:     false,
+			Method: auth.PingMethodTCP,
+			Error:  errors.New("unreachable"),
+		}
+	}
+	app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error {
+		return errors.New("ssh failed")
+	}
+
+	createBody := []byte(`{
+		"id":"cccccccc-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+		"kind":"vm",
+		"name":"unhealthy-node",
+		"host_ip":"10.20.30.41",
+		"group_name":"Administrators",
+		"generate":{"algorithm":"ed25519"}
+	}`)
+	createRequest := withSession(
+		httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
+		cookie,
+	)
+	createRecorder := httptest.NewRecorder()
+	router.ServeHTTP(createRecorder, createRequest)
+	if createRecorder.Code != http.StatusCreated {
+		t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
+	}
+
+	healthRequest := withSession(
+		httptest.NewRequest(
+			http.MethodPost,
+			"/api/v1/nodes/cccccccc-bbbb-4ccc-8ddd-eeeeeeeeeeee/health-check",
+			nil,
+		),
+		cookie,
+	)
+	healthRecorder := httptest.NewRecorder()
+	router.ServeHTTP(healthRecorder, healthRequest)
+	if healthRecorder.Code != http.StatusOK {
+		t.Fatalf("health status = %d body=%s", healthRecorder.Code, healthRecorder.Body.String())
+	}
+
+	var response nodeResponse
+	if err := json.Unmarshal(healthRecorder.Body.Bytes(), &response); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if response.Node.HealthOK == nil || *response.Node.HealthOK {
+		t.Fatalf("expected failure, got %#v", response.Node.HealthOK)
+	}
+}
+
+func TestNodesPatchHealthInterval(t *testing.T) {
+	configDir := t.TempDir()
+	cookie := seedCompletedSetup(t, configDir)
+	router := NewRouter(configDir)
+
+	createBody := []byte(`{
+		"id":"dddddddd-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+		"kind":"docker",
+		"name":"interval-node",
+		"host_ip":"10.20.30.42",
+		"group_name":"Administrators",
+		"generate":{"algorithm":"ed25519"}
+	}`)
+	createRequest := withSession(
+		httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
+		cookie,
+	)
+	createRecorder := httptest.NewRecorder()
+	router.ServeHTTP(createRecorder, createRequest)
+	if createRecorder.Code != http.StatusCreated {
+		t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
+	}
+
+	patchBody := []byte(`{"health_check_interval_seconds":120}`)
+	patchRequest := withSession(
+		httptest.NewRequest(
+			http.MethodPatch,
+			"/api/v1/nodes/dddddddd-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+			bytes.NewReader(patchBody),
+		),
+		cookie,
+	)
+	patchRecorder := httptest.NewRecorder()
+	router.ServeHTTP(patchRecorder, patchRequest)
+	if patchRecorder.Code != http.StatusOK {
+		t.Fatalf("patch status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String())
+	}
+
+	var response nodeResponse
+	if err := json.Unmarshal(patchRecorder.Body.Bytes(), &response); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if response.Node.HealthCheckIntervalSeconds != 120 {
+		t.Fatalf("interval = %d", response.Node.HealthCheckIntervalSeconds)
+	}
+}
+
+func TestNodesPatchHealthIntervalRejectsNegative(t *testing.T) {
+	configDir := t.TempDir()
+	cookie := seedCompletedSetup(t, configDir)
+	router := NewRouter(configDir)
+
+	createBody := []byte(`{
+		"id":"eeeeeeee-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+		"kind":"container",
+		"name":"bad-interval",
+		"host_ip":"10.20.30.43",
+		"group_name":"Administrators",
+		"generate":{"algorithm":"ed25519"}
+	}`)
+	createRequest := withSession(
+		httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
+		cookie,
+	)
+	createRecorder := httptest.NewRecorder()
+	router.ServeHTTP(createRecorder, createRequest)
+	if createRecorder.Code != http.StatusCreated {
+		t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
+	}
+
+	patchBody := []byte(`{"health_check_interval_seconds":-5}`)
+	patchRequest := withSession(
+		httptest.NewRequest(
+			http.MethodPatch,
+			"/api/v1/nodes/eeeeeeee-bbbb-4ccc-8ddd-eeeeeeeeeeee",
+			bytes.NewReader(patchBody),
+		),
+		cookie,
+	)
+	patchRecorder := httptest.NewRecorder()
+	router.ServeHTTP(patchRecorder, patchRequest)
+	if patchRecorder.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String())
+	}
+}