Add per-node health checks with status indicators and scheduled probes.

2026-07-19 22:40:21 +02:00
parent 5307ba1a7b
commit aac9e82766
20 changed files with 1674 additions and 65 deletions
@@ -0,0 +1,164 @@
+package health
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"codeberg.org/SquidSE/ClusterCanvas/service/internal/auth"
+	"codeberg.org/SquidSE/ClusterCanvas/service/internal/settings"
+)
+
+func TestIsHealthCheckDue(t *testing.T) {
+	now := time.Date(2026, 7, 19, 12, 0, 0, 0, time.UTC)
+
+	t.Run("disabled", func(t *testing.T) {
+		if IsHealthCheckDue(settings.Node{HealthCheckIntervalSeconds: 0}, now) {
+			t.Fatal("expected not due")
+		}
+	})
+
+	t.Run("never checked", func(t *testing.T) {
+		if !IsHealthCheckDue(settings.Node{HealthCheckIntervalSeconds: 60}, now) {
+			t.Fatal("expected due")
+		}
+	})
+
+	t.Run("interval elapsed", func(t *testing.T) {
+		checked := now.Add(-61 * time.Second)
+		node := settings.Node{
+			HealthCheckIntervalSeconds: 60,
+			HealthLastCheckedAt:        &checked,
+		}
+		if !IsHealthCheckDue(node, now) {
+			t.Fatal("expected due")
+		}
+	})
+
+	t.Run("interval not elapsed", func(t *testing.T) {
+		checked := now.Add(-30 * time.Second)
+		node := settings.Node{
+			HealthCheckIntervalSeconds: 60,
+			HealthLastCheckedAt:        &checked,
+		}
+		if IsHealthCheckDue(node, now) {
+			t.Fatal("expected not due")
+		}
+	})
+}
+
+func TestCheckNodeByIDPersistsResult(t *testing.T) {
+	dir := t.TempDir()
+	keyBytes := make([]byte, 32)
+	for index := range keyBytes {
+		keyBytes[index] = byte(index + 1)
+	}
+
+	nodeID := "11111111-2222-4333-8444-555555555555"
+	store := settings.NodeStore{
+		Nodes: []settings.Node{{
+			ID:        nodeID,
+			Kind:      settings.NodeKindContainer,
+			Name:      "ct-1",
+			HostIP:    "10.9.9.9",
+			Username:  "clustercanvas",
+			GroupName: "Administrators",
+			KeyAlgo:   "ed25519",
+			CreatedAt: time.Now().UTC(),
+		}},
+	}
+	if err := settings.SaveNodes(dir, store); err != nil {
+		t.Fatalf("SaveNodes: %v", err)
+	}
+	if err := settings.SaveNodeKeys(dir, settings.NodeKeyStore{
+		Keys: []settings.NodeKeyEntry{{
+			NodeID:     nodeID,
+			PrivateKey: "dummy-key",
+			Algorithm:  "ed25519",
+		}},
+	}, keyBytes); err != nil {
+		t.Fatalf("SaveNodeKeys: %v", err)
+	}
+
+	checker := NewChecker(dir, keyBytes)
+	checker.NowFn = func() time.Time {
+		return time.Date(2026, 7, 19, 15, 0, 0, 0, time.UTC)
+	}
+	checker.PingFn = func(hostIP string) auth.PingResult {
+		return auth.PingResult{OK: true, Method: auth.PingMethodTCP}
+	}
+	checker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error {
+		return nil
+	}
+
+	updated, err := checker.CheckNodeByID(nodeID, "tester")
+	if err != nil {
+		t.Fatalf("CheckNodeByID: %v", err)
+	}
+	if updated.HealthOK == nil || !*updated.HealthOK {
+		t.Fatalf("health_ok = %#v", updated.HealthOK)
+	}
+	if updated.HealthPingOK == nil || !*updated.HealthPingOK {
+		t.Fatal("expected ping ok")
+	}
+	if updated.HealthSSHOK == nil || !*updated.HealthSSHOK {
+		t.Fatal("expected ssh ok")
+	}
+	if updated.HealthMessage == "" {
+		t.Fatal("expected message")
+	}
+
+	loaded, err := settings.LoadNodes(dir)
+	if err != nil {
+		t.Fatalf("LoadNodes: %v", err)
+	}
+	if loaded.Nodes[0].HealthOK == nil || !*loaded.Nodes[0].HealthOK {
+		t.Fatal("persisted health_ok missing")
+	}
+}
+
+func TestCheckNodeByIDRecordsFailure(t *testing.T) {
+	dir := t.TempDir()
+	keyBytes := make([]byte, 32)
+	for index := range keyBytes {
+		keyBytes[index] = byte(index + 3)
+	}
+
+	nodeID := "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"
+	if err := settings.SaveNodes(dir, settings.NodeStore{
+		Nodes: []settings.Node{{
+			ID:       nodeID,
+			Kind:     settings.NodeKindVM,
+			Name:     "vm-1",
+			HostIP:   "10.8.8.8",
+			Username: "clustercanvas",
+		}},
+	}); err != nil {
+		t.Fatalf("SaveNodes: %v", err)
+	}
+	if err := settings.SaveNodeKeys(dir, settings.NodeKeyStore{
+		Keys: []settings.NodeKeyEntry{{
+			NodeID:     nodeID,
+			PrivateKey: "dummy-key",
+			Algorithm:  "ed25519",
+		}},
+	}, keyBytes); err != nil {
+		t.Fatalf("SaveNodeKeys: %v", err)
+	}
+
+	checker := NewChecker(dir, keyBytes)
+	checker.PingFn = func(hostIP string) auth.PingResult {
+		return auth.PingResult{OK: false, Method: auth.PingMethodTCP, Error: errors.New("unreachable")}
+	}
+	checker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error {
+		return errors.New("ssh dial failed")
+	}
+
+	updated, err := checker.CheckNodeByID(nodeID, "tester")
+	if err != nil {
+		t.Fatalf("CheckNodeByID: %v", err)
+	}
+	if updated.HealthOK == nil || *updated.HealthOK {
+		t.Fatalf("expected health failure, got %#v", updated.HealthOK)
+	}
+}