package api import ( "bytes" "encoding/json" "errors" "net/http" "net/http/httptest" "sync/atomic" "testing" "time" "codeberg.org/SquidSE/ClusterCanvas/service/internal/auth" ) func TestNodesHealthCheckPersistsStatus(t *testing.T) { configDir := t.TempDir() cookie := seedCompletedSetup(t, configDir) router, app := NewRouterWithApp(configDir) app.HealthChecker.PingFn = func(hostIP string) auth.PingResult { return auth.PingResult{OK: true, Method: auth.PingMethodICMP} } app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error { return nil } createBody := []byte(`{ "id":"bbbbbbbb-bbbb-4ccc-8ddd-eeeeeeeeeeee", "kind":"container", "name":"healthy-node", "host_ip":"10.20.30.40", "group_name":"Administrators", "generate":{"algorithm":"ed25519"} }`) createRequest := withSession( httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)), cookie, ) createRecorder := httptest.NewRecorder() router.ServeHTTP(createRecorder, createRequest) if createRecorder.Code != http.StatusCreated { t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String()) } healthRequest := withSession( httptest.NewRequest( http.MethodPost, "/api/v1/nodes/bbbbbbbb-bbbb-4ccc-8ddd-eeeeeeeeeeee/health-check", nil, ), cookie, ) healthRecorder := httptest.NewRecorder() router.ServeHTTP(healthRecorder, healthRequest) if healthRecorder.Code != http.StatusOK { t.Fatalf("health status = %d body=%s", healthRecorder.Code, healthRecorder.Body.String()) } var response nodeResponse if err := json.Unmarshal(healthRecorder.Body.Bytes(), &response); err != nil { t.Fatalf("decode: %v", err) } if response.Node.HealthOK == nil || !*response.Node.HealthOK { t.Fatalf("health_ok = %#v", response.Node.HealthOK) } if response.Node.HealthMessage == "" { t.Fatal("expected health message") } } func TestNodesHealthCheckRecordsFailure(t *testing.T) { configDir := t.TempDir() cookie := seedCompletedSetup(t, configDir) router, app := NewRouterWithApp(configDir) app.HealthChecker.PingFn = func(hostIP string) auth.PingResult { return auth.PingResult{ OK: false, Method: auth.PingMethodTCP, Error: errors.New("unreachable"), } } app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error { return errors.New("ssh failed") } createBody := []byte(`{ "id":"cccccccc-bbbb-4ccc-8ddd-eeeeeeeeeeee", "kind":"vm", "name":"unhealthy-node", "host_ip":"10.20.30.41", "group_name":"Administrators", "generate":{"algorithm":"ed25519"} }`) createRequest := withSession( httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)), cookie, ) createRecorder := httptest.NewRecorder() router.ServeHTTP(createRecorder, createRequest) if createRecorder.Code != http.StatusCreated { t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String()) } healthRequest := withSession( httptest.NewRequest( http.MethodPost, "/api/v1/nodes/cccccccc-bbbb-4ccc-8ddd-eeeeeeeeeeee/health-check", nil, ), cookie, ) healthRecorder := httptest.NewRecorder() router.ServeHTTP(healthRecorder, healthRequest) if healthRecorder.Code != http.StatusOK { t.Fatalf("health status = %d body=%s", healthRecorder.Code, healthRecorder.Body.String()) } var response nodeResponse if err := json.Unmarshal(healthRecorder.Body.Bytes(), &response); err != nil { t.Fatalf("decode: %v", err) } if response.Node.HealthOK == nil || *response.Node.HealthOK { t.Fatalf("expected failure, got %#v", response.Node.HealthOK) } } func TestNodesPatchHealthInterval(t *testing.T) { configDir := t.TempDir() cookie := seedCompletedSetup(t, configDir) router := NewRouter(configDir) createBody := []byte(`{ "id":"dddddddd-bbbb-4ccc-8ddd-eeeeeeeeeeee", "kind":"docker", "name":"interval-node", "host_ip":"10.20.30.42", "group_name":"Administrators", "generate":{"algorithm":"ed25519"} }`) createRequest := withSession( httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)), cookie, ) createRecorder := httptest.NewRecorder() router.ServeHTTP(createRecorder, createRequest) if createRecorder.Code != http.StatusCreated { t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String()) } patchBody := []byte(`{"health_check_interval_seconds":120}`) patchRequest := withSession( httptest.NewRequest( http.MethodPatch, "/api/v1/nodes/dddddddd-bbbb-4ccc-8ddd-eeeeeeeeeeee", bytes.NewReader(patchBody), ), cookie, ) patchRecorder := httptest.NewRecorder() router.ServeHTTP(patchRecorder, patchRequest) if patchRecorder.Code != http.StatusOK { t.Fatalf("patch status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String()) } var response nodeResponse if err := json.Unmarshal(patchRecorder.Body.Bytes(), &response); err != nil { t.Fatalf("decode: %v", err) } if response.Node.HealthCheckIntervalSeconds != 120 { t.Fatalf("interval = %d", response.Node.HealthCheckIntervalSeconds) } } func TestNodesPatchHealthIntervalTriggersBackgroundCheck(t *testing.T) { configDir := t.TempDir() cookie := seedCompletedSetup(t, configDir) router, app := NewRouterWithApp(configDir) app.HealthChecker.PingFn = func(hostIP string) auth.PingResult { return auth.PingResult{OK: true, Method: auth.PingMethodICMP} } app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error { return nil } const nodeID = "ffffffff-bbbb-4ccc-8ddd-eeeeeeeeeeee" createBody := []byte(`{ "id":"` + nodeID + `", "kind":"container", "name":"bg-check-node", "host_ip":"10.20.30.44", "group_name":"Administrators", "generate":{"algorithm":"ed25519"} }`) createRequest := withSession( httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)), cookie, ) createRecorder := httptest.NewRecorder() router.ServeHTTP(createRecorder, createRequest) if createRecorder.Code != http.StatusCreated { t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String()) } patchBody := []byte(`{"health_check_interval_seconds":60}`) patchRequest := withSession( httptest.NewRequest( http.MethodPatch, "/api/v1/nodes/"+nodeID, bytes.NewReader(patchBody), ), cookie, ) patchRecorder := httptest.NewRecorder() router.ServeHTTP(patchRecorder, patchRequest) if patchRecorder.Code != http.StatusOK { t.Fatalf("patch status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String()) } deadline := time.Now().Add(2 * time.Second) for { getRequest := withSession( httptest.NewRequest(http.MethodGet, "/api/v1/nodes/"+nodeID, nil), cookie, ) getRecorder := httptest.NewRecorder() router.ServeHTTP(getRecorder, getRequest) if getRecorder.Code != http.StatusOK { t.Fatalf("get status = %d body=%s", getRecorder.Code, getRecorder.Body.String()) } var response nodeResponse if err := json.Unmarshal(getRecorder.Body.Bytes(), &response); err != nil { t.Fatalf("decode: %v", err) } if response.Node.HealthOK != nil && *response.Node.HealthOK { return } if time.Now().After(deadline) { t.Fatalf("timed out waiting for background health check, node=%#v", response.Node) } time.Sleep(20 * time.Millisecond) } } func TestNodesPatchHealthIntervalZeroSkipsBackgroundCheck(t *testing.T) { configDir := t.TempDir() cookie := seedCompletedSetup(t, configDir) router, app := NewRouterWithApp(configDir) checked := atomic.Bool{} app.HealthChecker.PingFn = func(hostIP string) auth.PingResult { checked.Store(true) return auth.PingResult{OK: true, Method: auth.PingMethodICMP} } app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error { checked.Store(true) return nil } const nodeID = "aaaaaaaa-cccc-4ddd-8eee-ffffffffffff" createBody := []byte(`{ "id":"` + nodeID + `", "kind":"vm", "name":"no-bg-check-node", "host_ip":"10.20.30.45", "group_name":"Administrators", "generate":{"algorithm":"ed25519"} }`) createRequest := withSession( httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)), cookie, ) createRecorder := httptest.NewRecorder() router.ServeHTTP(createRecorder, createRequest) if createRecorder.Code != http.StatusCreated { t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String()) } patchBody := []byte(`{"health_check_interval_seconds":0}`) patchRequest := withSession( httptest.NewRequest( http.MethodPatch, "/api/v1/nodes/"+nodeID, bytes.NewReader(patchBody), ), cookie, ) patchRecorder := httptest.NewRecorder() router.ServeHTTP(patchRecorder, patchRequest) if patchRecorder.Code != http.StatusOK { t.Fatalf("patch status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String()) } time.Sleep(100 * time.Millisecond) getRequest := withSession( httptest.NewRequest(http.MethodGet, "/api/v1/nodes/"+nodeID, nil), cookie, ) getRecorder := httptest.NewRecorder() router.ServeHTTP(getRecorder, getRequest) if getRecorder.Code != http.StatusOK { t.Fatalf("get status = %d body=%s", getRecorder.Code, getRecorder.Body.String()) } var response nodeResponse if err := json.Unmarshal(getRecorder.Body.Bytes(), &response); err != nil { t.Fatalf("decode: %v", err) } if checked.Load() { t.Fatal("expected no background health check when interval is 0") } if response.Node.HealthOK != nil { t.Fatalf("expected health_ok unset, got %#v", response.Node.HealthOK) } if response.Node.HealthLastCheckedAt != nil { t.Fatalf("expected health_last_checked_at unset, got %#v", response.Node.HealthLastCheckedAt) } } func TestNodesPatchHealthIntervalRejectsNegative(t *testing.T) { configDir := t.TempDir() cookie := seedCompletedSetup(t, configDir) router := NewRouter(configDir) createBody := []byte(`{ "id":"eeeeeeee-bbbb-4ccc-8ddd-eeeeeeeeeeee", "kind":"container", "name":"bad-interval", "host_ip":"10.20.30.43", "group_name":"Administrators", "generate":{"algorithm":"ed25519"} }`) createRequest := withSession( httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)), cookie, ) createRecorder := httptest.NewRecorder() router.ServeHTTP(createRecorder, createRequest) if createRecorder.Code != http.StatusCreated { t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String()) } patchBody := []byte(`{"health_check_interval_seconds":-5}`) patchRequest := withSession( httptest.NewRequest( http.MethodPatch, "/api/v1/nodes/eeeeeeee-bbbb-4ccc-8ddd-eeeeeeeeeeee", bytes.NewReader(patchBody), ), cookie, ) patchRecorder := httptest.NewRecorder() router.ServeHTTP(patchRecorder, patchRequest) if patchRecorder.Code != http.StatusBadRequest { t.Fatalf("status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String()) } }