362 lines
11 KiB
Go
362 lines
11 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"codeberg.org/SquidSE/ClusterCanvas/service/internal/auth"
|
|
)
|
|
|
|
func TestNodesHealthCheckPersistsStatus(t *testing.T) {
|
|
configDir := t.TempDir()
|
|
cookie := seedCompletedSetup(t, configDir)
|
|
router, app := NewRouterWithApp(configDir)
|
|
|
|
app.HealthChecker.PingFn = func(hostIP string) auth.PingResult {
|
|
return auth.PingResult{OK: true, Method: auth.PingMethodICMP}
|
|
}
|
|
app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error {
|
|
return nil
|
|
}
|
|
|
|
createBody := []byte(`{
|
|
"id":"bbbbbbbb-bbbb-4ccc-8ddd-eeeeeeeeeeee",
|
|
"kind":"container",
|
|
"name":"healthy-node",
|
|
"host_ip":"10.20.30.40",
|
|
"group_name":"Administrators",
|
|
"generate":{"algorithm":"ed25519"}
|
|
}`)
|
|
createRequest := withSession(
|
|
httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
|
|
cookie,
|
|
)
|
|
createRecorder := httptest.NewRecorder()
|
|
router.ServeHTTP(createRecorder, createRequest)
|
|
if createRecorder.Code != http.StatusCreated {
|
|
t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
|
|
}
|
|
|
|
healthRequest := withSession(
|
|
httptest.NewRequest(
|
|
http.MethodPost,
|
|
"/api/v1/nodes/bbbbbbbb-bbbb-4ccc-8ddd-eeeeeeeeeeee/health-check",
|
|
nil,
|
|
),
|
|
cookie,
|
|
)
|
|
healthRecorder := httptest.NewRecorder()
|
|
router.ServeHTTP(healthRecorder, healthRequest)
|
|
if healthRecorder.Code != http.StatusOK {
|
|
t.Fatalf("health status = %d body=%s", healthRecorder.Code, healthRecorder.Body.String())
|
|
}
|
|
|
|
var response nodeResponse
|
|
if err := json.Unmarshal(healthRecorder.Body.Bytes(), &response); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
if response.Node.HealthOK == nil || !*response.Node.HealthOK {
|
|
t.Fatalf("health_ok = %#v", response.Node.HealthOK)
|
|
}
|
|
if response.Node.HealthMessage == "" {
|
|
t.Fatal("expected health message")
|
|
}
|
|
}
|
|
|
|
func TestNodesHealthCheckRecordsFailure(t *testing.T) {
|
|
configDir := t.TempDir()
|
|
cookie := seedCompletedSetup(t, configDir)
|
|
router, app := NewRouterWithApp(configDir)
|
|
|
|
app.HealthChecker.PingFn = func(hostIP string) auth.PingResult {
|
|
return auth.PingResult{
|
|
OK: false,
|
|
Method: auth.PingMethodTCP,
|
|
Error: errors.New("unreachable"),
|
|
}
|
|
}
|
|
app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error {
|
|
return errors.New("ssh failed")
|
|
}
|
|
|
|
createBody := []byte(`{
|
|
"id":"cccccccc-bbbb-4ccc-8ddd-eeeeeeeeeeee",
|
|
"kind":"vm",
|
|
"name":"unhealthy-node",
|
|
"host_ip":"10.20.30.41",
|
|
"group_name":"Administrators",
|
|
"generate":{"algorithm":"ed25519"}
|
|
}`)
|
|
createRequest := withSession(
|
|
httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
|
|
cookie,
|
|
)
|
|
createRecorder := httptest.NewRecorder()
|
|
router.ServeHTTP(createRecorder, createRequest)
|
|
if createRecorder.Code != http.StatusCreated {
|
|
t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
|
|
}
|
|
|
|
healthRequest := withSession(
|
|
httptest.NewRequest(
|
|
http.MethodPost,
|
|
"/api/v1/nodes/cccccccc-bbbb-4ccc-8ddd-eeeeeeeeeeee/health-check",
|
|
nil,
|
|
),
|
|
cookie,
|
|
)
|
|
healthRecorder := httptest.NewRecorder()
|
|
router.ServeHTTP(healthRecorder, healthRequest)
|
|
if healthRecorder.Code != http.StatusOK {
|
|
t.Fatalf("health status = %d body=%s", healthRecorder.Code, healthRecorder.Body.String())
|
|
}
|
|
|
|
var response nodeResponse
|
|
if err := json.Unmarshal(healthRecorder.Body.Bytes(), &response); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
if response.Node.HealthOK == nil || *response.Node.HealthOK {
|
|
t.Fatalf("expected failure, got %#v", response.Node.HealthOK)
|
|
}
|
|
}
|
|
|
|
func TestNodesPatchHealthInterval(t *testing.T) {
|
|
configDir := t.TempDir()
|
|
cookie := seedCompletedSetup(t, configDir)
|
|
router := NewRouter(configDir)
|
|
|
|
createBody := []byte(`{
|
|
"id":"dddddddd-bbbb-4ccc-8ddd-eeeeeeeeeeee",
|
|
"kind":"docker",
|
|
"name":"interval-node",
|
|
"host_ip":"10.20.30.42",
|
|
"group_name":"Administrators",
|
|
"generate":{"algorithm":"ed25519"}
|
|
}`)
|
|
createRequest := withSession(
|
|
httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
|
|
cookie,
|
|
)
|
|
createRecorder := httptest.NewRecorder()
|
|
router.ServeHTTP(createRecorder, createRequest)
|
|
if createRecorder.Code != http.StatusCreated {
|
|
t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
|
|
}
|
|
|
|
patchBody := []byte(`{"health_check_interval_seconds":120}`)
|
|
patchRequest := withSession(
|
|
httptest.NewRequest(
|
|
http.MethodPatch,
|
|
"/api/v1/nodes/dddddddd-bbbb-4ccc-8ddd-eeeeeeeeeeee",
|
|
bytes.NewReader(patchBody),
|
|
),
|
|
cookie,
|
|
)
|
|
patchRecorder := httptest.NewRecorder()
|
|
router.ServeHTTP(patchRecorder, patchRequest)
|
|
if patchRecorder.Code != http.StatusOK {
|
|
t.Fatalf("patch status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String())
|
|
}
|
|
|
|
var response nodeResponse
|
|
if err := json.Unmarshal(patchRecorder.Body.Bytes(), &response); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
if response.Node.HealthCheckIntervalSeconds != 120 {
|
|
t.Fatalf("interval = %d", response.Node.HealthCheckIntervalSeconds)
|
|
}
|
|
}
|
|
|
|
func TestNodesPatchHealthIntervalTriggersBackgroundCheck(t *testing.T) {
|
|
configDir := t.TempDir()
|
|
cookie := seedCompletedSetup(t, configDir)
|
|
router, app := NewRouterWithApp(configDir)
|
|
|
|
app.HealthChecker.PingFn = func(hostIP string) auth.PingResult {
|
|
return auth.PingResult{OK: true, Method: auth.PingMethodICMP}
|
|
}
|
|
app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error {
|
|
return nil
|
|
}
|
|
|
|
const nodeID = "ffffffff-bbbb-4ccc-8ddd-eeeeeeeeeeee"
|
|
createBody := []byte(`{
|
|
"id":"` + nodeID + `",
|
|
"kind":"container",
|
|
"name":"bg-check-node",
|
|
"host_ip":"10.20.30.44",
|
|
"group_name":"Administrators",
|
|
"generate":{"algorithm":"ed25519"}
|
|
}`)
|
|
createRequest := withSession(
|
|
httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
|
|
cookie,
|
|
)
|
|
createRecorder := httptest.NewRecorder()
|
|
router.ServeHTTP(createRecorder, createRequest)
|
|
if createRecorder.Code != http.StatusCreated {
|
|
t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
|
|
}
|
|
|
|
patchBody := []byte(`{"health_check_interval_seconds":60}`)
|
|
patchRequest := withSession(
|
|
httptest.NewRequest(
|
|
http.MethodPatch,
|
|
"/api/v1/nodes/"+nodeID,
|
|
bytes.NewReader(patchBody),
|
|
),
|
|
cookie,
|
|
)
|
|
patchRecorder := httptest.NewRecorder()
|
|
router.ServeHTTP(patchRecorder, patchRequest)
|
|
if patchRecorder.Code != http.StatusOK {
|
|
t.Fatalf("patch status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String())
|
|
}
|
|
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for {
|
|
getRequest := withSession(
|
|
httptest.NewRequest(http.MethodGet, "/api/v1/nodes/"+nodeID, nil),
|
|
cookie,
|
|
)
|
|
getRecorder := httptest.NewRecorder()
|
|
router.ServeHTTP(getRecorder, getRequest)
|
|
if getRecorder.Code != http.StatusOK {
|
|
t.Fatalf("get status = %d body=%s", getRecorder.Code, getRecorder.Body.String())
|
|
}
|
|
|
|
var response nodeResponse
|
|
if err := json.Unmarshal(getRecorder.Body.Bytes(), &response); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
if response.Node.HealthOK != nil && *response.Node.HealthOK {
|
|
return
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("timed out waiting for background health check, node=%#v", response.Node)
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func TestNodesPatchHealthIntervalZeroSkipsBackgroundCheck(t *testing.T) {
|
|
configDir := t.TempDir()
|
|
cookie := seedCompletedSetup(t, configDir)
|
|
router, app := NewRouterWithApp(configDir)
|
|
|
|
checked := atomic.Bool{}
|
|
app.HealthChecker.PingFn = func(hostIP string) auth.PingResult {
|
|
checked.Store(true)
|
|
return auth.PingResult{OK: true, Method: auth.PingMethodICMP}
|
|
}
|
|
app.HealthChecker.SSHFn = func(hostIP string, username string, privateKeyPEM string, passphrase string) error {
|
|
checked.Store(true)
|
|
return nil
|
|
}
|
|
|
|
const nodeID = "aaaaaaaa-cccc-4ddd-8eee-ffffffffffff"
|
|
createBody := []byte(`{
|
|
"id":"` + nodeID + `",
|
|
"kind":"vm",
|
|
"name":"no-bg-check-node",
|
|
"host_ip":"10.20.30.45",
|
|
"group_name":"Administrators",
|
|
"generate":{"algorithm":"ed25519"}
|
|
}`)
|
|
createRequest := withSession(
|
|
httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
|
|
cookie,
|
|
)
|
|
createRecorder := httptest.NewRecorder()
|
|
router.ServeHTTP(createRecorder, createRequest)
|
|
if createRecorder.Code != http.StatusCreated {
|
|
t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
|
|
}
|
|
|
|
patchBody := []byte(`{"health_check_interval_seconds":0}`)
|
|
patchRequest := withSession(
|
|
httptest.NewRequest(
|
|
http.MethodPatch,
|
|
"/api/v1/nodes/"+nodeID,
|
|
bytes.NewReader(patchBody),
|
|
),
|
|
cookie,
|
|
)
|
|
patchRecorder := httptest.NewRecorder()
|
|
router.ServeHTTP(patchRecorder, patchRequest)
|
|
if patchRecorder.Code != http.StatusOK {
|
|
t.Fatalf("patch status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String())
|
|
}
|
|
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
getRequest := withSession(
|
|
httptest.NewRequest(http.MethodGet, "/api/v1/nodes/"+nodeID, nil),
|
|
cookie,
|
|
)
|
|
getRecorder := httptest.NewRecorder()
|
|
router.ServeHTTP(getRecorder, getRequest)
|
|
if getRecorder.Code != http.StatusOK {
|
|
t.Fatalf("get status = %d body=%s", getRecorder.Code, getRecorder.Body.String())
|
|
}
|
|
|
|
var response nodeResponse
|
|
if err := json.Unmarshal(getRecorder.Body.Bytes(), &response); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
if checked.Load() {
|
|
t.Fatal("expected no background health check when interval is 0")
|
|
}
|
|
if response.Node.HealthOK != nil {
|
|
t.Fatalf("expected health_ok unset, got %#v", response.Node.HealthOK)
|
|
}
|
|
if response.Node.HealthLastCheckedAt != nil {
|
|
t.Fatalf("expected health_last_checked_at unset, got %#v", response.Node.HealthLastCheckedAt)
|
|
}
|
|
}
|
|
|
|
func TestNodesPatchHealthIntervalRejectsNegative(t *testing.T) {
|
|
configDir := t.TempDir()
|
|
cookie := seedCompletedSetup(t, configDir)
|
|
router := NewRouter(configDir)
|
|
|
|
createBody := []byte(`{
|
|
"id":"eeeeeeee-bbbb-4ccc-8ddd-eeeeeeeeeeee",
|
|
"kind":"container",
|
|
"name":"bad-interval",
|
|
"host_ip":"10.20.30.43",
|
|
"group_name":"Administrators",
|
|
"generate":{"algorithm":"ed25519"}
|
|
}`)
|
|
createRequest := withSession(
|
|
httptest.NewRequest(http.MethodPost, "/api/v1/nodes", bytes.NewReader(createBody)),
|
|
cookie,
|
|
)
|
|
createRecorder := httptest.NewRecorder()
|
|
router.ServeHTTP(createRecorder, createRequest)
|
|
if createRecorder.Code != http.StatusCreated {
|
|
t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
|
|
}
|
|
|
|
patchBody := []byte(`{"health_check_interval_seconds":-5}`)
|
|
patchRequest := withSession(
|
|
httptest.NewRequest(
|
|
http.MethodPatch,
|
|
"/api/v1/nodes/eeeeeeee-bbbb-4ccc-8ddd-eeeeeeeeeeee",
|
|
bytes.NewReader(patchBody),
|
|
),
|
|
cookie,
|
|
)
|
|
patchRecorder := httptest.NewRecorder()
|
|
router.ServeHTTP(patchRecorder, patchRequest)
|
|
if patchRecorder.Code != http.StatusBadRequest {
|
|
t.Fatalf("status = %d body=%s", patchRecorder.Code, patchRecorder.Body.String())
|
|
}
|
|
}
|