From 9eadd08a1fa33ccf03980890b2d08487233fc09f Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Tue, 12 Nov 2024 14:31:18 +0100 Subject: [PATCH] refactor: improve route healthcheck --- internal/handler.go | 29 ++++------- internal/healthCheck.go | 104 +++++++++++----------------------------- internal/helpers.go | 41 ++++++++++++++++ internal/types.go | 6 ++- internal/var.go | 2 +- 5 files changed, 83 insertions(+), 99 deletions(-) diff --git a/internal/handler.go b/internal/handler.go index ac5b0f5..4036850 100644 --- a/internal/handler.go +++ b/internal/handler.go @@ -20,7 +20,6 @@ import ( "encoding/json" "github.com/gorilla/mux" "github.com/jkaninda/goma-gateway/pkg/logger" - "github.com/jkaninda/goma-gateway/util" "net/http" "sync" ) @@ -70,30 +69,20 @@ func (heathRoute HealthCheckRoute) HealthCheckHandler(w http.ResponseWriter, r * wg := sync.WaitGroup{} wg.Add(len(heathRoute.Routes)) var routes []HealthCheckRouteResponse - for _, route := range heathRoute.Routes { + for _, health := range healthCheckRoutes(heathRoute.Routes) { go func() { defer wg.Done() - if route.HealthCheck.Path != "" { - timeout, _ := util.ParseDuration(route.HealthCheck.Timeout) - health := Health{ - URL: route.Destination + route.HealthCheck.Path, - TimeOut: timeout, - HealthyStatuses: route.HealthCheck.HealthyStatuses, - } - err := health.Check() - if err != nil { - if heathRoute.DisableRouteHealthCheckError { - routes = append(routes, HealthCheckRouteResponse{Name: route.Name, Status: "unhealthy", Error: "Route healthcheck errors disabled"}) - } - routes = append(routes, HealthCheckRouteResponse{Name: route.Name, Status: "unhealthy", Error: "Error: " + err.Error()}) - } else { - logger.Debug("Route %s is healthy", route.Name) - routes = append(routes, HealthCheckRouteResponse{Name: route.Name, Status: "healthy", Error: ""}) + err := health.Check() + if err != nil { + if heathRoute.DisableRouteHealthCheckError { + routes = append(routes, HealthCheckRouteResponse{Name: health.Name, Status: "unhealthy", Error: "Route healthcheck errors disabled"}) } + routes = append(routes, HealthCheckRouteResponse{Name: health.Name, Status: "unhealthy", Error: "Error: " + err.Error()}) } else { - logger.Debug("Route %s's healthCheck is undefined", route.Name) - routes = append(routes, HealthCheckRouteResponse{Name: route.Name, Status: "undefined", Error: ""}) + logger.Debug("Route %s is healthy", health.Name) + routes = append(routes, HealthCheckRouteResponse{Name: health.Name, Status: "healthy", Error: ""}) } + }() } diff --git a/internal/healthCheck.go b/internal/healthCheck.go index f3e3bfb..f84ae80 100644 --- a/internal/healthCheck.go +++ b/internal/healthCheck.go @@ -24,7 +24,6 @@ import ( "net/http" "net/url" "slices" - "time" ) func (health Health) Check() error { @@ -35,14 +34,14 @@ func (health Health) Check() error { // Create a new request for the route healthReq, err := http.NewRequest("GET", healthCheckURL.String(), nil) if err != nil { - return fmt.Errorf("error creating HealthCheck request: %v ", err) + return fmt.Errorf("error route %s: creating HealthCheck request: %v ", health.Name, err) } // Perform the request to the route's healthcheck client := &http.Client{Timeout: health.TimeOut} healthResp, err := client.Do(healthReq) if err != nil { - logger.Error("Error performing HealthCheck request: %v ", err) - return fmt.Errorf("error performing HealthCheck request: %v ", err) + logger.Error("Error route %s: performing HealthCheck request: %v ", health.Name, err) + return fmt.Errorf("Error route %s: performing HealthCheck request: %v ", health.Name, err) } defer func(Body io.ReadCloser) { err := Body.Close() @@ -51,90 +50,50 @@ func (health Health) Check() error { }(healthResp.Body) if len(health.HealthyStatuses) > 0 { if !slices.Contains(health.HealthyStatuses, healthResp.StatusCode) { - logger.Error("Error: health check failed with status code %d", healthResp.StatusCode) - return fmt.Errorf("health check failed with status code %v", healthResp.StatusCode) + logger.Error("Error: Route %s: health check failed with status code %d", health.Name, healthResp.StatusCode) + return fmt.Errorf("route %s health check failed with status code %d", health.Name, healthResp.StatusCode) } } else { if healthResp.StatusCode >= 400 { - logger.Error("Error: health check failed with status code %d", healthResp.StatusCode) - return fmt.Errorf("health check failed with status code %v", healthResp.StatusCode) + logger.Error("Error: Route %s: health check failed with status code %d", health.Name, healthResp.StatusCode) + return fmt.Errorf("route %s: health check failed with status code %d", health.Name, healthResp.StatusCode) } } return nil } func routesHealthCheck(routes []Route) { - for _, route := range routes { - if len(route.HealthCheck.Path) > 0 { - go func() { - interval := "30s" - timeout, _ := util.ParseDuration("") - if len(route.HealthCheck.Interval) > 0 { - interval = route.HealthCheck.Interval - } - expression := fmt.Sprintf("@every %s", interval) - if !util.IsValidCronExpression(expression) { - logger.Error("Health check interval is invalid: %s", interval) - logger.Info("Route health check ignored") - return - } - if len(route.HealthCheck.Timeout) > 0 { - d1, err1 := util.ParseDuration(route.HealthCheck.Timeout) - if err1 != nil { - logger.Error("Health check timeout is invalid: %s", route.HealthCheck.Timeout) - return - } - timeout = d1 + for _, health := range healthCheckRoutes(routes) { + go func() { + err := health.createHealthCheckJob() + if err != nil { + logger.Error("Error creating healthcheck job: %v ", err) + return + } - } - if n := len(route.Backends); len(route.Backends) > 0 { - for index, backend := range route.Backends { - if n > 1 { - go func() { - err := createHealthCheckJob(fmt.Sprintf("%s [%d]", route.Name, index), expression, backend+route.HealthCheck.Path, timeout, route.HealthCheck.HealthyStatuses) - if err != nil { - logger.Error("Error creating healthcheck job: %v ", err) - return - } - }() - } else { - err := createHealthCheckJob(fmt.Sprintf("%s [%d]", route.Name, index), expression, backend+route.HealthCheck.Path, timeout, route.HealthCheck.HealthyStatuses) - if err != nil { - logger.Error("Error creating healthcheck job: %v ", err) - return - } - } - - } - - } else { - err := createHealthCheckJob(route.Name, expression, route.Destination+route.HealthCheck.Path, timeout, route.HealthCheck.HealthyStatuses) - if err != nil { - logger.Error("Error creating cron expression: %v ", err) - return - } - } - - }() - } + }() } } -func createHealthCheckJob(name, expression string, healthURL string, timeout time.Duration, healthyStatuses []int) error { +func (health Health) createHealthCheckJob() error { + interval := "30s" + if len(health.Interval) > 0 { + interval = health.Interval + } + expression := fmt.Sprintf("@every %s", interval) + if !util.IsValidCronExpression(expression) { + logger.Error("Health check interval is invalid: %s", interval) + logger.Info("Route health check ignored") + return fmt.Errorf("health check interval is invalid: %s", interval) + } // Create a new cron instance c := cron.New() - _, err := c.AddFunc(expression, func() { - health := Health{ - URL: healthURL, - TimeOut: timeout, - HealthyStatuses: healthyStatuses, - } err := health.Check() if err != nil { - logger.Error("Route %s is unhealthy: error %v", name, err.Error()) + logger.Error("Route %s is unhealthy: error %v", health.Name, err.Error()) return } - logger.Info("Route %s is healthy", name) + logger.Info("Route %s is healthy", health.Name) }) if err != nil { return err @@ -144,10 +103,3 @@ func createHealthCheckJob(name, expression string, healthURL string, timeout tim defer c.Stop() select {} } - -type HealthCheck struct { - url string - interval string - timeout string - healthyStatuses []int -} diff --git a/internal/helpers.go b/internal/helpers.go index 2cefa70..7c47ecb 100644 --- a/internal/helpers.go +++ b/internal/helpers.go @@ -17,6 +17,7 @@ import ( "github.com/golang-jwt/jwt" "github.com/jedib0t/go-pretty/v6/table" "github.com/jkaninda/goma-gateway/pkg/logger" + "github.com/jkaninda/goma-gateway/util" "golang.org/x/oauth2" "net/http" "time" @@ -100,3 +101,43 @@ func createJWT(email, jwtSecret string) (string, error) { return signedToken, nil } + +// healthCheckRoutes creates []Health +func healthCheckRoutes(routes []Route) []Health { + var healthRoutes []Health + for _, route := range routes { + if len(route.HealthCheck.Path) > 0 { + timeout, _ := util.ParseDuration("") + if len(route.HealthCheck.Timeout) > 0 { + d1, err1 := util.ParseDuration(route.HealthCheck.Timeout) + if err1 != nil { + logger.Error("Health check timeout is invalid: %s", route.HealthCheck.Timeout) + } + timeout = d1 + } + if len(route.Backends) > 0 { + for index, backend := range route.Backends { + health := Health{ + Name: fmt.Sprintf("%s - [%d]", route.Name, index), + URL: backend + route.HealthCheck.Path, + TimeOut: timeout, + HealthyStatuses: route.HealthCheck.HealthyStatuses, + } + healthRoutes = append(healthRoutes, health) + } + + } else { + health := Health{ + Name: route.Name, + URL: route.Destination + route.HealthCheck.Path, + TimeOut: timeout, + HealthyStatuses: route.HealthCheck.HealthyStatuses, + } + healthRoutes = append(healthRoutes, health) + } + } else { + logger.Debug("Route %s's healthCheck is undefined", route.Name) + } + } + return healthRoutes +} diff --git a/internal/types.go b/internal/types.go index c042c99..34f53a4 100644 --- a/internal/types.go +++ b/internal/types.go @@ -147,11 +147,11 @@ type Route struct { // // Methods allowed method Methods []string `yaml:"methods"` - // HealthCheck Defines the backend is health - HealthCheck RouteHealthCheck `yaml:"healthCheck"` // Destination Defines backend URL Destination string `yaml:"destination"` Backends []string `yaml:"backends"` + // HealthCheck Defines the backend is health + HealthCheck RouteHealthCheck `yaml:"healthCheck"` // Cors contains the route cors headers Cors Cors `yaml:"cors"` RateLimit int `yaml:"rateLimit"` @@ -279,7 +279,9 @@ type JWTSecret struct { // Health represents the health check content for a route type Health struct { + Name string URL string TimeOut time.Duration + Interval string HealthyStatuses []int } diff --git a/internal/var.go b/internal/var.go index 4fdb6d4..baf9516 100644 --- a/internal/var.go +++ b/internal/var.go @@ -12,4 +12,4 @@ const OAuth = "oauth" // OAuth authentication middleware // Round-robin counter var counter uint32 -var routes *[]Route +var Routes *[]Route