feat: add auto route healthcheck
This commit is contained in:
1
go.mod
1
go.mod
@@ -29,6 +29,7 @@ require (
|
|||||||
github.com/prometheus/client_model v0.6.1 // indirect
|
github.com/prometheus/client_model v0.6.1 // indirect
|
||||||
github.com/prometheus/common v0.55.0 // indirect
|
github.com/prometheus/common v0.55.0 // indirect
|
||||||
github.com/prometheus/procfs v0.15.1 // indirect
|
github.com/prometheus/procfs v0.15.1 // indirect
|
||||||
|
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||||
github.com/spf13/pflag v1.0.5 // indirect
|
github.com/spf13/pflag v1.0.5 // indirect
|
||||||
google.golang.org/protobuf v1.34.2 // indirect
|
google.golang.org/protobuf v1.34.2 // indirect
|
||||||
|
|
||||||
|
|||||||
2
go.sum
2
go.sum
@@ -41,6 +41,8 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
|
|||||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||||
|
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||||
|
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
|
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
|
||||||
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
|
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
|
||||||
|
|||||||
@@ -204,8 +204,8 @@ func initConfig(configFile string) error {
|
|||||||
HealthCheck: RouteHealthCheck{
|
HealthCheck: RouteHealthCheck{
|
||||||
Path: "/health/live",
|
Path: "/health/live",
|
||||||
HealthyStatuses: []int{200, 404},
|
HealthyStatuses: []int{200, 404},
|
||||||
Interval: 30,
|
Interval: "30s",
|
||||||
Timeout: 10,
|
Timeout: "10s",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
"github.com/jkaninda/goma-gateway/pkg/logger"
|
"github.com/jkaninda/goma-gateway/pkg/logger"
|
||||||
|
"github.com/jkaninda/goma-gateway/util"
|
||||||
"net/http"
|
"net/http"
|
||||||
"sync"
|
"sync"
|
||||||
)
|
)
|
||||||
@@ -73,7 +74,13 @@ func (heathRoute HealthCheckRoute) HealthCheckHandler(w http.ResponseWriter, r *
|
|||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
if route.HealthCheck.Path != "" {
|
if route.HealthCheck.Path != "" {
|
||||||
err := healthCheck(route.Destination+route.HealthCheck.Path, route.HealthCheck.HealthyStatuses)
|
timeout, _ := util.ParseDuration(route.HealthCheck.Timeout)
|
||||||
|
health := Health{
|
||||||
|
URL: route.Destination + route.HealthCheck.Path,
|
||||||
|
TimeOut: timeout,
|
||||||
|
HealthyStatuses: route.HealthCheck.HealthyStatuses,
|
||||||
|
}
|
||||||
|
err := health.Check()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if heathRoute.DisableRouteHealthCheckError {
|
if heathRoute.DisableRouteHealthCheckError {
|
||||||
routes = append(routes, HealthCheckRouteResponse{Name: route.Name, Status: "unhealthy", Error: "Route healthcheck errors disabled"})
|
routes = append(routes, HealthCheckRouteResponse{Name: route.Name, Status: "unhealthy", Error: "Route healthcheck errors disabled"})
|
||||||
|
|||||||
@@ -18,14 +18,23 @@ limitations under the License.
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/jkaninda/goma-gateway/pkg/logger"
|
"github.com/jkaninda/goma-gateway/pkg/logger"
|
||||||
|
"github.com/jkaninda/goma-gateway/util"
|
||||||
|
"github.com/robfig/cron/v3"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"slices"
|
"slices"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func healthCheck(healthURL string, healthyStatuses []int) error {
|
type Health struct {
|
||||||
healthCheckURL, err := url.Parse(healthURL)
|
URL string
|
||||||
|
TimeOut time.Duration
|
||||||
|
HealthyStatuses []int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (health Health) Check() error {
|
||||||
|
healthCheckURL, err := url.Parse(health.URL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error parsing HealthCheck URL: %v ", err)
|
return fmt.Errorf("error parsing HealthCheck URL: %v ", err)
|
||||||
}
|
}
|
||||||
@@ -35,7 +44,7 @@ func healthCheck(healthURL string, healthyStatuses []int) error {
|
|||||||
return fmt.Errorf("error creating HealthCheck request: %v ", err)
|
return fmt.Errorf("error creating HealthCheck request: %v ", err)
|
||||||
}
|
}
|
||||||
// Perform the request to the route's healthcheck
|
// Perform the request to the route's healthcheck
|
||||||
client := &http.Client{}
|
client := &http.Client{Timeout: health.TimeOut}
|
||||||
healthResp, err := client.Do(healthReq)
|
healthResp, err := client.Do(healthReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Error("Error performing HealthCheck request: %v ", err)
|
logger.Error("Error performing HealthCheck request: %v ", err)
|
||||||
@@ -46,16 +55,94 @@ func healthCheck(healthURL string, healthyStatuses []int) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
}
|
}
|
||||||
}(healthResp.Body)
|
}(healthResp.Body)
|
||||||
if len(healthyStatuses) > 0 {
|
if len(health.HealthyStatuses) > 0 {
|
||||||
if !slices.Contains(healthyStatuses, healthResp.StatusCode) {
|
if !slices.Contains(health.HealthyStatuses, healthResp.StatusCode) {
|
||||||
logger.Error("Error performing HealthCheck request: %v ", err)
|
logger.Error("Error: health check failed with status code %d", healthResp.StatusCode)
|
||||||
return fmt.Errorf("health check failed with status code %v", healthResp.StatusCode)
|
return fmt.Errorf("health check failed with status code %v", healthResp.StatusCode)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if healthResp.StatusCode >= 400 {
|
if healthResp.StatusCode >= 400 {
|
||||||
logger.Debug("Error performing HealthCheck request: %v ", err)
|
logger.Error("Error: health check failed with status code %d", healthResp.StatusCode)
|
||||||
return fmt.Errorf("health check failed with status code %v", healthResp.StatusCode)
|
return fmt.Errorf("health check failed with status code %v", healthResp.StatusCode)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
func routesHealthCheck(routes []Route) {
|
||||||
|
for _, route := range routes {
|
||||||
|
go func() {
|
||||||
|
if len(route.HealthCheck.Path) > 0 {
|
||||||
|
interval := "30s"
|
||||||
|
timeout, _ := util.ParseDuration("")
|
||||||
|
if len(route.HealthCheck.Interval) > 0 {
|
||||||
|
interval = route.HealthCheck.Interval
|
||||||
|
}
|
||||||
|
expression := fmt.Sprintf("@every %s", interval)
|
||||||
|
if !util.IsValidCronExpression(expression) {
|
||||||
|
logger.Error("Health check interval is invalid: %s", interval)
|
||||||
|
logger.Info("Route health check ignored")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(route.HealthCheck.Timeout) > 0 {
|
||||||
|
d1, err1 := util.ParseDuration(route.HealthCheck.Timeout)
|
||||||
|
if err1 != nil {
|
||||||
|
logger.Error("Health check timeout is invalid: %s", route.HealthCheck.Timeout)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
timeout = d1
|
||||||
|
|
||||||
|
}
|
||||||
|
if len(route.Backends) > 0 {
|
||||||
|
for index, backend := range route.Backends {
|
||||||
|
err := createCron(fmt.Sprintf("%s [%d]", route.Name, index), expression, backend+route.HealthCheck.Path, timeout, route.HealthCheck.HealthyStatuses)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error creating cron expression: %v ", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
err := createCron(route.Name, expression, route.Destination+route.HealthCheck.Path, timeout, route.HealthCheck.HealthyStatuses)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Error creating cron expression: %v ", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func createCron(name, expression string, healthURL string, timeout time.Duration, healthyStatuses []int) error {
|
||||||
|
// Create a new cron instance
|
||||||
|
c := cron.New()
|
||||||
|
|
||||||
|
_, err := c.AddFunc(expression, func() {
|
||||||
|
health := Health{
|
||||||
|
URL: healthURL,
|
||||||
|
TimeOut: timeout,
|
||||||
|
HealthyStatuses: healthyStatuses,
|
||||||
|
}
|
||||||
|
err := health.Check()
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Route %s is unhealthy: error %v", name, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logger.Info("Route %s is healthy", name)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Start the cron scheduler
|
||||||
|
c.Start()
|
||||||
|
defer c.Stop()
|
||||||
|
select {}
|
||||||
|
}
|
||||||
|
|
||||||
|
type HealthCheck struct {
|
||||||
|
url string
|
||||||
|
interval string
|
||||||
|
timeout string
|
||||||
|
healthyStatuses []int
|
||||||
|
}
|
||||||
|
|||||||
@@ -72,12 +72,10 @@ func (proxyRoute ProxyRoute) ProxyHandler() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
// Update the headers to allow for SSL redirection
|
// Update the headers to allow for SSL redirection
|
||||||
if !proxyRoute.disableXForward {
|
if !proxyRoute.disableXForward {
|
||||||
r.URL.Host = targetURL.Host
|
|
||||||
r.URL.Scheme = targetURL.Scheme
|
r.URL.Scheme = targetURL.Scheme
|
||||||
r.Header.Set("X-Forwarded-Host", r.Header.Get("Host"))
|
r.Header.Set("X-Forwarded-Host", r.Header.Get("Host"))
|
||||||
r.Header.Set("X-Forwarded-For", getRealIP(r))
|
r.Header.Set("X-Forwarded-For", getRealIP(r))
|
||||||
r.Header.Set("X-Real-IP", getRealIP(r))
|
r.Header.Set("X-Real-IP", getRealIP(r))
|
||||||
r.Host = targetURL.Host
|
|
||||||
}
|
}
|
||||||
backendURL, _ := url.Parse(proxyRoute.destination)
|
backendURL, _ := url.Parse(proxyRoute.destination)
|
||||||
if len(proxyRoute.backends) > 0 {
|
if len(proxyRoute.backends) > 0 {
|
||||||
|
|||||||
@@ -35,6 +35,8 @@ func init() {
|
|||||||
func (gatewayServer GatewayServer) Initialize() *mux.Router {
|
func (gatewayServer GatewayServer) Initialize() *mux.Router {
|
||||||
gateway := gatewayServer.gateway
|
gateway := gatewayServer.gateway
|
||||||
middlewares := gatewayServer.middlewares
|
middlewares := gatewayServer.middlewares
|
||||||
|
//Routes background healthcheck
|
||||||
|
routesHealthCheck(gateway.Routes)
|
||||||
r := mux.NewRouter()
|
r := mux.NewRouter()
|
||||||
heath := HealthCheckRoute{
|
heath := HealthCheckRoute{
|
||||||
DisableRouteHealthCheckError: gateway.DisableRouteHealthCheckError,
|
DisableRouteHealthCheckError: gateway.DisableRouteHealthCheckError,
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Start starts the server
|
||||||
func (gatewayServer GatewayServer) Start(ctx context.Context) error {
|
func (gatewayServer GatewayServer) Start(ctx context.Context) error {
|
||||||
logger.Info("Initializing routes...")
|
logger.Info("Initializing routes...")
|
||||||
route := gatewayServer.Initialize()
|
route := gatewayServer.Initialize()
|
||||||
@@ -64,7 +65,6 @@ func (gatewayServer GatewayServer) Start(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
// Set KeepAlive
|
// Set KeepAlive
|
||||||
httpServer.SetKeepAlivesEnabled(!gatewayServer.gateway.DisableKeepAlive)
|
httpServer.SetKeepAlivesEnabled(!gatewayServer.gateway.DisableKeepAlive)
|
||||||
httpsServer.SetKeepAlivesEnabled(!gatewayServer.gateway.DisableKeepAlive)
|
|
||||||
go func() {
|
go func() {
|
||||||
logger.Info("Starting HTTP server listen=0.0.0.0:8080")
|
logger.Info("Starting HTTP server listen=0.0.0.0:8080")
|
||||||
if err := httpServer.ListenAndServe(); err != nil {
|
if err := httpServer.ListenAndServe(); err != nil {
|
||||||
|
|||||||
@@ -208,8 +208,8 @@ type Gateway struct {
|
|||||||
|
|
||||||
type RouteHealthCheck struct {
|
type RouteHealthCheck struct {
|
||||||
Path string `yaml:"path"`
|
Path string `yaml:"path"`
|
||||||
Interval int `yaml:"interval"`
|
Interval string `yaml:"interval"`
|
||||||
Timeout int `yaml:"timeout"`
|
Timeout string `yaml:"timeout"`
|
||||||
HealthyStatuses []int `yaml:"healthyStatuses"`
|
HealthyStatuses []int `yaml:"healthyStatuses"`
|
||||||
}
|
}
|
||||||
type GatewayConfig struct {
|
type GatewayConfig struct {
|
||||||
|
|||||||
@@ -11,3 +11,5 @@ const JWTAuth = "jwt" // JWT authentication middleware
|
|||||||
const OAuth = "oauth" // OAuth authentication middleware
|
const OAuth = "oauth" // OAuth authentication middleware
|
||||||
// Round-robin counter
|
// Round-robin counter
|
||||||
var counter uint32
|
var counter uint32
|
||||||
|
|
||||||
|
var routes *[]Route
|
||||||
|
|||||||
@@ -10,11 +10,13 @@ You may get a copy of the License at
|
|||||||
http://www.apache.org/licenses/LICENSE-2.0
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
*/
|
*/
|
||||||
import (
|
import (
|
||||||
|
"github.com/robfig/cron/v3"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FileExists checks if the file does exist
|
// FileExists checks if the file does exist
|
||||||
@@ -122,3 +124,21 @@ func UrlParsePath(uri string) string {
|
|||||||
func HasWhitespace(s string) bool {
|
func HasWhitespace(s string) bool {
|
||||||
return regexp.MustCompile(`\s`).MatchString(s)
|
return regexp.MustCompile(`\s`).MatchString(s)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IsValidCronExpression verify cronExpression and returns boolean
|
||||||
|
func IsValidCronExpression(cronExpr string) bool {
|
||||||
|
// Parse the cron expression
|
||||||
|
_, err := cron.ParseStandard(cronExpr)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ParseDuration(durationStr string) (time.Duration, error) {
|
||||||
|
if durationStr == "" {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
duration, err := time.ParseDuration(durationStr)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return duration, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -1 +1,35 @@
|
|||||||
package util
|
package util
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestConExpression(t *testing.T) {
|
||||||
|
cronExpression := "@every 30s"
|
||||||
|
if !IsValidCronExpression(cronExpression) {
|
||||||
|
t.Fatal("Cron expression should be valid")
|
||||||
|
}
|
||||||
|
log.Println(" Cron is valid")
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseDuration(t *testing.T) {
|
||||||
|
d1, err1 := ParseDuration("20s")
|
||||||
|
if err1 != nil {
|
||||||
|
t.Error("Error:", err1)
|
||||||
|
} else {
|
||||||
|
log.Printf("Parsed duration: %d", d1)
|
||||||
|
log.Printf("Time out: %s\n", time.Now().Add(d1))
|
||||||
|
|
||||||
|
}
|
||||||
|
d2, err2 := ParseDuration("10m")
|
||||||
|
if err2 != nil {
|
||||||
|
t.Errorf("Error: %v", err2)
|
||||||
|
} else {
|
||||||
|
log.Printf("Parsed duration: %d\n", d2)
|
||||||
|
log.Printf("Time out: %s\n", time.Now().Add(d2))
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user