✨ Add grace period to reach LAPI without blocking further queries (#153)

* ✨ Add grace period to reach LAPI without blocking further queries

* 🐛 Fix config validation for maxFailedStreamUpdate

* 🚨 Fix some lint issue

* 🚨 Bypass lint complexity on ServeHTTP

* 🍱 fix and improve

* 🚨 Fix lint

* 🚨 Fix lint

* 🐛 Fix logic for update max failure

* 📝 Update doc and docker compose local reset

* 🍱 fix log nightmare

* 🍱 fix

---------

Co-authored-by: max.lerebourg <max.lerebourg@monisnap.com>

README.md

@@ -396,6 +396,10 @@ Only one instance of the plugin is *possible*.
   - int64
   - default: 60
   - Used only in `stream` mode, the interval between requests to fetch blacklisted IPs from LAPI
+- UpdateMaxFailure
+  - int64
+  - default: 0
+  - Used only in `stream` and `alone` mode, the maximum number of time we can not reach Crowdsec before blocking traffic (set -1 to never block)
 - DefaultDecisionSeconds
   - int64
   - default: 60
@@ -475,6 +479,7 @@ http:
           enabled: false
           logLevel: DEBUG
           updateIntervalSeconds: 60
+          updateMaxFailure: 0
           defaultDecisionSeconds: 60
           httpTimeoutSeconds: 10
           crowdsecMode: live

bouncer.go

@@ -43,6 +43,7 @@ const (
 var (
 	isStartup               = true
 	isCrowdsecStreamHealthy = true
+	updateFailure           = 0
 	ticker                  chan bool
 )
 
@@ -69,6 +70,7 @@ type Bouncer struct {
 	crowdsecPassword       string
 	crowdsecScenarios      []string
 	updateInterval         int64
+	updateMaxFailure       int
 	defaultDecisionTimeout int64
 	customHeader           string
 	crowdsecStreamRoute    string
@@ -150,6 +152,7 @@ func New(ctx context.Context, next http.Handler, config *configuration.Config, n
 		crowdsecPassword:       config.CrowdsecCapiPassword,
 		crowdsecScenarios:      config.CrowdsecCapiScenarios,
 		updateInterval:         config.UpdateIntervalSeconds,
+		updateMaxFailure:       config.UpdateMaxFailure,
 		customHeader:           config.ForwardedHeadersCustomName,
 		defaultDecisionTimeout: config.DefaultDecisionSeconds,
 		banTemplateString:      banTemplateString,
@@ -282,7 +285,7 @@ func (bouncer *Bouncer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 		if isCrowdsecStreamHealthy {
 			handleNextServeHTTP(bouncer, remoteIP, rw, req)
 		} else {
-			bouncer.log.Debug(fmt.Sprintf("ServeHTTP isCrowdsecStreamHealthy:false ip:%s", remoteIP))
+			bouncer.log.Debug(fmt.Sprintf("ServeHTTP isCrowdsecStreamHealthy:false ip:%s updateFailure:%d", remoteIP, updateFailure))
 			handleBanServeHTTP(bouncer, rw)
 		}
 	} else {
@@ -358,10 +361,15 @@ func handleNextServeHTTP(bouncer *Bouncer, remoteIP string, rw http.ResponseWrit
 
 func handleStreamTicker(bouncer *Bouncer) {
 	if err := handleStreamCache(bouncer); err != nil {
-		isCrowdsecStreamHealthy = false
-		bouncer.log.Error(err.Error())
+		bouncer.log.Debug(fmt.Sprintf("handleStreamTicker updateFailure:%d isCrowdsecStreamHealthy:%t %s", updateFailure, isCrowdsecStreamHealthy, err.Error()))
+		if bouncer.updateMaxFailure != -1 && updateFailure >= bouncer.updateMaxFailure && isCrowdsecStreamHealthy {
+			isCrowdsecStreamHealthy = false
+			bouncer.log.Error(fmt.Sprintf("handleStreamTicker:error updateFailure:%d %s", updateFailure, err.Error()))
+		}
+		updateFailure++
 	} else {
 		isCrowdsecStreamHealthy = true
+		updateFailure = 0
 	}
 }
 
@@ -457,7 +465,6 @@ func getToken(bouncer *Bouncer) error {
 	var login Login
 	err = json.Unmarshal(body, &login)
 	if err != nil {
-		isCrowdsecStreamHealthy = false
 		return fmt.Errorf("getToken:parsingBody %w", err)
 	}
 	if login.Code == 200 && len(login.Token) > 0 {
@@ -516,7 +523,6 @@ func handleStreamCache(bouncer *Bouncer) error {
 		bouncer.cacheClient.Delete(decision.Value)
 	}
 	bouncer.log.Debug("handleStreamCache:updated")
-	isCrowdsecStreamHealthy = true
 	return nil
 }
 

pkg/configuration/configuration.go

@@ -57,6 +57,7 @@ type Config struct {
 	CrowdsecCapiPasswordFile                 string   `json:"crowdsecCapiPasswordFile,omitempty"`
 	CrowdsecCapiScenarios                    []string `json:"crowdsecCapiScenarios,omitempty"`
 	UpdateIntervalSeconds                    int64    `json:"updateIntervalSeconds,omitempty"`
+	UpdateMaxFailure                         int      `json:"updateMaxFailure,omitempty"`
 	DefaultDecisionSeconds                   int64    `json:"defaultDecisionSeconds,omitempty"`
 	HTTPTimeoutSeconds                       int64    `json:"httpTimeoutSeconds,omitempty"`
 	ForwardedHeadersCustomName               string   `json:"forwardedHeadersCustomName,omitempty"`
@@ -100,6 +101,7 @@ func New() *Config {
 		CrowdsecLapiKey:               "",
 		CrowdsecLapiTLSInsecureVerify: false,
 		UpdateIntervalSeconds:         60,
+		UpdateMaxFailure:              0,
 		DefaultDecisionSeconds:        60,
 		HTTPTimeoutSeconds:            10,
 		CaptchaProvider:               "",
@@ -318,6 +320,10 @@ func validateParamsRequired(config *Config) error {
 			return fmt.Errorf("%v: cannot be less than 1", key)
 		}
 	}
+	if config.UpdateMaxFailure < -1 {
+		return fmt.Errorf("UpdateMaxFailure: cannot be less than -1")
+	}
+
 	if !contains([]string{NoneMode, LiveMode, StreamMode, AloneMode, AppsecMode}, config.CrowdsecMode) {
 		return fmt.Errorf("CrowdsecMode: must be one of 'none', 'live', 'stream', 'alone' or 'appsec'")
 	}