✨ Add variable to not block if redis is unreachable (#214)

* ✨ Add variable to not block if redis is unreachable

* 🚨 fix lint

* 📝 Update README.md

.github/dependabot.yml

@@ -0,0 +1,24 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  # Maintain dependencies for Go
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  # Maintain dependencies for build tools
+  - package-ecosystem: "gomod"
+    directory: "/tools"
+    schedule:
+      interval: "weekly"
+
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/main.yml

@@ -12,8 +12,8 @@ jobs:
     name: Main Process
     runs-on: ubuntu-latest
     env:
-      GO_VERSION: 1.22
+      GO_VERSION: 1.23
-      GOLANGCI_LINT_VERSION: v1.57.2
+      GOLANGCI_LINT_VERSION: v1.63.4
       YAEGI_VERSION: v0.16.1
       CGO_ENABLED: 0
     defaults:
@@ -24,20 +24,20 @@ jobs:
 
       # https://github.com/marketplace/actions/setup-go-environment
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GO_VERSION }}
 
       # https://github.com/marketplace/actions/checkout
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           path: go/src/github.com/${{ github.repository }}
           fetch-depth: 0
 
       # https://github.com/marketplace/actions/cache
       - name: Cache Go modules
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ${{ github.workspace }}/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

.golangci.yml

@@ -50,20 +50,8 @@ linters-settings:
 linters:
   enable-all: true
   disable:
-    - deadcode # deprecated
-    - exhaustivestruct # deprecated
-    - golint # deprecated
-    - ifshort # deprecated
-    - interfacer # deprecated
-    - maligned # deprecated
-    - nosnakecase # deprecated
-    - scopelint # deprecated
-    - scopelint # deprecated
-    - structcheck # deprecated
-    - varcheck # deprecated
     - sqlclosecheck # not relevant (SQL)
     - rowserrcheck # not relevant (SQL)
-    - execinquery # not relevant (SQL)
     - cyclop # duplicate of gocyclo
     - bodyclose # Too many false positives: https://github.com/timakin/bodyclose/issues/30
     - dupl
@@ -74,17 +62,16 @@ linters:
     - wsl
     - exhaustive
     - exhaustruct
-    - goerr113
+    - err113
     - wrapcheck
-    - ifshort
     - noctx
     - lll
-    - gomnd
     - forbidigo
     - varnamelen
     - wastedassign # is disabled because of generics
     - gofumpt
     - gci
+    - mnd
 
 issues:
   exclude-use-default: false

README.md

@@ -310,6 +310,9 @@ make run
 *This means if an IP is banned, all services which are protected by an instance of the plugin will deny requests from that IP*
 Only one instance of the plugin is *possible*.
 
+**/!\ Appsec maximum body limit is defaulted to 10MB**
+*By careful when you upgrade to >1.4.x*
+
 ### Variables
 - Enabled
   - bool
@@ -329,6 +332,10 @@ Only one instance of the plugin is *possible*.
   - string
   - default: "crowdsec:7422"
   - Crowdsec Appsec Server available on which host and port. The scheme will be handled by the CrowdsecLapiScheme var.
+- CrowdsecAppsecPath
+  - string
+  - default: "/"
+  - Crowdsec Appsec Server available on this path. Will be appended to CrowdsecAppsecHost. Need to finish with "/".
 - CrowdsecAppsecFailureBlock
   - bool
   - default: true
@@ -337,6 +344,10 @@ Only one instance of the plugin is *possible*.
   - bool
   - default: true
   - Block request when Crowdsec Appsec Server is unreachable.
+- CrowdsecAppsecBodyLimit
+  - int64
+  - default: 10485760 (= 10MB)
+  - Transmit only the first number of bytes to Crowdsec Appsec Server.
 - CrowdsecLapiScheme
   - string
   - default: `http`, expected values are: `http`, `https`
@@ -344,6 +355,10 @@ Only one instance of the plugin is *possible*.
   - string
   - default: "crowdsec:8080"
   - Crowdsec LAPI available on which host and port.
+- CrowdsecLapiPath
+  - string
+  - default: "/"
+  - Crowdsec LAPI Server available on this path. Will be appended to CrowdsecLapiHost. Need to finish with "/".
 - CrowdsecLapiKey
   - string
   - default: ""
@@ -396,6 +411,10 @@ Only one instance of the plugin is *possible*.
   - string 
   - default: ""
   - Database selection for the Redis service
+- RedisUnreachableBlock
+  - bool
+  - default: true
+  - Block request when Redis is unreachable (if Redis is unreachable, 1-second delay is added to each request)
 - HTTPTimeoutSeconds
   - int64
   - default: 10
@@ -493,12 +512,15 @@ http:
           crowdsecMode: live
           crowdsecAppsecEnabled: false
           crowdsecAppsecHost: crowdsec:7422
+          crowdsecAppsecPath: "/"
           crowdsecAppsecFailureBlock: true
           crowdsecAppsecUnreachableBlock: true
+          crowdsecAppsecBodyLimit: 10485760
           crowdsecLapiKey: privateKey-foo
           crowdsecLapiKeyFile: /etc/traefik/cs-privateKey-foo
-          crowdsecLapiHost: crowdsec:8080
           crowdsecLapiScheme: http
+          crowdsecLapiHost: crowdsec:8080
+          crowdsecLapiPath: "/"
           crowdsecLapiTLSInsecureVerify: false
           crowdsecCapiMachineId: login
           crowdsecCapiPassword: password
@@ -517,6 +539,7 @@ http:
           redisCacheHost: "redis:6379"
           redisCachePassword: password
           redisCacheDatabase: "5"
+          redisCacheUnreachableBlock: true
           crowdsecLapiTLSCertificateAuthority: |-
             -----BEGIN CERTIFICATE-----
             MIIEBzCCAu+gAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwgZQxCzAJBgNVBAYTAlVT

bouncer.go

@@ -62,10 +62,13 @@ type Bouncer struct {
 	enabled                 bool
 	appsecEnabled           bool
 	appsecHost              string
+	appsecPath              string
 	appsecFailureBlock      bool
 	appsecUnreachableBlock  bool
+	appsecBodyLimit         int64
 	crowdsecScheme          string
 	crowdsecHost            string
+	crowdsecPath            string
 	crowdsecKey             string
 	crowdsecMode            string
 	crowdsecMachineID       string
@@ -78,6 +81,7 @@ type Bouncer struct {
 	forwardedCustomHeader   string
 	crowdsecStreamRoute     string
 	crowdsecHeader          string
+	redisUnreachableBlock   bool
 	banTemplateString       string
 	clientPoolStrategy      *ip.PoolStrategy
 	serverPoolStrategy      *ip.PoolStrategy
@@ -105,8 +109,10 @@ func New(_ context.Context, next http.Handler, config *configuration.Config, nam
 	if config.CrowdsecMode == configuration.AloneMode {
 		config.CrowdsecCapiMachineID, _ = configuration.GetVariable(config, "CrowdsecCapiMachineID")
 		config.CrowdsecCapiPassword, _ = configuration.GetVariable(config, "CrowdsecCapiPassword")
+		config.CrowdsecLapiScheme = configuration.HTTPS
 		config.CrowdsecLapiHost = crowdsecCapiHost
-		config.CrowdsecLapiScheme = "https"
+		config.CrowdsecLapiPath = "/"
+		config.CrowdsecAppsecEnabled = false
 		config.UpdateIntervalSeconds = 7200 // 2 hours
 		crowdsecStreamRoute = crowdsecCapiStreamRoute
 		crowdsecHeader = crowdsecCapiHeader
@@ -121,7 +127,7 @@ func New(_ context.Context, next http.Handler, config *configuration.Config, nam
 		apiKey, errAPIKey := configuration.GetVariable(config, "CrowdsecLapiKey")
 		if errAPIKey != nil && len(tlsConfig.Certificates) == 0 {
 			log.Error("New:crowdsecLapiKey fail to get CrowdsecLapiKey and no client certificate setup " + errAPIKey.Error())
-			return nil, err
+			return nil, errAPIKey
 		}
 		config.CrowdsecLapiKey = apiKey
 	}
@@ -147,10 +153,13 @@ func New(_ context.Context, next http.Handler, config *configuration.Config, nam
 		crowdsecMode:            config.CrowdsecMode,
 		appsecEnabled:           config.CrowdsecAppsecEnabled,
 		appsecHost:              config.CrowdsecAppsecHost,
+		appsecPath:              config.CrowdsecAppsecPath,
 		appsecFailureBlock:      config.CrowdsecAppsecFailureBlock,
 		appsecUnreachableBlock:  config.CrowdsecAppsecUnreachableBlock,
+		appsecBodyLimit:         config.CrowdsecAppsecBodyLimit,
 		crowdsecScheme:          config.CrowdsecLapiScheme,
 		crowdsecHost:            config.CrowdsecLapiHost,
+		crowdsecPath:            config.CrowdsecLapiPath,
 		crowdsecKey:             config.CrowdsecLapiKey,
 		crowdsecMachineID:       config.CrowdsecCapiMachineID,
 		crowdsecPassword:        config.CrowdsecCapiPassword,
@@ -160,6 +169,7 @@ func New(_ context.Context, next http.Handler, config *configuration.Config, nam
 		remediationCustomHeader: config.RemediationHeadersCustomName,
 		forwardedCustomHeader:   config.ForwardedHeadersCustomName,
 		defaultDecisionTimeout:  config.DefaultDecisionSeconds,
+		redisUnreachableBlock:   config.RedisCacheUnreachableBlock,
 		banTemplateString:       banTemplateString,
 		crowdsecStreamRoute:     crowdsecStreamRoute,
 		crowdsecHeader:          crowdsecHeader,
@@ -232,7 +242,7 @@ func New(_ context.Context, next http.Handler, config *configuration.Config, nam
 
 // ServeHTTP principal function of plugin.
 //
-//nolint:nestif
+//nolint:nestif,gocyclo
 func (bouncer *Bouncer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	if !bouncer.enabled {
 		bouncer.next.ServeHTTP(rw, req)
@@ -270,6 +280,11 @@ func (bouncer *Bouncer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 		if cacheErr != nil {
 			cacheErrString := cacheErr.Error()
 			bouncer.log.Debug(fmt.Sprintf("ServeHTTP:Get ip:%s isBanned:false %s", remoteIP, cacheErrString))
+			if !bouncer.redisUnreachableBlock && cacheErrString == cache.CacheUnreachable {
+				bouncer.log.Error(fmt.Sprintf("ServeHTTP:Get ip:%s redisUnreachable=true", remoteIP))
+				handleNextServeHTTP(bouncer, remoteIP, rw, req)
+				return
+			}
 			if cacheErrString != cache.CacheMiss {
 				bouncer.log.Error(fmt.Sprintf("ServeHTTP:Get ip:%s %s", remoteIP, cacheErrString))
 				handleBanServeHTTP(bouncer, rw)
@@ -344,7 +359,10 @@ func handleBanServeHTTP(bouncer *Bouncer, rw http.ResponseWriter) {
 		rw.Header().Set(bouncer.remediationCustomHeader, "ban")
 	}
 	rw.WriteHeader(http.StatusForbidden)
-	fmt.Fprint(rw, bouncer.banTemplateString)
+	_, err := fmt.Fprint(rw, bouncer.banTemplateString)
+	if err != nil {
+		bouncer.log.Error("handleBanServeHTTP could not write template to ResponseWriter")
+	}
 }
 
 func handleRemediationServeHTTP(bouncer *Bouncer, remoteIP, remediation string, rw http.ResponseWriter, req *http.Request) {
@@ -408,7 +426,7 @@ func handleNoStreamCache(bouncer *Bouncer, remoteIP string) (string, error) {
 	routeURL := url.URL{
 		Scheme:   bouncer.crowdsecScheme,
 		Host:     bouncer.crowdsecHost,
-		Path:     crowdsecLapiRoute,
+		Path:     bouncer.crowdsecPath + crowdsecLapiRoute,
 		RawQuery: fmt.Sprintf("ip=%v&banned=true", remoteIP),
 	}
 	body, err := crowdsecQuery(bouncer, routeURL.String(), false)
@@ -504,7 +522,7 @@ func handleStreamCache(bouncer *Bouncer) error {
 	streamRouteURL := url.URL{
 		Scheme:   bouncer.crowdsecScheme,
 		Host:     bouncer.crowdsecHost,
-		Path:     bouncer.crowdsecStreamRoute,
+		Path:     bouncer.crowdsecPath + bouncer.crowdsecStreamRoute,
 		RawQuery: fmt.Sprintf("startup=%t", !isCrowdsecStreamHealthy || isStartup),
 	}
 	body, err := crowdsecQuery(bouncer, streamRouteURL.String(), false)
@@ -584,15 +602,19 @@ func appsecQuery(bouncer *Bouncer, ip string, httpReq *http.Request) error {
 	routeURL := url.URL{
 		Scheme: bouncer.crowdsecScheme,
 		Host:   bouncer.appsecHost,
-		Path:   "/",
+		Path:   bouncer.appsecPath,
 	}
 	var req *http.Request
-	if httpReq.Body != nil && httpReq.ContentLength > 0 {
+	if bouncer.appsecBodyLimit > 0 && httpReq.Body != nil && httpReq.ContentLength > 0 {
-		bodyBytes, err := io.ReadAll(httpReq.Body)
+		var bodyBuffer bytes.Buffer
+		limitedReader := io.LimitReader(httpReq.Body, bouncer.appsecBodyLimit)
+		teeReader := io.TeeReader(limitedReader, &bodyBuffer)
+		bodyBytes, err := io.ReadAll(teeReader)
 		if err != nil {
 			return fmt.Errorf("appsecQuery:GetBody %w", err)
 		}
-		httpReq.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
+		// Conserve body intact after reading it for other middlewares and service
+		httpReq.Body = io.NopCloser(io.MultiReader(&bodyBuffer, httpReq.Body))
 		req, _ = http.NewRequest(http.MethodPost, routeURL.String(), bytes.NewBuffer(bodyBytes))
 	} else {
 		req, _ = http.NewRequest(http.MethodGet, routeURL.String(), nil)

examples/standalone-mode/README.md

@@ -3,7 +3,9 @@ You need to create a crowdsec API credentials for the CAPI.
 You can follow the documentation here: https://docs.crowdsec.net/docs/central_api/intro
 
 ```bash
-curl -X POST "https://api.crowdsec.net/v2/watchers" -H  "accept: application/json" -H  "Content-Type: application/json" -d "{ \"password\": \"PASSWORD\",  \"machine_id\": \"LOGIN\"}"
+LOGIN=...
+PASSWORD=...
+curl -X POST "https://api.crowdsec.net/v2/watchers" -H "accept: application/json" -H "Content-Type: application/json" -d "{ \"password\": \"$PASSWORD\",  \"machine_id\": \"$LOGIN\"}"
 ```
 
 These CAPI credentials must be set in your docker-compose.yml or in your config files

examples/standalone-mode/docker-compose.yml

@@ -35,8 +35,7 @@ services:
 
       - "traefik.http.middlewares.crowdsec.plugin.bouncer.enabled=true"
       # - "traefik.http.middlewares.crowdsec.plugin.bouncer.loglevel=DEBUG"
-      - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdsecmode=alone"
+      - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdsecMode=alone"
-      - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdseclapikey=40796d93c2958f9e58345514e67740e5"
       - "traefik.http.middlewares.crowdsec.plugin.bouncer.CrowdsecCapiMachineId=FIXME"
       - "traefik.http.middlewares.crowdsec.plugin.bouncer.CrowdsecCapiPassword=FIXME"
       - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdseccapiscenarios=crowdsecurity/sshd,crowdsecurity/asterisk_bf,crowdsecurity/asterisk_user_enum,crowdsecurity/base-http-scenarios"

pkg/cache/cache.go

@@ -23,6 +23,8 @@ const (
 	CaptchaDoneValue = "d"
 	// CacheMiss error string when cache is miss.
 	CacheMiss = "cache:miss"
+	// CacheUnreachable error string when cache is unreachable.
+	CacheUnreachable = "cache:unreachable"
 )
 
 //nolint:gochecknoglobals
@@ -60,9 +62,13 @@ func (redisCache) get(key string) (string, error) {
 	if err == nil && len(valueString) > 0 {
 		return valueString, nil
 	}
-	if err.Error() == simpleredis.RedisMiss {
+	errRedisMessage := err.Error()
+	if errRedisMessage == simpleredis.RedisMiss {
 		return "", errors.New(CacheMiss)
 	}
+	if errRedisMessage == simpleredis.RedisUnreachable {
+		return "", errors.New(CacheUnreachable)
+	}
 	return "", err
 }
 

pkg/captcha/captcha.go

@@ -138,7 +138,7 @@ func (c *Client) Validate(r *http.Request) (bool, error) {
 			c.log.Error("captcha:Validate " + err.Error())
 		}
 	}()
-	if !strings.Contains(res.Header.Get("content-type"), "application/json") {
+	if !strings.Contains(res.Header.Get("Content-Type"), "application/json") {
 		c.log.Debug("captcha:Validate responseType:noJson")
 		return false, nil
 	}

pkg/configuration/configuration.go

@@ -40,10 +40,13 @@ type Config struct {
 	CrowdsecMode                             string   `json:"crowdsecMode,omitempty"`
 	CrowdsecAppsecEnabled                    bool     `json:"crowdsecAppsecEnabled,omitempty"`
 	CrowdsecAppsecHost                       string   `json:"crowdsecAppsecHost,omitempty"`
+	CrowdsecAppsecPath                       string   `json:"crowdsecAppsecPath,omitempty"`
 	CrowdsecAppsecFailureBlock               bool     `json:"crowdsecAppsecFailureBlock,omitempty"`
 	CrowdsecAppsecUnreachableBlock           bool     `json:"crowdsecAppsecUnreachableBlock,omitempty"`
+	CrowdsecAppsecBodyLimit                  int64    `json:"crowdsecAppsecBodyLimit,omitempty"`
 	CrowdsecLapiScheme                       string   `json:"crowdsecLapiScheme,omitempty"`
 	CrowdsecLapiHost                         string   `json:"crowdsecLapiHost,omitempty"`
+	CrowdsecLapiPath                         string   `json:"crowdsecLapiPath,omitempty"`
 	CrowdsecLapiKey                          string   `json:"crowdsecLapiKey,omitempty"`
 	CrowdsecLapiKeyFile                      string   `json:"crowdsecLapiKeyFile,omitempty"`
 	CrowdsecLapiTLSInsecureVerify            bool     `json:"crowdsecLapiTlsInsecureVerify,omitempty"`
@@ -71,6 +74,7 @@ type Config struct {
 	RedisCachePassword                       string   `json:"redisCachePassword,omitempty"`
 	RedisCachePasswordFile                   string   `json:"redisCachePasswordFile,omitempty"`
 	RedisCacheDatabase                       string   `json:"redisCacheDatabase,omitempty"`
+	RedisCacheUnreachableBlock               bool     `json:"redisCacheUnreachableBlock,omitempty"`
 	BanHTMLFilePath                          string   `json:"banHtmlFilePath,omitempty"`
 	CaptchaHTMLFilePath                      string   `json:"captchaHtmlFilePath,omitempty"`
 	CaptchaProvider                          string   `json:"captchaProvider,omitempty"`
@@ -98,10 +102,13 @@ func New() *Config {
 		CrowdsecMode:                   LiveMode,
 		CrowdsecAppsecEnabled:          false,
 		CrowdsecAppsecHost:             "crowdsec:7422",
+		CrowdsecAppsecPath:             "/",
 		CrowdsecAppsecFailureBlock:     true,
 		CrowdsecAppsecUnreachableBlock: true,
+		CrowdsecAppsecBodyLimit:        10485760,
 		CrowdsecLapiScheme:             HTTP,
 		CrowdsecLapiHost:               "crowdsec:8080",
+		CrowdsecLapiPath:               "/",
 		CrowdsecLapiKey:                "",
 		CrowdsecLapiTLSInsecureVerify:  false,
 		UpdateIntervalSeconds:          60,
@@ -122,6 +129,7 @@ func New() *Config {
 		RedisCacheHost:                 "redis:6379",
 		RedisCachePassword:             "",
 		RedisCacheDatabase:             "",
+		RedisCacheUnreachableBlock:     true,
 	}
 }
 
@@ -217,11 +225,11 @@ func ValidateParams(config *Config) error {
 		}
 	}
 
-	if err := validateURL("CrowdsecLapi", config.CrowdsecLapiScheme, config.CrowdsecLapiHost); err != nil {
+	if err := validateURL("CrowdsecLapi", config.CrowdsecLapiScheme, config.CrowdsecLapiHost, config.CrowdsecLapiPath); err != nil {
 		return err
 	}
 
-	if err := validateURL("CrowdsecAppsec", config.CrowdsecLapiScheme, config.CrowdsecAppsecHost); err != nil {
+	if err := validateURL("CrowdsecAppsec", config.CrowdsecLapiScheme, config.CrowdsecAppsecHost, config.CrowdsecAppsecPath); err != nil {
 		return err
 	}
 
@@ -257,11 +265,11 @@ func ValidateParams(config *Config) error {
 	return nil
 }
 
-func validateURL(variable, scheme, host string) error {
+func validateURL(variable, scheme, host, path string) error {
-	// This only check that the format of the URL scheme://host is correct and do not make requests
+	// This only check that the format of the URL scheme://host/path is correct and do not make requests
-	testURL := url.URL{Scheme: scheme, Host: host}
+	testURL := url.URL{Scheme: scheme, Host: host, Path: path}
 	if _, err := http.NewRequest(http.MethodGet, testURL.String(), nil); err != nil {
-		return fmt.Errorf("CrowdsecLapiScheme://%sHost: '%v://%v' must be an URL", variable, scheme, host)
+		return fmt.Errorf("CrowdsecLapiScheme://%sHost: '%v://%v%v' must be a valid URL", variable, scheme, host, path)
 	}
 	return nil
 }
@@ -328,6 +336,9 @@ func validateParamsRequired(config *Config) error {
 	if config.UpdateMaxFailure < -1 {
 		return errors.New("UpdateMaxFailure: cannot be less than -1")
 	}
+	if config.CrowdsecAppsecBodyLimit < 0 {
+		return errors.New("CrowdsecAppsecBodyLimit: cannot be less than 0")
+	}
 
 	if !contains([]string{NoneMode, LiveMode, StreamMode, AloneMode, AppsecMode}, config.CrowdsecMode) {
 		return errors.New("CrowdsecMode: must be one of 'none', 'live', 'stream', 'alone' or 'appsec'")

pkg/logger/logger.go

@@ -34,15 +34,15 @@ func New(logLevel string) *Log {
 
 // Info log to Stdout.
 func (l *Log) Info(str string) {
-	l.logInfo.Printf(str)
+	l.logInfo.Printf("%s", str)
 }
 
 // Debug log to Stdout.
 func (l *Log) Debug(str string) {
-	l.logDebug.Printf(str)
+	l.logDebug.Printf("%s", str)
 }
 
 // Error log to Stderr.
 func (l *Log) Error(str string) {
-	l.logError.Printf(str)
+	l.logError.Printf("%s", str)
 }