✨ Separate TLS conf for LAPI and Appsec (#293)

* ✨  Separate TLS conf for LAPI and Appsec

* 🍱 fix lint

* 🍱 fix test

* 🍱 fix lint

* 🍱 fix lint

* 🍱 fix lint

* 🍱 fix lint ?

* 🍱 fix lint

* 🍱 Add default for traceCustomHeaders

.github/workflows/main.yml

@@ -30,7 +30,7 @@ jobs:
 
       # https://github.com/marketplace/actions/checkout
       - name: Check out code
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           path: go/src/github.com/${{ github.repository }}
           fetch-depth: 0

README.md

@@ -497,6 +497,10 @@ make run
   - string
   - default: ""
   - Path where the ban html file is stored (default empty ""=disabled)
+- TraceHeadersCustomName
+  - string
+  - default: ""
+  - Request Header name whose value to inject in ban HTML response (default empty ""=disabled)
 
 ### Configuration
 
@@ -550,6 +554,7 @@ http:
           httpTimeoutSeconds: 10
           crowdsecMode: live
           crowdsecAppsecEnabled: false
+          crowdsecAppsecScheme: ""
           crowdsecAppsecHost: crowdsec:7422
           crowdsecAppsecPath: "/"
           crowdsecAppsecFailureBlock: true
@@ -603,6 +608,7 @@ http:
           captchaGracePeriodSeconds: 1800
           captchaHTMLFilePath: /captcha.html
           banHTMLFilePath: /ban.html
+          traceHeadersCustomName: X-Request-ID
           metricsUpdateIntervalSeconds: 600
 ```
 

bouncer.go

@@ -85,8 +85,10 @@ type Bouncer struct {
 
 	enabled                 bool
 	appsecEnabled           bool
+	appsecScheme            string
 	appsecHost              string
 	appsecPath              string
+	appsecKey               string
 	appsecFailureBlock      bool
 	appsecUnreachableBlock  bool
 	appsecBodyLimit         int64
@@ -108,9 +110,11 @@ type Bouncer struct {
 	crowdsecHeader          string
 	redisUnreachableBlock   bool
 	banTemplate             *htmltemplate.Template
+	traceCustomHeader       string
 	clientPoolStrategy      *ip.PoolStrategy
 	serverPoolStrategy      *ip.PoolStrategy
 	httpClient              *http.Client
+	httpAppsecClient        *http.Client
 	cacheClient             *cache.Client
 	captchaClient           *captcha.Client
 	log                     *logger.Log
@@ -130,6 +134,17 @@ func New(_ context.Context, next http.Handler, config *configuration.Config, nam
 
 	serverChecker, _ := ip.NewChecker(log, config.ForwardedHeadersTrustedIPs)
 	clientChecker, _ := ip.NewChecker(log, config.ClientTrustedIPs)
+	tlsAppsecConfig, err := configuration.GetTLSConfigCrowdsec(config, log, true)
+	if err != nil {
+		log.Error("New:getTLSConfigCrowdsec fail to get tlsAppsecConfig " + err.Error())
+		return nil, err
+	}
+	apiAppsecKey, errAppsecKey := configuration.GetVariable(config, "CrowdsecAppsecKey")
+	if errAppsecKey != nil && len(tlsAppsecConfig.Certificates) == 0 {
+		log.Error("New:crowdsecLapiKey fail to get CrowdsecAppsecKey and no client certificate setup " + errAppsecKey.Error())
+		return nil, errAppsecKey
+	}
+	config.CrowdsecAppsecKey = apiAppsecKey
 
 	var tlsConfig *tls.Config
 	crowdsecStreamRoute := ""
@@ -140,22 +155,22 @@ func New(_ context.Context, next http.Handler, config *configuration.Config, nam
 		config.CrowdsecLapiScheme = configuration.HTTPS
 		config.CrowdsecLapiHost = crowdsecCapiHost
 		config.CrowdsecLapiPath = "/"
-		config.CrowdsecAppsecEnabled = false
+		config.CrowdsecAppsecEnabled = config.CrowdsecAppsecEnabled && config.CrowdsecAppsecScheme != ""
 		config.UpdateIntervalSeconds = 7200 // 2 hours
 		crowdsecStreamRoute = crowdsecCapiStreamRoute
 		crowdsecHeader = crowdsecCapiHeader
 	} else {
 		crowdsecStreamRoute = crowdsecLapiStreamRoute
 		crowdsecHeader = crowdsecLapiHeader
-		tlsConfig, err = configuration.GetTLSConfigCrowdsec(config, log)
+		tlsConfig, err = configuration.GetTLSConfigCrowdsec(config, log, false)
 		if err != nil {
 			log.Error("New:getTLSConfigCrowdsec fail to get tlsConfig " + err.Error())
 			return nil, err
 		}
-		apiKey, errAPIKey := configuration.GetVariable(config, "CrowdsecLapiKey")
+		apiKey, errKey := configuration.GetVariable(config, "CrowdsecLapiKey")
-		if errAPIKey != nil && len(tlsConfig.Certificates) == 0 {
+		if errKey != nil && len(tlsConfig.Certificates) == 0 {
-			log.Error("New:crowdsecLapiKey fail to get CrowdsecLapiKey and no client certificate setup " + errAPIKey.Error())
+			log.Error("New:crowdsecLapiKey fail to get CrowdsecLapiKey and no client certificate setup " + errKey.Error())
-			return nil, errAPIKey
+			return nil, errKey
 		}
 		config.CrowdsecLapiKey = apiKey
 	}
@@ -173,8 +188,10 @@ func New(_ context.Context, next http.Handler, config *configuration.Config, nam
 		enabled:                 config.Enabled,
 		crowdsecMode:            config.CrowdsecMode,
 		appsecEnabled:           config.CrowdsecAppsecEnabled,
+		appsecScheme:            config.CrowdsecAppsecScheme,
 		appsecHost:              config.CrowdsecAppsecHost,
 		appsecPath:              config.CrowdsecAppsecPath,
+		appsecKey:               config.CrowdsecAppsecKey,
 		appsecFailureBlock:      config.CrowdsecAppsecFailureBlock,
 		appsecUnreachableBlock:  config.CrowdsecAppsecUnreachableBlock,
 		appsecBodyLimit:         config.CrowdsecAppsecBodyLimit,
@@ -193,6 +210,7 @@ func New(_ context.Context, next http.Handler, config *configuration.Config, nam
 		remediationStatusCode:   config.RemediationStatusCode,
 		redisUnreachableBlock:   config.RedisCacheUnreachableBlock,
 		banTemplate:             banTemplate,
+		traceCustomHeader:       config.TraceHeadersCustomName,
 		crowdsecStreamRoute:     crowdsecStreamRoute,
 		crowdsecHeader:          crowdsecHeader,
 		log:                     log,
@@ -210,6 +228,14 @@ func New(_ context.Context, next http.Handler, config *configuration.Config, nam
 			},
 			Timeout: time.Duration(config.HTTPTimeoutSeconds) * time.Second,
 		},
+		httpAppsecClient: &http.Client{
+			Transport: &http.Transport{
+				MaxIdleConns:    10,
+				IdleConnTimeout: 30 * time.Second,
+				TLSClientConfig: tlsAppsecConfig,
+			},
+			Timeout: time.Duration(config.HTTPTimeoutSeconds) * time.Second,
+		},
 		cacheClient:   &cache.Client{},
 		captchaClient: &captcha.Client{},
 	}
@@ -402,7 +428,21 @@ func (bouncer *Bouncer) handleBanServeHTTP(rw http.ResponseWriter, req *http.Req
 	if req.Method == http.MethodHead {
 		return
 	}
-	err := bouncer.banTemplate.Execute(rw, map[string]string{"RemediationReason": reason, "ClientIP": remoteIP})
+	templateData := map[string]string{
+		"RemediationReason": reason,
+		"ClientIP":          remoteIP,
+	}
+
+	if bouncer.traceCustomHeader != "" {
+		headerVal := req.Header.Get(bouncer.traceCustomHeader)
+
+		if headerVal != "" {
+			templateData["TraceID"] = headerVal
+		}
+	}
+
+	err := bouncer.banTemplate.Execute(rw, templateData)
+
 	if err != nil {
 		bouncer.log.Error("handleBanServeHTTP banTemplateServe " + err.Error())
 	}
@@ -656,7 +696,7 @@ func crowdsecQuery(bouncer *Bouncer, stringURL string, data []byte) ([]byte, err
 
 func appsecQuery(bouncer *Bouncer, ip string, httpReq *http.Request) error {
 	routeURL := url.URL{
-		Scheme: bouncer.crowdsecScheme,
+		Scheme: bouncer.appsecScheme,
 		Host:   bouncer.appsecHost,
 		Path:   bouncer.appsecPath,
 	}
@@ -681,14 +721,14 @@ func appsecQuery(bouncer *Bouncer, ip string, httpReq *http.Request) error {
 			req.Header.Add(key, value)
 		}
 	}
-	req.Header.Set(crowdsecAppsecHeader, bouncer.crowdsecKey)
+	req.Header.Set(crowdsecAppsecHeader, bouncer.appsecKey)
 	req.Header.Set(crowdsecAppsecIPHeader, ip)
 	req.Header.Set(crowdsecAppsecVerbHeader, httpReq.Method)
 	req.Header.Set(crowdsecAppsecHostHeader, httpReq.Host)
 	req.Header.Set(crowdsecAppsecURIHeader, httpReq.URL.String())
 	req.Header.Set(crowdsecAppsecUserAgent, httpReq.Header.Get("User-Agent"))
 
-	res, err := bouncer.httpClient.Do(req)
+	res, err := bouncer.httpAppsecClient.Do(req)
 	if err != nil {
 		bouncer.log.Error("appsecQuery:unreachable")
 		if bouncer.appsecUnreachableBlock {

examples/custom-ban-page/README.md

@@ -50,8 +50,10 @@ make run_custom_ban_page
 In the html of the ban page, you can use:  
 - {{ .ClientIP }} to display the IP used to ban the request.  
 - {{ .RemediationReason }} that convert on runtime into why the ban page is served. It's an enum with "APPSEC", "LAPI", "TECHNICAL_ISSUE" and it is useful to help user understand why the request is blocked.  
+- {{ .CustomHeader }} value of the specified Request Header (for example X-Request-ID)
 ```
 <script>var remediation = "{{ .RemediationReason }}"</script>
 <script>var clientIp = "{{ .ClientIP }}"</script>
+<script>var traceID = "{{ .TraceID }}"</script>
 ```
 With the above tweak and some other js, you can customize your ban page on runtime.

examples/tls-auth/README.md

@@ -89,4 +89,8 @@ make run_tlsauth
 ```
 
 Note:
-> Traefik need to be restart if certificates are regenerated after his launch
+> Traefik need to be restarted if certificates are regenerated after his launch
+
+## Separate LAPI and Appsec HTTP/S config
+To separate TLS config for LAPI and Appsec, you can use all the TLS LAPI variable beginning with `CrowdsecLapi...` into `CrowdsecAppsec...`.
+Don't forget to set `CrowdsecAppsecScheme: HTTP` or `HTTPS` to trigger the separate setup.

pkg/configuration/configuration.go

@@ -47,8 +47,17 @@ type Config struct {
 	LogFilePath                               string   `json:"logFilePath,omitempty"`
 	CrowdsecMode                              string   `json:"crowdsecMode,omitempty"`
 	CrowdsecAppsecEnabled                     bool     `json:"crowdsecAppsecEnabled,omitempty"`
+	CrowdsecAppsecScheme                      string   `json:"crowdsecAppsecScheme,omitempty"`
 	CrowdsecAppsecHost                        string   `json:"crowdsecAppsecHost,omitempty"`
 	CrowdsecAppsecPath                        string   `json:"crowdsecAppsecPath,omitempty"`
+	CrowdsecAppsecKey                         string   `json:"crowdsecAppsecKey,omitempty"`
+	CrowdsecAppsecKeyFile                     string   `json:"crowdsecAppsecKeyFile,omitempty"`
+	CrowdsecAppsecTLSInsecureVerify           bool     `json:"crowdsecAppsecTlsInsecureVerify,omitempty"`
+	CrowdsecAppsecTLSCertificateAuthority     string   `json:"crowdsecAppsecTlsCertificateAuthority,omitempty"`
+	CrowdsecAppsecTLSCertificateAuthorityFile string   `json:"crowdsecAppsecTlsCertificateAuthorityFile,omitempty"`
+	CrowdsecAppsecTLSCertificateBouncer       string   `json:"crowdsecAppsecTlsCertificateBouncer,omitempty"`
+	CrowdsecAppsecTLSCertificateBouncerFile   string   `json:"crowdsecAppsecTlsCertificateBouncerFile,omitempty"`
+	CrowdsecAppsecTLSCertificateBouncerKey    string   `json:"crowdsecAppsecTlsCertificateBouncerKey,omitempty"`
 	CrowdsecAppsecFailureBlock                bool     `json:"crowdsecAppsecFailureBlock,omitempty"`
 	CrowdsecAppsecUnreachableBlock            bool     `json:"crowdsecAppsecUnreachableBlock,omitempty"`
 	CrowdsecAppsecBodyLimit                   int64    `json:"crowdsecAppsecBodyLimit,omitempty"`
@@ -75,6 +84,7 @@ type Config struct {
 	DefaultDecisionSeconds                    int64    `json:"defaultDecisionSeconds,omitempty"`
 	RemediationStatusCode                     int      `json:"remediationStatusCode,omitempty"`
 	HTTPTimeoutSeconds                        int64    `json:"httpTimeoutSeconds,omitempty"`
+	TraceHeadersCustomName                    string   `json:"traceHeadersCustomName,omitempty"`
 	RemediationHeadersCustomName              string   `json:"remediationHeadersCustomName,omitempty"`
 	ForwardedHeadersCustomName                string   `json:"forwardedHeadersCustomName,omitempty"`
 	ForwardedHeadersTrustedIPs                []string `json:"forwardedHeadersTrustedIps,omitempty"`
@@ -116,11 +126,14 @@ func New() *Config {
 		LogFilePath:                     "",
 		CrowdsecMode:                    LiveMode,
 		CrowdsecAppsecEnabled:           false,
-		CrowdsecAppsecHost:             "crowdsec:7422",
-		CrowdsecAppsecPath:             "/",
 		CrowdsecAppsecFailureBlock:      true,
 		CrowdsecAppsecUnreachableBlock:  true,
 		CrowdsecAppsecBodyLimit:         10485760,
+		CrowdsecAppsecScheme:            "",
+		CrowdsecAppsecHost:              "crowdsec:7422",
+		CrowdsecAppsecPath:              "/",
+		CrowdsecAppsecKey:               "",
+		CrowdsecAppsecTLSInsecureVerify: false,
 		CrowdsecLapiScheme:              HTTP,
 		CrowdsecLapiHost:                "crowdsec:8080",
 		CrowdsecLapiPath:                "/",
@@ -142,6 +155,7 @@ func New() *Config {
 		CaptchaGracePeriodSeconds:       1800,
 		CaptchaHTMLFilePath:             "/captcha.html",
 		BanHTMLFilePath:                 "",
+		TraceHeadersCustomName:          "",
 		RemediationHeadersCustomName:    "",
 		ForwardedHeadersCustomName:      "X-Forwarded-For",
 		ForwardedHeadersTrustedIPs:      []string{},
@@ -412,26 +426,27 @@ func validateParamsRequired(config *Config) error {
 	if !contains([]string{HTTP, HTTPS}, config.CrowdsecLapiScheme) {
 		return errors.New("CrowdsecLapiScheme: must be one of 'http' or 'https'")
 	}
+	if !contains([]string{HTTP, HTTPS, ""}, config.CrowdsecAppsecScheme) {
+		return errors.New("CrowdsecAppsecScheme: must be one of 'http' or 'https'")
+	}
 	return nil
 }
 
-// GetTLSConfigCrowdsec get TLS config from Config.
+func getTLSConfig(config *Config, log *logger.Log, prefix, scheme string, insecureVerify bool) (*tls.Config, error) {
-//
-//nolint:nestif
-func GetTLSConfigCrowdsec(config *Config, log *logger.Log) (*tls.Config, error) {
 	tlsConfig := new(tls.Config)
 	tlsConfig.RootCAs = x509.NewCertPool()
-	//nolint:gocritic
+	if scheme != HTTPS {
-	if config.CrowdsecLapiScheme != HTTPS {
+		log.Debug("getTLSConfigCrowdsec:" + prefix + "Scheme https:no")
-		log.Debug("getTLSConfigCrowdsec:CrowdsecLapiScheme https:no")
 		return tlsConfig, nil
-	} else if config.CrowdsecLapiTLSInsecureVerify {
+	}
+	//nolint:nestif
+	if insecureVerify {
 		tlsConfig.InsecureSkipVerify = true
-		log.Debug("getTLSConfigCrowdsec:CrowdsecLapiTLSInsecureVerify tlsInsecure:true")
+		log.Debug("getTLSConfigCrowdsec:" + prefix + "TLSInsecureVerify tlsInsecure:true")
 		// If we return here and still want to use client auth this won't work
 		// return tlsConfig, nil
 	} else {
-		certAuthority, err := GetVariable(config, "CrowdsecLapiTLSCertificateAuthority")
+		certAuthority, err := GetVariable(config, prefix+"TLSCertificateAuthority")
 		if err != nil {
 			return nil, err
 		}
@@ -439,17 +454,16 @@ func GetTLSConfigCrowdsec(config *Config, log *logger.Log) (*tls.Config, error)
 			if !tlsConfig.RootCAs.AppendCertsFromPEM([]byte(certAuthority)) {
 				// here we return because if CrowdsecLapiTLSInsecureVerify is false
 				// and CA not load, we can't communicate with https
-				return nil, errors.New("getTLSConfigCrowdsec:cannot load CA and verify cert is enabled")
+				return nil, errors.New("getTLSConfigCrowdsec:" + prefix + "cannot load CA and verify cert is enabled")
 			}
-			log.Debug("getTLSConfigCrowdsec:CrowdsecLapiTLSCertificateAuthority CA added successfully")
+			log.Debug("getTLSConfigCrowdsec:" + prefix + "TLSCertificateAuthority CA added successfully")
 		}
 	}
-
+	certBouncer, err := GetVariable(config, prefix+"TLSCertificateBouncer")
-	certBouncer, err := GetVariable(config, "CrowdsecLapiTLSCertificateBouncer")
 	if err != nil {
 		return nil, err
 	}
-	certBouncerKey, err := GetVariable(config, "CrowdsecLapiTLSCertificateBouncerKey")
+	certBouncerKey, err := GetVariable(config, prefix+"TLSCertificateBouncerKey")
 	if err != nil {
 		return nil, err
 	}
@@ -464,3 +478,14 @@ func GetTLSConfigCrowdsec(config *Config, log *logger.Log) (*tls.Config, error)
 
 	return tlsConfig, nil
 }
+
+// GetTLSConfigCrowdsec get TLS config from Config.
+func GetTLSConfigCrowdsec(config *Config, log *logger.Log, isAppsec bool) (*tls.Config, error) {
+	var prefix string
+	if isAppsec && config.CrowdsecAppsecScheme != "" {
+		prefix = "CrowdsecAppsec"
+		return getTLSConfig(config, log, prefix, config.CrowdsecAppsecScheme, config.CrowdsecAppsecTLSInsecureVerify)
+	}
+	prefix = "CrowdsecLapi"
+	return getTLSConfig(config, log, prefix, config.CrowdsecLapiScheme, config.CrowdsecLapiTLSInsecureVerify)
+}

pkg/configuration/configuration_test.go

@@ -242,7 +242,7 @@ func Test_GetTLSConfigCrowdsec(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := GetTLSConfigCrowdsec(tt.args.config, logger.New("INFO", ""))
+			got, err := GetTLSConfigCrowdsec(tt.args.config, logger.New("INFO", ""), false)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("getTLSConfigCrowdsec() error = %v, wantErr %v", err, tt.wantErr)
 				return