diff --git a/Makefile b/Makefile index 5dcec89..83db57f 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,9 @@ run_tlsauth: run_appsec: docker compose -f examples/appsec-enabled/docker-compose.yml up -d --remove-orphans +run_bot_detection: + docker compose -f examples/bot-detection/docker-compose.yml up -d --remove-orphans + run_custom_captcha: docker compose -f examples/custom-captcha/docker-compose.yml up -d --remove-orphans @@ -109,6 +112,7 @@ clean_all_docker: docker compose -f examples/trusted-ips/docker-compose.yml down --remove-orphans docker compose -f examples/tls-auth/docker-compose.yml down --remove-orphans docker compose -f examples/appsec-enabled/docker-compose.yml down --remove-orphans + docker compose -f examples/bot-detection/docker-compose.yml down --remove-orphans docker compose -f examples/captcha/docker-compose.yml down --remove-orphans docker compose -f examples/custom-captcha/docker-compose.yml down --remove-orphans docker compose -f examples/custom-ban-page/docker-compose.yml down --remove-orphans diff --git a/README.md b/README.md index 339cb77..6abe84e 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ When used with Crowdsec it will leverage the local API which will analyze Traefi Appsec feature is supported from plugin version 1.2.0 and Crowdsec 1.6.0. +Appsec bot detection is supported from plugin version 1.8.0 and Crowdsec 1.8.0. + The AppSec Component offers: - Low-effort virtual patching capabilities. @@ -428,7 +430,7 @@ make run - RemediationHeadersCustomName - string - default: "" - - Name of the header you want in response when request are handled by plugin (possible value of the header `ban`, `captcha` or `solved-captcha`) + - Name of the header you want in response when request are handled by plugin (possible value of the header `ban`, `challenge`, `captcha` or `solved-captcha`) - ForwardedHeadersCustomName - string - default: "X-Forwarded-For" @@ -756,6 +758,29 @@ Set `crowdsecAppsecScheme` to `https`. Same three options as for the LAPI, prefi Currently AppSec does not support mTLS authentication for the AppSec Component. +#### AppSec bot detection: route `/crowdsec-internal` + +When AppSec bot detection is enabled, the challenge page it returns loads its fingerprint +script from `/crowdsec-internal/challenge/fpscanner.js`, an absolute path. Any router +protected by this middleware therefore has to match that prefix as well, otherwise the +script 404s, the proof-of-work never runs, and the client is stuck on the challenge page +with no error anywhere: + +```yaml + - "traefik.http.routers.my-router.rule=PathPrefix(`/my-app`) || PathPrefix(`/crowdsec-internal`)" +``` + +The backend service never sees these requests: the plugin forwards them to the AppSec +component and returns its response directly, so the prefix only needs to reach a router +carrying the middleware. + +CrowdSec documents the same requirement, that the bouncer must forward +`/crowdsec-internal/challenge/*` unchanged: see +[enabling bot detection](https://docs.crowdsec.net/docs/appsec/bot_detection/enable) and the +[challenge protocol](https://docs.crowdsec.net/docs/appsec/bot_detection/challenge_protocol). + +See [examples/bot-detection/README.md](https://github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin/blob/main/examples/bot-detection/README.md). + #### Manually add an IP to the blocklist (for testing purposes) ```bash @@ -790,6 +815,8 @@ docker exec crowdsec cscli decisions remove --ip 10.0.0.10 -t captcha #### 11. Using Traefik with Custom Captcha Whiketkeeper[examples/custom-captcha/README.md](https://github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin/blob/main/examples/custom-captcha/README.md) +#### 12. Using Traefik with AppSec bot detection enabled [examples/bot-detection/README.md](https://github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin/blob/main/examples/bot-detection/README.md) + ### Local Mode Traefik also offers a developer mode that can be used for temporary testing of plugins not hosted on GitHub. diff --git a/bouncer.go b/bouncer.go index 17ddd66..7006025 100644 --- a/bouncer.go +++ b/bouncer.go @@ -42,6 +42,9 @@ const ( crowdsecCapiLoginRoute = "v2/watchers/login" crowdsecCapiStreamRoute = "v2/decisions/stream" cacheTimeoutKey = "updated" + appsecAllowAction = "allow" + appsecChallengeAction = "challenge" + appsecResponseBodyLimit = 1 << 20 // 1 MiB ) // ############################################################## @@ -122,6 +125,15 @@ type Bouncer struct { log *slog.Logger } +// AppSecResponse is the structured remediation Appsec returns for a request. +type AppSecResponse struct { + Action string `json:"action"` + HTTPStatus int `json:"http_status"` //nolint:tagliatelle + UserBodyContent string `json:"user_body_content,omitempty"` //nolint:tagliatelle + UserCookies []string `json:"user_cookies,omitempty"` //nolint:tagliatelle + UserHeaders map[string][]string `json:"user_headers,omitempty"` //nolint:tagliatelle +} + // New creates the crowdsec bouncer plugin. // //nolint:nestif,gocyclo,gocognit,funlen,maintidx @@ -489,15 +501,48 @@ func (bouncer *Bouncer) handleRemediationServeHTTP(rw http.ResponseWriter, req * func (bouncer *Bouncer) handleNextServeHTTP(rw http.ResponseWriter, req *http.Request, remoteIP string) { if bouncer.appsecEnabled { - if err := appsecQuery(bouncer, remoteIP, req); err != nil { + decision, err := appsecQuery(bouncer, remoteIP, req) + if err != nil { bouncer.log.Debug(fmt.Sprintf("handleNextServeHTTP ip:%s isWaf:true %s", remoteIP, err.Error())) bouncer.handleBanServeHTTP(rw, req, remoteIP, configuration.ReasonAPPSEC) return } + if decision != nil && decision.Action != appsecAllowAction { + if decision.Action == appsecChallengeAction { + bouncer.handleAppsecResponseServeHTTP(rw, req, decision) + return + } + bouncer.handleBanServeHTTP(rw, req, remoteIP, configuration.ReasonAPPSEC) + return + } } bouncer.next.ServeHTTP(rw, req) } +func (bouncer *Bouncer) handleAppsecResponseServeHTTP(rw http.ResponseWriter, req *http.Request, decision *AppSecResponse) { + atomic.AddInt64(&blockedRequests, 1) + + for name, values := range decision.UserHeaders { + for _, value := range values { + rw.Header().Add(name, value) + } + } + for _, cookie := range decision.UserCookies { + rw.Header().Add("Set-Cookie", cookie) + } + if bouncer.remediationCustomHeader != "" { + rw.Header().Set(bouncer.remediationCustomHeader, "challenge") + } + rw.WriteHeader(decision.HTTPStatus) + + if req.Method == http.MethodHead || decision.UserBodyContent == "" { + return + } + if _, err := rw.Write([]byte(decision.UserBodyContent)); err != nil { + bouncer.log.Warn("handleAppsecResponseServeHTTP could not write appsec response: " + err.Error()) + } +} + func handleStreamTicker(bouncer *Bouncer) { if err := handleStreamCache(bouncer); err != nil { bouncer.log.Warn(fmt.Sprintf("handleStreamTicker updateFailure:%d isCrowdsecStreamHealthy:%t %s", updateFailure, isCrowdsecStreamHealthy, err.Error())) @@ -622,7 +667,7 @@ func getToken(bouncer *Bouncer) error { var login Login err = json.Unmarshal(body, &login) if err != nil { - return fmt.Errorf("getToken:parsingBody %w", err) + return fmt.Errorf("getToken:parseBody %w", err) } if login.Code == http.StatusOK && len(login.Token) > 0 { bouncer.crowdsecKey = login.Token @@ -665,7 +710,7 @@ func handleStreamCache(bouncer *Bouncer) error { var stream Stream err = json.Unmarshal(body, &stream) if err != nil { - return fmt.Errorf("handleStreamCache:parsingBody %w", err) + return fmt.Errorf("handleStreamCache:parseBody %w", err) } for _, decision := range stream.New { duration, err := time.ParseDuration(decision.Duration) @@ -756,7 +801,8 @@ func isMethodWithBody(method string) bool { } } -func appsecQuery(bouncer *Bouncer, ip string, httpReq *http.Request) error { +//nolint:nilnil,gocyclo,funlen,gocognit +func appsecQuery(bouncer *Bouncer, ip string, httpReq *http.Request) (*AppSecResponse, error) { routeURL := url.URL{ Scheme: bouncer.appsecScheme, Host: bouncer.appsecHost, @@ -766,7 +812,7 @@ func appsecQuery(bouncer *Bouncer, ip string, httpReq *http.Request) error { switch { case isBodyUnreadable(httpReq): if bouncer.appsecUnreadableBodyBlock && isMethodWithBody(httpReq.Method) { - return errors.New("appsecQuery:unreadableBody dropped") + return nil, errors.New("appsecQuery:unreadableBody dropped") } req, _ = http.NewRequest(http.MethodGet, routeURL.String(), nil) case bouncer.appsecBodyLimit > 0 && httpReq.Body != nil: @@ -775,7 +821,7 @@ func appsecQuery(bouncer *Bouncer, ip string, httpReq *http.Request) error { teeReader := io.TeeReader(limitedReader, &bodyBuffer) bodyBytes, err := io.ReadAll(teeReader) if err != nil { - return fmt.Errorf("appsecQuery:GetBody %w", err) + return nil, fmt.Errorf("appsecQuery:GetBody %w", err) } // Conserve body intact after reading it for other middlewares and service httpReq.Body = io.NopCloser(io.MultiReader(&bodyBuffer, httpReq.Body)) @@ -801,9 +847,9 @@ func appsecQuery(bouncer *Bouncer, ip string, httpReq *http.Request) error { if err != nil || isReverseProxyError(res.StatusCode) { bouncer.log.Error("appsecQuery:unreachable") if bouncer.appsecUnreachableBlock { - return fmt.Errorf("appsecQuery:unreachable %w", err) + return nil, fmt.Errorf("appsecQuery:unreachable %w", err) } - return nil + return nil, nil } defer func() { // net/http returns a conn to the idle pool once its body has been read to EOF, closing early discards it. @@ -811,22 +857,44 @@ func appsecQuery(bouncer *Bouncer, ip string, httpReq *http.Request) error { if _, errDrain := io.Copy(io.Discard, res.Body); errDrain != nil { bouncer.log.Debug("appsecQuery:drainBody " + errDrain.Error()) } - if err = res.Body.Close(); err != nil { - bouncer.log.Error("appsecQuery:closeBody " + err.Error()) + if errClose := res.Body.Close(); errClose != nil { + bouncer.log.Error("appsecQuery:closeBody " + errClose.Error()) } }() - if res.StatusCode == http.StatusInternalServerError { + switch res.StatusCode { + case http.StatusInternalServerError: bouncer.log.Info("appsecQuery:failure") if bouncer.appsecFailureBlock { - return errors.New("appsecQuery statusCode:500") + return nil, errors.New("appsecQuery:failure statusCode:500") + } + return nil, nil + case http.StatusOK: + return nil, nil + case http.StatusForbidden: + body, err := io.ReadAll(io.LimitReader(res.Body, appsecResponseBodyLimit+1)) + if err != nil { + return nil, fmt.Errorf("appsecQuery:readBody %w", err) + } + if int64(len(body)) > appsecResponseBodyLimit { + bouncer.log.Debug("appsecQuery:responseBodyTooLarge") + return nil, errors.New("appsecQuery:responseBodyTooLarge statusCode:403") + } + body = bytes.TrimSpace(body) + if len(body) == 0 { + return nil, errors.New("appsecQuery:responseBodyMissing statusCode:403") } - return nil - } - if res.StatusCode != http.StatusOK { - return fmt.Errorf("appsecQuery statusCode:%d", res.StatusCode) - } - return nil + var decision AppSecResponse + if err := json.Unmarshal(body, &decision); err != nil { + return nil, fmt.Errorf("appsecQuery:parseBody %w", err) + } + if decision.Action == "" || decision.HTTPStatus == 0 { + return nil, errors.New("appsecQuery:responseAppsecKeysMissing") + } + return &decision, nil + default: + return nil, fmt.Errorf("appsecQuery: statusCode:%d", res.StatusCode) + } } func reportMetrics(bouncer *Bouncer) error { diff --git a/bouncer_test.go b/bouncer_test.go index 386f4ae..f523a5d 100644 --- a/bouncer_test.go +++ b/bouncer_test.go @@ -472,6 +472,13 @@ func Test_isBodyUnreadable(t *testing.T) { } } +func newUnreadableRequest(method string, done <-chan struct{}) *http.Request { + req, _ := http.NewRequest(method, "http://localhost/api/admin/reservations/8fff14a2", blockingBody{done: done}) + req.ProtoMajor = 3 + req.ContentLength = -1 + return req +} + // newStreamingRequest builds an HTTP/2 request whose body never reaches EOF, // like a bidirectional gRPC stream (issue #323). func newStreamingRequest(done <-chan struct{}) *http.Request { @@ -509,7 +516,8 @@ func Test_appsecQuery_streamingDoesNotBlock(t *testing.T) { finished := make(chan error, 1) go func() { - finished <- appsecQuery(bouncer, "1.2.3.4", newStreamingRequest(done)) + _, err := appsecQuery(bouncer, "1.2.3.4", newStreamingRequest(done)) + finished <- err }() select { @@ -542,57 +550,182 @@ func Test_appsecQuery_dropUnreadableBody(t *testing.T) { log: logger.New("INFO", ""), } - done := make(chan struct{}) - defer close(done) + type bodyReq func(done <-chan struct{}) *http.Request + tests := []struct { + name string + req bodyReq + wantErr bool + }{ + // a streaming body that never reaches EOF is unreadable: the request is dropped. + {name: "streaming request is dropped", req: newStreamingRequest, wantErr: true}, + // issue #351 regression: an HTTP/3 GET without content-length has no body to read, so it must not be dropped. + {name: "bodyless HTTP/3 GET is not dropped", req: func(done <-chan struct{}) *http.Request { return newUnreadableRequest(http.MethodGet, done) }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + done := make(chan struct{}) + defer close(done) - finished := make(chan error, 1) - go func() { - finished <- appsecQuery(bouncer, "1.2.3.4", newStreamingRequest(done)) - }() + finished := make(chan error, 1) + go func() { + _, err := appsecQuery(bouncer, "1.2.3.4", tc.req(done)) + finished <- err + }() - select { - case err := <-finished: - if err == nil { - t.Error("appsecQuery() expected an error to block the request, got nil") - } - case <-time.After(2 * time.Second): - t.Fatal("appsecQuery() blocked on a streaming request body (issue #323 regression)") + select { + case err := <-finished: + if tc.wantErr && err == nil { + t.Error("appsecQuery() expected the request to be dropped, got nil") + } + if !tc.wantErr && err != nil { + t.Errorf("appsecQuery() returned error: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("appsecQuery() blocked while handling the request body") + } + }) } } -// Test_appsecQuery_unreadableBodyGetNotDropped is a regression test for issue #351 -func Test_appsecQuery_unreadableBodyGetNotDropped(t *testing.T) { - appsecServer := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) { - rw.WriteHeader(http.StatusOK) +func Test_appsecQuery_oversizedResponse(t *testing.T) { + testCases := []struct { + name string + statusCode int + wantErr bool + }{ + {name: "OK passes", statusCode: http.StatusOK, wantErr: false}, + {name: "Forbidden blocks", statusCode: http.StatusForbidden, wantErr: true}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + appsecServer := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) { + rw.WriteHeader(tc.statusCode) + _, _ = io.WriteString(rw, strings.Repeat("x", int(appsecResponseBodyLimit)+1)) + })) + defer appsecServer.Close() + + appsecURL, _ := url.Parse(appsecServer.URL) + bouncer := &Bouncer{ + appsecScheme: appsecURL.Scheme, + appsecHost: appsecURL.Host, + appsecPath: "/", + appsecBodyLimit: 1000, + httpAppsecClient: appsecServer.Client(), + log: logger.New("INFO", ""), + } + + decision, err := appsecQuery(bouncer, "1.2.3.4", httptest.NewRequest(http.MethodGet, "http://localhost/", nil)) + if tc.wantErr && err == nil { + t.Fatal("appsecQuery() expected error, got nil") + } + if !tc.wantErr && err != nil { + t.Fatalf("appsecQuery() returned error: %v", err) + } + if decision != nil { + t.Fatalf("appsecQuery() returned decision: %v", decision) + } + }) + } +} + +func TestHandleNextServeHTTPRelaysStructuredAppsecChallenge(t *testing.T) { + appsecServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{ + "action":"challenge", + "http_status":200, + "user_body_content":"challenge", + "user_cookies":["__crowdsec_challenge=value; Path=/; HttpOnly"], + "user_headers":{ + "Content-Type":["text/html"], + "Cache-Control":["no-store"] + } + }`)) })) defer appsecServer.Close() - appsecURL, _ := url.Parse(appsecServer.URL) - bouncer := &Bouncer{ - appsecScheme: appsecURL.Scheme, - appsecHost: appsecURL.Host, - appsecPath: "/", - appsecBodyLimit: 10485760, - appsecUnreadableBodyBlock: true, - httpAppsecClient: appsecServer.Client(), - log: logger.New("INFO", ""), + appsecURL, err := url.Parse(appsecServer.URL) + if err != nil { + t.Fatal(err) } - done := make(chan struct{}) - defer close(done) + nextCalled := false + bouncer := &Bouncer{ + next: http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + nextCalled = true + }), + appsecEnabled: true, + appsecScheme: appsecURL.Scheme, + appsecHost: appsecURL.Host, + appsecPath: "/", + httpAppsecClient: appsecServer.Client(), + remediationStatusCode: http.StatusForbidden, + remediationCustomHeader: "X-Remediation", + log: logger.New("DEBUG", ""), + } - finished := make(chan error, 1) - go func() { - finished <- appsecQuery(bouncer, "1.2.3.4", newUnreadableRequest(http.MethodGet, done)) - }() + recorder := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "http://example.com/protected", nil) + bouncer.handleNextServeHTTP(recorder, req, "192.0.2.10") - select { - case err := <-finished: - if err != nil { - t.Errorf("appsecQuery() on an HTTP/3 GET without content-length returned error: %v", err) - } - case <-time.After(2 * time.Second): - t.Fatal("appsecQuery() blocked on an HTTP/3 GET request body (issue #351 regression)") + if nextCalled { + t.Fatal("next handler should not be called for appsec challenge") + } + if recorder.Code != http.StatusOK { + t.Fatalf("expected challenge status 200, got %d", recorder.Code) + } + if got := recorder.Body.String(); got != "challenge" { + t.Fatalf("expected appsec challenge body, got %q", got) + } + if got := recorder.Header().Get("Content-Type"); got != "text/html" { + t.Fatalf("expected Content-Type relayed, got %q", got) + } + if got := recorder.Header().Get("Set-Cookie"); got != "__crowdsec_challenge=value; Path=/; HttpOnly" { + t.Fatalf("expected Set-Cookie relayed, got %q", got) + } + if got := recorder.Header().Get("X-Remediation"); got != "challenge" { + t.Fatalf("expected custom remediation header challenge, got %q", got) + } +} + +func TestHandleNextServeHTTPLegacyAppsecForbiddenFallsBackToBan(t *testing.T) { + appsecServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + defer appsecServer.Close() + + appsecURL, err := url.Parse(appsecServer.URL) + if err != nil { + t.Fatal(err) + } + + nextCalled := false + bouncer := &Bouncer{ + next: http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + nextCalled = true + }), + appsecEnabled: true, + appsecScheme: appsecURL.Scheme, + appsecHost: appsecURL.Host, + appsecPath: "/", + httpAppsecClient: appsecServer.Client(), + remediationStatusCode: http.StatusForbidden, + remediationCustomHeader: "X-Remediation", + log: logger.New("DEBUG", ""), + } + + recorder := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "http://example.com/protected", nil) + bouncer.handleNextServeHTTP(recorder, req, "192.0.2.10") + + if nextCalled { + t.Fatal("next handler should not be called for appsec forbidden") + } + if recorder.Code != http.StatusForbidden { + t.Fatalf("expected fallback ban status 403, got %d", recorder.Code) + } + if got := recorder.Header().Get("X-Remediation"); got != "ban" { + t.Fatalf("expected fallback remediation header ban, got %q", got) } } @@ -628,7 +761,7 @@ func Test_appsecQuery_reusesConnection(t *testing.T) { const calls = 10 for i := 0; i < calls; i++ { //nolint:intrange req, _ := http.NewRequest(http.MethodGet, "http://localhost/", nil) - _ = appsecQuery(bouncer, "1.2.3.4", req) + _, _ = appsecQuery(bouncer, "1.2.3.4", req) } mu.Lock() @@ -640,11 +773,60 @@ func Test_appsecQuery_reusesConnection(t *testing.T) { } } -func newUnreadableRequest(method string, done <-chan struct{}) *http.Request { - req, _ := http.NewRequest(method, "http://localhost/api/admin/reservations/8fff14a2", blockingBody{done: done}) - req.ProtoMajor = 3 - req.ContentLength = -1 - return req +func TestHandleNextServeHTTPStructuredBanKeepsBanTemplate(t *testing.T) { + appsecServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"action":"ban","http_status":403,"user_body_content":"appsec default page"}`)) + })) + defer appsecServer.Close() + + appsecURL, err := url.Parse(appsecServer.URL) + if err != nil { + t.Fatal(err) + } + + banTemplate, err := template.New("ban").Parse("BAN ip={{.ClientIP}} reason={{.RemediationReason}}") + if err != nil { + t.Fatal(err) + } + + nextCalled := false + bouncer := &Bouncer{ + next: http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + nextCalled = true + }), + appsecEnabled: true, + appsecScheme: appsecURL.Scheme, + appsecHost: appsecURL.Host, + appsecPath: "/", + httpAppsecClient: appsecServer.Client(), + remediationStatusCode: http.StatusForbidden, + remediationCustomHeader: "X-Remediation", + banTemplate: banTemplate, + banTemplateContentType: "text/html; charset=utf-8", + log: logger.New("ERROR", ""), + } + + recorder := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "http://example.com/protected", nil) + bouncer.handleNextServeHTTP(recorder, req, "192.0.2.10") + + if nextCalled { + t.Fatal("next handler should not be called for an appsec ban") + } + if recorder.Code != http.StatusForbidden { + t.Fatalf("expected 403, got %d", recorder.Code) + } + want := "BAN ip=192.0.2.10 reason=APPSEC" + if got := recorder.Body.String(); got != want { + t.Fatalf("appsec ban must keep the configured ban template, got %q want %q", got, want) + } + if got := recorder.Header().Get("Content-Type"); got != "text/html; charset=utf-8" { + t.Fatalf("expected banTemplateContentType, got %q", got) + } + if got := recorder.Header().Get("X-Remediation"); got != "ban" { + t.Fatalf("expected remediation header ban, got %q", got) + } } func Test_appsecQuery_unreadableBodyMethods(t *testing.T) { @@ -685,7 +867,8 @@ func Test_appsecQuery_unreadableBodyMethods(t *testing.T) { finished := make(chan error, 1) go func() { - finished <- appsecQuery(bouncer, "1.2.3.4", newUnreadableRequest(tt.method, done)) + _, err := appsecQuery(bouncer, "1.2.3.4", newUnreadableRequest(tt.method, done)) + finished <- err }() select { diff --git a/examples/bot-detection/README.md b/examples/bot-detection/README.md new file mode 100644 index 0000000..19c5de3 --- /dev/null +++ b/examples/bot-detection/README.md @@ -0,0 +1,74 @@ +# Example +## Bot detection with the AppSec challenge + +AppSec bot detection, added in CrowdSec 1.8.0, answers a request with a challenge page +instead of a plain block. The browser solves a proof of work and gets a cookie that lets the +following requests through. + +That needs the bouncer to relay what AppSec sends back: a ban is a status code, a challenge +is a status code plus a body, cookies and headers. + +Docs: [intro](https://docs.crowdsec.net/docs/appsec/bot_detection/intro), +[enable](https://docs.crowdsec.net/docs/appsec/bot_detection/enable), +[challenge protocol](https://docs.crowdsec.net/docs/appsec/bot_detection/challenge_protocol). + +### Two things that fail silently + +**The router must also match `/crowdsec-internal`.** The challenge page loads its fingerprint +script and proof-of-work worker from there by absolute path. Scope the router to your +application prefix alone and they 404, so the challenge can never be solved and nothing is +logged: + +```yaml + - "traefik.http.routers.router-foo.rule=PathPrefix(`/foo`) || PathPrefix(`/crowdsec-internal`)" +``` + +**The acquisition must load the config that calls `SendChallenge()`**, which is +`appsec-bot-challenge-scoring`; the `-balanced` variant only carries the rejection threshold. +The wildcard below is the form from the enable guide, and it also picks up the exclusion +configs the collection ships: + +```yaml +appsec_configs: + - crowdsecurity/appsec-default + - crowdsecurity/appsec-bot-* +``` + +> **Warning:** leave `crowdsecAppsecBodyLimit` above 0. At 0 the browser's solved challenge is +> never forwarded to AppSec, so it can never be validated and the client loops on the page. + +`WAF challenge runtime initialized` in the CrowdSec logs means the configuration is right. + +### Running it + +```bash +make run_bot_detection +``` + +Open http://localhost:8000/foo in a browser: challenge page, then whoami. curl gets the same +page but cannot solve it, which is the point. + +A client that fails the fingerprint scoring is told so rather than silently blocked: + +![Challenge rejected](image_challenge_rejected.png) + +Relaying the AppSec remediation landed in plugin 1.8.0, so that is the version the compose +file pins. To run it against the working tree instead, swap in the `localplugins` lines that +are there commented out. + +### Troubleshooting + +```bash +docker exec crowdsec cscli metrics show appsec +``` + +`Ch. Requested` at 0 while `Processed` climbs means requests reach AppSec but nothing +challenges them, so the acquisition config is wrong rather than the bouncer. + +![cscli metrics show appsec](image_metrics_appsec.png) + +[Output of cscli metrics appsec](../../../../../..) + +[Failed challenge Crowdsec bot detected](../../../../../..) +> Bot detection is flagged **Alpha** by CrowdSec. The challenge needs SSE4.1 and +> writable-executable memory, so hardened or older clients cannot solve it. diff --git a/examples/bot-detection/acquis.yaml b/examples/bot-detection/acquis.yaml new file mode 100644 index 0000000..b6824a8 --- /dev/null +++ b/examples/bot-detection/acquis.yaml @@ -0,0 +1,16 @@ +filenames: + - /var/log/traefik/access.log +labels: + type: traefik + +--- +listen_addr: 0.0.0.0:7422 +# The wildcard matches installed appsec-configs only, so it picks up exactly what the +# collection provides: the scoring engine, its threshold, and every exclusion config. +appsec_configs: + - crowdsecurity/appsec-default + - crowdsecurity/appsec-bot-* +name: myBotDetectionComponent +source: appsec +labels: + type: appsec diff --git a/examples/bot-detection/docker-compose.yml b/examples/bot-detection/docker-compose.yml new file mode 100644 index 0000000..5e6b40a --- /dev/null +++ b/examples/bot-detection/docker-compose.yml @@ -0,0 +1,71 @@ +services: + traefik: + image: "traefik:v3.7.13" + container_name: "traefik" + restart: unless-stopped + command: + # - "--log.level=DEBUG" + - "--accesslog" + - "--accesslog.filepath=/var/log/traefik/access.log" + - "--api.insecure=true" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + + # Relaying the AppSec remediation landed in 1.8.0. + # Uncomment the line below, and the volume mount, to run the working tree instead. + # - "--experimental.localplugins.bouncer.modulename=github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin" + - "--experimental.plugins.bouncer.modulename=github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin" + - "--experimental.plugins.bouncer.version=v1.8.0" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - logs-bot-detection:/var/log/traefik + # - ./../../:/plugins-local/src/github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin + ports: + - 8000:80 + - 8080:8080 + depends_on: + - crowdsec + + whoami1: + image: traefik/whoami + container_name: "simple-service-foo" + restart: unless-stopped + labels: + - "traefik.enable=true" + # Definition of the router + # /crowdsec-internal is where the challenge page loads its fingerprint script from. + # Without it the browser gets a 404 and the challenge can never be solved. + - "traefik.http.routers.router-foo.rule=PathPrefix(`/foo`) || PathPrefix(`/crowdsec-internal`)" + - "traefik.http.routers.router-foo.entrypoints=web" + - "traefik.http.routers.router-foo.middlewares=crowdsec@docker" + # Definition of the service + - "traefik.http.services.service-foo.loadbalancer.server.port=80" + # Definition of the middleware + - "traefik.http.middlewares.crowdsec.plugin.bouncer.enabled=true" + - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdseclapikey=40796d93c2958f9e58345514e67740e5" + - "traefik.http.middlewares.crowdsec.plugin.bouncer.loglevel=DEBUG" + # Enable AppSec, which is what serves the bot challenge + - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdsecappsecenabled=true" + - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdsecappsechost=crowdsec:7422" + + crowdsec: + image: crowdsecurity/crowdsec:v1.8.1 + container_name: "crowdsec" + restart: unless-stopped + environment: + COLLECTIONS: crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules crowdsecurity/appsec-bot-challenge + CUSTOM_HOSTNAME: crowdsec + BOUNCER_KEY_TRAEFIK_DEV: 40796d93c2958f9e58345514e67740e5 + volumes: + - ./acquis.yaml:/etc/crowdsec/acquis.yaml:ro + - logs-bot-detection:/var/log/traefik:ro + - crowdsec-db-bot-detection:/var/lib/crowdsec/data/ + - crowdsec-config-bot-detection:/etc/crowdsec/ + labels: + - "traefik.enable=false" + +volumes: + logs-bot-detection: + crowdsec-db-bot-detection: + crowdsec-config-bot-detection: diff --git a/examples/bot-detection/image_challenge_rejected.png b/examples/bot-detection/image_challenge_rejected.png new file mode 100644 index 0000000..7ec657c Binary files /dev/null and b/examples/bot-detection/image_challenge_rejected.png differ diff --git a/examples/bot-detection/image_metrics_appsec.png b/examples/bot-detection/image_metrics_appsec.png new file mode 100644 index 0000000..dffddc6 Binary files /dev/null and b/examples/bot-detection/image_metrics_appsec.png differ