From 68adb31065c1ad452897ccc58d3cd9f77f7f6314 Mon Sep 17 00:00:00 2001 From: Max Lerebourg Date: Sat, 1 Oct 2022 19:09:22 +0200 Subject: [PATCH 1/5] :sparkles: fix loop --- .gitignore | 5 ++++- acquis.yaml | 2 +- bouncer.go | 30 +++++++++++++------------- docker-compose.local.yml | 46 +++++++++++++++++++++++++--------------- 4 files changed, 49 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 68729d6..4dc052c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ .idea/ -.DS_Store \ No newline at end of file +.DS_Store +config +db +logs \ No newline at end of file diff --git a/acquis.yaml b/acquis.yaml index 12f95af..7684d56 100644 --- a/acquis.yaml +++ b/acquis.yaml @@ -1,4 +1,4 @@ filenames: - - /var/log/traefik/*.log + - /var/log/traefik/traefik.log labels: type: traefik \ No newline at end of file diff --git a/bouncer.go b/bouncer.go index 327e316..352a466 100644 --- a/bouncer.go +++ b/bouncer.go @@ -58,8 +58,8 @@ func CreateConfig() *Config { CrowdsecCapiLogin: "", CrowdsecCapiPwd: "", CrowdsecCapiScenarios: []string{}, - UpdateIntervalSeconds: 60, - DefaultDecisionSeconds: 60, + UpdateIntervalSeconds: 10, + DefaultDecisionSeconds: 10, } } @@ -71,6 +71,7 @@ type Bouncer struct { enabled bool crowdsecStreamHealthy bool + needInit bool crowdsecScheme string crowdsecHost string crowdsecKey string @@ -116,14 +117,16 @@ func New(ctx context.Context, next http.Handler, config *Config, name string) (h }, cache: ttl_map.New(), } - // if we are on a stream mode, we fetch in a go routine every minute the new decisions. - if config.CrowdsecMode == streamMode { - go handleStreamCache(bouncer, true) - } else if config.CrowdsecMode == aloneMode { - getToken(bouncer) - time.AfterFunc(10*time.Second, func() { - handleStreamCache(bouncer, false) - }) + if config.CrowdsecMode == streamMode || config.CrowdsecMode == aloneMode { + if config.CrowdsecMode == aloneMode { + getToken(bouncer) + } + ticker := time.NewTicker(time.Duration(config.UpdateIntervalSeconds) * time.Second) + go func() { + for range ticker.C { + go handleStreamCache(bouncer) + } + }() } return bouncer, nil } @@ -265,18 +268,15 @@ func handleNoStreamCache(a *Bouncer, rw http.ResponseWriter, req *http.Request, setDecision(a.cache, remoteHost, true, int64(duration.Seconds())) } -func handleStreamCache(a *Bouncer, initialized bool) { +func handleStreamCache(a *Bouncer) { // TODO clean properly on exit. - time.AfterFunc(time.Duration(a.updateInterval)*time.Second, func() { - handleStreamCache(a, false) - }) var rawQuery string var path string if a.crowdsecMode == aloneMode { rawQuery = "" path = crowdsecCapiDecisions } else { - rawQuery = fmt.Sprintf("startup=%t", initialized) + rawQuery = fmt.Sprintf("startup=%t", !a.crowdsecStreamHealthy) path = crowdsecLapiStreamRoute } streamRouteURL := url.URL{ diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 5b63c13..f7250ff 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -15,28 +15,40 @@ services: - "--experimental.localplugins.bouncer.modulename=github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin" volumes: - - "/var/run/docker.sock:/var/run/docker.sock:ro" - - "logs:/var/log/traefik" + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./logs:/var/log/traefik - ./:/plugins-local/src/github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin - ports: - - 8000:80 - - 8080:8080 - depends_on: - - 'crowdsec' - - whoami: - image: traefik/whoami - container_name: "simple-service" labels: - "traefik.enable=true" - - "traefik.http.routers.whoami.rule=Host(`localhost`)" - - "traefik.http.routers.whoami.entrypoints=web" - - "traefik.http.routers.whoami.middlewares=crowdsec@docker" - "traefik.http.middlewares.crowdsec.plugin.bouncer.enabled=true" - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdseclapikey=40796d93c2958f9e58345514e67740e5" - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdseccapilogin=3829a6c9870e4726a377d8951ebb64a1m8psGvMaq1ykJ3zX" - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdseccapipwd=Q5pgN8bRNInHGdx6QCksdPOJVLeLQ7ipJntSeuP3r8088zXzRVs4G8liXAKfI1k6" - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdseccapiscenarios=crowdsecurity/http-backdoors-attempts,baudneo/zoneminder-bf" + ports: + - 8000:80 + - 8080:8080 + depends_on: + - crowdsec + + whoami1: + image: traefik/whoami + container_name: "simple-service1" + labels: + - "traefik.enable=true" + - "traefik.http.routers.whoami.rule=Host(`localhost`)" + - "traefik.http.routers.whoami.entrypoints=web" + - "traefik.http.routers.whoami.middlewares=crowdsec@docker" + - "traefik.http.services.whoami.loadbalancer.server.port=80" + whoami2: + image: traefik/whoami + container_name: "simple-service2" + labels: + - "traefik.enable=true" + - "traefik.http.routers.whoami.rule=Host(`localhost`)" + - "traefik.http.routers.whoami.entrypoints=web" + - "traefik.http.routers.whoami.middlewares=crowdsec@docker" + - "traefik.http.services.whoami.loadbalancer.server.port=80" crowdsec: image: crowdsecurity/crowdsec:v1.4.1 @@ -48,9 +60,9 @@ services: BOUNCER_KEY_TRAEFIK: 40796d93c2958f9e58345514e67740e5 volumes: - ./acquis.yaml:/etc/crowdsec/acquis.yaml:ro - - logs:/var/log/traefik:ro - - crowdsec-db:/var/lib/crowdsec/data/ - - crowdsec-config:/etc/crowdsec/ + - ./logs:/var/log/traefik:ro + - ./db:/var/lib/crowdsec/data/ + - ./config:/etc/crowdsec/ volumes: logs: From 89e8b5631e0f8c59cd863e239f6d07813b688f60 Mon Sep 17 00:00:00 2001 From: Max Lerebourg Date: Sat, 1 Oct 2022 19:12:48 +0200 Subject: [PATCH 2/5] :sparkles: fix loop --- bouncer.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bouncer.go b/bouncer.go index 352a466..31aa919 100644 --- a/bouncer.go +++ b/bouncer.go @@ -58,8 +58,8 @@ func CreateConfig() *Config { CrowdsecCapiLogin: "", CrowdsecCapiPwd: "", CrowdsecCapiScenarios: []string{}, - UpdateIntervalSeconds: 10, - DefaultDecisionSeconds: 10, + UpdateIntervalSeconds: 60, + DefaultDecisionSeconds: 60, } } @@ -71,7 +71,6 @@ type Bouncer struct { enabled bool crowdsecStreamHealthy bool - needInit bool crowdsecScheme string crowdsecHost string crowdsecKey string From 1e4508f0917b336fec06e29c301d983135bd8019 Mon Sep 17 00:00:00 2001 From: Max Lerebourg Date: Sat, 1 Oct 2022 19:18:12 +0200 Subject: [PATCH 3/5] :sparkles: fix log --- bouncer.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/bouncer.go b/bouncer.go index 31aa919..41fd99b 100644 --- a/bouncer.go +++ b/bouncer.go @@ -140,7 +140,7 @@ func (a *Bouncer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { // TODO Make sur remote address does not include the port. remoteHost, _, err := net.SplitHostPort(req.RemoteAddr) if err != nil { - log.Printf("failed to extract ip from remote address: %v", err) + logger(fmt.Sprintf("failed to extract ip from remote address: %v", err)) a.next.ServeHTTP(rw, req) return } @@ -197,6 +197,10 @@ type Login struct { Expire string `json:"expire"` } +func logger(str string) { + log.Printf("Crowdsec Bouncer Traefik Plugin - %s", str) +} + func contains(source []string, target string) bool { for _, a := range source { if a == target { @@ -219,7 +223,7 @@ func getDecision(cache *ttl_map.Heap, clientIP string) (bool, error) { func setDecision(cache *ttl_map.Heap, clientIP string, isBanned bool, duration int64) { if isBanned { - log.Printf("%v banned", clientIP) + logger(fmt.Sprintf("%v banned", clientIP)) cache.Set(clientIP, cacheBannedValue, duration) } else { cache.Set(clientIP, cacheNoBannedValue, duration) @@ -247,7 +251,7 @@ func handleNoStreamCache(a *Bouncer, rw http.ResponseWriter, req *http.Request, var decisions []Decision err := json.Unmarshal(body, &decisions) if err != nil { - log.Printf("failed to parse body: %s", err) + logger(fmt.Sprintf("failed to parse body: %s", err)) rw.WriteHeader(http.StatusForbidden) return } @@ -261,7 +265,7 @@ func handleNoStreamCache(a *Bouncer, rw http.ResponseWriter, req *http.Request, rw.WriteHeader(http.StatusForbidden) duration, err := time.ParseDuration(decisions[0].Duration) if err != nil { - log.Printf("failed to parse duration: %s", err) + logger(fmt.Sprintf("failed to parse duration: %s", err)) return } setDecision(a.cache, remoteHost, true, int64(duration.Seconds())) @@ -288,7 +292,7 @@ func handleStreamCache(a *Bouncer) { var stream Stream err := json.Unmarshal(body, &stream) if err != nil { - log.Printf("error while parsing body: %s", err) + logger(fmt.Sprintf("error while parsing body: %s", err)) a.crowdsecStreamHealthy = false return } @@ -314,7 +318,7 @@ func getToken(a *Bouncer) { var login Login err := json.Unmarshal(body, &login) if err != nil { - log.Printf("error while parsing body: %s", err) + logger(fmt.Sprintf("error while parsing body: %s", err)) a.crowdsecStreamHealthy = false return } @@ -343,7 +347,7 @@ func crowdsecQuery(a *Bouncer, stringURL string, isPost bool) []byte { } res, err := a.client.Do(req) if err != nil { - log.Printf("error while fetching %v: %s", stringURL, err) + logger(fmt.Sprintf("error while fetching %v: %s", stringURL, err)) a.crowdsecStreamHealthy = false return nil } @@ -357,19 +361,19 @@ func crowdsecQuery(a *Bouncer, stringURL string, isPost bool) []byte { return crowdsecQuery(a, stringURL, false) } if res.StatusCode != http.StatusOK { - log.Printf("error while fetching %v, status code: %d", stringURL, res.StatusCode) + logger(fmt.Sprintf("error while fetching %v, status code: %d", stringURL, res.StatusCode)) a.crowdsecStreamHealthy = false return nil } defer func(body io.ReadCloser) { err = body.Close() if err != nil { - log.Printf("failed to close body reader: %s", err) + logger(fmt.Sprintf("failed to close body reader: %s", err)) } }(res.Body) body, err := ioutil.ReadAll(res.Body) if err != nil { - log.Printf("error while reading body: %s", err) + logger(fmt.Sprintf("error while reading body: %s", err)) a.crowdsecStreamHealthy = false return nil } From dd704d427424f478663f65c42fdb3363dddd4809 Mon Sep 17 00:00:00 2001 From: Max Lerebourg Date: Sat, 1 Oct 2022 19:23:20 +0200 Subject: [PATCH 4/5] :sparkles: fix --- acquis.yaml | 2 +- bouncer.go | 1 + docker-compose.local.yml | 2 +- docker-compose.yml | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/acquis.yaml b/acquis.yaml index 7684d56..ae7a63f 100644 --- a/acquis.yaml +++ b/acquis.yaml @@ -1,4 +1,4 @@ filenames: - - /var/log/traefik/traefik.log + - /var/log/traefik/access.log labels: type: traefik \ No newline at end of file diff --git a/bouncer.go b/bouncer.go index 41fd99b..d480b22 100644 --- a/bouncer.go +++ b/bouncer.go @@ -122,6 +122,7 @@ func New(ctx context.Context, next http.Handler, config *Config, name string) (h } ticker := time.NewTicker(time.Duration(config.UpdateIntervalSeconds) * time.Second) go func() { + go handleStreamCache(bouncer) for range ticker.C { go handleStreamCache(bouncer) } diff --git a/docker-compose.local.yml b/docker-compose.local.yml index f7250ff..a179bea 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -7,7 +7,7 @@ services: command: # - "--log.level=DEBUG" - "--accesslog" - - "--accesslog.filepath=/var/log/traefik/traefik.log" + - "--accesslog.filepath=/var/log/traefik/access.log" - "--api.insecure=true" - "--providers.docker=true" - "--providers.docker.exposedbydefault=false" diff --git a/docker-compose.yml b/docker-compose.yml index 0e61236..14e0e69 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: command: # - "--log.level=DEBUG" - "--accesslog" - - "--accesslog.filepath=/var/log/traefik/traefik.log" + - "--accesslog.filepath=/var/log/traefik/access.log" - "--api.insecure=true" - "--providers.docker=true" - "--providers.docker.exposedbydefault=false" From 37c76488540156edd5755412c7da11189b3867a2 Mon Sep 17 00:00:00 2001 From: MathieuHa Date: Sat, 1 Oct 2022 19:26:26 +0200 Subject: [PATCH 5/5] Clean docker-compose and aquis.yaml --- .gitignore | 3 ++- docker-compose.local.yml | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 4dc052c..4c1871f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ .DS_Store config db -logs \ No newline at end of file +logs +docker-compose.dev.yml \ No newline at end of file diff --git a/docker-compose.local.yml b/docker-compose.local.yml index a179bea..8c1431e 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -16,7 +16,7 @@ services: - "--experimental.localplugins.bouncer.modulename=github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin" volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - - ./logs:/var/log/traefik + - logs:/var/log/traefik - ./:/plugins-local/src/github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin labels: - "traefik.enable=true" @@ -60,9 +60,9 @@ services: BOUNCER_KEY_TRAEFIK: 40796d93c2958f9e58345514e67740e5 volumes: - ./acquis.yaml:/etc/crowdsec/acquis.yaml:ro - - ./logs:/var/log/traefik:ro - - ./db:/var/lib/crowdsec/data/ - - ./config:/etc/crowdsec/ + - logs:/var/log/traefik:ro + - db:/var/lib/crowdsec/data/ + - config:/etc/crowdsec/ volumes: logs: