Merge pull request #31 from maxlerebourg/30-significant-slow-down-after-updating-to-v111-and-configuring-to-use-redis

✨ rewrite redis communication

.gitignore

@@ -1,2 +1,6 @@
 .idea/
-.DS_Store
+.DS_Store
+config
+db
+logs
+docker-compose.dev.yml

Makefile

@@ -17,4 +17,47 @@ vendor:
 	go mod vendor
 
 clean:
-	rm -rf ./vendor
+	rm -rf ./vendor
+
+run_dev:
+	docker-compose -f docker-compose.dev.yml up -d --remove-orphans
+
+run_local:
+	docker-compose -f docker-compose.local.yml up -d --remove-orphans
+
+run_behindproxy:
+	docker-compose -f exemples/behind-proxy/docker-compose.cloudflare.yml up -d --remove-orphans
+
+run_cacheredis:
+	docker-compose -f exemples/redis-cache/docker-compose.redis.yml up -d --remove-orphans
+
+run:
+	docker-compose -f docker-compose.yml up -d --remove-orphans
+
+restart_dev:
+	docker-compose -f docker-compose.dev.yml restart
+
+restart_local:
+	docker-compose -f docker-compose.local.yml restart
+
+restart:
+	docker-compose -f docker-compose.yml restart
+
+show_logs:
+	docker-compose -f docker-compose.yml restart
+
+show_local_logs:
+	docker-compose -f docker-compose.local.yml logs -f
+
+show_dev_logs:
+	docker-compose -f docker-compose.dev.yml logs -f
+
+clean_all_docker:
+	docker-compose -f exemples/behind-proxy/docker-compose.cloudflare.yml down --remove-orphans
+	docker-compose -f exemples/behind-proxy/docker-compose.redis.yml down --remove-orphans
+	docker-compose -f docker-compose.local.yml down --remove-orphans
+	docker-compose -f docker-compose.yml down --remove-orphans
+
+show_metrics:
+	docker exec crowdsec cscli metrics
+

README.md

@@ -16,61 +16,80 @@ The crowdsec utility will provide the community blocklist which contains highly
 
 When used with crowdsec it will leverage the local API which will analyze traefik logs and take decisions on the requests made by users/bots. Malicious actors will be banned based on patterns against your website.
 
-There are 4 operating modes (CrowdsecMode) for this plugin:
+There are 3 operating modes (CrowdsecMode) for this plugin:
-- none -> If the client IP is on ban list, it will get a http code 403 response.
-    Otherwise, request will continue as usual. All request call the Crowdsec LAPI
 
-- live -> If the client IP is on ban list, it will get a http code 403 response.
+| Mode | Description |
-    Otherwise, request will continue as usual.
+|------|------|
-    The bouncer can leverage use of a local cache in order to reduce the number
+| none | If the client IP is on ban list, it will get a http code 403 response. Otherwise, request will continue as usual. All request call the Crowdsec LAPI |
-    of requests made to the Crowdsec LAPI. It will keep in cache the status for
+| live | If the client IP is on ban list, it will get a http code 403 response. Otherwise, request will continue as usual.    The bouncer can leverage use of a local cache in order to reduce the number of requests made to the Crowdsec LAPI. It will keep in cache the status for  each IP that makes queries. |
-    each IP that makes queries.
+| stream | Stream Streaming mode allows you to keep in the local cache only the Banned IPs, every requests that does not hit the cache is authorized. Every minute, the cache is updated with news from the Crowdsec LAPI. |
-
-- stream -> Stream Streaming mode allows you to keep in the local cache only the Banned IPs,
-    every requests that does not hit the cache is authorized.
-    Every minute, the cache is updated with news from the Crowdsec LAPI.
-
-- alone -> Streaming mode but the blacklisted IPs are fetched on the CAPI.
-    Every 2 hours, the cache is updated with news from the Crowdsec CAPI.
 
 The recommanded mode for performance is the streaming mode, decisions are updated every 60 sec by default and that's the only communication between traefik and crowdsec. Every requests that happens hits the cache for quick decisions.
 
+The cache can be local to the Traefik instance using the filesystem or use of a separated redis instance.  
+The redis instance is currently in beta and support Redis 7.0.X version
+
 ## Usage
 
+To get started, use the `docker-compose.yml` file.
+
+You can run it with:
+```bash
+make run
+```
+
+### Note
+
+**/!\ Since Release 1.10, cache is no longer duplicated but shared by all services**
+*This lowers the overhead of the cache in memory and the numbers of cache to fetch it from crowdsec in situation with many services*
+
+
 ### Variables
 - Enabled
   - bool
   - enable the plugin
+  - default: false
+- LogLevel
+  - string
+  - default: `INFO`, expected value are: `INFO`, `DEBUG`
 - CrowdsecMode
   - string
-  - default: `stream`, expected value are: `none`, `live`, `stream`, `alone`
+  - default: `live`, expected value are: `none`, `live`, `stream`
 - CrowdsecLapiScheme
   - string
   - default: `http`, expected value are: `http`, `https`
 - CrowdsecLapiHost
   - string
   - default: "crowdsec:8080"
-  - Crowdsec LAPI available on which host.
+  - Crowdsec LAPI available on which host and port.
 - CrowdsecLapiKey
   - string
-  - Crowdsec LAPI generated key for the bouncer. 
+  - Crowdsec LAPI generated key for the bouncer : **must be unique by service**. 
-- CrowdsecCapiLogin
-  - string
-  - Used only in `alone` mode, login for Crowdsec CAPI
-- CrowdsecCapiPwd
-  - string
-  - Used only in `alone` mode, password for Crowdsec CAPI
-- CrowdsecCapiScenarios
-  - []string
-  - Used only in `alone` mode, login for Crowdsec CAPI
 - UpdateIntervalSeconds
   - int64
   - default: 60
-  - User only in `stream` mode, interval of time between fetching blacklisted IPs from LAPI
+  - Used only in `stream` mode, interval between fetching blacklisted IPs from LAPI
 - DefaultDecisionSeconds
   - int64
   - default: 60
-  - User only in `live` mode, decision duration of accepted IPs
+  - Used only in `live` mode, decision duration of accepted IPs
+- ForwardedHeadersTrustedIPs
+  - []string
+  - default: []
+  - List of IPs of trusted Proxies that are in front of traefik (ex: Cloudflare)
+- ForwardedHeadersCustomName
+  - string
+  - default: "X-Forwarded-For"
+  - Name of the header where the real IP of the client should be retrieved
+- RedisCacheEnabled
+  - bool
+  - default: false
+  - enable redis cache instead of filesystem cache
+- RedisCacheHost
+  - string 
+  - default: "redis:6379"
+  - hostname and port for the redis service
+
 
 ### Configuration
 
@@ -113,21 +132,20 @@ http:
           enabled: false
           updateIntervalSeconds: 60
           defaultDecisionSeconds: 60
-          crowdsecMode: stream
+          crowdsecMode: live
           crowdsecLapiKey: privateKey
           crowdsecLapiHost: crowdsec:8080
           crowdsecLapiScheme: http
-          crowdsecCapiLogin: login
+          forwardedHeadersTrustedIPs: 
-          crowdsecCapiPwd: password
+            - 10.0.10.23/32
-          crowdsecCapiScenarios:
+            - 10.0.20.0/24
-            - scenario1
+          forwardedHeadersCustomName: X-Custom-Header
-            - scenario2
+          redisCacheEnabled: false
-            ...
+          redisCacheHost: "redis:6379"
-
 ```
-Except for the crowdsecLapiKey, crowdsecCapiLogin, crowdsecCapiPwd, crowdsecCapiScenarios, these are the default value of the plugin.
+These are the default values of the plugin except for LapiKey.
 
-#### Generate LAPI KEY (exept for `alone` mode)
+#### Generate LAPI KEY
 You need to generate a crowdsec API key for the LAPI.
 You can follow the documentation here: https://docs.crowdsec.net/docs/user_guides/lapi_mgmt/
 
@@ -156,36 +174,6 @@ You can then run all the containers:
 docker-compose up -d
 ```
 
-#### Generate CAPI credentials (only for `alone` mode)
-You need to create a crowdsec API credentials for the CAPI.
-You can follow the documentation here: https://docs.crowdsec.net/docs/central_api/intro
-
-```bash
-curl -X POST "https://api.crowdsec.net/v2/watchers" -H  "accept: application/json" -H  "Content-Type: application/json" -d "{ \"password\": \"PASSWORD\",  \"machine_id\": \"LOGIN\"}"
-```
-
-These CAPI credentials must be set in your docker-compose.yml or in your config files
-```yaml
-...
-traefik:
-  command:
-    ...
-    - "--experimental.plugins.bouncer.modulename=github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin"
-    - "--experimental.plugins.bouncer.version=v1.0.0"
-    ...
-whoami:
-  labels:
-    - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdseccapilogin=LOGIN"
-    - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdseccapipwd=PASSWORD"
-    - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdseccapiscenarios=scenario1, scenario2, ..."
-    - "traefik.http.middlewares.crowdsec.plugin.bouncer.enabled=true"
-```
-
-You can then run all the containers:
-```bash
-docker-compose up -d
-```
-
 #### Add manually an IP to the blocklist (testing purpose)
 
 ```bash
@@ -223,14 +211,53 @@ For local developpement a docker-compose.local.yml is provided and reproduce the
 ```bash
 docker-compose -f docker-compose.local.yml up -d
 ```
+Equivalent to
+```bash
+make run_local
+```
+
+### Exemples
+
+1. Behind another proxy service (ex: clouflare)
+
+You need to configure your Traefik to trust Forwarded headers by your front proxy
+In the exemple we use another instance of traefik with the container named cloudflare to simulate a front proxy
+
+The "internal" Traefik instance is configured to trust the cloudflare forward headers
+```yaml
+  - "--entrypoints.web.forwardedheaders.trustedips=172.21.0.5"
+```
+
+We configure the middleware to trust as well the IP:
+```yaml
+    - "traefik.http.middlewares.crowdsec1.plugin.bouncer.forwardedheaderstrustedips=172.21.0.5"
+```
+
+To run the environnement run:
+```bash
+make run_behindproxy
+```
+
+2. With Redis as an external shared cache
+
+The plugin must be configured to connect to a redis instance
+```yaml
+  redisCacheHost: "redis:6379"
+```
+Here **redis** is the hostname of a container located in the same network as Traefik and **6379** the default port of redis
+
+To run the demo environnement run:
+```bash
+make run_cacheredis
+```
 
 ### About
 
-Me and [mhanotaux](https://github.com/mhanotaux) have been using traefik since 2020 at [Primadviz](https://primadviz.com).
+Me and [mathieuHa](https://github.com/mathieuHa) have been using traefik since 2020 at [Primadviz](https://primadviz.com).
 We come from web developper and security engineer background and wanted to add the power of a very promesing technology (Crowdsec) into the edge router we love.
 
 We initially run into this project: https://github.com/fbonalair/traefik-crowdsec-bouncer
 It was using traefik and forward auth middleware to verify every requests.
 They had to go through a webserver which then contacts of another webservice (the crowdsec LAPI) to make a decision based on the source IP.
 We initially proposed some improvement by implementing a streaming mode and a local cache.
-With the Traefik hackathon we deciced to implement our solution directly as a traefik plugin which could be found by every one on plugins.traefik.io and be more performant.
+With the Traefik hackathon we deciced to implement our solution directly as a traefik plugin which could be found by every one on plugins.traefik.io and be more performant.

acquis.yaml

@@ -1,4 +1,4 @@
 filenames:
- - /var/log/traefik/*.log
+ - /var/log/traefik/access.log
 labels:
-  type: traefik
+  type: traefik

bouncer.go

@@ -7,106 +7,110 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
-	"net"
 	"net/http"
 	"net/url"
-	"strings"
 	"text/template"
 	"time"
 
-	ttl_map "github.com/leprosus/golang-ttl-map"
+	cache "github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin/pkg/cache"
+	ip "github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin/pkg/ip"
+	logger "github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin/pkg/logger"
 )
 
 const (
-	aloneMode               = "alone"
 	streamMode              = "stream"
 	liveMode                = "live"
 	noneMode                = "none"
 	crowdsecLapiHeader      = "X-Api-Key"
-	crowdsecCapiHeader      = "Authorization"
 	crowdsecLapiRoute       = "v1/decisions"
 	crowdsecLapiStreamRoute = "v1/decisions/stream"
-	crowdsecCapiLogin       = "v2/watchers/login"
+	cacheTimeoutKey         = "updated"
-	crowdsecCapiDecisions   = "v2/decisions/stream"
+)
-	cacheBannedValue        = "t"
+
-	cacheNoBannedValue      = "f"
+var (
+	crowdsecStreamHealthy = false
+	ticker                chan bool
 )
 
 // Config the plugin configuration.
 type Config struct {
-	Enabled                bool     `json:"enabled,omitempty"`
+	Enabled                    bool     `json:"enabled,omitempty"`
-	CrowdsecMode           string   `json:"crowdsecMode,omitempty"`
+	LogLevel                   string   `json:"logLevel,omitempty"`
-	CrowdsecLapiScheme     string   `json:"crowdsecLapiScheme,omitempty"`
+	CrowdsecMode               string   `json:"crowdsecMode,omitempty"`
-	CrowdsecLapiHost       string   `json:"crowdsecLapiHost,omitempty"`
+	CrowdsecLapiScheme         string   `json:"crowdsecLapiScheme,omitempty"`
-	CrowdsecLapiKey        string   `json:"crowdsecLapiKey,omitempty"`
+	CrowdsecLapiHost           string   `json:"crowdsecLapiHost,omitempty"`
-	CrowdsecCapiLogin      string   `json:"crowdsecCapiLogin,omitempty"`
+	CrowdsecLapiKey            string   `json:"crowdsecLapiKey,omitempty"`
-	CrowdsecCapiPwd        string   `json:"crowdsecCapiPwd,omitempty"`
+	ForwardedHeadersCustomName string   `json:"forwardedheaderscustomheader,omitempty"`
-	CrowdsecCapiScenarios  []string `json:"crowdsecCapiScenarios,omitempty"`
+	UpdateIntervalSeconds      int64    `json:"updateIntervalSeconds,omitempty"`
-	UpdateIntervalSeconds  int64    `json:"updateIntervalSeconds,omitempty"`
+	DefaultDecisionSeconds     int64    `json:"defaultDecisionSeconds,omitempty"`
-	DefaultDecisionSeconds int64    `json:"defaultDecisionSeconds,omitempty"`
+	ForwardedHeadersTrustedIPs []string `json:"forwardedHeadersTrustedIps,omitempty"`
+	RedisCacheEnabled          bool     `json:"redisCacheEnabled,omitempty"`
+	RedisCacheHost             string   `json:"redisCacheHost,omitempty"`
 }
 
 // CreateConfig creates the default plugin configuration.
 func CreateConfig() *Config {
 	return &Config{
-		Enabled:                false,
+		Enabled:                    false,
-		CrowdsecMode:           streamMode,
+		LogLevel:                   "INFO",
-		CrowdsecLapiScheme:     "http",
+		CrowdsecMode:               liveMode,
-		CrowdsecLapiHost:       "crowdsec:8080",
+		CrowdsecLapiScheme:         "http",
-		CrowdsecLapiKey:        "",
+		CrowdsecLapiHost:           "crowdsec:8080",
-		CrowdsecCapiLogin:      "",
+		CrowdsecLapiKey:            "",
-		CrowdsecCapiPwd:        "",
+		UpdateIntervalSeconds:      60,
-		CrowdsecCapiScenarios:  []string{},
+		DefaultDecisionSeconds:     60,
-		UpdateIntervalSeconds:  60,
+		ForwardedHeadersTrustedIPs: []string{},
-		DefaultDecisionSeconds: 60,
+		ForwardedHeadersCustomName: "X-Forwarded-For",
+		RedisCacheEnabled:          false,
+		RedisCacheHost:             "redis:6379",
 	}
 }
 
-// Bouncer a Bouncer plugin.
+// Bouncer a Bouncer struct.
 type Bouncer struct {
 	next     http.Handler
 	name     string
 	template *template.Template
 
 	enabled                bool
-	crowdsecStreamHealthy  bool
 	crowdsecScheme         string
 	crowdsecHost           string
 	crowdsecKey            string
 	crowdsecMode           string
 	updateInterval         int64
 	defaultDecisionTimeout int64
-	crowdsecLogin          string
+	customHeader           string
-	crowdsecPwd            string
+	poolStrategy           *ip.PoolStrategy
-	crowdsecScenarios      []string
 	client                 *http.Client
-	cache                  *ttl_map.Heap
 }
 
 // New creates the crowdsec bouncer plugin.
 func New(ctx context.Context, next http.Handler, config *Config, name string) (http.Handler, error) {
+	logger.Init(config.LogLevel)
 	err := validateParams(config)
-	if (err != nil) {
+	if err != nil {
+		logger.Info(fmt.Sprintf("%w", err))
 		return nil, err
 	}
 
+	checker, _ := ip.NewChecker(config.ForwardedHeadersTrustedIPs)
+
 	bouncer := &Bouncer{
 		next:     next,
 		name:     name,
 		template: template.New("CrowdsecBouncer").Delims("[[", "]]"),
 
 		enabled:                config.Enabled,
-		crowdsecStreamHealthy:  false,
 		crowdsecMode:           config.CrowdsecMode,
 		crowdsecScheme:         config.CrowdsecLapiScheme,
 		crowdsecHost:           config.CrowdsecLapiHost,
 		crowdsecKey:            config.CrowdsecLapiKey,
-		crowdsecLogin:          config.CrowdsecCapiLogin,
-		crowdsecPwd:            config.CrowdsecCapiPwd,
-		crowdsecScenarios:      config.CrowdsecCapiScenarios,
 		updateInterval:         config.UpdateIntervalSeconds,
+		customHeader:           config.ForwardedHeadersCustomName,
 		defaultDecisionTimeout: config.DefaultDecisionSeconds,
+		poolStrategy: &ip.PoolStrategy{
+			Checker: checker,
+		},
 		client: &http.Client{
 			Transport: &http.Transport{
 				MaxIdleConns:    10,
@@ -114,56 +118,57 @@ func New(ctx context.Context, next http.Handler, config *Config, name string) (h
 			},
 			Timeout: 5 * time.Second,
 		},
-		cache: ttl_map.New(),
 	}
-	// if we are on a stream mode, we fetch in a go routine every minute the new decisions.
+	if config.RedisCacheEnabled {
-	if config.CrowdsecMode == streamMode {
+		cache.InitRedisClient(config.RedisCacheHost)
-		go handleStreamCache(bouncer, true)
+	}
-	} else if config.CrowdsecMode == aloneMode {
+	if config.CrowdsecMode == streamMode && ticker == nil {
-		getToken(bouncer)
+		ticker = startTicker(config, func() {
-		time.AfterFunc(10*time.Second, func() {
+			handleStreamCache(bouncer)
-			handleStreamCache(bouncer, false)
 		})
+		go handleStreamCache(bouncer)
 	}
+
 	return bouncer, nil
 }
 
 // ServeHTTP principal function of plugin.
-func (a *Bouncer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
+func (bouncer *Bouncer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
-	if !a.enabled {
+	if !bouncer.enabled {
-		a.next.ServeHTTP(rw, req)
+		bouncer.next.ServeHTTP(rw, req)
 		return
 	}
 
-	// TODO Make sur remote address does not include the port.
+	remoteHost, err := ip.GetRemoteIP(req, bouncer.poolStrategy, bouncer.customHeader)
-	remoteHost, _, err := net.SplitHostPort(req.RemoteAddr)
 	if err != nil {
-		log.Printf("failed to extract ip from remote address: %v", err)
+		logger.Info(fmt.Sprintf("%w", err))
-		a.next.ServeHTTP(rw, req)
+		bouncer.next.ServeHTTP(rw, req)
 		return
 	}
+	logger.Debug(fmt.Sprintf("ServeHTTP ip:%v", remoteHost))
 
-	if a.crowdsecMode == streamMode || a.crowdsecMode == liveMode {
+	if bouncer.crowdsecMode != noneMode {
-		isBanned, err := getDecision(a.cache, remoteHost)
+		isBanned, err := cache.GetDecision(remoteHost)
 		if err == nil {
+			logger.Debug(fmt.Sprintf("ServeHTTP cacheHit isBanned:%v", isBanned))
 			if isBanned {
 				rw.WriteHeader(http.StatusForbidden)
 			} else {
-				a.next.ServeHTTP(rw, req)
+				bouncer.next.ServeHTTP(rw, req)
 			}
 			return
 		}
 	}
 
 	// Right here if we cannot join the stream we forbid the request to go on.
-	if a.crowdsecMode == streamMode {
+	if bouncer.crowdsecMode == streamMode {
-		if a.crowdsecStreamHealthy {
+		if crowdsecStreamHealthy {
-			a.next.ServeHTTP(rw, req)
+			bouncer.next.ServeHTTP(rw, req)
 		} else {
 			rw.WriteHeader(http.StatusForbidden)
 		}
 	} else {
-		handleNoStreamCache(a, rw, req, remoteHost)
+		handleNoStreamCache(bouncer, rw, req, remoteHost)
 	}
 }
 
@@ -188,226 +193,160 @@ type Stream struct {
 	New     []Decision `json:"new"`
 }
 
-// Login Body returned from Crowdsec Login CAPI.
-type Login struct {
-	Code   int    `json:"code"`
-	Token  string `json:"token"`
-	Expire string `json:"expire"`
-}
-
 func contains(source []string, target string) bool {
-	for _, a := range source {
+	for _, item := range source {
-		if a == target {
+		if item == target {
 			return true
 		}
 	}
 	return false
 }
 
-// Get Decision check in the cache if the IP has the banned / not banned value.
+func startTicker(config *Config, work func()) chan bool {
-// Otherwise return with an error to add the IP in cache if we are on.
+	ticker := time.NewTicker(time.Duration(config.UpdateIntervalSeconds) * time.Second)
-func getDecision(cache *ttl_map.Heap, clientIP string) (bool, error) {
+	stop := make(chan bool, 1)
-	banned, isCached := cache.Get(clientIP)
+	go func() {
-	bannedString, isValid := banned.(string)
+		defer logger.Debug("ticker:stopped")
-	if isCached && isValid && len(bannedString) > 0 {
+		for {
-		return bannedString == cacheBannedValue, nil
+			select {
-	}
+			case <-ticker.C:
-	return false, fmt.Errorf("no cache data")
+				go work()
+			case <-stop:
+				return
+			}
+		}
+	}()
+	return stop
 }
 
-func setDecision(cache *ttl_map.Heap, clientIP string, isBanned bool, duration int64) {
+// We are now in none or live mode.
-	if isBanned {
+func handleNoStreamCache(bouncer *Bouncer, rw http.ResponseWriter, req *http.Request, remoteHost string) {
-		log.Printf("%v banned", clientIP)
-		cache.Set(clientIP, cacheBannedValue, duration)
-	} else {
-		cache.Set(clientIP, cacheNoBannedValue, duration)
-	}
-}
-
-func handleNoStreamCache(a *Bouncer, rw http.ResponseWriter, req *http.Request, remoteHost string) {
-	// We are now in none or live mode.
 	routeURL := url.URL{
-		Scheme:   a.crowdsecScheme,
+		Scheme:   bouncer.crowdsecScheme,
-		Host:     a.crowdsecHost,
+		Host:     bouncer.crowdsecHost,
 		Path:     crowdsecLapiRoute,
 		RawQuery: fmt.Sprintf("ip=%v&banned=true", remoteHost),
 	}
-	body := crowdsecQuery(a, routeURL.String(), false)
+	body, err := crowdsecQuery(bouncer, routeURL.String())
+	if err != nil {
+		logger.Info(fmt.Sprintf("%w", err))
+		rw.WriteHeader(http.StatusForbidden)
+		return
+	}
 
 	if bytes.Equal(body, []byte("null")) {
-		if a.crowdsecMode == liveMode {
+		if bouncer.crowdsecMode == liveMode {
-			setDecision(a.cache, remoteHost, false, a.defaultDecisionTimeout)
+			cache.SetDecision(remoteHost, false, bouncer.defaultDecisionTimeout)
 		}
-		a.next.ServeHTTP(rw, req)
+		bouncer.next.ServeHTTP(rw, req)
 		return
 	}
 
 	var decisions []Decision
-	err := json.Unmarshal(body, &decisions)
+	err = json.Unmarshal(body, &decisions)
 	if err != nil {
-		log.Printf("failed to parse body: %s", err)
+		logger.Info(fmt.Sprintf("failed to parse body: %s", err))
 		rw.WriteHeader(http.StatusForbidden)
 		return
 	}
 	if len(decisions) == 0 {
-		if a.crowdsecMode == liveMode {
+		if bouncer.crowdsecMode == liveMode {
-			setDecision(a.cache, remoteHost, false, a.defaultDecisionTimeout)
+			cache.SetDecision(remoteHost, false, bouncer.defaultDecisionTimeout)
 		}
-		a.next.ServeHTTP(rw, req)
+		bouncer.next.ServeHTTP(rw, req)
 		return
 	}
 	rw.WriteHeader(http.StatusForbidden)
 	duration, err := time.ParseDuration(decisions[0].Duration)
 	if err != nil {
-		log.Printf("failed to parse duration: %s", err)
+		logger.Info(fmt.Sprintf("failed to parse duration: %s", err))
 		return
 	}
-	setDecision(a.cache, remoteHost, true, int64(duration.Seconds()))
+	if bouncer.crowdsecMode == liveMode {
+		cache.SetDecision(remoteHost, true, int64(duration.Seconds()))
+	}
 }
 
-func handleStreamCache(a *Bouncer, initialized bool) {
+func handleStreamCache(bouncer *Bouncer) {
 	// TODO clean properly on exit.
-	time.AfterFunc(time.Duration(a.updateInterval)*time.Second, func() {
+	// Instead of blocking the goroutine interval for all the secondary node,
-		handleStreamCache(a, false)
+	// if the master service is shut down, other goroutine can take the lead
-	})
+	// because updated routine information is in the cache
-	var rawQuery string
+	logger.Debug("handleStreamCache")
-	var path string
+	_, err := cache.GetDecision(cacheTimeoutKey)
-	if a.crowdsecMode == aloneMode {
+	if err == nil {
-		rawQuery = ""
+		return
-		path = crowdsecCapiDecisions
-	} else {
-		rawQuery = fmt.Sprintf("startup=%t", initialized)
-		path = crowdsecLapiStreamRoute
 	}
+	cache.SetDecision(cacheTimeoutKey, false, bouncer.updateInterval-1)
 	streamRouteURL := url.URL{
-		Scheme:   a.crowdsecScheme,
+		Scheme:   bouncer.crowdsecScheme,
-		Host:     a.crowdsecHost,
+		Host:     bouncer.crowdsecHost,
-		Path:     path,
+		Path:     crowdsecLapiStreamRoute,
-		RawQuery: rawQuery,
+		RawQuery: fmt.Sprintf("startup=%t", !crowdsecStreamHealthy),
 	}
-	body := crowdsecQuery(a, streamRouteURL.String(), false)
+	body, err := crowdsecQuery(bouncer, streamRouteURL.String())
-	var stream Stream
-	err := json.Unmarshal(body, &stream)
 	if err != nil {
-		log.Printf("error while parsing body: %s", err)
+		logger.Info(fmt.Sprintf("%w", err))
-		a.crowdsecStreamHealthy = false
+		crowdsecStreamHealthy = false
+		return
+	}
+	var stream Stream
+	err = json.Unmarshal(body, &stream)
+	if err != nil {
+		logger.Info(fmt.Sprintf("error while parsing body: %s", err))
+		crowdsecStreamHealthy = false
 		return
 	}
 	for _, decision := range stream.New {
 		duration, err := time.ParseDuration(decision.Duration)
 		if err == nil {
-			setDecision(a.cache, decision.Value, true, int64(duration.Seconds()))
+			cache.SetDecision(decision.Value, true, int64(duration.Seconds()))
 		}
 	}
 	for _, decision := range stream.Deleted {
-		a.cache.Del(decision.Value)
+		cache.DeleteDecision(decision.Value)
 	}
-	a.crowdsecStreamHealthy = true
+	crowdsecStreamHealthy = true
 }
 
-func getToken(a *Bouncer) {
+func crowdsecQuery(bouncer *Bouncer, stringURL string) ([]byte, error) {
-	loginURL := url.URL{
-		Scheme: a.crowdsecScheme,
-		Host:   a.crowdsecHost,
-		Path:   crowdsecCapiLogin,
-	}
-	body := crowdsecQuery(a, loginURL.String(), true)
-	var login Login
-	err := json.Unmarshal(body, &login)
-	if err != nil {
-		log.Printf("error while parsing body: %s", err)
-		a.crowdsecStreamHealthy = false
-		return
-	}
-	if login.Code == 200 && len(login.Token) > 0 {
-		a.crowdsecKey = login.Token
-	}
-}
-
-func crowdsecQuery(a *Bouncer, stringURL string, isPost bool) []byte {
 	var req *http.Request
-	if isPost {
+	req, _ = http.NewRequest(http.MethodGet, stringURL, nil)
-		data := []byte(fmt.Sprintf(
+	req.Header.Add(crowdsecLapiHeader, bouncer.crowdsecKey)
-			`{"machine_id": "%v","password": "%v","scenarios": ["%v"]}`,
+	res, err := bouncer.client.Do(req)
-			a.crowdsecLogin,
-			a.crowdsecPwd,
-			strings.Join(a.crowdsecScenarios, `","`),
-		))
-		req, _ = http.NewRequest(http.MethodPost, stringURL, bytes.NewBuffer(data))
-	} else {
-		req, _ = http.NewRequest(http.MethodGet, stringURL, nil)
-	}
-	if a.crowdsecMode == aloneMode {
-		req.Header.Add(crowdsecCapiHeader, a.crowdsecKey)
-	} else {
-		req.Header.Add(crowdsecLapiHeader, a.crowdsecKey)
-	}
-	res, err := a.client.Do(req)
 	if err != nil {
-		log.Printf("error while fetching %v: %s", stringURL, err)
+		return nil, fmt.Errorf("error while fetching %v: %s", stringURL, err)
-		a.crowdsecStreamHealthy = false
-		return nil
-	}
-	if res.StatusCode == http.StatusUnauthorized && a.crowdsecMode == aloneMode {
-		oldToken := a.crowdsecKey
-		getToken(a)
-		if oldToken == a.crowdsecKey {
-			a.crowdsecStreamHealthy = false
-			return nil
-		}
-		return crowdsecQuery(a, stringURL, false)
 	}
 	if res.StatusCode != http.StatusOK {
-		log.Printf("error while fetching %v, status code: %d", stringURL, res.StatusCode)
+		return nil, fmt.Errorf("error while fetching %v, status code: %d", stringURL, res.StatusCode)
-		a.crowdsecStreamHealthy = false
-		return nil
 	}
 	defer func(body io.ReadCloser) {
 		err = body.Close()
 		if err != nil {
-			log.Printf("failed to close body reader: %s", err)
+			logger.Info(fmt.Sprintf("failed to close body reader: %s", err))
 		}
 	}(res.Body)
 	body, err := ioutil.ReadAll(res.Body)
 	if err != nil {
-		log.Printf("error while reading body: %s", err)
+		return nil, fmt.Errorf("error while reading body: %s", err)
-		a.crowdsecStreamHealthy = false
-		return nil
 	}
-	return body
+	return body, nil
 }
 
 func validateParams(config *Config) error {
-	var requiredStrings map[string]string
+	requiredStrings := map[string]string{
-	if config.CrowdsecMode == aloneMode {
+		"CrowdsecLapiScheme": config.CrowdsecLapiScheme,
-		requiredStrings = map[string]string{
+		"CrowdsecLapiHost":   config.CrowdsecLapiHost,
-			"CrowdsecCapiLogin": config.CrowdsecLapiScheme,
+		"CrowdsecLapiKey":    config.CrowdsecLapiKey,
-			"CrowdsecCapiPwd":   config.CrowdsecLapiHost,
+		"CrowdsecMode":       config.CrowdsecMode,
-		}
+	}
-		for _, val := range config.CrowdsecCapiScenarios {
+	requiredInt := map[string]int64{
-			if len(val) == 0 {
+		"UpdateIntervalSeconds":  config.UpdateIntervalSeconds,
-				return fmt.Errorf("CrowdsecCapiScenarios: one or more scenario are empty")
+		"DefaultDecisionSeconds": config.DefaultDecisionSeconds,
-			}
+	}
-		}
+	for key, val := range requiredInt {
-		config.UpdateIntervalSeconds = 7200
+		if val < 1 {
-		config.CrowdsecLapiKey = ""
+			return fmt.Errorf("%v: cannot be less than 1", key)
-		config.CrowdsecLapiScheme = "https"
-		config.CrowdsecLapiHost = "api.crowdsec.net"
-	} else {
-		requiredStrings = map[string]string{
-			"CrowdsecLapiScheme": config.CrowdsecLapiScheme,
-			"CrowdsecLapiHost":   config.CrowdsecLapiHost,
-			"CrowdsecLapiKey":    config.CrowdsecLapiKey,
-			"CrowdsecMode":       config.CrowdsecMode,
-		}
-		requiredInt := map[string]int64{
-			"UpdateIntervalSeconds":  config.UpdateIntervalSeconds,
-			"DefaultDecisionSeconds": config.DefaultDecisionSeconds,
-		}
-		for key, val := range requiredInt {
-			if val < 1 {
-				return fmt.Errorf("%v: cannot be less than 1", key)
-			}
 		}
 	}
 	for key, val := range requiredStrings {
@@ -415,7 +354,7 @@ func validateParams(config *Config) error {
 			return fmt.Errorf("%v: cannot be empty", key)
 		}
 	}
-	if !contains([]string{noneMode, liveMode, streamMode, aloneMode}, config.CrowdsecMode) {
+	if !contains([]string{noneMode, liveMode, streamMode}, config.CrowdsecMode) {
 		return fmt.Errorf("CrowdsecMode: must be one of 'none', 'live' or 'stream'")
 	}
 	if !contains([]string{"http", "https"}, config.CrowdsecLapiScheme) {
@@ -429,5 +368,14 @@ func validateParams(config *Config) error {
 	if err != nil {
 		return fmt.Errorf("CrowdsecLapiScheme://CrowdsecLapiHost: '%v://%v' must be an URL", config.CrowdsecLapiScheme, config.CrowdsecLapiHost)
 	}
+	if len(config.ForwardedHeadersTrustedIPs) > 0 {
+		_, err = ip.NewChecker(config.ForwardedHeadersTrustedIPs)
+		if err != nil {
+			return fmt.Errorf("ForwardedHeadersTrustedIPs must be a list of IP/CIDR :%w", err)
+		}
+	} else {
+		logger.Debug("No IP provided for ForwardedHeadersTrustedIPs")
+	}
+
 	return nil
 }

bouncer_test.go

@@ -1,22 +1,20 @@
-package crowdsec_bouncer_traefik_plugin_test
+package crowdsec_bouncer_traefik_plugin
 
 import (
 	"context"
 	"net/http"
 	"net/http/httptest"
 	"testing"
-
-	crowdsec_bouncer_traefik_plugin "github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin"
 )
 
 func TestCrowdSec(t *testing.T) {
-	cfg := crowdsec_bouncer_traefik_plugin.CreateConfig()
+	cfg := CreateConfig()
-	cfg.CrowdsecLapiKey = "caca"
+	cfg.CrowdsecLapiKey = "test"
 
 	ctx := context.Background()
 	next := http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {})
 
-	handler, err := crowdsec_bouncer_traefik_plugin.New(ctx, next, cfg, "demo-plugin")
+	handler, err := New(ctx, next, cfg, "demo-plugin")
 	if err != nil {
 		t.Fatal(err)
 	}

docker-compose.local.yml

@@ -2,12 +2,12 @@ version: "3.8"
 
 services:
   traefik:
-    image: "traefik:v2.8.7"
+    image: "traefik:v2.9.4"
     container_name: "traefik"
     command:
       # - "--log.level=DEBUG"
       - "--accesslog"
-      - "--accesslog.filepath=/var/log/traefik/traefik.log"
+      - "--accesslog.filepath=/var/log/traefik/access.log"
       - "--api.insecure=true"
       - "--providers.docker=true"
       - "--providers.docker.exposedbydefault=false"
@@ -15,44 +15,55 @@ services:
 
       - "--experimental.localplugins.bouncer.modulename=github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin"
     volumes:
-      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+      - /var/run/docker.sock:/var/run/docker.sock:ro
-      - "logs:/var/log/traefik"
+      - logs-local:/var/log/traefik
       - ./:/plugins-local/src/github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin
     ports:
-      - 8000:80
+      - 80:80
       - 8080:8080
     depends_on:
-      - 'crowdsec'
+      - crowdsec
 
-  whoami:
+  whoami1:
     image: traefik/whoami
-    container_name: "simple-service"
+    container_name: "simple-service1"
     labels:
       - "traefik.enable=true"
-      - "traefik.http.routers.whoami.rule=Host(`localhost`)"
+      - "traefik.http.routers.router1.rule=Host(`localhost`) && Path(`/foo`)"
-      - "traefik.http.routers.whoami.entrypoints=web"
+      - "traefik.http.routers.router1.entrypoints=web"
-      - "traefik.http.routers.whoami.middlewares=crowdsec@docker"
+      - "traefik.http.routers.router1.middlewares=crowdsec1@docker" 
-      - "traefik.http.middlewares.crowdsec.plugin.bouncer.enabled=true"
+      - "traefik.http.services.service1.loadbalancer.server.port=80"
-      - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdseclapikey=40796d93c2958f9e58345514e67740e5"
+      - "traefik.http.middlewares.crowdsec1.plugin.bouncer.enabled=true"
-      - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdseccapilogin=3829a6c9870e4726a377d8951ebb64a1m8psGvMaq1ykJ3zX"
+      - "traefik.http.middlewares.crowdsec1.plugin.bouncer.crowdseclapikey=40796d93c2958f9e58345514e67740e5"
-      - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdseccapipwd=Q5pgN8bRNInHGdx6QCksdPOJVLeLQ7ipJntSeuP3r8088zXzRVs4G8liXAKfI1k6"
+
-      - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdseccapiscenarios=crowdsecurity/http-backdoors-attempts,baudneo/zoneminder-bf"
+  whoami2:
+    image: traefik/whoami
+    container_name: "simple-service2"
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.router2.rule=Host(`localhost`) && Path(`/bar`)"
+      - "traefik.http.routers.router2.entrypoints=web"
+      - "traefik.http.routers.router2.middlewares=crowdsec2@docker"
+      - "traefik.http.services.service2.loadbalancer.server.port=80"
+      - "traefik.http.middlewares.crowdsec2.plugin.bouncer.enabled=true"
+      - "traefik.http.middlewares.crowdsec2.plugin.bouncer.crowdseclapikey=44c36dac5c4140af9f06f397508e82c7"
 
   crowdsec:
     image: crowdsecurity/crowdsec:v1.4.1
     container_name: "crowdsec"
-    command: rm -rf /etc/crowdsec/acquis.yaml
     environment:
       COLLECTIONS: crowdsecurity/traefik
       CUSTOM_HOSTNAME: crowdsec
-      BOUNCER_KEY_TRAEFIK: 40796d93c2958f9e58345514e67740e5
+      BOUNCER_KEY_TRAEFIK_1: 40796d93c2958f9e58345514e67740e5
+      BOUNCER_KEY_TRAEFIK_2: 44c36dac5c4140af9f06f397508e82c7
     volumes:
       - ./acquis.yaml:/etc/crowdsec/acquis.yaml:ro
-      - logs:/var/log/traefik:ro
+      - logs-local:/var/log/traefik:ro
-      - crowdsec-db:/var/lib/crowdsec/data/
+      - crowdsec-db-local:/var/lib/crowdsec/data/
-      - crowdsec-config:/etc/crowdsec/
+      - crowdsec-config-local:/etc/crowdsec/
-
+    labels:
+      - "traefik.enable=false"
 volumes:
-  logs:
+  logs-local:
-  crowdsec-db:
+  crowdsec-db-local:
-  crowdsec-config:
+  crowdsec-config-local:

docker-compose.yml

@@ -2,18 +2,18 @@ version: "3.8"
 
 services:
   traefik:
-    image: "traefik:v2.8.7"
+    image: "traefik:v2.9.4"
     container_name: "traefik"
     command:
-      # - "--log.level=DEBUG"
       - "--accesslog"
-      - "--accesslog.filepath=/var/log/traefik/traefik.log"
+      - "--accesslog.filepath=/var/log/traefik/access.log"
       - "--api.insecure=true"
       - "--providers.docker=true"
       - "--providers.docker.exposedbydefault=false"
       - "--entrypoints.web.address=:80"
 
       - "--experimental.plugins.bouncer.modulename=github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin"
+      - "--experimental.plugins.bouncer.version=v1.1.0"
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock:ro"
       - "logs:/var/log/traefik"
@@ -23,32 +23,60 @@ services:
     depends_on:
       - 'crowdsec'
 
-  whoami:
+  whoami1:
     image: traefik/whoami
-    container_name: "simple-service"
+    container_name: "simple-service-1"
     labels:
       - "traefik.enable=true"
-      - "traefik.http.routers.whoami.rule=Host(`localhost`)"
+      # Definition of the router
-      - "traefik.http.routers.whoami.entrypoints=web"
+      - "traefik.http.routers.router1.rule=Path(`/foo`)"
-      - "traefik.http.routers.whoami.middlewares=crowdsec@docker"
+      - "traefik.http.routers.router1.entrypoints=web"
-      - "traefik.http.middlewares.crowdsec.plugin.bouncer.enabled=true"
+      - "traefik.http.routers.router1.middlewares=crowdsec2@docker"
-      - "traefik.http.middlewares.crowdsec.plugin.bouncer.crowdseclapikey=FIXME-LAPI-KEY"
+      # Definition of the service
+      - "traefik.http.services.service1.loadbalancer.server.port=80"
+      # Definition of the middleware
+      - "traefik.http.middlewares.crowdsec1.plugin.bouncer.enabled=true"
+      # crowdseclapikey must be unique to the middleware attached to the service
+      - "traefik.http.middlewares.crowdsec1.plugin.bouncer.crowdseclapikey=FIXME-LAPI-KEY-1"
+      # forwardedheaderstrustedips should be the IP of the proxy that is in front of traefik (if any)
+      - "traefik.http.middlewares.crowdsec1.plugin.bouncer.forwardedheaderstrustedips=172.21.0.5"
+  
+  whoami2:
+    image: traefik/whoami
+    container_name: "simple-service-2"
+    labels:
+      - "traefik.enable=true"
+      # Definition of the router
+      - "traefik.http.routers.router2.rule=Path(`/bar`)"
+      - "traefik.http.routers.router2.entrypoints=web"
+      - "traefik.http.routers.router2.middlewares=crowdsec2@docker"
+      # Definition of the service
+      - "traefik.http.services.service2.loadbalancer.server.port=80"
+      # Definitin of the middleware
+      - "traefik.http.middlewares.crowdsec2.plugin.bouncer.enabled=true"
+      # crowdseclapikey must be unique to the middleware attached to the service
+      - "traefik.http.middlewares.crowdsec2.plugin.bouncer.crowdseclapikey=FIXME-LAPI-KEY-2"
+      # forwardedheaderstrustedips should be the IP of the proxy that is in front of traefik (if any)
+      - "traefik.http.middlewares.crowdsec2.plugin.bouncer.forwardedheaderstrustedips=172.21.0.5"
 
   crowdsec:
     image: crowdsecurity/crowdsec:v1.4.1
     container_name: "crowdsec"
-    command: rm -rf /etc/crowdsec/acquis.yaml
     environment:
       COLLECTIONS: crowdsecurity/traefik
       CUSTOM_HOSTNAME: crowdsec
-      BOUNCER_KEY_TRAEFIK: FIXME-LAPI-KEY
+      # We need to register one api key per service we will use
+      BOUNCER_KEY_TRAEFIK_1: FIXME-LAPI-KEY-1
+      BOUNCER_KEY_TRAEFIK_2: FIXME-LAPI-KEY-2
     volumes:
       - ./acquis.yaml:/etc/crowdsec/acquis.yaml:ro
       - logs:/var/log/traefik:ro
       - crowdsec-db:/var/lib/crowdsec/data/
       - crowdsec-config:/etc/crowdsec/
+    labels:
+      - "traefik.enable=false"
 
 volumes:
   logs:
   crowdsec-db:
-  crowdsec-config:
+  crowdsec-config:

exemples/behind-proxy/acquis.yaml

@@ -0,0 +1,4 @@
+filenames:
+ - /var/log/traefik/access.log
+labels:
+  type: traefik

exemples/behind-proxy/cloudflare-exemple.yaml

@@ -0,0 +1,15 @@
+http:
+  # Add the router
+  routers:
+    router0:
+      entryPoints:
+      - web
+      service: service-foo
+      rule: Path(`/foo`)
+
+  # Add the service
+  services:
+    service-foo:
+      loadBalancer:
+        servers:
+        - url: http://traefik/foo:80

exemples/behind-proxy/docker-compose.cloudflare.yml

@@ -0,0 +1,86 @@
+version: "3.8"
+
+services:
+  cloudflare:
+    image: "traefik:v2.9.1"
+    container_name: "cloudflare"
+    command:
+      # - "--log.level=DEBUG"
+      - "--accesslog"
+      - "--accesslog.filepath=/var/log/traefik/access.log"
+      - "--api.insecure=true"
+      - "--entrypoints.web.address=:80"
+      - "--providers.file.filename=/cloud.yaml"
+      - "--experimental.localplugins.bouncer.modulename=github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ./cloudflare-exemple.yaml:/cloud.yaml:ro
+      - logs-cloudflare:/var/log/traefik
+    ports:
+      - 80:80
+      - 8080:8080
+
+  traefik:
+    image: "traefik:v2.9.1"
+    container_name: "traefik"
+    command:
+      # - "--log.level=DEBUG"
+      - "--accesslog"
+      - "--accesslog.filepath=/var/log/traefik/access.log"
+      - "--api.insecure=true"
+      - "--providers.docker=true"
+      - "--providers.docker.exposedbydefault=false"
+      - "--entrypoints.web.address=:80"
+      - "--entrypoints.web.forwardedheaders.trustedips=172.21.0.5"
+
+      - "--experimental.plugins.bouncer.modulename=github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin"
+      - "--experimental.plugins.bouncer.version=v1.1.0"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - logs-dev:/var/log/traefik
+    ports:
+      - 90:80
+      - 9080:8080
+    depends_on:
+      - crowdsec
+
+  whoami1:
+    image: traefik/whoami
+    container_name: "simple-service1"
+    labels:
+      - "traefik.enable=true"
+      # Definition of the router
+      - "traefik.http.routers.router1.rule=Path(`/foo`)"
+      - "traefik.http.routers.router1.entrypoints=web"
+      - "traefik.http.routers.router1.middlewares=crowdsec1@docker" 
+      # Definition of the service
+      - "traefik.http.services.service1.loadbalancer.server.port=80"
+      # Definitin of the middleware
+      - "traefik.http.middlewares.crowdsec1.plugin.bouncer.enabled=true"
+      # crowdseclapikey must be uniq to the middleware attached to the service
+      - "traefik.http.middlewares.crowdsec1.plugin.bouncer.crowdseclapikey=40796d93c2958f9e58345514e67740e5"
+      - "traefik.http.middlewares.crowdsec1.plugin.bouncer.crowdsecmode=live"
+      - "traefik.http.middlewares.crowdsec1.plugin.bouncer.forwardedheaderstrustedips=172.21.0.5"
+  
+
+  crowdsec:
+    image: crowdsecurity/crowdsec:v1.4.1
+    container_name: "crowdsec"
+    environment:
+      COLLECTIONS: crowdsecurity/traefik
+      CUSTOM_HOSTNAME: crowdsec
+      BOUNCER_KEY_TRAEFIK_DEV_1: 40796d93c2958f9e58345514e67740e5
+      BOUNCER_KEY_TRAEFIK_DEV_2: 44c36dac5c4140af9f06f397508e82c7
+    volumes:
+      - ./acquis.yaml:/etc/crowdsec/acquis.yaml:ro
+      - logs-dev:/var/log/traefik:ro
+      - crowdsec-db-dev:/var/lib/crowdsec/data/
+      - crowdsec-config-dev:/etc/crowdsec/
+    labels:
+      - "traefik.enable=false"
+
+volumes:
+  logs-dev:
+  logs-cloudflare:
+  crowdsec-db-dev:
+  crowdsec-config-dev:

exemples/redis-cache/acquis.yaml

@@ -0,0 +1,4 @@
+filenames:
+ - /var/log/traefik/access.log
+labels:
+  type: traefik

exemples/redis-cache/docker-compose.redis.yml

@@ -0,0 +1,96 @@
+version: "3.8"
+
+services:
+  traefik:
+    image: "traefik:v2.9.4"
+    container_name: "traefik"
+    command:
+      # - "--log.level=DEBUG"
+      - "--accesslog"
+      - "--accesslog.filepath=/var/log/traefik/access.log"
+      - "--api.insecure=true"
+      - "--providers.docker=true"
+      - "--providers.docker.exposedbydefault=false"
+      - "--entrypoints.web.address=:80"
+
+      #- "--experimental.plugins.bouncer.modulename=github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin"
+      #- "--experimental.plugins.bouncer.version=v1.0.9"
+      - "--experimental.localplugins.bouncer.modulename=github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - logs-redis:/var/log/traefik
+      - ./../../:/plugins-local/src/github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin
+    ports:
+      - 80:80
+      - 8080:8080
+    depends_on:
+      - crowdsec
+      - redis
+
+  whoami1:
+    image: traefik/whoami
+    container_name: "simple-service1"
+    labels:
+      - "traefik.enable=true"
+      # Definition of the router
+      - "traefik.http.routers.router1.rule=Path(`/foo`)"
+      - "traefik.http.routers.router1.entrypoints=web"
+      - "traefik.http.routers.router1.middlewares=crowdsec1@docker" 
+      # Definition of the service
+      - "traefik.http.services.service1.loadbalancer.server.port=80"
+      # Definition of the middleware
+      - "traefik.http.middlewares.crowdsec1.plugin.bouncer.enabled=true"
+      # crowdseclapikey must be uniq to the middleware attached to the service
+      - "traefik.http.middlewares.crowdsec1.plugin.bouncer.crowdseclapikey=40796d93c2958f9e58345514e67740e5"
+      - "traefik.http.middlewares.crowdsec1.plugin.bouncer.rediscacheenabled=true"
+
+  whoami2:
+    image: 4206969/spiderfoot
+    container_name: "simple-service2"
+    volumes:
+      - ./spiderfoot-data:/var/lib/spiderfoot
+    labels:
+      - "traefik.enable=true"
+      # Definition of the router
+      - "traefik.http.routers.router2.rule=PathPrefix(`/`)"
+      - "traefik.http.routers.router2.entrypoints=web"
+      - "traefik.http.routers.router2.middlewares=crowdsec1@docker" 
+      # Definition of the service
+      - "traefik.http.services.service2.loadbalancer.server.port=5001"
+      # Definition of the middleware
+      - "traefik.http.middlewares.crowdsec2.plugin.bouncer.enabled=true"
+      # crowdseclapikey must be uniq to the middleware attached to the service
+      - "traefik.http.middlewares.crowdsec2.plugin.bouncer.crowdseclapikey=44c36dac5c4140af9f06f397508e82c7"
+      - "traefik.http.middlewares.crowdsec2.plugin.bouncer.rediscacheenabled=true"
+
+
+  crowdsec:
+    image: crowdsecurity/crowdsec:v1.4.1
+    container_name: "crowdsec"
+    environment:
+      COLLECTIONS: crowdsecurity/traefik
+      CUSTOM_HOSTNAME: crowdsec
+      BOUNCER_KEY_TRAEFIK_DEV_1: 40796d93c2958f9e58345514e67740e5
+      BOUNCER_KEY_TRAEFIK_DEV_2: 44c36dac5c4140af9f06f397508e82c7
+    volumes:
+      - ./acquis.yaml:/etc/crowdsec/acquis.yaml:ro
+      - logs-redis:/var/log/traefik:ro
+      - crowdsec-db-redis:/var/lib/crowdsec/data/
+      - crowdsec-config-redis:/etc/crowdsec/
+    labels:
+      - "traefik.enable=false"
+  
+  redis:
+    image: "redis:7.0.5-alpine"
+    container_name: "redis"
+    command: "redis-server --save 60 1"
+    volumes:
+      - redis-data:/data
+    ports:
+      - 6379:6379
+
+volumes:
+  logs-redis:
+  crowdsec-db-redis:
+  crowdsec-config-redis:
+  redis-data:

go.mod

@@ -3,3 +3,5 @@ module github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin
 go 1.17
 
 require github.com/leprosus/golang-ttl-map v1.1.7
+
+require github.com/tehnerd/goUtils v0.0.0-20150515130609-5a2d8fb2ded8 // indirect

go.sum

@@ -1,2 +1,4 @@
 github.com/leprosus/golang-ttl-map v1.1.7 h1:cF4AAFDDnJTFSV+/42sKLhmMluvLdRlCGS2UaifH6UM=
 github.com/leprosus/golang-ttl-map v1.1.7/go.mod h1:4QWHJPeVBbrkhOhXdhCv9IEiyj/YzkO04/iexy4vSe0=
+github.com/tehnerd/goUtils v0.0.0-20150515130609-5a2d8fb2ded8 h1:/b777evAfRRdUJHasZLgQ/w8D/s1HtbaeDXsCVsV0B0=
+github.com/tehnerd/goUtils v0.0.0-20150515130609-5a2d8fb2ded8/go.mod h1:UuuqaOb+pZOxJZtjF1mBWTo8HYa7HQCbNkwUEaG9uU0=

pkg/cache/cache.go

@@ -0,0 +1,98 @@
+package cache
+
+import (
+	"fmt"
+
+	ttl_map "github.com/leprosus/golang-ttl-map"
+
+	logger "github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin/pkg/logger"
+	simpleredis "github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin/pkg/redis"
+)
+
+const (
+	cacheBannedValue   = "t"
+	cacheNoBannedValue = "f"
+)
+
+var cache = ttl_map.New()
+var redis simpleredis.SimpleRedis
+
+var redisEnabled = false
+
+// CLASSIC
+
+func getDecisionLocalCache(clientIP string) (bool, error) {
+	banned, isCached := cache.Get(clientIP)
+	bannedString, isValid := banned.(string)
+	if isCached && isValid && len(bannedString) > 0 {
+		return bannedString == cacheBannedValue, nil
+	}
+	return false, fmt.Errorf("no cache data")
+}
+
+func setDecisionLocalCache(clientIP string, value string, duration int64) {
+	cache.Set(clientIP, value, duration)
+}
+
+func deleteDecisionLocalCache(clientIP string) {
+	cache.Del(clientIP)
+}
+
+// REDIS
+
+func getDecisionRedisCache(clientIP string) (bool, error) {
+	banned, err := redis.Get(clientIP)
+	bannedString := string(banned)
+	if err == nil && len(bannedString) > 0 {
+		return bannedString == cacheBannedValue, nil
+	}
+	return false, fmt.Errorf("no cache data")
+}
+
+func setDecisionRedisCache(clientIP string, value string, duration int64) {
+	redis.Set(clientIP, []byte(value), duration)
+}
+
+func deleteDecisionRedisCache(clientIP string) {
+	redis.Del(clientIP)
+}
+
+// DeleteDecision delete decision in cache
+func DeleteDecision(clientIP string) {
+	if redisEnabled {
+		deleteDecisionRedisCache(clientIP)
+	} else {
+		deleteDecisionLocalCache(clientIP)
+	}
+}
+
+// GetDecision check in the cache if the IP has the banned / not banned value.
+// Otherwise return with an error to add the IP in cache if we are on.
+func GetDecision(clientIP string) (bool, error) {
+	if redisEnabled {
+		return getDecisionRedisCache(clientIP)
+	} else {
+		return getDecisionLocalCache(clientIP)
+	}
+}
+
+func SetDecision(clientIP string, isBanned bool, duration int64) {
+	var value string
+	if isBanned {
+		logger.Debug(fmt.Sprintf("%v banned", clientIP))
+		value = cacheBannedValue
+	} else {
+		value = cacheNoBannedValue
+	}
+	if redisEnabled {
+		setDecisionRedisCache(clientIP, value, duration)
+	} else {
+		setDecisionLocalCache(clientIP, value, duration)
+	}
+}
+
+func InitRedisClient(host string) {
+	redisEnabled = true
+	redis.Init(host)
+	logger.Debug("Redis initialized")
+}

pkg/ip/ip.go

@@ -0,0 +1,127 @@
+package ip
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+)
+
+// CHECKER
+
+// Checker allows to check that addresses are in a trusted IPs.
+type Checker struct {
+	authorizedIPs    []*net.IP
+	authorizedIPsNet []*net.IPNet
+}
+
+// NewChecker builds a new Checker given a list of CIDR-Strings to trusted IPs.
+func NewChecker(trustedIPs []string) (*Checker, error) {
+	if len(trustedIPs) == 0 {
+		return nil, errors.New("no trusted IPs provided")
+	}
+
+	checker := &Checker{}
+
+	for _, ipMask := range trustedIPs {
+		if ipAddr := net.ParseIP(ipMask); ipAddr != nil {
+			checker.authorizedIPs = append(checker.authorizedIPs, &ipAddr)
+			continue
+		}
+
+		_, ipAddr, err := net.ParseCIDR(ipMask)
+		if err != nil {
+			return nil, fmt.Errorf("parsing CIDR trusted IPs %s: %w", ipAddr, err)
+		}
+		checker.authorizedIPsNet = append(checker.authorizedIPsNet, ipAddr)
+	}
+
+	return checker, nil
+}
+
+// Contains checks if provided address is in the trusted IPs.
+func (ip *Checker) Contains(addr string) (bool, error) {
+	if len(addr) == 0 {
+		return false, errors.New("empty IP address")
+	}
+
+	ipAddr, err := parseIP(addr)
+	if err != nil {
+		return false, fmt.Errorf("unable to parse address: %s: %w", addr, err)
+	}
+
+	return ip.ContainsIP(ipAddr), nil
+}
+
+// ContainsIP checks if provided address is in the trusted IPs.
+func (ip *Checker) ContainsIP(addr net.IP) bool {
+	for _, authorizedIP := range ip.authorizedIPs {
+		if authorizedIP.Equal(addr) {
+			return true
+		}
+	}
+
+	for _, authorizedNet := range ip.authorizedIPsNet {
+		if authorizedNet.Contains(addr) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func parseIP(addr string) (net.IP, error) {
+	userIP := net.ParseIP(addr)
+	if userIP == nil {
+		return nil, fmt.Errorf("can't parse IP from address %s", addr)
+	}
+
+	return userIP, nil
+}
+
+// STRATEGY
+
+// PoolStrategy is a strategy based on an IP Checker.
+// It allows to check whether addresses are in a given pool of IPs.
+type PoolStrategy struct {
+	Checker *Checker
+}
+
+// GetIP checks the list of Forwarded IPs (most recent first) against the
+// Checker pool of IPs. It returns the first IP that is not in the pool, or the
+// empty string otherwise.
+func (s *PoolStrategy) getIP(req *http.Request, customHeader string) string {
+	if s.Checker == nil {
+		return ""
+	}
+
+	xff := req.Header.Get(customHeader)
+
+	xffs := strings.Split(xff, ",")
+
+	for i := len(xffs) - 1; i >= 0; i-- {
+		xffTrimmed := strings.TrimSpace(xffs[i])
+		if len(xffTrimmed) == 0 {
+			continue
+		}
+		if contain, _ := s.Checker.Contains(xffTrimmed); !contain {
+			return xffTrimmed
+		}
+	}
+
+	return ""
+}
+
+// GetRemoteIP It returns the first IP that is not in the pool, or the empty string otherwise.
+func GetRemoteIP(req *http.Request, strategy *PoolStrategy, customHeader string) (string, error) {
+	remoteIP := strategy.getIP(req, customHeader)
+	if len(remoteIP) != 0 {
+		return remoteIP, nil
+	}
+	remoteIP, _, err := net.SplitHostPort(req.RemoteAddr)
+	if err != nil {
+		return "", fmt.Errorf("failed to extract ip from remote address: %w", err)
+	}
+	return remoteIP, nil
+}

pkg/logger/logger.go

@@ -0,0 +1,35 @@
+package logger
+
+import (
+	"io"
+	"log"
+	"os"
+)
+
+var (
+	loggerInfo  = log.New(io.Discard, "INFO: CrowdsecBouncerTraefikPlugin: ", log.Ldate|log.Ltime)
+	loggerDebug = log.New(io.Discard, "DEBUG: CrowdsecBouncerTraefikPlugin: ", log.Ldate|log.Ltime)
+)
+
+// Init Set Default log level to info in case log level to defined
+func Init(logLevel string) {
+	switch logLevel {
+	case "INFO":
+		loggerInfo.SetOutput(os.Stdout)
+	case "DEBUG":
+		loggerInfo.SetOutput(os.Stdout)
+		loggerDebug.SetOutput(os.Stdout)
+	default:
+		loggerInfo.SetOutput(os.Stdout)
+	}
+}
+
+// Info Log info
+func Info(str string) {
+	loggerInfo.Printf(str)
+}
+
+// Info Log debug
+func Debug(str string) {
+	loggerDebug.Printf(str)
+}

pkg/redis/redis.go

@@ -0,0 +1,114 @@
+package simpleredis
+
+import (
+	"bufio"
+	"fmt"
+	"net"
+	"net/textproto"
+	"strings"
+	"time"
+
+	logger "github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin/pkg/logger"
+)
+
+type RedisCmd struct {
+	Command  string
+	Name     string
+	Data     []byte
+	Duration int64
+	Error    error
+}
+
+type SimpleRedis struct {
+	redisHost string
+}
+
+func genRedisArray(params ...[]byte) []byte {
+	MSG := ""
+	for cntr := 0; cntr < len(params); cntr++ {
+		MSG = strings.Join([]string{MSG, string(params[cntr])}, " ")
+	}
+	MSG = strings.Trim(MSG, " ")
+	MSG = strings.Join([]string{MSG, "\r\n"}, "")
+	return []byte(MSG)
+}
+
+func askRedis(hostnamePort string, cmd RedisCmd, channel chan RedisCmd) {
+	conn, err := net.Dial("tcp", hostnamePort)
+	if err != nil {
+		return
+	}
+	defer conn.Close()
+
+	writer := textproto.NewWriter(bufio.NewWriter(conn))
+	reader := textproto.NewReader(bufio.NewReader(conn))
+
+	switch cmd.Command {
+	case "SET":
+		data := genRedisArray([]byte("SET"), []byte(cmd.Name), []byte(cmd.Data), []byte("EX"), []byte(fmt.Sprintf("%v", cmd.Duration)))
+		writer.PrintfLine(string(data))
+		logger.Info("set")
+	case "DEL":
+		data := genRedisArray([]byte("DEL"), []byte(cmd.Name))
+		writer.PrintfLine(string(data))
+		logger.Info("del")
+	case "GET":
+		data := genRedisArray([]byte("GET"), []byte(cmd.Name))
+		writer.PrintfLine(string(data))
+		logger.Info("get")
+		for {
+			select {
+			case <-time.After(time.Second * 1):
+				channel <- RedisCmd{Error: fmt.Errorf("timeout")}
+				return
+			default:
+				read, _ := reader.ReadLineBytes()
+				if string(read) != "$1" {
+					channel <- RedisCmd{Error: fmt.Errorf("miss")}
+					return
+				}
+				read, _ = reader.ReadLineBytes()
+				channel <- RedisCmd{Data: read}
+				return
+			}
+		}
+	}
+}
+
+func (sr *SimpleRedis) Init(redisHost string) {
+	sr.redisHost = redisHost
+}
+
+func (sr *SimpleRedis) Get(name string) ([]byte, error) {
+	redisCmd := RedisCmd{
+		Command: "GET",
+		Name:    name,
+	}
+	channel := make(chan RedisCmd)
+	go askRedis(sr.redisHost, redisCmd, channel)
+	resp := <-channel
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+	return resp.Data, nil
+}
+
+func (sr *SimpleRedis) Set(name string, data []byte, duration int64) error {
+	redisCmd := RedisCmd{
+		Command:  "SET",
+		Name:     name,
+		Data:     data,
+		Duration: duration,
+	}
+	go askRedis(sr.redisHost, redisCmd, nil)
+	return nil
+}
+
+func (sr *SimpleRedis) Del(name string) error {
+	redisCmd := RedisCmd{
+		Command: "DEL",
+		Name:    name,
+	}
+	go askRedis(sr.redisHost, redisCmd, nil)
+	return nil
+}

vendor/github.com/tehnerd/goUtils/netutils/netutils.go

@@ -0,0 +1,525 @@
+package netutils
+
+import (
+	"math/rand"
+	"net"
+	"strconv"
+	"strings"
+	"sync"
+	//"sync/atomic"
+	"time"
+)
+
+//Receive msg from tcp socket and send it as a []byte to readChan
+func ReadFromTCP2(sock *net.TCPConn, msgBuf []byte, readChan chan []byte,
+	feedbackChanFromSocket chan int) {
+	loop := 1
+	for loop == 1 {
+		bytes, err := sock.Read(msgBuf)
+		if err != nil {
+			feedbackChanFromSocket <- 1
+			loop = 0
+			continue
+		}
+		b := make([]byte, 0)
+		b = append(b, msgBuf[:bytes]...)
+		readChan <- b
+	}
+}
+
+//Receive msg from tcp socket and send it as a []byte to readChan
+func ReadFromTCP(sock *net.TCPConn, msgBuf []byte, readChan chan []byte,
+	feedbackChanFromSocket chan int) {
+	feedbackToSocket := make(chan bool)
+	feedbackFromSocket := make(chan bool)
+	reuseBufferChan := make(chan []byte, 1)
+	go readFromTCP(sock, readChan, feedbackFromSocket,
+		feedbackToSocket, reuseBufferChan)
+	go func() {
+		<-feedbackFromSocket
+		feedbackChanFromSocket <- 1
+	}()
+}
+
+/*simple write to TCP, for oneway connections only (no communitcation w/ "read" part of the socket
+in terms of error propogation)*/
+func WriteToTCPw2(sock *net.TCPConn, writeChan chan []byte,
+	feedbackChan chan int) {
+	loop := 1
+	for loop == 1 {
+		select {
+		case msg := <-writeChan:
+			_, err := sock.Write(msg)
+			if err != nil {
+				feedbackChan <- 1
+				continue
+			}
+		case <-feedbackChan:
+			loop = 0
+		}
+	}
+}
+
+/*simple write to TCP, for oneway connections only (no communitcation w/ "read" part of the socket
+in terms of error propogation)*/
+func WriteToTCPw(sock *net.TCPConn, writeChan chan []byte,
+	feedbackChan chan int) {
+	feedbackToSocket := make(chan bool)
+	feedbackFromSocket := make(chan bool)
+	go writeToTCP(sock, writeChan, feedbackFromSocket, feedbackToSocket)
+	go func() {
+		<-feedbackFromSocket
+		feedbackChan <- 1
+	}()
+}
+
+//simple write to tcp w/ erorr propagation to/from "read" part of the socket
+func WriteToTCPrw2(sock *net.TCPConn, writeChan chan []byte,
+	feedbackChanFromSocket, feedbackChanToSocket chan int) {
+	loop := 1
+	for loop == 1 {
+		select {
+		case msg := <-writeChan:
+			_, err := sock.Write(msg)
+			if err != nil {
+				select {
+				case feedbackChanFromSocket <- 1:
+					continue
+				case loop = <-feedbackChanToSocket:
+					loop = 0
+					continue
+				}
+			}
+		case <-feedbackChanToSocket:
+			loop = 0
+		}
+	}
+}
+
+//simple write to tcp w/ erorr propagation to/from "read" part of the socket
+func WriteToTCPrw(sock *net.TCPConn, writeChan chan []byte,
+	feedbackChanFromSocket, feedbackChanToSocket chan int) {
+	feedbackToSocket := make(chan bool)
+	feedbackFromSocket := make(chan bool)
+	go writeToTCP(sock, writeChan, feedbackFromSocket, feedbackToSocket)
+	go func() {
+		select {
+		case <-feedbackFromSocket:
+			feedbackChanFromSocket <- 1
+		case <-feedbackChanToSocket:
+			feedbackToSocket <- true
+
+		}
+	}()
+
+}
+
+func makeBuffer(reuseBufferChan chan []byte) []byte {
+	select {
+	case buf := <-reuseBufferChan:
+		return buf
+	default:
+		return make([]byte, 10000)
+	}
+}
+
+//Receive msg from tcp socket and send it as a []byte to readChan,with buffer reuse
+func readFromTCP(sock *net.TCPConn, readChan chan []byte,
+	feedbackFromSocket, feedbackToSocket chan bool,
+	reuseBufferChan chan []byte) {
+	loop := 1
+	var buf []byte
+	for loop == 1 {
+		buf = makeBuffer(reuseBufferChan)
+		bytes, err := sock.Read(buf)
+		if err != nil {
+			select {
+			case feedbackFromSocket <- true:
+				loop = 0
+				continue
+			case <-feedbackToSocket:
+				loop = 0
+				continue
+			}
+		}
+		select {
+		case readChan <- buf[:bytes]:
+		case <-feedbackToSocket:
+			loop = 0
+			continue
+
+		}
+	}
+}
+
+//TCP's write routine, with feedback's chans
+func writeToTCP(sock *net.TCPConn, writeChan chan []byte,
+	feedbackFromSocket, feedbackToSocket chan bool) {
+	loop := 1
+	for loop == 1 {
+		select {
+		case msg := <-writeChan:
+			_, err := sock.Write(msg)
+			if err != nil {
+				select {
+				case feedbackFromSocket <- true:
+				case <-feedbackToSocket:
+				}
+				loop = 0
+				continue
+			}
+		case <-feedbackToSocket:
+			loop = 0
+			continue
+		}
+	}
+}
+
+//reconnecting to remote host for both read and write purpose
+func ReconnectTCPRW(ladr, radr *net.TCPAddr, msgBuf []byte, writeChan chan []byte,
+	readChan chan []byte, feedbackChanToSocket, feedbackChanFromSocket chan int,
+	init_msg []byte) {
+	loop := 1
+	for loop == 1 {
+		sock, err := net.DialTCP("tcp", ladr, radr)
+		if err != nil {
+			time.Sleep(time.Duration(20+rand.Intn(15)) * time.Second)
+			continue
+		}
+		//testing health of the new socket. GO sometimes doesnt rise the error when
+		// we receive RST from remote side
+		_, err = sock.Write(init_msg)
+		if err != nil {
+			sock.Close()
+			time.Sleep(time.Duration(20+rand.Intn(15)) * time.Second)
+			continue
+		}
+		loop = 0
+		go ReadFromTCP(sock, msgBuf, readChan, feedbackChanFromSocket)
+		go WriteToTCPrw(sock, writeChan, feedbackChanFromSocket, feedbackChanToSocket)
+	}
+}
+
+func ReconnectTCPRWReuse(ladr, radr *net.TCPAddr,
+	readChan, writeChan, reuseBufferChan chan []byte,
+	readFeedbackFrom, readFeedbackTo chan bool,
+	writeFeedbackFrom, writeFeedbackTo chan bool) {
+	loop := 1
+	for loop == 1 {
+		sock, err := net.DialTCP("tcp", ladr, radr)
+		if err != nil {
+			time.Sleep(time.Duration(20+rand.Intn(15)) * time.Second)
+			continue
+		}
+		loop = 0
+		go readFromTCP(sock, readChan, readFeedbackFrom, readFeedbackTo,
+			reuseBufferChan)
+		go writeToTCP(sock, writeChan, writeFeedbackFrom, writeFeedbackTo)
+	}
+}
+
+func AutoRecoonectedTCP(ladr, radr *net.TCPAddr, msgBuf, initMsg []byte,
+	writeChan, readChan chan []byte, flushChan chan int) {
+	feedbackChanFromSocket := make(chan int)
+	feedbackChanToSocket := make(chan int)
+	go ReconnectTCPRW(ladr, radr, msgBuf, writeChan, readChan, feedbackChanToSocket,
+		feedbackChanFromSocket, initMsg)
+	for {
+		select {
+		case feedbackFromSocket := <-feedbackChanFromSocket:
+			feedbackChanToSocket <- feedbackFromSocket
+			flushChan <- 1
+			go ReconnectTCPRW(ladr, radr, msgBuf, writeChan,
+				readChan, feedbackChanToSocket,
+				feedbackChanFromSocket, initMsg)
+
+		}
+	}
+
+}
+
+func AutoRecoonectedTCPReuse(ladr, radr *net.TCPAddr,
+	readChan, writeChan chan []byte,
+	reuseChan chan []byte,
+	flushChan chan bool) {
+	readFeedbackFrom := make(chan bool)
+	readFeedbackTo := make(chan bool)
+	writeFeedbackFrom := make(chan bool)
+	writeFeedbackTo := make(chan bool)
+	go ReconnectTCPRWReuse(ladr, radr, readChan, writeChan, reuseChan,
+		readFeedbackFrom, readFeedbackTo,
+		writeFeedbackFrom, writeFeedbackTo)
+	for {
+		select {
+		case <-readFeedbackFrom:
+			writeFeedbackTo <- true
+		case <-writeFeedbackFrom:
+			readFeedbackTo <- true
+		}
+		flushChan <- true
+		go ReconnectTCPRWReuse(ladr, radr, readChan, writeChan, reuseChan,
+			readFeedbackFrom, readFeedbackTo,
+			writeFeedbackFrom, writeFeedbackTo)
+	}
+
+}
+
+//reconnecting to remote host for write only
+func ReconnectTCPW(radr net.TCPAddr, writeChan chan []byte, feedbackChan chan int) {
+	loop := 1
+	for loop == 1 {
+		time.Sleep(time.Duration(20+rand.Intn(15)) * time.Second)
+		sock, err := net.DialTCP("tcp", nil, &radr)
+		if err != nil {
+			continue
+		}
+		//testing health of the new socket. GO sometimes doesnt rise the error when
+		// we receive RST from remote side
+		_, err = sock.Write([]byte{1})
+		if err != nil {
+			sock.Close()
+			continue
+		}
+		loop = 0
+		go WriteToTCPw(sock, writeChan, feedbackChan)
+	}
+}
+
+/* --------------------- CONNECTION MANAGER -------------------------
+Connection manager will allow send data and receive data from remote hosts.
+it will have single ConnectionMsg (see below) read chan and single
+ConnectionMsg write chan toward it's clients
+as well as single read chan from sockets, but multiple write sockets.
+it will route msgs according to Host field in connectionManager struct
+(if it received from client, it will send this msg toward Host's sockets;
+if recved from socket, will proxy it toward client(and client will know from which remote host
+it was received)
+-------------------------------------------------------------------- */
+/*
+MsgType's could be:
+from Api's client to ConnectionManager:
+"Data" - msg with Data to Host
+"Connect" - connect to new Host
+...
+from ConnectionManager to Api's client:
+"BufferFlush" - notification that connection to remote Host not longer working.
+advice to flush all the msg buffers assosiated with remote host
+*/
+type ConnectionMsg struct {
+	Host string
+	Data []byte
+	Type string
+}
+
+/*
+Receive msg from tcp socket and send it as a ConnectionMsg to readChan
+TODO: think about more generic version to be more DRYer (to work in both CM and []byte chans
+*/
+func CMReadFromTCP(sock *net.TCPConn, readChan chan ConnectionMsg,
+	peerAddress string) {
+	msgBuf := make([]byte, 65000)
+	loop := 1
+	var msg ConnectionMsg
+	msg.Host = peerAddress
+	msg.Type = "Data"
+	for loop == 1 {
+		bytes, err := sock.Read(msgBuf)
+		if err != nil {
+			msg.Type = "ReadError"
+			readChan <- msg
+			loop = 0
+			continue
+		}
+		b := make([]byte, 0)
+		b = append(b, msgBuf[:bytes]...)
+		msg.Data = b
+		readChan <- msg
+	}
+}
+
+/*
+ConnectionManager write instance to tcp w/ erorr propagation to/from "read" part of the socket
+TODO: think about more generic version to be more DRYer (to work in both CM and []byte chans
+*/
+func CMWriteToTCP(sock *net.TCPConn, writeChan, readChan chan ConnectionMsg,
+	peerAddress string) {
+	loop := 1
+	var errorMsg ConnectionMsg
+	errorMsg.Host = peerAddress
+	errorMsg.Type = "WriteError"
+	for loop == 1 {
+		select {
+		case msg := <-writeChan:
+			switch msg.Type {
+			case "Data":
+				_, err := sock.Write(msg.Data)
+				if err != nil {
+					for loop == 1 {
+						select {
+						case readChan <- errorMsg:
+						case errorMsg := <-writeChan:
+							if errorMsg.Type != "ConnectionError" {
+								continue
+							}
+						}
+						loop = 0
+					}
+				}
+			case "ConnectionError":
+				loop = 0
+			default:
+				continue
+			}
+		}
+	}
+}
+
+func StartConnection(tcpConn *net.TCPConn, writeChan,
+	readChan chan ConnectionMsg, peerAddress string) {
+	go CMReadFromTCP(tcpConn, readChan, peerAddress)
+	go CMWriteToTCP(tcpConn, writeChan, readChan, peerAddress)
+}
+
+func CMListenForConnection(mutex *sync.RWMutex, localPort int,
+	writeChanMap map[string]chan ConnectionMsg,
+	connectionStateMap map[string]int,
+	readChan chan ConnectionMsg) {
+	laddr := strings.Join([]string{":", strconv.Itoa(localPort)}, "")
+	tcpLaddr, err := net.ResolveTCPAddr("tcp", laddr)
+	if err != nil {
+		panic("cant resolve local address for binding")
+	}
+	tcpListener, err := net.ListenTCP("tcp", tcpLaddr)
+	if err != nil {
+		panic("cant listen on local address for binding")
+	}
+	for {
+		tcpConn, err := tcpListener.AcceptTCP()
+		if err == nil {
+			radr := strings.Split(tcpConn.RemoteAddr().String(), ":")[0]
+			// check if we already has connection to remote peer as a client
+			mutex.Lock()
+			if val, exist := connectionStateMap[radr]; exist && val == 1 {
+				tcpConn.Close()
+				mutex.Unlock()
+				continue
+			}
+			connectionStateMap[radr] = 1
+			if writeChan, exist := writeChanMap[radr]; exist {
+				mutex.Unlock()
+				go StartConnection(tcpConn, writeChan, readChan, radr)
+			} else {
+				writeChanMap[radr] = make(chan ConnectionMsg)
+				mutex.Unlock()
+				go StartConnection(tcpConn, writeChanMap[radr], readChan, radr)
+			}
+		}
+	}
+}
+
+func CMConnectToRemotePeer(mutex *sync.RWMutex, peerTcpAddr *net.TCPAddr,
+	radr string,
+	writeChan chan ConnectionMsg,
+	readChan chan ConnectionMsg,
+	connectionStateMap map[string]int) {
+	connectLoop := 1
+	for connectLoop == 1 {
+		mutex.RLock()
+		if connectionStateMap[radr] == 1 {
+			connectLoop = 0
+			mutex.RUnlock()
+			continue
+		}
+		mutex.RUnlock()
+		tcpConn, err := net.DialTCP("tcp", nil, peerTcpAddr)
+		if err != nil {
+			time.Sleep(time.Second * time.Duration(rand.Int63n(15)))
+			continue
+		}
+		mutex.Lock()
+		if connectionStateMap[radr] == 0 {
+			connectionStateMap[radr] = 1
+			go StartConnection(tcpConn, writeChan, readChan, radr)
+			mutex.Unlock()
+			connectLoop = 0
+			continue
+		} else {
+			mutex.Unlock()
+			tcpConn.Close()
+			connectLoop = 0
+			continue
+		}
+	}
+}
+
+func ConnectionManager(msgChan chan ConnectionMsg, localPort int) {
+	writeChanMap := make(map[string]chan ConnectionMsg)
+	connectionStateMap := make(map[string]int)
+	readChan := make(chan ConnectionMsg)
+	var connectionMutex sync.RWMutex
+	go CMListenForConnection(&connectionMutex, localPort, writeChanMap,
+		connectionStateMap, readChan)
+	for {
+		select {
+		case msgToPeer := <-msgChan:
+			switch msgToPeer.Type {
+			case "Data":
+				if state, exists := connectionStateMap[msgToPeer.Host]; exists && state == 1 {
+					/*FIXME/THINK: There could be deadlock if connectin closes before we will
+					be able to send to the chan */
+					writeChan := writeChanMap[msgToPeer.Host]
+					writeChan <- msgToPeer
+				} else {
+					msgChan <- ConnectionMsg{Type: "ConnectionNotExist"}
+				}
+			case "Connect":
+				if len(strings.Split(msgToPeer.Host, ":")) > 1 {
+					radr := strings.Split(msgToPeer.Host, ":")[0]
+					connectionMutex.Lock()
+					if _, exist := writeChanMap[radr]; !exist {
+						writeChanMap[radr] = make(chan ConnectionMsg)
+					}
+					connectionMutex.Unlock()
+					peerTcpAddr, err := net.ResolveTCPAddr("tcp", msgToPeer.Host)
+					if err != nil {
+						//XXX: think about , mb make something less drastic
+						panic("cant resolve remote address")
+					}
+					go CMConnectToRemotePeer(&connectionMutex, peerTcpAddr, radr,
+						writeChanMap[radr], readChan, connectionStateMap)
+				} else {
+					connectionMutex.Lock()
+					if _, exist := writeChanMap[msgToPeer.Host]; !exist {
+						writeChanMap[msgToPeer.Host] = make(chan ConnectionMsg)
+					}
+					connectionMutex.Unlock()
+					remoteAddr := strings.Join([]string{msgToPeer.Host, strconv.Itoa(localPort)}, ":")
+					peerTcpAddr, err := net.ResolveTCPAddr("tcp", remoteAddr)
+					if err != nil {
+						//XXX: again panic could be overkill
+						panic("cant resolve remote address")
+					}
+					go CMConnectToRemotePeer(&connectionMutex, peerTcpAddr, msgToPeer.Host,
+						writeChanMap[msgToPeer.Host], readChan, connectionStateMap)
+				}
+			}
+		case msgFromPeer := <-readChan:
+			switch msgFromPeer.Type {
+			case "Data":
+				msgChan <- msgFromPeer
+			case "WriteError", "ReadError":
+				connectionMutex.Lock()
+				connectionStateMap[msgFromPeer.Host] = 0
+				connectionMutex.Unlock()
+				if msgFromPeer.Type == "ReadError" {
+					writeChanMap[msgFromPeer.Host] <- ConnectionMsg{Type: "ConnectionError"}
+				}
+				var msgToApiClient ConnectionMsg
+				msgToApiClient.Host = msgFromPeer.Host
+				msgToApiClient.Type = "BufferFlush"
+				msgChan <- msgToApiClient
+
+			}
+		}
+	}
+}

vendor/github.com/tehnerd/goUtils/netutils/tcp_listen.go

@@ -0,0 +1,42 @@
+package netutils
+
+import (
+	"errors"
+	"net"
+)
+
+/*
+	we need to provide a function, which will read/write to/from socket and read/write to/from sockets feedback chans
+*/
+func ListenForConnection(port string, fn func(chan []byte, chan []byte, chan int, chan int)) error {
+	addr := ":" + port
+	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
+	if err != nil {
+		return errors.New("cant resolve local tcp address")
+	}
+	loop := 1
+	servSock, err := net.ListenTCP("tcp", tcpAddr)
+	if err != nil {
+		return errors.New("cant bind to local tcp address")
+	}
+
+	for loop == 1 {
+		sock, err := servSock.AcceptTCP()
+		if err != nil {
+			continue
+		}
+		go ServeTcpConn(sock, fn)
+	}
+	return nil
+}
+
+func ServeTcpConn(sock *net.TCPConn, fn func(chan []byte, chan []byte, chan int, chan int)) {
+	readChan := make(chan []byte)
+	writeChan := make(chan []byte)
+	feedbackFrom := make(chan int, 1)
+	feedbackTo := make(chan int, 1)
+	buf := make([]byte, 65535)
+	go ReadFromTCP(sock, buf, readChan, feedbackFrom)
+	go WriteToTCPrw(sock, writeChan, feedbackFrom, feedbackTo)
+	fn(readChan, writeChan, feedbackFrom, feedbackTo)
+}

vendor/modules.txt

@@ -1,3 +1,6 @@
 # github.com/leprosus/golang-ttl-map v1.1.7
 ## explicit; go 1.15
 github.com/leprosus/golang-ttl-map
+# github.com/tehnerd/goUtils v0.0.0-20150515130609-5a2d8fb2ded8
+## explicit
+github.com/tehnerd/goUtils/netutils