diff --git a/Makefile b/Makefile index 2581487..11a2f61 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ export GO111MODULE=on # Binary/mock suite (Traefik binary + mock LAPI). This is what CI runs. # The local Docker suite (make e2e) lives in a separate PR/branch. -E2E_MOCK_SCENARIOS := stream-mode live-mode none-mode trusted-ips custom-ban-page captcha appsec +E2E_MOCK_SCENARIOS := stream-mode live-mode none-mode trusted-ips custom-ban-page captcha appsec tls-system-ca default: lint test diff --git a/README.md b/README.md index 41b069b..62c21ca 100644 --- a/README.md +++ b/README.md @@ -362,7 +362,7 @@ make run - CrowdsecAppsecTlsCertificateAuthority - string - default: "" - - PEM-encoded Certificate Authority of Appsec + - PEM-encoded Certificate Authority used to verify Appsec's server certificate. When empty (and `crowdsecAppsecTlsInsecureVerify` is `false`), the host's system trust store is used. - CrowdsecAppsecScheme - string - default: value of `CrowdsecLapiScheme`, expected values are: `http`, `https` @@ -408,7 +408,7 @@ make run - CrowdsecLapiTlsCertificateAuthority - string - default: "" - - PEM-encoded Certificate Authority of the Crowdsec LAPI + - PEM-encoded Certificate Authority used to verify the LAPI's server certificate. When empty (and `crowdsecLapiTlsInsecureVerify` is `false`), the host's system trust store is used. - CrowdsecLapiTlsCertificateBouncer - string - default: "" @@ -727,16 +727,18 @@ A script is available to generate certificates in `examples/tls-auth/gencerts.sh #### Use HTTPS to communicate with the LAPI -To communicate with the LAPI in HTTPS you need to either accept any certificates by setting the `crowdsecLapiTLSInsecureVerify` to true or add the CA used by the server certificate of Crowdsec using `crowdsecLapiTLSCertificateAuthority` or `crowdsecLapiTLSCertificateAuthorityFile`. -Set the `crowdsecLapiScheme` to https. +Set `crowdsecLapiScheme` to `https`. The plugin then validates Crowdsec's server certificate. Three options: + +- **Publicly trusted certificate** (e.g. Let's Encrypt behind a reverse proxy): leave `crowdsecLapiTLSCertificateAuthority` empty and `crowdsecLapiTLSInsecureVerify` `false`. The plugin falls back to the host's system trust store (the `traefik` image ships `ca-certificates`). +- **Private/self-signed CA**: set `crowdsecLapiTLSCertificateAuthority` (or `…File`) to the PEM-encoded CA that signed Crowdsec's server cert. +- **Skip verification entirely** (not recommended for production): set `crowdsecLapiTLSInsecureVerify` to `true`. Crowdsec must be listening in HTTPS for this to work. Please see the [tls-auth example](https://github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin/blob/main/examples/tls-auth/README.md) or the official documentation: [docs.crowdsec.net/docs/local_api/tls_auth/](https://docs.crowdsec.net/docs/local_api/tls_auth/) #### Use HTTPS to communicate with the Appsec -To communicate with the Appsec in HTTPS you need to either accept any certificates by setting the `crowdsecAppsecTLSInsecureVerify` to true or add the CA used by the server certificate of Crowdsec using `crowdsecAppsecTLSCertificateAuthority` or `crowdsecAppsecTLSCertificateAuthorityFile`. -Set the `crowdsecAppsecScheme` to https. +Set `crowdsecAppsecScheme` to `https`. Same three options as for the LAPI, prefixed `crowdsecAppsec…` instead of `crowdsecLapi…`: empty CA + secure verify falls back to the system trust store, a custom CA pins to your private PKI, and `crowdsecAppsecTLSInsecureVerify=true` skips verification altogether. Currently AppSec does not support mTLS authentication for the AppSec Component. diff --git a/pkg/configuration/configuration.go b/pkg/configuration/configuration.go index 8cae113..b8411e5 100644 --- a/pkg/configuration/configuration.go +++ b/pkg/configuration/configuration.go @@ -361,7 +361,8 @@ func validateParamsTLS(config *Config) error { return err } if certAuth == "" { - return errors.New("CrowdsecLapiTLSCertificateAuthority must be specified when CrowdsecLapiScheme='https' and CrowdsecLapiTLSInsecureVerify=false") + // No custom CA — runtime will fall back to the system trust store. + return nil } tlsConfig := new(tls.Config) tlsConfig.RootCAs = x509.NewCertPool() @@ -453,29 +454,31 @@ func validateParamsRequired(config *Config) error { func getTLSConfig(config *Config, log *slog.Logger, prefix, scheme string, insecureVerify bool) (*tls.Config, error) { tlsConfig := new(tls.Config) - tlsConfig.RootCAs = x509.NewCertPool() if scheme != HTTPS { log.Debug("getTLSConfig:" + prefix + "Scheme https:no") return tlsConfig, nil } + // RootCAs is intentionally left nil unless a custom CA is provided: + // crypto/tls then falls back to x509.SystemCertPool(), which is what we + // want when the LAPI is exposed behind a reverse proxy with a publicly + // trusted certificate (e.g. Let's Encrypt). //nolint:nestif if insecureVerify { tlsConfig.InsecureSkipVerify = true log.Debug("getTLSConfig:" + prefix + "TLSInsecureVerify tlsInsecure:true") - // If we return here and still want to use client auth this won't work - // return tlsConfig, nil } else { certAuthority, err := GetVariable(config, prefix+"TLSCertificateAuthority") if err != nil { return nil, err } if len(certAuthority) > 0 { + tlsConfig.RootCAs = x509.NewCertPool() if !tlsConfig.RootCAs.AppendCertsFromPEM([]byte(certAuthority)) { - // here we return because if CrowdsecLapiTLSInsecureVerify is false - // and CA not load, we can't communicate with https return nil, errors.New("getTLSConfig:" + prefix + " cannot load CA and verify cert is enabled") } log.Debug("getTLSConfig:" + prefix + "TLSCertificateAuthority CA added successfully") + } else { + log.Debug("getTLSConfig:" + prefix + " no CA provided, using system trust store") } } certBouncer, err := GetVariable(config, prefix+"TLSCertificateBouncer") diff --git a/pkg/configuration/configuration_test.go b/pkg/configuration/configuration_test.go index 5a735aa..e1c4e61 100644 --- a/pkg/configuration/configuration_test.go +++ b/pkg/configuration/configuration_test.go @@ -1,13 +1,25 @@ package configuration import ( - "crypto/tls" - "reflect" "testing" logger "github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin/pkg/logger" ) +// validPEM is a minimal self-signed certificate accepted by AppendCertsFromPEM, +// shared by the TLS tests below. +const validPEM = `-----BEGIN CERTIFICATE----- +MIIBhTCCASugAwIBAgIQIRi6zePL6mKjOipn+dNuaTAKBggqhkjOPQQDAjASMRAw +DgYDVQQKEwdBY21lIENvMB4XDTE3MTAyMDE5NDMwNloXDTE4MTAyMDE5NDMwNlow +EjEQMA4GA1UEChMHQWNtZSBDbzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABD0d +7VNhbWvZLWPuj/RtHFjvtJBEwOkhbN/BnnE8rnZR8+sbwnc/KhCk3FhnpHZnQz7B +5aETbbIgmuvewdjvSBSjYzBhMA4GA1UdDwEB/wQEAwICpDATBgNVHSUEDDAKBggr +BgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MCkGA1UdEQQiMCCCDmxvY2FsaG9zdDo1 +NDUzgg4xMjcuMC4wLjE6NTQ1MzAKBggqhkjOPQQDAgNIADBFAiEA2zpJEPQyz6/l +Wf86aX6PepsntZv2GYlA5UpabfT2EZICICpJ5h/iI+i341gBmLiAFQOyTDT+/wQc +6MF9+Yw1Yy0t +-----END CERTIFICATE-----` + func getMinimalConfig() *Config { cfg := New() cfg.CrowdsecLapiKey = "test" @@ -111,7 +123,7 @@ func Test_ValidateParams(t *testing.T) { {name: "Not validate a bad clients ips", args: args{config: cfg5}, wantErr: true}, // HTTPS enabled {name: "Validate https config with insecure verify", args: args{config: cfg6}, wantErr: false}, - {name: "Not validate https without cert authority", args: args{config: cfg7}, wantErr: true}, + {name: "Validate https without cert authority (falls back to system trust store)", args: args{config: cfg7}, wantErr: false}, {name: "Valid log level uppercase INFO", args: args{config: cfg8}, wantErr: false}, {name: "Valid log level lowercase info", args: args{config: cfg9}, wantErr: false}, {name: "Invalid log level Warning", args: args{config: cfg10}, wantErr: true}, @@ -126,19 +138,24 @@ func Test_ValidateParams(t *testing.T) { } func Test_validateParamsTLS(t *testing.T) { - type args struct { - config *Config - } + cfgEmpty := getMinimalConfig() + cfgValid := getMinimalConfig() + cfgValid.CrowdsecLapiTLSCertificateAuthority = validPEM + cfgInvalidCA := getMinimalConfig() + cfgInvalidCA.CrowdsecLapiTLSCertificateAuthority = "not a pem" + tests := []struct { name string - args args + config *Config wantErr bool }{ - // TODO: Add test cases. + {name: "Empty CA is accepted (system trust store used at runtime)", config: cfgEmpty, wantErr: false}, + {name: "Valid PEM CA is accepted", config: cfgValid, wantErr: false}, + {name: "Invalid CA is rejected", config: cfgInvalidCA, wantErr: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if err := validateParamsTLS(tt.args.config); (err != nil) != tt.wantErr { + if err := validateParamsTLS(tt.config); (err != nil) != tt.wantErr { t.Errorf("validateParamsTLS() error = %v, wantErr %v", err, tt.wantErr) } }) @@ -233,26 +250,53 @@ func Test_validateParamsAPIKey(t *testing.T) { func Test_GetTLSConfigCrowdsec(t *testing.T) { log := logger.New("INFO", "") - type args struct { - config *Config - } + + httpCfg := getMinimalConfig() + httpCfg.CrowdsecLapiScheme = HTTP + + httpsSystemCA := getMinimalConfig() + httpsSystemCA.CrowdsecLapiScheme = HTTPS + + httpsCustomCA := getMinimalConfig() + httpsCustomCA.CrowdsecLapiScheme = HTTPS + httpsCustomCA.CrowdsecLapiTLSCertificateAuthority = validPEM + + httpsInsecure := getMinimalConfig() + httpsInsecure.CrowdsecLapiScheme = HTTPS + httpsInsecure.CrowdsecLapiTLSInsecureVerify = true + + httpsBadCA := getMinimalConfig() + httpsBadCA.CrowdsecLapiScheme = HTTPS + httpsBadCA.CrowdsecLapiTLSCertificateAuthority = "not a pem" + tests := []struct { - name string - args args - want *tls.Config - wantErr bool + name string + config *Config + wantErr bool + wantRootCAsNil bool + wantInsecureSkip bool }{ - // TODO: Add test cases. + {name: "HTTP scheme returns empty tls.Config", config: httpCfg, wantRootCAsNil: true}, + {name: "HTTPS without CA leaves RootCAs nil (system trust store)", config: httpsSystemCA, wantRootCAsNil: true}, + {name: "HTTPS with custom CA populates RootCAs", config: httpsCustomCA, wantRootCAsNil: false}, + {name: "HTTPS with insecure verify sets InsecureSkipVerify", config: httpsInsecure, wantRootCAsNil: true, wantInsecureSkip: true}, + {name: "HTTPS with garbage CA is rejected", config: httpsBadCA, wantErr: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := GetTLSConfigCrowdsec(tt.args.config, log, false) + got, err := GetTLSConfigCrowdsec(tt.config, log, false) if (err != nil) != tt.wantErr { - t.Errorf("getTLSConfigCrowdsec() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetTLSConfigCrowdsec() error = %v, wantErr %v", err, tt.wantErr) return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("getTLSConfigCrowdsec() = %v, want %v", got, tt.want) + if tt.wantErr { + return + } + if (got.RootCAs == nil) != tt.wantRootCAsNil { + t.Errorf("GetTLSConfigCrowdsec() RootCAs nil = %v, want nil = %v", got.RootCAs == nil, tt.wantRootCAsNil) + } + if got.InsecureSkipVerify != tt.wantInsecureSkip { + t.Errorf("GetTLSConfigCrowdsec() InsecureSkipVerify = %v, want %v", got.InsecureSkipVerify, tt.wantInsecureSkip) } }) } diff --git a/tests/e2e/mock/README.md b/tests/e2e/mock/README.md index 86999e1..8c05d5e 100644 --- a/tests/e2e/mock/README.md +++ b/tests/e2e/mock/README.md @@ -30,7 +30,7 @@ that lives upstream in Crowdsec. |-----------|-----| | Traefik | Binary `v3.7.1`, downloaded into `.cache/` (reused across local runs; re-downloaded on fresh CI runners) | | Plugin | Loaded via `experimental.localPlugins` from the repo root (symlinked into `plugins-local/`) | -| LAPI | `mocklapi` — a stdlib-only Go command (its own nested module), compiled and cached under `.cache/`, driven through `/admin` endpoints instead of `cscli` | +| LAPI | `mocklapi` — a stdlib-only Go command (its own nested module), compiled and cached under `.cache/`, driven through `/admin` endpoints instead of `cscli`. Serves plain HTTP, or HTTPS when `--lapi-tls-cert/--lapi-tls-key` are passed (the `tls-system-ca` scenario) | | AppSec | WAF stand-in built into the mock — blocks URIs containing `rpc2`, allows the rest | | Backend | A plain HTTP responder built into the mock | @@ -39,9 +39,16 @@ backend `8091`, AppSec `8092`. ## Running locally -Prerequisites: `bash`, `curl`, `go`, `tar`. On first use the Traefik binary is -fetched and the mock is compiled into `.cache/`. That cache is reused across -local runs; CI runs on fresh runners, so both are recreated on every CI run. +Prerequisites: `bash`, `curl`, `go`, `tar` (plus `openssl` for the +`tls-system-ca` scenario, which mints a throwaway CA at runtime). On first use +the Traefik binary is fetched and the mock is compiled into `.cache/`. That +cache is reused across local runs; CI runs on fresh runners, so both are +recreated on every CI run. + +The `tls-system-ca` scenario verifies that, with no custom CA configured, the +bouncer falls back to the OS/system trust store for an HTTPS LAPI: it serves the +mock over TLS and points the Traefik process's `SSL_CERT_FILE` at the test CA +(trusted → 200) or an empty bundle (untrusted → 403, proving it still verifies). ```bash # one scenario diff --git a/tests/e2e/mock/lib/common.sh b/tests/e2e/mock/lib/common.sh index 1431bd1..d32e85d 100644 --- a/tests/e2e/mock/lib/common.sh +++ b/tests/e2e/mock/lib/common.sh @@ -190,16 +190,29 @@ start_stack() { -e "s|@@SCENARIO_DIR@@|${scenario_dir}|g" \ "$scenario_dir/dynamic.yml" > "$WORKDIR/dynamic.yml" + # Opt-in HTTPS LAPI: a scenario exports LAPI_TLS_CERT/LAPI_TLS_KEY to serve the + # LAPI over TLS (used by tls-system-ca). Default empty -> plaintext as before. + local mock_tls_args=() lapi_scheme=http lapi_curl=() + if [[ -n "${LAPI_TLS_CERT:-}" && -n "${LAPI_TLS_KEY:-}" ]]; then + mock_tls_args=(--lapi-tls-cert "$LAPI_TLS_CERT" --lapi-tls-key "$LAPI_TLS_KEY") + lapi_scheme=https + lapi_curl=(-k) # the readiness probe ignores trust; the bouncer's trust is what we test + fi + "$mock_bin" \ --lapi-addr "127.0.0.1:${LAPI_PORT}" \ --backend-addr "127.0.0.1:${BACKEND_PORT}" \ - --appsec-addr "127.0.0.1:${APPSEC_PORT}" >"$WORKDIR/mock.log" 2>&1 & + --appsec-addr "127.0.0.1:${APPSEC_PORT}" \ + "${mock_tls_args[@]}" >"$WORKDIR/mock.log" 2>&1 & MOCK_PID=$! - ( cd "$WORKDIR" && exec "$traefik_bin" --configfile=traefik.yml ) >"$WORKDIR/traefik.log" 2>&1 & + # Opt-in trust store for the Traefik process: a scenario exports + # TRAEFIK_SSL_CERT_FILE to point Go's x509.SystemCertPool() at a specific CA + # bundle. Empty -> Go's default system store (unchanged behaviour). + ( cd "$WORKDIR" && SSL_CERT_FILE="${TRAEFIK_SSL_CERT_FILE:-}" exec "$traefik_bin" --configfile=traefik.yml ) >"$WORKDIR/traefik.log" 2>&1 & TRAEFIK_PID=$! - wait_for_status "http://127.0.0.1:${LAPI_PORT}/health" 200 30 + wait_for_status "${lapi_scheme}://127.0.0.1:${LAPI_PORT}/health" 200 30 "${lapi_curl[@]}" # AppSec stand-in: a bare GET carries no "rpc2" URI, so it answers 200 (allow). wait_for_status "http://127.0.0.1:${APPSEC_PORT}/" 200 30 # /ping is served by Traefik itself once it is up (plugin compilation included). diff --git a/tests/e2e/mock/mocklapi/main.go b/tests/e2e/mock/mocklapi/main.go index 32a3631..a2804bd 100644 --- a/tests/e2e/mock/mocklapi/main.go +++ b/tests/e2e/mock/mocklapi/main.go @@ -52,6 +52,11 @@ func main() { backendAddr := flag.String("backend-addr", "127.0.0.1:8091", "address for the stub upstream service") // AppSec WAF stand-in (the real engine listens on :7422). Not a CRS engine. appsecAddr := flag.String("appsec-addr", "127.0.0.1:8092", "address for the AppSec mock") + // Optional TLS for the LAPI: when both are set the LAPI is served over HTTPS + // (cert signed by the scenario's throwaway CA) so the suite can exercise the + // bouncer's system-trust-store path. Backend and AppSec stay plaintext. + lapiTLSCert := flag.String("lapi-tls-cert", "", "PEM cert to serve the LAPI over HTTPS (optional)") + lapiTLSKey := flag.String("lapi-tls-key", "", "PEM key for --lapi-tls-cert") flag.Parse() go func() { @@ -130,6 +135,10 @@ func main() { } }) + if *lapiTLSCert != "" && *lapiTLSKey != "" { + log.Printf("mocklapi: LAPI on %s (TLS), backend on %s, appsec on %s", *lapiAddr, *backendAddr, *appsecAddr) + log.Fatal(http.ListenAndServeTLS(*lapiAddr, *lapiTLSCert, *lapiTLSKey, mux)) + } log.Printf("mocklapi: LAPI on %s, backend on %s, appsec on %s", *lapiAddr, *backendAddr, *appsecAddr) log.Fatal(http.ListenAndServe(*lapiAddr, mux)) } diff --git a/tests/e2e/mock/scenarios/tls-system-ca/dynamic.yml b/tests/e2e/mock/scenarios/tls-system-ca/dynamic.yml new file mode 100644 index 0000000..bfc391e --- /dev/null +++ b/tests/e2e/mock/scenarios/tls-system-ca/dynamic.yml @@ -0,0 +1,28 @@ +http: + routers: + r: + rule: "PathPrefix(`/foo`)" + entryPoints: + - web + service: backend + middlewares: + - bouncer + services: + backend: + loadBalancer: + servers: + - url: "@@BACKEND_URL@@" + middlewares: + bouncer: + plugin: + bouncer: + enabled: "true" + crowdsecMode: live + defaultDecisionSeconds: "2" + # HTTPS LAPI with NO custom CA configured: the bouncer must fall back to + # the OS/system trust store (which the scenario controls via SSL_CERT_FILE). + crowdsecLapiScheme: https + crowdsecLapiHost: "@@LAPI_HOST@@" + crowdsecLapiKey: "@@APIKEY@@" + forwardedHeadersTrustedIps: + - "127.0.0.1/32" diff --git a/tests/e2e/mock/scenarios/tls-system-ca/run.sh b/tests/e2e/mock/scenarios/tls-system-ca/run.sh new file mode 100755 index 0000000..4370a71 --- /dev/null +++ b/tests/e2e/mock/scenarios/tls-system-ca/run.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Scenario: HTTPS LAPI with no custom CA configured -> the bouncer must fall back +# to the OS/system trust store (PR #331). In the binary suite the "system trust +# store" is whatever Go's x509.SystemCertPool() reads, which honours SSL_CERT_FILE +# on the Traefik process. We mint a throwaway CA, serve the mock LAPI over HTTPS +# with a cert signed by it, and run the stack twice: +# +# positive: SSL_CERT_FILE = our CA -> LAPI trusted -> 200 +# negative: SSL_CERT_FILE = empty bundle -> LAPI not trusted -> 403 +# +# live mode is fail-closed, so a TLS error becomes a 403. The negative run proves +# the patch still VERIFIES (it is not an insecure skip). +# +# Extra dependency vs other scenarios: openssl. +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=../../lib/common.sh +source "$HERE/../../lib/common.sh" + +SCENARIO=tls-system-ca +SCENARIO_NAME="$SCENARIO" +SCENARIO_LOG="/tmp/e2e-mock-${SCENARIO}.log" +CERT_DIR="$(mktemp -d)" + +cleanup() { + local rc=$? + if (( rc != 0 )); then + dump_diagnostics > "$SCENARIO_LOG" 2>&1 || true + echo "[$SCENARIO] failed. Logs written to $SCENARIO_LOG" >&2 + fi + stop_stack + rm -rf "$CERT_DIR" + exit $rc +} +trap cleanup EXIT + +echo "[$SCENARIO] minting throwaway CA + LAPI cert (SAN=IP:127.0.0.1)..." +openssl ecparam -name prime256v1 -genkey -noout -out "$CERT_DIR/ca.key" 2>/dev/null +openssl req -x509 -new -key "$CERT_DIR/ca.key" -sha256 -days 3650 \ + -subj "/CN=crowdsec-bouncer e2e test CA" -out "$CERT_DIR/ca.crt" 2>/dev/null +openssl ecparam -name prime256v1 -genkey -noout -out "$CERT_DIR/lapi.key" 2>/dev/null +openssl req -new -key "$CERT_DIR/lapi.key" -subj "/CN=lapi" -out "$CERT_DIR/lapi.csr" 2>/dev/null +openssl x509 -req -in "$CERT_DIR/lapi.csr" -CA "$CERT_DIR/ca.crt" -CAkey "$CERT_DIR/ca.key" \ + -CAcreateserial -days 3650 -sha256 -out "$CERT_DIR/lapi.crt" \ + -extfile <(printf "subjectAltName=IP:127.0.0.1\nbasicConstraints=CA:FALSE\nkeyUsage=digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth") 2>/dev/null +: > "$CERT_DIR/empty.crt" # an empty bundle = a system store that trusts nothing + +# The mock serves the same CA-signed cert in both runs; only Traefik's trust differs. +export LAPI_TLS_CERT="$CERT_DIR/lapi.crt" LAPI_TLS_KEY="$CERT_DIR/lapi.key" + +echo "[$SCENARIO] === positive: CA in the system trust store ===" +export TRAEFIK_SSL_CERT_FILE="$CERT_DIR/ca.crt" +start_stack "$HERE" +echo "[$SCENARIO] HTTPS LAPI verifies via system trust store -> request passes (200)" +assert_status "http://127.0.0.1:${WEB_PORT}/foo" 200 -H "X-Forwarded-For: 1.2.3.4" +stop_stack + +echo "[$SCENARIO] === negative: CA absent from the system trust store ===" +export TRAEFIK_SSL_CERT_FILE="$CERT_DIR/empty.crt" +start_stack "$HERE" +echo "[$SCENARIO] LAPI cert not trusted -> TLS fails, fail-closed (403)" +assert_status "http://127.0.0.1:${WEB_PORT}/foo" 403 -H "X-Forwarded-For: 1.2.3.4" +stop_stack + +echo "[$SCENARIO] OK"