Add headless Chrome as an HTTP client to send requests (#246)

* Refactoring

* Fix placeholders

* Refactoring, add Chrome support as HTTP client

* Fix README.md

* Bump required golang version

* Fix bugs

* Fix chromedp logs

* Fix bug with gRPC placeholder

* Fix config for tests

* Add integration tests for Chrome backend

* Change default client back to Chrome

* Fix vulnerable dependency

* go mod tidy

.gitignore

@@ -1,6 +1,6 @@
 .idea
 .DS_store
-gotestwaf
-main
-reports/*
+/gotestwaf
+/main
+/reports/*
 /modsec_stat_*.txt

Dockerfile

@@ -1,14 +1,19 @@
 # syntax=docker/dockerfile:1
 
 # Build Stage ==================================================================
-FROM golang:1.19-alpine AS build
+FROM golang:1.22-alpine AS build
 
 RUN apk --no-cache add git
 
 WORKDIR /app
-COPY . .
 
-RUN go build -o gotestwaf -ldflags "-X github.com/wallarm/gotestwaf/internal/version.Version=$(git describe --tags)" ./cmd/
+COPY ./go.mod ./go.sum ./
+RUN go mod download
+
+COPY . .
+RUN go build -o gotestwaf \
+ -ldflags "-X github.com/wallarm/gotestwaf/internal/version.Version=$(git describe --tags)" \
+ ./cmd/gotestwaf
 
 
 # Main Stage ===================================================================

Makefile

@@ -4,7 +4,9 @@ gotestwaf:
  DOCKER_BUILDKIT=1 docker build --force-rm -t gotestwaf .
 
 gotestwaf_bin:
- go build -o gotestwaf -ldflags "-X github.com/wallarm/gotestwaf/internal/version.Version=$(GOTESTWAF_VERSION)" ./cmd
+ go build -o gotestwaf \
+ -ldflags "-X github.com/wallarm/gotestwaf/internal/version.Version=$(GOTESTWAF_VERSION)" \
+ ./cmd/gotestwaf
 
 modsec:
  docker pull mendhak/http-https-echo:31

README.md

@@ -1,7 +1,7 @@
 # GoTestWAF [![Black Hat Arsenal USA 2022](https://img.shields.io/badge/Black%20Hat%20Arsenal-USA%202022-blue)](https://www.blackhat.com/us-22/arsenal/schedule/index.html#gotestwaf---well-known-open-source-waf-tester-now-supports-api-security-hacking-27986)
 
 GoTestWAF is a tool for API and OWASP attack simulation that supports a wide range of API protocols including
-REST, GraphQL, gRPC, WebSockets, SOAP, XMLRPC, and others.
+REST, GraphQL, gRPC, SOAP, XMLRPC, and others.
 
 It was designed to evaluate web application security solutions, such as API security proxies, Web Application Firewalls,
 IPS, API gateways, and others.
@@ -26,7 +26,6 @@ The results of the security solution evaluation are recorded in the report file
 Default conditions for request generation are defined in the `testcases` folder in the YAML files of the following format:
 
 ```yaml
----
 payload:
  - '"union select -7431.1, name, @aaa from u_base--w-'
  - "'or 123.22=123.22"
@@ -40,8 +39,7 @@ placeholder:
  - UrlParam
  - JSUnicode
  - Header
-type: "SQL Injection"
-...
+type: SQL Injection
 ```
 
 * `payload` is a malicious attack sample (e.g XSS payload like ```<script>alert(111)</script>``` or something more sophisticated).
@@ -70,10 +68,6 @@ Since the format of the YAML string is required for payloads, they must be [enco
  * XMLBody
  * URLParam
  * URLPath
- * NonCrudUrlPath
- * NonCrudUrlParam
- * NonCRUDHeader
- * NonCRUDRequestBody
  * RawRequest
 
  The `RawRequest` placeholder will allow you to do an arbitrary HTTP request. The payload is substituted by replacing the string `{{payload}}` in the URL path, Headers or body. Fields of `RawRequest` placeholder:
@@ -90,7 +84,6 @@ Since the format of the YAML string is required for payloads, they must be [enco
  Example:
 
  ```yaml
- ---
  payload:
  - test
  encoder:
@@ -113,8 +106,7 @@ Since the format of the YAML string is required for payloads, they must be [enco
  Knock knock.
  {{payload}}
  --boundary--
- type: "RawRequest test"
- ...
+ type: RawRequest test
  ```
 
 * `type` is a name of entire group of the payloads in file. It can be arbitrary, but should reflect the type of attacks in the file.
@@ -376,15 +368,16 @@ Options:
  --blockStatusCodes ints HTTP status code that WAF uses while blocking requests (default [403])
  --configPath string Path to the config file (default "config.yaml")
  --email string E-mail to which the report will be sent
- --followCookies If true, use cookies sent by the server. May work only with --maxIdleConns=1
+ --followCookies If true, use cookies sent by the server. May work only with --maxIdleConns=1 (gohttp only)
  --grpcPort uint16 gRPC port to check
- --idleConnTimeout int The maximum amount of time a keep-alive connection will live (default 2)
+ --httpClient string Which HTTP client use to send requests: chrome, gohttp (default "gohttp")
+ --idleConnTimeout int The maximum amount of time a keep-alive connection will live (gohttp only) (default 2)
  --ignoreUnresolved If true, unresolved test cases will be considered as bypassed (affect score and results)
  --includePayloads If true, payloads will be included in HTML/PDF report
  --logFormat string Set logging format: text, json (default "text")
  --logLevel string Logging level: panic, fatal, error, warn, info, debug, trace (default "info")
- --maxIdleConns int The maximum number of keep-alive connections (default 2)
- --maxRedirects int The maximum number of handling redirects (default 50)
+ --maxIdleConns int The maximum number of keep-alive connections (gohttp only) (default 2)
+ --maxRedirects int The maximum number of handling redirects (gohttp only) (default 50)
  --noEmailReport Save report locally
  --nonBlockedAsPassed If true, count requests that weren't blocked as passed. If false, requests that don't satisfy to PassStatusCodes/PassRegExp as blocked
  --openapiFile string Path to openAPI file
@@ -393,7 +386,7 @@ Options:
  --proxy string Proxy URL to use
  --quiet If true, disable verbose logging
  --randomDelay int Random delay in ms in addition to the delay between requests (default 400)
- --renewSession Renew cookies before each test. Should be used with --followCookies flag
+ --renewSession Renew cookies before each test. Should be used with --followCookies flag (gohttp only)
  --reportFormat string Export report to one of the following formats: none, pdf, html, json (default "pdf")
  --reportName string Report file name. Supports `time' package template format (default "waf-evaluation-report-2006-January-02-15-04-05")
  --reportPath string A directory to store reports (default "reports")
@@ -408,7 +401,6 @@ Options:
  --version Show GoTestWAF version and exit
  --wafName string Name of the WAF product (default "generic")
  --workers int The number of workers to scan (default 5)
- --wsURL string WebSocket URL to check
 ```
 
 The listed options can be passed to GoTestWAF as follows:

cmd/flags.go → cmd/gotestwaf/flags.go

@@ -34,8 +34,7 @@ const (
  jsonLogFormat = "json"
 
  cliDescription = `GoTestWAF is a tool for API and OWASP attack simulation that supports a
-wide range of API protocols including REST, GraphQL, gRPC, WebSockets,
-SOAP, XMLRPC, and others.
+wide range of API protocols including REST, GraphQL, gRPC, SOAP, XMLRPC, and others.
 Homepage: https://github.com/wallarm/gotestwaf
 
 Usage: %s [OPTIONS] --url <URL>
@@ -67,21 +66,44 @@ func parseFlags() (args []string, err error) {
 
  flag.Usage = usage
 
+ // General parameters
  flag.StringVar(&configPath, "configPath", defaultConfigPath, "Path to the config file")
  flag.BoolVar(&quiet, "quiet", false, "If true, disable verbose logging")
  logLvl := flag.String("logLevel", "info", "Logging level: panic, fatal, error, warn, info, debug, trace")
  flag.StringVar(&logFormat, "logFormat", textLogFormat, "Set logging format: text, json")
+ showVersion := flag.Bool("version", false, "Show GoTestWAF version and exit")
 
+ // Target settings
  urlParam := flag.String("url", "", "URL to check")
- wsURL := flag.String("wsURL", "", "WebSocket URL to check")
  flag.Uint16("grpcPort", 0, "gRPC port to check")
- flag.String("proxy", "", "Proxy URL to use")
+ openapiFile := flag.String("openapiFile", "", "Path to openAPI file")
+
+ // Test cases settings
+ flag.String("testCase", "", "If set then only this test case will be run")
+ flag.String("testCasesPath", testCasesPath, "Path to a folder with test cases")
+ flag.String("testSet", "", "If set then only this test set's cases will be run")
+
+ // HTTP client settings
+ httpClient := flag.String("httpClient", "gohttp", "Which HTTP client use to send requests: chrome, gohttp")
  flag.Bool("tlsVerify", false, "If true, the received TLS certificate will be verified")
- flag.Int("maxIdleConns", 2, "The maximum number of keep-alive connections")
- flag.Int("maxRedirects", 50, "The maximum number of handling redirects")
- flag.Int("idleConnTimeout", 2, "The maximum amount of time a keep-alive connection will live")
- flag.Bool("followCookies", false, "If true, use cookies sent by the server. May work only with --maxIdleConns=1")
- flag.Bool("renewSession", false, "Renew cookies before each test. Should be used with --followCookies flag")
+ flag.String("proxy", "", "Proxy URL to use")
+ flag.String("addHeader", "", "An HTTP header to add to requests")
+ flag.Bool("addDebugHeader", false, "Add header with a hash of the test information in each request")
+
+ // GoHTTP client only settings
+ flag.Int("maxIdleConns", 2, "The maximum number of keep-alive connections (gohttp only)")
+ flag.Int("maxRedirects", 50, "The maximum number of handling redirects (gohttp only)")
+ flag.Int("idleConnTimeout", 2, "The maximum amount of time a keep-alive connection will live (gohttp only)")
+ flag.Bool("followCookies", false, "If true, use cookies sent by the server. May work only with --maxIdleConns=1 (gohttp only)")
+ flag.Bool("renewSession", false, "Renew cookies before each test. Should be used with --followCookies flag (gohttp only)")
+
+ // Performance settings
+ flag.Int("workers", 5, "The number of workers to scan")
+ flag.Int("sendDelay", 400, "Delay in ms between requests")
+ flag.Int("randomDelay", 400, "Random delay in ms in addition to the delay between requests")
+
+ // Analysis settings
+ flag.Bool("skipWAFBlockCheck", false, "If true, WAF detection tests will be skipped")
  flag.Bool("skipWAFIdentification", false, "Skip WAF identification")
  flag.IntSlice("blockStatusCodes", []int{403}, "HTTP status code that WAF uses while blocking requests")
  flag.IntSlice("passStatusCodes", []int{200, 404}, "HTTP response status code that WAF uses while passing requests")
@@ -91,26 +113,18 @@ func parseFlags() (args []string, err error) {
  "Regex to a detect normal (not blocked) web page with the same HTTP status code as a blocked request")
  flag.Bool("nonBlockedAsPassed", false,
  "If true, count requests that weren't blocked as passed. If false, requests that don't satisfy to PassStatusCodes/PassRegExp as blocked")
- flag.Int("workers", 5, "The number of workers to scan")
- flag.Int("sendDelay", 400, "Delay in ms between requests")
- flag.Int("randomDelay", 400, "Random delay in ms in addition to the delay between requests")
- flag.String("testCase", "", "If set then only this test case will be run")
- flag.String("testSet", "", "If set then only this test set's cases will be run")
+ flag.Bool("ignoreUnresolved", false, "If true, unresolved test cases will be considered as bypassed (affect score and results)")
+ flag.Bool("blockConnReset", false, "If true, connection resets will be considered as block")
+
+ // Report settings
+ flag.String("wafName", wafName, "Name of the WAF product")
+ flag.Bool("includePayloads", false, "If true, payloads will be included in HTML/PDF report")
  flag.String("reportPath", reportPath, "A directory to store reports")
  reportName := flag.String("reportName", defaultReportName, "Report file name. Supports `time' package template format")
  flag.String("reportFormat", "pdf", "Export report to one of the following formats: none, pdf, html, json")
- flag.Bool("includePayloads", false, "If true, payloads will be included in HTML/PDF report")
  noEmailReport := flag.Bool("noEmailReport", false, "Save report locally")
  email := flag.String("email", "", "E-mail to which the report will be sent")
- flag.String("testCasesPath", testCasesPath, "Path to a folder with test cases")
- flag.String("wafName", wafName, "Name of the WAF product")
- flag.Bool("ignoreUnresolved", false, "If true, unresolved test cases will be considered as bypassed (affect score and results)")
- flag.Bool("blockConnReset", false, "If true, connection resets will be considered as block")
- flag.Bool("skipWAFBlockCheck", false, "If true, WAF detection tests will be skipped")
- flag.String("addHeader", "", "An HTTP header to add to requests")
- flag.Bool("addDebugHeader", false, "Add header with a hash of the test information in each request")
- flag.String("openapiFile", "", "Path to openAPI file")
- showVersion := flag.Bool("version", false, "Show GoTestWAF version and exit")
+
  flag.Parse()
 
  if len(os.Args) == 1 {
@@ -165,16 +179,10 @@ func parseFlags() (args []string, err error) {
 
  *urlParam = validURL.String()
 
- // format WebSocket URL from given HTTP URL
- if *wsURL == "" {
- wsScheme := "ws"
- if validURL.Scheme == "https" {
- wsScheme = "wss"
- }
- validURL.Scheme = wsScheme
- validURL.Path = ""
-
- *wsURL = validURL.String()
+ // Force GoHTTP to be used as the HTTP client
+ // when scanning against the OpenAPI spec.
+ if openapiFile != nil && len(*openapiFile) > 0 {
+ *httpClient = "gohttp"
  }
 
  if *blockRegex != "" {