Move use of --xvfb into settings

.github/workflows/regression-test-404.yml

@@ -51,7 +51,7 @@ jobs:
     - if: ${{ matrix.os == 'ubuntu-latest' }}
       name: RUNNING TEST - LINUX
       run: |
-         python default.py -t ${{ matrix.version }} -r -i sites.json -o data/testresult-${{ matrix.version }}.json --setting ${{ matrix.config }}
+         python default.py -t ${{ matrix.version }} -r -i sites.json -o data/testresult-${{ matrix.version }}.json --setting ${{ matrix.config }} --setting tests.sitespeed.xvfb=true
          python tools/verify_result.py -t ${{ matrix.version }}
     - if: ${{ matrix.os == 'windows-latest' }}
       name: RUNNING TEST - WINDOWS

.github/workflows/regression-test-css.yml

@@ -53,7 +53,7 @@ jobs:
     - if: ${{ matrix.os == 'ubuntu-latest' }}
       name: RUNNING TEST - LINUX
       run: |
-         python default.py -t ${{ matrix.version }} -r -i sites.json -o data/testresult-${{ matrix.version }}.json --setting ${{ matrix.config }}
+         python default.py -t ${{ matrix.version }} -r -i sites.json -o data/testresult-${{ matrix.version }}.json --setting ${{ matrix.config }} --setting tests.sitespeed.xvfb=true
          python tools/verify_result.py -t ${{ matrix.version }}
     - if: ${{ matrix.os == 'windows-latest' }}
       name: RUNNING TEST - WINDOWS

.github/workflows/regression-test-html.yml

@@ -53,7 +53,7 @@ jobs:
     - if: ${{ matrix.os == 'ubuntu-latest' }}
       name: RUNNING TEST - LINUX
       run: |
-         python default.py -t ${{ matrix.version }} -r -i sites.json -o data/testresult-${{ matrix.version }}.json --setting ${{ matrix.config }}
+         python default.py -t ${{ matrix.version }} -r -i sites.json -o data/testresult-${{ matrix.version }}.json --setting ${{ matrix.config }} --setting tests.sitespeed.xvfb=true
          python tools/verify_result.py -t ${{ matrix.version }}
     - if: ${{ matrix.os == 'windows-latest' }}
       name: RUNNING TEST - WINDOWS

.github/workflows/regression-test-http.yml

@@ -51,7 +51,7 @@ jobs:
     - if: ${{ matrix.os == 'ubuntu-latest' }}
       name: RUNNING TEST - LINUX
       run: |
-         python default.py -t ${{ matrix.version }} -r -i sites.json -o data/testresult-${{ matrix.version }}.json --setting ${{ matrix.config }}
+         python default.py -t ${{ matrix.version }} -r -i sites.json -o data/testresult-${{ matrix.version }}.json --setting ${{ matrix.config }} --setting tests.sitespeed.xvfb=true
          python tools/verify_result.py -t ${{ matrix.version }}
     - if: ${{ matrix.os == 'windows-latest' }}
       name: RUNNING TEST - WINDOWS

.github/workflows/regression-test-sitespeed.yml

@@ -67,7 +67,7 @@ jobs:
     - if: ${{ matrix.os == 'ubuntu-latest' }}
       name: RUNNING TEST - LINUX
       run: |
-         python default.py -t ${{ matrix.version }} -r -i sites.json -o data/testresult-${{ matrix.version }}.json --setting ${{ matrix.config }}
+         python default.py -t ${{ matrix.version }} -r -i sites.json -o data/testresult-${{ matrix.version }}.json --setting ${{ matrix.config }} --setting tests.sitespeed.xvfb=true
          python tools/verify_result.py -t ${{ matrix.version }}
     - if: ${{ matrix.os == 'windows-latest' }}
       name: RUNNING TEST - WINDOWS

.github/workflows/regression-test-software.yml

@@ -48,7 +48,7 @@ jobs:
     - if: ${{ matrix.os == 'ubuntu-latest' }}
       name: RUNNING TEST - LINUX
       run: |
-         python default.py -t ${{ matrix.version }} -r -i sites.json -o data/testresult-${{ matrix.version }}.json --setting ${{ matrix.config }}
+         python default.py -t ${{ matrix.version }} -r -i sites.json -o data/testresult-${{ matrix.version }}.json --setting ${{ matrix.config }} --setting tests.sitespeed.xvfb=true
          python tools/verify_result.py -t ${{ matrix.version }}
     - if: ${{ matrix.os == 'windows-latest' }}
       name: RUNNING TEST - WINDOWS

.github/workflows/regression-test-tracking.yml

@@ -103,7 +103,7 @@ jobs:
     - if: ${{ matrix.os == 'ubuntu-latest' }}
       name: RUNNING TEST - LINUX
       run: |
-         python default.py -t ${{ matrix.version }} -r -i sites.json -o data/testresult-${{ matrix.version }}.json --setting ${{ matrix.config }}
+         python default.py -t ${{ matrix.version }} -r -i sites.json -o data/testresult-${{ matrix.version }}.json --setting ${{ matrix.config }} --setting tests.sitespeed.xvfb=true
          python tools/verify_result.py -t ${{ matrix.version }}
     - if: ${{ matrix.os == 'windows-latest' }}
       name: RUNNING TEST - WINDOWS

Dockerfile

@@ -69,7 +69,7 @@ RUN python3.12 -m pip install -r requirements.txt --break-system-packages && \
     python3.12 -m pip install --upgrade setuptools --break-system-packages && \
     python3.12 -m pip install pyssim Pillow image --break-system-packages
 
-RUN python3.12 default.py --setting tests.lighthouse.disable-sandbox=true --save-setting settings.json
+RUN python3.12 default.py --setting tests.lighthouse.disable-sandbox=true --setting tests.sitespeed.xvfb=true --save-setting settings.json
 
 ENTRYPOINT []
 

defaults/settings.json

@@ -41,7 +41,8 @@
                 "use": false
             },
             "timeout": 30,
-            "iterations": 2
+            "iterations": 2,
+            "xvfb": false
         },
         "software": {
             "advisory": {

docs/settings-json.md

@@ -159,17 +159,22 @@ This is needed when using it in our docker image (IF not used in interactive mod
 This variable tells sitespeed based test(s) to use docker image version instead of NPM version.
 Please read more about this on [SiteSpeed test section](tests/sitespeed.md).
 
+### tests.sitespeed.iterations `(Default = 2)`
+
+This variable tells sitespeed based test(s) how many iterations it should do against the url to get the best measurement.
+Please read more about this on [SiteSpeed test section](tests/sitespeed.md).
+
 ### tests.sitespeed.timeout `(Default = 300 ms)`
 
 This variable tells sitespeed based test(s) how long it should wait for a url to load.
 Setting this to a lower value may improve overall test speed if many urls are being tested and
 it is not important if one or two tests fail.
 Please read more about this on [SiteSpeed test section](tests/sitespeed.md).
 
-### tests.sitespeed.iterations `(Default = 2)`
+### tests.sitespeed.xvfb `(Default = false)`
 
-This variable tells sitespeed based test(s) how many iterations it should do against the url to get the best measurement.
-Please read more about this on [SiteSpeed test section](tests/sitespeed.md).
+This variable tells sitespeed based test(s) to start xvfb before the browser is started.
+This is only relevant for linux based os.
 
 ### test.software.advisory.path `(Default = "")`
 This variable is ONLY used to generate a CVE and security related info for software.

helpers/setting_helper.py

@@ -74,6 +74,9 @@
         "tests.sitespeed.iterations",
         "sitespeed_iterations",
         "SITESPEED_ITERATIONS"): "int|tests.sitespeed.iterations",
+    (
+        "sitespeedxvfb",
+        "tests.sitespeed.xvfb"): "bool|tests.sitespeed.xvfb",
     (
         "csponly",
         "tests.http.csp-only",

tests/http_validator.py

@@ -850,7 +850,7 @@ def get_website_support_from_sitespeed(url, org_domain, configuration, browser,
 
     sitespeed_arg = f'--shm-size=1g {sitespeed_arg}'
 
-    if not ('nt' in os.name or 'Darwin' in os.uname().sysname):
+    if get_config('tests.sitespeed.xvfb'):
         sitespeed_arg += ' --xvfb'
 
     (_, filename) = get_result(

tests/page_not_found.py

@@ -65,7 +65,7 @@ def get_http_content_with_status(url):
             '--browsertime.chrome.includeResponseBodies all --utc true '
             '--browsertime.chrome.args ignore-certificate-errors '
             f'-n {sitespeed_iterations}')
-    if not ('nt' in os.name or 'Darwin' in os.uname().sysname):
+    if get_config('tests.sitespeed.xvfb'):
         sitespeed_arg += ' --xvfb'
 
     sitespeed_arg += ' --postScript chrome-cookies.cjs --postScript chrome-versions.cjs'

tests/performance_sitespeed_io.py

@@ -208,7 +208,7 @@ def validate_on_mobile_using_validator(url, validator_config):
         f'{url}'
         f'{browertime_plugin_options}'
         )
-    if not ('nt' in os.name or 'Darwin' in os.uname().sysname):
+    if get_config('tests.sitespeed.xvfb'):
         arg = '--xvfb ' + arg
 
     result_dict = get_result_dict(get_result(arg), validator_config['name'])
@@ -283,7 +283,7 @@ def validate_on_desktop_using_validator(url, validator_config):
         f'{url}'
         f'{browertime_plugin_options}'
         )
-    if not ('nt' in os.name or 'Darwin' in os.uname().sysname):
+    if get_config('tests.sitespeed.xvfb'):
         arg = '--xvfb ' + arg
 
     result_dict = get_result_dict(get_result(arg), validator_config['name'])
@@ -316,7 +316,7 @@ def validate_on_desktop(url):
         '--preScript chrome-custom.cjs '
         f'{url}'
         )
-    if not ('nt' in os.name or 'Darwin' in os.uname().sysname):
+    if get_config('tests.sitespeed.xvfb'):
         arg = '--xvfb ' + arg
 
     result_dict = get_result_dict(get_result(arg), 'desktop')
@@ -348,7 +348,7 @@ def validate_on_mobile(url):
         '--preScript chrome-custom.cjs '
         f'{url}'
         )
-    if not ('nt' in os.name or 'Darwin' in os.uname().sysname):
+    if get_config('tests.sitespeed.xvfb'):
         arg = '--xvfb ' + arg
 
     result_dict = get_result_dict(get_result(arg), 'mobile')

tests/software.py

@@ -90,7 +90,7 @@ def get_rating_from_sitespeed(url, local_translation, global_translation):
 
     sitespeed_arg = f'--shm-size=1g {sitespeed_arg}'
 
-    if not ('nt' in os.name or 'Darwin' in os.uname().sysname):
+    if get_config('tests.sitespeed.xvfb'):
         sitespeed_arg += ' --xvfb'
 
     sitespeed_arg += ' --postScript chrome-cookies.cjs --postScript chrome-versions.cjs'

tests/tracking_validator.py

@@ -767,7 +767,7 @@ def get_rating_from_sitespeed(url, local_translation, global_translation):
 
     sitespeed_arg = '--shm-size=1g -b chrome --plugins.remove screenshot --plugins.remove html --plugins.remove metrics --browsertime.screenshot false --screenshot false --screenshotLCP false --browsertime.screenshotLCP false --chrome.cdp.performance false --browsertime.chrome.timeline false --videoParams.createFilmstrip false --visualMetrics false --visualMetricsPerceptual false --visualMetricsContentful false --browsertime.headless true --browsertime.chrome.includeResponseBodies all --utc true --browsertime.chrome.args ignore-certificate-errors -n {0}'.format(
         sitespeed_iterations)
-    if not ('nt' in os.name or 'Darwin' in os.uname().sysname):
+    if get_config('tests.sitespeed.xvfb'):
         sitespeed_arg += ' --xvfb'
 
     sitespeed_arg += ' --postScript chrome-cookies.cjs --postScript chrome-versions.cjs'

tests/w3c_base.py

@@ -108,7 +108,7 @@ def get_data_for_url(url):
             '--browsertime.chrome.includeResponseBodies all --utc true '
             '--browsertime.chrome.args ignore-certificate-errors '
             f'-n {sitespeed_iterations}')
-    if not ('nt' in os.name or 'Darwin' in os.uname().sysname):
+    if get_config('tests.sitespeed.xvfb'):
         sitespeed_arg += ' --xvfb'
 
     sitespeed_arg += ' --postScript chrome-cookies.cjs --postScript chrome-versions.cjs'

tools/dependency.py

@@ -7,6 +7,7 @@
 import sys
 from datetime import datetime
 import packaging.version
+from setting_helper import get_config
 
 def test_cmd(command):
     process_failsafe_timeout = 600
@@ -336,7 +337,7 @@ def check_browser(browser):
 
     sitespeed_arg = f'--shm-size=1g {sitespeed_arg}'
 
-    if not ('nt' in os.name or 'Darwin' in os.uname().sysname):
+    if get_config('tests.sitespeed.xvfb'):
         sitespeed_arg += ' --xvfb'
 
     sitespeed_arg += ' https://webperf.se'