From 88e4841cc290e6b8f2a652205fb185b2609ee3ba Mon Sep 17 00:00:00 2001 From: amimas Date: Mon, 16 Feb 2026 22:16:12 -0500 Subject: [PATCH 1/7] docs(testing): improve tox test execution and document workflow This improves the developer experience by making the testing workflow more discoverable and flexible. - Updates `tox.ini` to use `posargs` with default values for the test environments. This allows developers to run the full test suite by default, or easily specify a single test file to run. - Adds `description` fields to the unit and functional test environments in `tox.ini`. This makes instructions on how to run specific tests visible in the output of `tox list`. - Updates `CONTRIBUTING.rst` with concrete examples for running specific unit, API functional, and CLI functional tests, aligning the documentation with the new `tox` configuration. --- CONTRIBUTING.rst | 9 +++++++++ tox.ini | 14 +++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 9b07ada11..b0f55a52a 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -111,6 +111,9 @@ You need to install ``tox`` (``pip3 install tox``) to run tests and lint checks # run unit tests in one python environment only (useful for quick testing during development): tox -e py311 + # run a specific unit test file: + tox -e py311 -- tests/unit/objects/test_projects.py + # run unit and smoke tests in one python environment only tox -e py312,smoke @@ -148,9 +151,15 @@ To run these tests: # run the CLI tests: tox -e cli_func_v4 + # run a specific CLI functional test file: + tox -e cli_func_v4 -- tests/functional/cli/test_cli_v4.py + # run the python API tests: tox -e api_func_v4 + # run a specific API functional test file: + tox -e api_func_v4 -- tests/functional/api/test_projects.py + When developing tests it can be a little frustrating to wait for GitLab to spin up every run. To prevent the containers from being cleaned up afterwards, pass ``--keep-containers`` to pytest, i.e.: diff --git a/tox.ini b/tox.ini index 0ba295692..ad32d85ed 100644 --- a/tox.ini +++ b/tox.ini @@ -26,6 +26,7 @@ passenv = NO_COLOR PWD PY_COLORS + USER setenv = DOCS_SOURCE = docs DOCS_BUILD = build/sphinx/html @@ -35,10 +36,13 @@ usedevelop = True install_command = pip install {opts} {packages} -e . isolated_build = True + +[testenv:py{314,313,312,311,310}] +description = Runs unit tests. For a specific test: tox -e py311 -- deps = -r{toxinidir}/requirements.txt -r{toxinidir}/requirements-test.txt commands = - pytest tests/unit {posargs} + pytest {posargs:tests/unit} [testenv:black] basepython = python3 @@ -131,14 +135,18 @@ exclude_lines = return NotImplemented [testenv:cli_func_v4] +description = + Runs CLI functional tests. For a specific test: tox -e cli_func_v4 -- deps = -r{toxinidir}/requirements-docker.txt commands = - pytest --script-launch-mode=subprocess --cov --cov-report xml tests/functional/cli {posargs} + pytest --script-launch-mode=subprocess --cov --cov-report xml {posargs:tests/functional/cli} [testenv:api_func_v4] +description = + Runs API functional tests. For a specific test: tox -e api_func_v4 -- deps = -r{toxinidir}/requirements-docker.txt commands = - pytest --cov --cov-report xml tests/functional/api {posargs} + pytest --cov --cov-report xml {posargs:tests/functional/api} [testenv:smoke] deps = -r{toxinidir}/requirements-test.txt From bd6b425e865530b9f8d57aeeb602fb0695e113f4 Mon Sep 17 00:00:00 2001 From: amimas Date: Tue, 10 Mar 2026 11:59:36 -0400 Subject: [PATCH 2/7] fix(ci): install dependencies for coverage environment The `cover` environment was failing in CI with a "No such file or directory: 'pytest'" error. This occurred because the environment did not have a `deps` section to install the testing requirements. While this might work locally due to `tox` reusing environments, it fails in a clean CI run. This change adds the required dependencies to the `[testenv:cover]` section, ensuring it is self-contained and runs reliably. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index ad32d85ed..eec3d8e99 100644 --- a/tox.ini +++ b/tox.ini @@ -119,6 +119,8 @@ deps = -r{toxinidir}/requirements-docs.txt commands = sphinx-autobuild {env:DOCS_SOURCE} {env:DOCS_BUILD} --open-browser --port 8000 [testenv:cover] +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/requirements-test.txt commands = pytest --cov --cov-report term --cov-report html \ --cov-report xml tests/unit {posargs} From 389f2c99ef84da2cd24898c3ae76faa05f182ed0 Mon Sep 17 00:00:00 2001 From: amimas Date: Tue, 10 Mar 2026 14:19:58 -0400 Subject: [PATCH 3/7] fix(ci): install test dependencies for functional test envs The functional test environments (`cli_func_v4`, `api_func_v4`) were failing in CI with a `ValueError: no option named 'keep_containers'`. This occurred because the environments were missing the `requirements-test.txt` dependency, so `pytest` and its plugins (like `pytest-docker`) were not installed. A previous refactoring of `tox.ini` made the environments more isolated, exposing this latent dependency issue. This change adds the required test dependencies to the functional test environments, ensuring they are self-contained and run reliably. --- tox.ini | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index eec3d8e99..bfc11a969 100644 --- a/tox.ini +++ b/tox.ini @@ -139,14 +139,16 @@ exclude_lines = [testenv:cli_func_v4] description = Runs CLI functional tests. For a specific test: tox -e cli_func_v4 -- -deps = -r{toxinidir}/requirements-docker.txt +deps = -r{toxinidir}/requirements-test.txt + -r{toxinidir}/requirements-docker.txt commands = pytest --script-launch-mode=subprocess --cov --cov-report xml {posargs:tests/functional/cli} [testenv:api_func_v4] description = Runs API functional tests. For a specific test: tox -e api_func_v4 -- -deps = -r{toxinidir}/requirements-docker.txt +deps = -r{toxinidir}/requirements-test.txt + -r{toxinidir}/requirements-docker.txt commands = pytest --cov --cov-report xml {posargs:tests/functional/api} From 3ebd3ce2d84008693ca5ebb18e315dcd134f514d Mon Sep 17 00:00:00 2001 From: amimas Date: Sun, 22 Mar 2026 11:19:11 -0400 Subject: [PATCH 4/7] Revert "fix(ci): install dependencies for coverage environment" This reverts commit bd6b425e865530b9f8d57aeeb602fb0695e113f4. --- tox.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/tox.ini b/tox.ini index bfc11a969..0872d3355 100644 --- a/tox.ini +++ b/tox.ini @@ -119,8 +119,6 @@ deps = -r{toxinidir}/requirements-docs.txt commands = sphinx-autobuild {env:DOCS_SOURCE} {env:DOCS_BUILD} --open-browser --port 8000 [testenv:cover] -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/requirements-test.txt commands = pytest --cov --cov-report term --cov-report html \ --cov-report xml tests/unit {posargs} From 741325e39309b3c653b821c808699aabc725f5ce Mon Sep 17 00:00:00 2001 From: amimas Date: Sun, 22 Mar 2026 11:19:24 -0400 Subject: [PATCH 5/7] Revert "fix(ci): install test dependencies for functional test envs" This reverts commit 389f2c99ef84da2cd24898c3ae76faa05f182ed0. --- tox.ini | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 0872d3355..ad32d85ed 100644 --- a/tox.ini +++ b/tox.ini @@ -137,16 +137,14 @@ exclude_lines = [testenv:cli_func_v4] description = Runs CLI functional tests. For a specific test: tox -e cli_func_v4 -- -deps = -r{toxinidir}/requirements-test.txt - -r{toxinidir}/requirements-docker.txt +deps = -r{toxinidir}/requirements-docker.txt commands = pytest --script-launch-mode=subprocess --cov --cov-report xml {posargs:tests/functional/cli} [testenv:api_func_v4] description = Runs API functional tests. For a specific test: tox -e api_func_v4 -- -deps = -r{toxinidir}/requirements-test.txt - -r{toxinidir}/requirements-docker.txt +deps = -r{toxinidir}/requirements-docker.txt commands = pytest --cov --cov-report xml {posargs:tests/functional/api} From 1d49a9cb092b4f906200baeee668b89b57c87d94 Mon Sep 17 00:00:00 2001 From: amimas Date: Sun, 22 Mar 2026 11:22:31 -0400 Subject: [PATCH 6/7] revert: remove changes made to tox config No changes are needed to `tox` config to be able to run individual tests. --- tox.ini | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tox.ini b/tox.ini index ad32d85ed..0ba295692 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,6 @@ passenv = NO_COLOR PWD PY_COLORS - USER setenv = DOCS_SOURCE = docs DOCS_BUILD = build/sphinx/html @@ -36,13 +35,10 @@ usedevelop = True install_command = pip install {opts} {packages} -e . isolated_build = True - -[testenv:py{314,313,312,311,310}] -description = Runs unit tests. For a specific test: tox -e py311 -- deps = -r{toxinidir}/requirements.txt -r{toxinidir}/requirements-test.txt commands = - pytest {posargs:tests/unit} + pytest tests/unit {posargs} [testenv:black] basepython = python3 @@ -135,18 +131,14 @@ exclude_lines = return NotImplemented [testenv:cli_func_v4] -description = - Runs CLI functional tests. For a specific test: tox -e cli_func_v4 -- deps = -r{toxinidir}/requirements-docker.txt commands = - pytest --script-launch-mode=subprocess --cov --cov-report xml {posargs:tests/functional/cli} + pytest --script-launch-mode=subprocess --cov --cov-report xml tests/functional/cli {posargs} [testenv:api_func_v4] -description = - Runs API functional tests. For a specific test: tox -e api_func_v4 -- deps = -r{toxinidir}/requirements-docker.txt commands = - pytest --cov --cov-report xml {posargs:tests/functional/api} + pytest --cov --cov-report xml tests/functional/api {posargs} [testenv:smoke] deps = -r{toxinidir}/requirements-test.txt From 8bc75e380e12afd226e9c2bc87e94fc5b4a2ea75 Mon Sep 17 00:00:00 2001 From: amimas Date: Sun, 22 Mar 2026 13:29:50 -0400 Subject: [PATCH 7/7] docs: add instructions for passing pytest options Added a section into the contributing docs to show how to pass options to `pytest`. This can be useful during local development for selecting specific tests. Running entire test suite is very time consuming. Removed earlier instructions for the same purpose as they are no longer valid. --- CONTRIBUTING.rst | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index b0f55a52a..4da710499 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -111,9 +111,6 @@ You need to install ``tox`` (``pip3 install tox``) to run tests and lint checks # run unit tests in one python environment only (useful for quick testing during development): tox -e py311 - # run a specific unit test file: - tox -e py311 -- tests/unit/objects/test_projects.py - # run unit and smoke tests in one python environment only tox -e py312,smoke @@ -151,15 +148,9 @@ To run these tests: # run the CLI tests: tox -e cli_func_v4 - # run a specific CLI functional test file: - tox -e cli_func_v4 -- tests/functional/cli/test_cli_v4.py - # run the python API tests: tox -e api_func_v4 - # run a specific API functional test file: - tox -e api_func_v4 -- tests/functional/api/test_projects.py - When developing tests it can be a little frustrating to wait for GitLab to spin up every run. To prevent the containers from being cleaned up afterwards, pass ``--keep-containers`` to pytest, i.e.: @@ -203,6 +194,32 @@ To cleanup the environment delete the container: docker rm -f gitlab-test docker rm -f gitlab-runner-test +Pass options to ``pytest`` +-------------------------- + +Options to ``pytest`` can be passed by adding them after ``--`` when running ``tox``: + +.. code-block:: bash + + tox -e api_func_v4 -- . + +For example, you can use this to run a specific test. Running all tests can be time-consuming, +so this allows you to focus on just the tests relevant to your changes. You can do this by passing +the ``-k`` flag to ``pytest`` and setting a relevant expression to select the tests to run. For example: + +.. code-block:: bash + + # Run all API functional tests from the ``test_projects.py`` file: + tox -e api_func_v4 -- --keep-containers -k test_projects.py + + # Run only the ``test_get_project`` test method from the ``test_projects.py`` file: + tox -e api_func_v4 -- --keep-containers -k "test_projects.py and test_create_project" + + # The above will select all test methods start with ``test_create_project`` from the ``test_projects.py`` file. + # To select only the ``test_create_project`` method, you can exclude other methods by using the ``not`` operator: + tox -e api_func_v4 -- --keep-containers -k "test_projects.py and test_create_project and not test_create_project_" + + Rerunning failed CI workflows -----------------------------