diff --git a/ipatests/azure/azure-pipelines.yml b/ipatests/azure/azure-pipelines.yml index 89f81b87f..6d75956bd 100644 --- a/ipatests/azure/azure-pipelines.yml +++ b/ipatests/azure/azure-pipelines.yml @@ -128,8 +128,8 @@ jobs: - template: templates/test-jobs.yml parameters: - jobName: Base - jobTitle: Base tests + jobName: BASE_XMLRPC + jobTitle: BASE and XMLRPC tests testsToRun: - test_cmdline - test_install @@ -138,20 +138,12 @@ jobs: - test_ipaplatform - test_ipapython - test_ipatests_plugins - testsToIgnore: - - test_integration - - test_webui - - test_ipapython/test_keyring.py - taskToRun: run-tests - -- template: templates/test-jobs.yml - parameters: - jobName: XMLRPC - jobTitle: XMLRPC tests - testsToRun: - test_xmlrpc testsToIgnore: - test_integration - test_webui - test_ipapython/test_keyring.py + testsToDedicate: + - test_xmlrpc/test_dns_plugin.py taskToRun: run-tests + tasksParallel: 3 diff --git a/ipatests/azure/azure-run-tests.sh b/ipatests/azure/azure-run-tests.sh index aef13568e..c66e59028 100755 --- a/ipatests/azure/azure-run-tests.sh +++ b/ipatests/azure/azure-run-tests.sh @@ -6,6 +6,9 @@ server_password=Secret123 # Normalize spacing and expand the list afterwards. Remove {} for the single list element case tests_to_run=$(eval "echo {$(echo $TESTS_TO_RUN | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}') tests_to_ignore=$(eval "echo --ignore\ {$(echo $TESTS_TO_IGNORE | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}') +tests_to_dedicate= +[[ -n "$TESTS_TO_DEDICATE" ]] && \ +tests_to_dedicate=$(eval "echo --slice-dedicated={$(echo $TESTS_TO_DEDICATE | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}') systemctl --now enable firewalld echo "Installing FreeIPA master for the domain ${server_domain} and realm ${server_realm}" @@ -39,7 +42,11 @@ if [ "$install_result" -eq 0 ] ; then ipa-test-task --help ipa-run-tests --help - ipa-run-tests ${tests_to_ignore} --verbose --with-xunit '-k not test_dns_soa' ${tests_to_run} + ipa-run-tests ${tests_to_ignore} \ + ${tests_to_dedicate} \ + --slices=${SYSTEM_TOTALJOBSINPHASE:-1} \ + --slice-num=${SYSTEM_JOBPOSITIONINPHASE:-1} \ + --verbose --with-xunit '-k not test_dns_soa' ${tests_to_run} tests_result=$? else echo "ipa-server-install failed with code ${save_result}, skip IPA tests" diff --git a/ipatests/azure/templates/run-test.yml b/ipatests/azure/templates/run-test.yml index 1e4498419..301115f16 100644 --- a/ipatests/azure/templates/run-test.yml +++ b/ipatests/azure/templates/run-test.yml @@ -5,6 +5,7 @@ parameters: taskToRun: 'run-tests' testsToRun: '' testsToIgnore: '' + testsToDedicate: '' steps: - script: | @@ -22,7 +23,10 @@ steps: set -e docker exec --env TESTS_TO_RUN="${{ parameters.testsToRun }}" \ --env TESTS_TO_IGNORE="${{ parameters.testsToIgnore }}" \ + --env TESTS_TO_DEDICATE="${{ parameters.testsToDedicate }}" \ --env CI_RUNNER_LOGS_DIR="${{ parameters.logsPath }}" \ + --env SYSTEM_TOTALJOBSINPHASE=$(System.TotalJobsInPhase) \ + --env SYSTEM_JOBPOSITIONINPHASE=$(System.JobPositionInPhase) \ --privileged -t \ $(createContainer.containerName) \ /bin/bash --noprofile --norc -x /freeipa/ipatests/azure/azure-${{parameters.taskToRun}}.sh diff --git a/ipatests/azure/templates/test-jobs.yml b/ipatests/azure/templates/test-jobs.yml index c79058c9e..5b0173fdf 100644 --- a/ipatests/azure/templates/test-jobs.yml +++ b/ipatests/azure/templates/test-jobs.yml @@ -3,7 +3,9 @@ parameters: jobTitle: '' testsToIgnore: [] testsToRun: [] + testsToDedicate: [] taskToRun: '' + tasksParallel: 1 jobs: - job: ${{ parameters.jobName }} @@ -12,6 +14,8 @@ jobs: condition: succeeded() pool: vmImage: 'Ubuntu-16.04' + strategy: + parallel: ${{ parameters.tasksParallel }} steps: - template: setup-test-environment.yml - template: run-test.yml @@ -21,6 +25,7 @@ jobs: taskToRun: ${{ parameters.taskToRun}} testsToRun: ${{ join(' ', parameters.testsToRun ) }} testsToIgnore: ${{ join(' ', parameters.testsToIgnore ) }} + testsToDedicate: ${{ join(' ', parameters.testsToDedicate ) }} - task: PublishTestResults@2 inputs: testResultsFiles: $(CI_RUNNER_LOGS_DIR)/nosetests.xml @@ -28,4 +33,4 @@ jobs: condition: succeededOrFailed() - template: save-test-artifacts.yml parameters: - logsArtifact: logs-${{parameters.jobName}}-$(Build.BuildId)-$(Agent.OS)-$(Agent.OSArchitecture) + logsArtifact: logs-${{parameters.jobName}}-$(Build.BuildId)-$(System.JobPositionInPhase)-$(Agent.OS)-$(Agent.OSArchitecture) diff --git a/ipatests/conftest.py b/ipatests/conftest.py index ae523c88a..62045de31 100644 --- a/ipatests/conftest.py +++ b/ipatests/conftest.py @@ -25,6 +25,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) pytest_plugins = [ 'ipatests.pytest_ipa.additional_config', + 'ipatests.pytest_ipa.slicing', 'ipatests.pytest_ipa.beakerlib', 'ipatests.pytest_ipa.declarative', 'ipatests.pytest_ipa.nose_compat', diff --git a/ipatests/pytest_ipa/slicing.py b/ipatests/pytest_ipa/slicing.py new file mode 100644 index 000000000..172208514 --- /dev/null +++ b/ipatests/pytest_ipa/slicing.py @@ -0,0 +1,200 @@ +# +# Copyright (C) 2019 FreeIPA Contributors see COPYING for license +# + +""" +The main purpose of this plugin is to slice a test suite into +several pieces to run each within its own test environment(for example, +an Agent of Azure Pipelines). + +Tests within a slice are grouped by test modules because not all of the tests +within the module are independent from each other. + +Slices are balanced by the number of tests within test module. +* Actually, tests should be grouped by the execution duration. +This could be achieved by the caching of tests results. Azure Pipelines +caching is in development. * +To workaround slow tests a dedicated slice is added. + +:param slices: A total number of slices to split the test suite into +:param slice-num: A number of slice to run +:param slice-dedicated: A file path to the module to run in its own slice + +**Examples** + +Inputs: +ipa-run-tests test_cmdline --collectonly -qq +... +test_cmdline/test_cli.py: 39 +test_cmdline/test_help.py: 7 +test_cmdline/test_ipagetkeytab.py: 16 +... + +* Split tests into 2 slices and run the first one: + +ipa-run-tests --slices=2 --slice-num=1 test_cmdline + +The outcome would be: +... +Running slice: 1 (46 tests) +Modules: +test_cmdline/test_cli.py: 39 +test_cmdline/test_help.py: 7 +... + +* Split tests into 2 slices, move one module out to its own slice +and run the second one + +ipa-run-tests --slices=2 --slice-dedicated=test_cmdline/test_cli.py \ + --slice-num=2 test_cmdline + +The outcome would be: +... +Running slice: 2 (23 tests) +Modules: +test_cmdline/test_ipagetkeytab.py: 16 +test_cmdline/test_help.py: 7 +... + +""" +import pytest + + +def pytest_addoption(parser): + group = parser.getgroup("slicing") + group.addoption( + '--slices', dest='slices_num', type=int, + help='The number of slices to split the test suite into') + group.addoption( + '--slice-num', dest='slice_num', type=int, + help='The specific number of slice to run') + group.addoption( + '--slice-dedicated', action="append", dest='slices_dedicated', + help='The file path to the module to run in dedicated slice') + + +@pytest.hookimpl(hookwrapper=True) +def pytest_collection_modifyitems(session, config, items): + yield + slice_count = config.getoption('slices_num') + slice_id = config.getoption('slice_num') + modules_dedicated = config.getoption('slices_dedicated') + # deduplicate + if modules_dedicated: + modules_dedicated = list(set(modules_dedicated)) + + # sanity check + if not slice_count or not slice_id: + return + + # nothing to do + if slice_count == 1: + return + + if modules_dedicated and len(modules_dedicated) > slice_count: + raise ValueError( + "Dedicated slice number({}) shouldn't be greater than the number " + "of slices({})".format(len(modules_dedicated), slice_count)) + + if slice_id > slice_count: + raise ValueError( + "Slice number({}) shouldn't be greater than the number of slices" + "({})".format(slice_id, slice_count)) + + modules = [] + # Calculate modules within collection + # Note: modules within pytest collection could be placed in not consecutive + # order + for number, item in enumerate(items): + name = item.nodeid.split("::", 1)[0] + if not modules or name != modules[-1]["name"]: + modules.append({"name": name, "begin": number, "end": number}) + else: + modules[-1]["end"] = number + + if slice_count > len(modules): + raise ValueError( + "Total number of slices({}) shouldn't be greater than the number " + "of Python test modules({})".format(slice_count, len(modules))) + + slices_dedicated = [] + if modules_dedicated: + slices_dedicated = [ + [m] for m in modules for x in modules_dedicated if x in m["name"] + ] + if modules_dedicated and len(slices_dedicated) != len(modules_dedicated): + raise ValueError( + "The number of dedicated slices({}) should be equal to the " + "number of dedicated modules({})".format( + slices_dedicated, modules_dedicated)) + + if (slices_dedicated and len(slices_dedicated) == slice_count and + len(slices_dedicated) != len(modules)): + raise ValueError( + "The total number of slices({}) is not sufficient to run dedicated" + " modules({}) as well as usual ones({})".format( + slice_count, len(slices_dedicated), + len(modules) - len(slices_dedicated))) + + # remove dedicated modules from usual ones + for s in slices_dedicated: + for m in s: + if m in modules: + modules.remove(m) + + avail_slice_count = slice_count - len(slices_dedicated) + # initialize slices with empty lists + slices = [[] for i in range(slice_count)] + + # initialize slices with dedicated ones + for sn, s in enumerate(slices_dedicated): + slices[sn] = s + + # initial reverse sort by the number of tests in a test module + modules.sort(reverse=True, key=lambda x: x["end"] - x["begin"] + 1) + reverse = True + while modules: + for sslice_num, sslice in enumerate(sorted( + modules[:avail_slice_count], + reverse=reverse, key=lambda x: x["end"] - x["begin"] + 1)): + slices[len(slices_dedicated) + sslice_num].append(sslice) + + modules[:avail_slice_count] = [] + reverse = not reverse + + calc_ntests = sum(x["end"] - x["begin"] + 1 for s in slices for x in s) + assert calc_ntests == len(items) + assert len(slices) == slice_count + + # the range of the given argument `slice_id` begins with 1(one) + sslice = slices[slice_id - 1] + + new_items = [] + for m in sslice: + new_items += items[m["begin"]:m["end"] + 1] + items[:] = new_items + + tw = config.get_terminal_writer() + if tw: + tw.line() + tw.write( + "Running slice: {} ({} tests)\n".format( + slice_id, + len(items), + ), + cyan=True, + bold=True, + ) + tw.write( + "Modules:\n", + yellow=True, + bold=True, + ) + for module in sslice: + tw.write( + "{}: {}\n".format( + module["name"], + module["end"] - module["begin"] + 1), + yellow=True, + ) + tw.line() diff --git a/ipatests/test_ipatests_plugins/test_slicing.py b/ipatests/test_ipatests_plugins/test_slicing.py new file mode 100644 index 000000000..e21af33a1 --- /dev/null +++ b/ipatests/test_ipatests_plugins/test_slicing.py @@ -0,0 +1,127 @@ +# +# Copyright (C) 2019 FreeIPA Contributors see COPYING for license +# + +import glob + +import pytest + +MOD_NAME = "test_module_{}" +FUNC_NAME = "test_func_{}" +PYTEST_INTERNAL_ERROR = 3 +MODS_NUM = 5 + + +@pytest.fixture +def ipatestdir(testdir): + """ + Create MODS_NUM test modules within testdir. + Each module contains 1 test function. + """ + testdir.makeconftest( + """ + pytest_plugins = ["ipatests.pytest_ipa.slicing"] + """ + ) + for i in range(MODS_NUM): + testdir.makepyfile( + **{MOD_NAME.format(i): + """ + def {func}(): + pass + """.format(func=FUNC_NAME.format(i)) + } + ) + return testdir + + +@pytest.mark.parametrize( + "nslices,nslices_d,groups", + [(2, 0, [[x for x in range(MODS_NUM) if x % 2 == 0], + [x for x in range(MODS_NUM) if x % 2 != 0]]), + (2, 1, [[0], [x for x in range(1, MODS_NUM)]]), + (1, 0, [[x for x in range(MODS_NUM)]]), + (1, 1, [[x for x in range(MODS_NUM)]]), + (MODS_NUM, MODS_NUM, [[x] for x in range(MODS_NUM)]), + ]) +def test_slicing(ipatestdir, nslices, nslices_d, groups): + """ + Positive tests. + + Run `nslices` slices, including `nslices_d` dedicated slices. + The `groups` is an expected result of slices grouping. + + For example, there are 5 test modules. If one runs them in + two slices (without dedicated ones) the expected result will + be [[0, 2, 4], [1, 3]]. This means, that first slice will run + modules 0, 2, 4, second one - 1 and 3. + + Another example, there are 5 test modules. We want to run them + in two slices. Also we specify module 0 as dedicated. + The expected result will be [[0], [1, 2, 3, 4]], which means, that + first slice will run module 0, second one - 1, 2, 3, 4. + + If the given slice count is one, then this plugin does nothing. + """ + for sl in range(nslices): + args = [ + "-v", + "--slices={}".format(nslices), + "--slice-num={}".format(sl + 1) + ] + for dslice in range(nslices_d): + args.append( + "--slice-dedicated={}.py".format(MOD_NAME.format(dslice))) + result = ipatestdir.runpytest(*args) + assert result.ret == 0 + result.assert_outcomes(passed=len(groups[sl])) + for mod_num in groups[sl]: + result.stdout.fnmatch_lines(["*{mod}.py::{func} PASSED*".format( + mod=MOD_NAME.format(mod_num), + func=FUNC_NAME.format(mod_num))]) + + +@pytest.mark.parametrize( + "nslices,nslices_d,nslice,dmod,err_message", + [(2, 3, 1, None, + "Dedicated slice number({}) shouldn't be greater than" + " the number of slices({})".format(3, 2)), + (MODS_NUM, 0, MODS_NUM + 1, None, + "Slice number({}) shouldn't be greater than the number of slices" + "({})".format( + MODS_NUM + 1, MODS_NUM)), + (MODS_NUM + 1, 1, 1, None, + "Total number of slices({}) shouldn't be greater" + " than the number of Python test modules({})".format( + MODS_NUM + 1, MODS_NUM)), + (MODS_NUM, MODS_NUM, 1, "notexisted_module", + "The number of dedicated slices({}) should be equal to the " + "number of dedicated modules({})".format( + [], ["notexisted_module.py"])), + (MODS_NUM - 1, MODS_NUM - 1, 1, None, + "The total number of slices({}) is not sufficient to" + " run dedicated modules({}) as well as usual ones({})".format( + MODS_NUM - 1, MODS_NUM - 1, 1)), + ]) +def test_slicing_negative(ipatestdir, nslices, nslices_d, nslice, dmod, + err_message): + """ + Negative scenarios + """ + args = [ + "-v", + "--slices={}".format(nslices), + "--slice-num={}".format(nslice) + ] + if dmod is None: + for dslice in range(nslices_d): + args.append( + "--slice-dedicated={}.py".format(MOD_NAME.format(dslice))) + else: + args.append( + "--slice-dedicated={}.py".format(dmod)) + result = ipatestdir.runpytest(*args) + assert result.ret == PYTEST_INTERNAL_ERROR + result.assert_outcomes() + result.stdout.fnmatch_lines(["*ValueError: {err_message}*".format( + err_message=glob.escape(err_message))])