diff --git a/ipatests/azure/Dockerfile.build.fedora b/ipatests/azure/Dockerfiles/Dockerfile.build.fedora similarity index 100% rename from ipatests/azure/Dockerfile.build.fedora rename to ipatests/azure/Dockerfiles/Dockerfile.build.fedora diff --git a/ipatests/azure/docker-compose.yml b/ipatests/azure/Dockerfiles/docker-compose.yml similarity index 94% rename from ipatests/azure/docker-compose.yml rename to ipatests/azure/Dockerfiles/docker-compose.yml index e077db29d..778bfcc80 100644 --- a/ipatests/azure/docker-compose.yml +++ b/ipatests/azure/Dockerfiles/docker-compose.yml @@ -11,7 +11,7 @@ services: volumes: - /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd - ./ipa-test-config.yaml:/root/.ipa/ipa-test-config.yaml:ro - - ${BUILD_REPOSITORY_LOCALPATH}:/freeipa + - ${BUILD_REPOSITORY_LOCALPATH}:${IPA_TESTS_REPO_PATH} networks: - ${IPA_NETWORK} diff --git a/ipatests/azure/azure-pipelines.yml b/ipatests/azure/azure-pipelines.yml index 2be8bea8e..e35a822be 100644 --- a/ipatests/azure/azure-pipelines.yml +++ b/ipatests/azure/azure-pipelines.yml @@ -27,16 +27,36 @@ jobs: displayName: Quick code style check condition: eq(variables['Build.Reason'], 'PullRequest') - template: templates/${{ variables.BUILD_TEMPLATE }} + - template: templates/publish-build.yml + parameters: + artifactName: 'packages-$(Build.BuildId)-$(Agent.OS)-$(Agent.OSArchitecture)' + targetPath: $(Build.Repository.LocalPath)/dist + displayName: Publish packages + - script: | set -e mkdir container cp -pr dist container/ - cp ipatests/azure/$(DOCKER_DOCKERFILE) container/Dockerfile + cp $(IPA_TESTS_DOCKERFILES)/$(DOCKER_DOCKERFILE) container/Dockerfile cd container docker build -t freeipa-azure-builder . docker save freeipa-azure-builder | gzip > '$(builddir)/freeipa-azure-builder-container.tar.gz' displayName: Create container image for test - template: templates/publish-build.yml + parameters: + artifactName: 'image-$(Build.BuildId)-$(Agent.OS)-$(Agent.OSArchitecture)' + targetPath: $(Build.Repository.LocalPath)/freeipa-azure-builder-container.tar.gz + displayName: Publish container image + - template: templates/generate-matrix.yml + parameters: + definition: 'ipatests/azure/azure_definitions/gating.yml' + displayName: Generate Matrix for Gating tests + name: gating_matrix + - template: templates/generate-matrix.yml + parameters: + definition: 'ipatests/azure/azure_definitions/base.yml' + displayName: Generate Matrix for Base tests + name: base_matrix - job: Lint pool: @@ -121,27 +141,10 @@ jobs: dependsOn: Build condition: succeeded() strategy: - parallel: 3 + matrix: $[ dependencies.Build.outputs['base_matrix.matrix'] ] steps: - - template: templates/test-jobs.yml - parameters: - testsToRun: - - test_cmdline - - test_install - - test_ipaclient - - test_ipalib - - test_ipaplatform - - test_ipapython - - test_ipaserver - - test_ipatests_plugins - - test_xmlrpc - testsToIgnore: - - test_integration - - test_webui - - test_ipapython/test_keyring.py - testsToDedicate: - - test_xmlrpc/test_dns_plugin.py - taskToRun: run-tests + - template: templates/generate-job-variables.yml + - template: templates/test-jobs.yml - job: GATING pool: @@ -149,25 +152,8 @@ jobs: dependsOn: Build condition: succeeded() strategy: - matrix: - installation_TestInstallMaster: - testsToRun: test_integration/test_installation.py::TestInstallMaster - kerberos_flags: - testsToRun: test_integration/test_kerberos_flags.py - testsClients: 1 + matrix: $[ dependencies.Build.outputs['gating_matrix.matrix'] ] timeoutInMinutes: 90 - steps: - - script: | - set -e - echo "##vso[task.setvariable variable=ntestsClients]${TESTSCLIENTS:-0}" - echo "##vso[task.setvariable variable=ntestsReplicas]${TESTSREPLICAS:-0}" - displayName: Generate variables - - - template: templates/test-jobs.yml - parameters: - taskToRun: run-integration-tests - testsToRun: "$(testsToRun)" - topology: - clients: "$(ntestsClients)" - replicas: "$(ntestsReplicas)" + - template: templates/generate-job-variables.yml + - template: templates/test-jobs.yml diff --git a/ipatests/azure/azure-run-integration-tests.sh b/ipatests/azure/azure-run-integration-tests.sh deleted file mode 100755 index 55933a8b7..000000000 --- a/ipatests/azure/azure-run-integration-tests.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -ex - -# Normalize spacing and expand the list afterwards. Remove {} for the single list element case -tests_to_run=$(eval "echo {$(echo $TESTS_TO_RUN | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}') -tests_to_ignore= -[[ -n "$TESTS_TO_IGNORE" ]] && \ -tests_to_ignore=$(eval "echo --ignore\ {$(echo $TESTS_TO_IGNORE | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}') -tests_to_dedicate= -[[ -n "$TESTS_TO_DEDICATE" ]] && \ -tests_to_dedicate=$(eval "echo --slice-dedicated={$(echo $TESTS_TO_DEDICATE | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}') - -tests_dir="/freeipa/$CI_RUNNER_LOGS_DIR" -mkdir -p "$tests_dir" -cd "$tests_dir" - -export IPATEST_YAML_CONFIG=~/.ipa/ipa-test-config.yaml - -echo "Run IPA tests" -ipa-run-tests \ - ${tests_to_ignore} \ - ${tests_to_dedicate} \ - --slices=${SYSTEM_TOTALJOBSINPHASE:-1} \ - --slice-num=${SYSTEM_JOBPOSITIONINPHASE:-1} \ - --logging-level=debug \ - --logfile-dir="$tests_dir" \ - --verbose --with-xunit ${tests_to_run} - -tests_result=$? -find "$tests_dir" -mindepth 1 -maxdepth 1 -not -name '.*' -type d \ - -exec tar --remove-files -czf {}.tar.gz {} \; - -exit $tests_result diff --git a/ipatests/azure/azure-run-tests.sh b/ipatests/azure/azure-run-tests.sh deleted file mode 100755 index aefc5c970..000000000 --- a/ipatests/azure/azure-run-tests.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash -ex - -server_realm=IPA.TEST -server_domain=ipa.test -server_password=Secret123 - -# Normalize spacing and expand the list afterwards. Remove {} for the single list element case -tests_to_run=$(eval "echo {$(echo $TESTS_TO_RUN | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}') -tests_to_ignore= -[[ -n "$TESTS_TO_IGNORE" ]] && \ -tests_to_ignore=$(eval "echo --ignore\ {$(echo $TESTS_TO_IGNORE | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}') -tests_to_dedicate= -[[ -n "$TESTS_TO_DEDICATE" ]] && \ -tests_to_dedicate=$(eval "echo --slice-dedicated={$(echo $TESTS_TO_DEDICATE | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}') - -systemctl --now enable firewalld -echo "Installing FreeIPA master for the domain ${server_domain} and realm ${server_realm}" -ipa-server-install -U --domain ${server_domain} --realm ${server_realm} \ - -p ${server_password} -a ${server_password} \ - --setup-dns --setup-kra --auto-forwarders - -install_result=$? - -tests_result=1 - -mkdir -p /freeipa/$CI_RUNNER_LOGS_DIR -cd /freeipa/$CI_RUNNER_LOGS_DIR - -if [ "$install_result" -eq 0 ] ; then - echo "Run IPA tests" - echo "Installation complete. Performance of individual steps:" - grep 'service duration:' /var/log/ipaserver-install.log | sed -e 's/DEBUG //g' - - sed -ri "s/mode = production/mode = development/" /etc/ipa/default.conf - systemctl restart httpd.service - firewall-cmd --add-service={freeipa-ldap,freeipa-ldaps,dns} - - echo ${server_password} | kinit admin && ipa ping - mkdir -p ~/.ipa - cp -r /etc/ipa/* ~/.ipa/ - echo ${server_password} > ~/.ipa/.dmpw - echo 'wait_for_dns=5' >> ~/.ipa/default.conf - - ipa-test-config --help - ipa-test-task --help - ipa-run-tests --help - - ipa-run-tests \ - ${tests_to_ignore} \ - ${tests_to_dedicate} \ - --slices=${SYSTEM_TOTALJOBSINPHASE:-1} \ - --slice-num=${SYSTEM_JOBPOSITIONINPHASE:-1} \ - --verbose --with-xunit '-k not test_dns_soa' ${tests_to_run} - tests_result=$? -else - echo "ipa-server-install failed with code ${save_result}, skip IPA tests" -fi - -echo "Potential Python 3 incompatibilities in the IPA framework:" -grep -n -C5 BytesWarning /var/log/httpd/error_log || echo "Good, none detected" - -echo "State of the directory server instance, httpd databases, PKI CA database:" -ls -laZ /etc/dirsrv/slapd-*/ /etc/httpd/alias/ /var/lib/ /etc/pki/pki-tomcat/alias/ || true -ls -laZ /var/lib/ipa/certs/ /var/lib/ipa/passwds/ /var/lib/ipa/private/ || true - -echo "Uninstall the server" -ipa-server-install --uninstall -U -# second uninstall to verify that --uninstall without installation works -ipa-server-install --uninstall -U - - -if [ "$install_result" -eq 0 ] ; then - firewall-cmd --remove-service={freeipa-ldap,freeipa-ldaps,dns} -fi - -echo "Collect the logs" -journalctl -b --no-pager > systemd_journal.log -tar --ignore-failed-read --remove-files -czf var_log.tar.gz \ - /var/log/dirsrv \ - /var/log/httpd \ - /var/log/ipa* \ - /var/log/krb5kdc.log \ - /var/log/pki \ - /var/log/samba \ - /var/named/data \ - systemd_journal.log - -# Final result depends on the exit code of the ipa-run-tests -test "$tests_result" -eq 0 -a "$install_result" -eq 0 diff --git a/ipatests/azure/azure_definitions/base-fedora.yml b/ipatests/azure/azure_definitions/base-fedora.yml new file mode 100644 index 000000000..37c86693d --- /dev/null +++ b/ipatests/azure/azure_definitions/base-fedora.yml @@ -0,0 +1,23 @@ +vms: +- vm_jobs: + - container_job: base + tests: + - test_cmdline + - test_install + - test_ipaclient + - test_ipalib + - test_ipaplatform + - test_ipapython + - test_ipaserver + - test_ipatests_plugins + - test_xmlrpc/test_dns_plugin.py + ignore: + - test_ipapython/test_keyring.py + type: base + + - container_job: xmlrpc + tests: + - test_xmlrpc + ignore: + - test_xmlrpc/test_dns_plugin.py + type: base diff --git a/ipatests/azure/azure_definitions/base.yml b/ipatests/azure/azure_definitions/base.yml new file mode 120000 index 000000000..d156a512f --- /dev/null +++ b/ipatests/azure/azure_definitions/base.yml @@ -0,0 +1 @@ +base-fedora.yml \ No newline at end of file diff --git a/ipatests/azure/azure_definitions/gating-fedora.yml b/ipatests/azure/azure_definitions/gating-fedora.yml new file mode 100644 index 000000000..72312009f --- /dev/null +++ b/ipatests/azure/azure_definitions/gating-fedora.yml @@ -0,0 +1,84 @@ +vms: +- vm_jobs: + - container_job: InstallMaster + tests: + - test_integration/test_installation.py::TestInstallMaster + + - container_job: kerberos_flags + containers: + clients: 1 + tests: + - test_integration/test_kerberos_flags.py + + - container_job: forced_client_reenrollment + containers: + replicas: 1 + clients: 1 + tests: + - test_integration/test_forced_client_reenrollment.py + + - container_job: ExternalCAInstall + tests: + - test_integration/test_external_ca.py::TestExternalCAInstall + +- vm_jobs: + - container_job: InstallDNSSECFirst + containers: + replicas: 1 + tests: + - test_integration/test_dnssec.py::TestInstallDNSSECFirst + + - container_job: simple_replication + containers: + replicas: 1 + tests: + - test_integration/test_simple_replication.py + + - container_job: service_permissions + tests: + - test_integration/test_service_permissions.py + + - container_job: netgroup + tests: + - test_integration/test_netgroup.py + +- vm_jobs: + - container_job: sudo + containers: + clients: 1 + tests: + - test_integration/test_sudo.py + + - container_job: ExternalCA + containers: + replicas: 1 + clients: 1 + tests: + - test_integration/test_external_ca.py::TestExternalCA + + # requires nothing, installs nothing + - container_job: topologies_and_testconfig + tests: + - test_integration/test_topologies.py + - test_integration/test_testconfig.py + + - container_job: SelfExternalSelf + tests: + - test_integration/test_external_ca.py::TestSelfExternalSelf + +- vm_jobs: + - container_job: commands + tests: + - test_integration/test_commands.py + + - container_job: ServerReplicaCALessToCAFull + tests: + - test_integration/test_caless.py::TestServerReplicaCALessToCAFull + containers: + replicas: 1 + + - container_job: SubCAkeyReplication + containers: + replicas: 1 + tests: + - test_integration/test_replica_promotion.py::TestSubCAkeyReplication diff --git a/ipatests/azure/azure_definitions/gating.yml b/ipatests/azure/azure_definitions/gating.yml new file mode 120000 index 000000000..d5bb1269c --- /dev/null +++ b/ipatests/azure/azure_definitions/gating.yml @@ -0,0 +1 @@ +gating-fedora.yml \ No newline at end of file diff --git a/ipatests/azure/scripts/azure-run-base-tests.sh b/ipatests/azure/scripts/azure-run-base-tests.sh new file mode 100755 index 000000000..6c80af1bb --- /dev/null +++ b/ipatests/azure/scripts/azure-run-base-tests.sh @@ -0,0 +1,96 @@ +#!/bin/bash -eux + +# this script is intended to be run within container +# +# distro-specifics +source "${IPA_TESTS_SCRIPTS}/variables.sh" + +server_password=Secret123 + +echo "Installing FreeIPA master for the domain ${IPA_TESTS_DOMAIN} and realm ${IPA_TESTS_REALM}" + +install_result=1 +{ ipa-server-install -U \ + --domain "$IPA_TESTS_DOMAIN" \ + --realm "$IPA_TESTS_REALM" \ + -p "$server_password" -a "$server_password" \ + --setup-dns --setup-kra --auto-forwarders && install_result=0 ; } || \ + install_result=$? + +rm -rf "$IPA_TESTS_LOGSDIR" +mkdir "$IPA_TESTS_LOGSDIR" +pushd "$IPA_TESTS_LOGSDIR" +tests_result=1 + +if [ "$install_result" -eq 0 ] ; then + echo "Run IPA tests" + echo "Installation complete. Performance of individual steps:" + grep 'service duration:' /var/log/ipaserver-install.log | sed -e 's/DEBUG //g' + + sed -ri "s/mode = production/mode = development/" /etc/ipa/default.conf + systemctl restart "$HTTPD_SYSTEMD_NAME" + firewalld_cmd --add-service={freeipa-ldap,freeipa-ldaps,dns} + + echo ${server_password} | kinit admin && ipa ping + mkdir -p ~/.ipa + cp -r /etc/ipa/* ~/.ipa/ + echo ${server_password} > ~/.ipa/.dmpw + echo 'wait_for_dns=5' >> ~/.ipa/default.conf + + ipa-test-config --help + ipa-test-task --help + ipa-run-tests --help + + { ipa-run-tests \ + --logging-level=debug \ + --logfile-dir="$IPA_TESTS_LOGSDIR" \ + --verbose \ + --with-xunit \ + '-k not test_dns_soa' \ + $IPA_TESTS_TO_IGNORE \ + $IPA_TESTS_TO_RUN && tests_result=0 ; } || \ + tests_result=$? +else + echo "ipa-server-install failed with code ${install_result}, skip IPA tests" +fi + +echo "Potential Python 3 incompatibilities in the IPA framework:" +grep -n -C5 BytesWarning "$HTTPD_ERRORLOG" || echo "Good, none detected" + +echo "State of the directory server instance, httpd databases, PKI CA database:" +ls -laZ \ + /etc/dirsrv/slapd-*/ \ + "${HTTPD_ALIASDIR}/" \ + /var/lib/ \ + /etc/pki/pki-tomcat/alias/ \ + ||: +ls -laZ \ + /var/lib/ipa/certs/ \ + /var/lib/ipa/passwds/ \ + /var/lib/ipa/private/ \ + ||: + +echo "Uninstall the server" +ipa-server-install --uninstall -U +# second uninstall to verify that --uninstall without installation works +ipa-server-install --uninstall -U + + +if [ "$install_result" -eq 0 ] ; then + firewalld_cmd --remove-service={freeipa-ldap,freeipa-ldaps,dns} +fi + +echo "Collect the logs" +journalctl -b --no-pager > systemd_journal.log +tar --ignore-failed-read --remove-files -czf var_log.tar.gz \ + /var/log/dirsrv \ + "$HTTPD_LOGDIR" \ + /var/log/ipa* \ + /var/log/krb5kdc.log \ + /var/log/pki \ + /var/log/samba \ + "$BIND_DATADIR" \ + systemd_journal.log + +# Final result depends on the exit code of the ipa-run-tests +test "$tests_result" -eq 0 -a "$install_result" -eq 0 diff --git a/ipatests/azure/scripts/azure-run-integration-tests.sh b/ipatests/azure/scripts/azure-run-integration-tests.sh new file mode 100755 index 000000000..bf1114050 --- /dev/null +++ b/ipatests/azure/scripts/azure-run-integration-tests.sh @@ -0,0 +1,29 @@ +#!/bin/bash -eux + +# this script is intended to be run within container +# +# distro-specifics +source "${IPA_TESTS_SCRIPTS}/variables.sh" + +rm -rf "$IPA_TESTS_LOGSDIR" +mkdir "$IPA_TESTS_LOGSDIR" +pushd "$IPA_TESTS_LOGSDIR" + +tests_result=1 +{ IPATEST_YAML_CONFIG=~/.ipa/ipa-test-config.yaml \ + ipa-run-tests \ + --logging-level=debug \ + --logfile-dir="$IPA_TESTS_LOGSDIR" \ + --with-xunit \ + --verbose \ + $IPA_TESTS_TO_IGNORE \ + $IPA_TESTS_TO_RUN && tests_result=0 ; } || \ + tests_result=$? + +# fix permissions on logs to be readable by Azure's user (vsts) +chmod -R o+rX "$IPA_TESTS_LOGSDIR" + +find "$IPA_TESTS_LOGSDIR" -mindepth 1 -maxdepth 1 -not -name '.*' -type d \ + -exec tar --remove-files -czf {}.tar.gz {} \; + +exit $tests_result diff --git a/ipatests/azure/scripts/azure-run-tests.sh b/ipatests/azure/scripts/azure-run-tests.sh new file mode 100755 index 000000000..d221e283d --- /dev/null +++ b/ipatests/azure/scripts/azure-run-tests.sh @@ -0,0 +1,96 @@ +#!/bin/bash -eux + +if [ $# -ne 1 ]; then + echo "Docker environment ID is not provided" + exit 1 +fi + +PROJECT_ID="$1" +BUILD_REPOSITORY_LOCALPATH="${BUILD_REPOSITORY_LOCALPATH:-$(realpath .)}" + +IPA_TESTS_TO_RUN_VARNAME="IPA_TESTS_TO_RUN_${PROJECT_ID}" +IPA_TESTS_TO_RUN="${!IPA_TESTS_TO_RUN_VARNAME:-}" +# in case of missing explicit list of tests to be run the Pytest run all the +# discovered tests, this is an error for this CI +[ -z "$IPA_TESTS_TO_RUN" ] && { echo 'Nothing to test'; exit 1; } + +IPA_TESTS_ENV_NAME_VARNAME="IPA_TESTS_ENV_NAME_${PROJECT_ID}" +IPA_TESTS_ENV_NAME="${!IPA_TESTS_ENV_NAME_VARNAME:-}" +[ -z "$IPA_TESTS_ENV_NAME" ] && \ + { echo "Project name is not set for project:${PROJECT_ID}"; exit 1 ;} + +IPA_TESTS_TYPE_VARNAME="IPA_TESTS_TYPE_${PROJECT_ID}" +IPA_TESTS_TYPE="${!IPA_TESTS_TYPE_VARNAME:-integration}" + +# Normalize spacing and expand the list afterwards. Remove {} for the single list element case +IPA_TESTS_TO_RUN=$(eval "echo {$(echo $IPA_TESTS_TO_RUN | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}') + +IPA_TESTS_TO_IGNORE_VARNAME="IPA_TESTS_TO_IGNORE_${PROJECT_ID}" +IPA_TESTS_TO_IGNORE="${!IPA_TESTS_TO_IGNORE_VARNAME:-}" +[ -n "$IPA_TESTS_TO_IGNORE" ] && \ +IPA_TESTS_TO_IGNORE=$(eval "echo --ignore\ {$(echo $IPA_TESTS_TO_IGNORE | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}') + +IPA_TESTS_CLIENTS_VARNAME="IPA_TESTS_CLIENTS_${PROJECT_ID}" +IPA_TESTS_CLIENTS="${!IPA_TESTS_CLIENTS_VARNAME:-0}" + +IPA_TESTS_REPLICAS_VARNAME="IPA_TESTS_REPLICAS_${PROJECT_ID}" +IPA_TESTS_REPLICAS="${!IPA_TESTS_REPLICAS_VARNAME:-0}" + +IPA_TESTS_CONTROLLER="${PROJECT_ID}_master_1" +IPA_TESTS_LOGSDIR="${IPA_TESTS_REPO_PATH}/ipa_envs/${IPA_TESTS_ENV_NAME}/${CI_RUNNER_LOGS_DIR}" + +IPA_TESTS_DOMAIN="${IPA_TESTS_DOMAIN:-ipa.test}" +# bash4 +IPA_TESTS_REALM="${IPA_TESTS_DOMAIN^^}" + +# for base tests only 1 master is needed even if another was specified +if [ "$IPA_TESTS_TYPE" == "base" ]; then + IPA_TESTS_CLIENTS="0" + IPA_TESTS_REPLICAS="0" +fi + +project_dir="${IPA_TESTS_ENV_WORKING_DIR}/${IPA_TESTS_ENV_NAME}" +ln -sfr \ + "${IPA_TESTS_DOCKERFILES}/docker-compose.yml" \ + "$project_dir"/ + +# will be generated later in setup_containers.py +touch "${project_dir}"/ipa-test-config.yaml + +pushd "$project_dir" + +BUILD_REPOSITORY_LOCALPATH="$BUILD_REPOSITORY_LOCALPATH" \ +IPA_DOCKER_IMAGE="${IPA_DOCKER_IMAGE:-freeipa-azure-builder}" \ +IPA_NETWORK="${IPA_NETWORK:-ipanet}" \ +IPA_IPV6_SUBNET="2001:db8:1:${PROJECT_ID}::/64" \ +docker-compose -p "$PROJECT_ID" up \ + --scale replica="$IPA_TESTS_REPLICAS" \ + --scale client="$IPA_TESTS_CLIENTS" \ + --force-recreate --remove-orphans -d + +popd + +IPA_TESTS_CLIENTS="$IPA_TESTS_CLIENTS" \ +IPA_TESTS_REPLICAS="$IPA_TESTS_REPLICAS" \ +IPA_TESTS_ENV_ID="$PROJECT_ID" \ +IPA_TESTS_ENV_WORKING_DIR="$IPA_TESTS_ENV_WORKING_DIR" \ +IPA_TESTS_ENV_NAME="$IPA_TESTS_ENV_NAME" \ +IPA_TEST_CONFIG_TEMPLATE="${BUILD_REPOSITORY_LOCALPATH}/ipatests/azure/templates/ipa-test-config-template.yaml" \ +IPA_TESTS_REPO_PATH="$IPA_TESTS_REPO_PATH" \ +IPA_TESTS_DOMAIN="$IPA_TESTS_DOMAIN" \ +python3 setup_containers.py + +# path to runner within container +tests_runner="${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}/azure-run-${IPA_TESTS_TYPE}-tests.sh" + +docker exec -t \ + --env IPA_TESTS_SCRIPTS="${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}" \ + --env IPA_PLATFORM="$IPA_PLATFORM" \ + --env IPA_TESTS_DOMAIN="$IPA_TESTS_DOMAIN" \ + --env IPA_TESTS_REALM="$IPA_TESTS_REALM" \ + --env IPA_TESTS_LOGSDIR="$IPA_TESTS_LOGSDIR" \ + --env IPA_TESTS_TO_RUN="$IPA_TESTS_TO_RUN" \ + --env IPA_TESTS_TO_IGNORE="$IPA_TESTS_TO_IGNORE" \ + "$IPA_TESTS_CONTROLLER" \ + /bin/bash --noprofile --norc \ + -eux "$tests_runner" diff --git a/ipatests/azure/scripts/generate-matrix.py b/ipatests/azure/scripts/generate-matrix.py new file mode 100644 index 000000000..864e746d1 --- /dev/null +++ b/ipatests/azure/scripts/generate-matrix.py @@ -0,0 +1,41 @@ +import argparse +import json + +import yaml + +parser = argparse.ArgumentParser(description='Generate Azure jobs matrix.') +parser.add_argument('azure_template', help='path to Azure template') + +args = parser.parse_args() + +with open(args.azure_template) as f: + data = yaml.safe_load(f) + matrix_jobs = {} + for vm in data['vms']: + jobs = {} + job_name = '' + for job_id, vm_job in enumerate(vm['vm_jobs'], 1): + if not job_name: + job_name = f'{vm_job["container_job"]}_{job_id}' + jobs[f'ipa_tests_env_name_{job_id}'] = vm_job['container_job'] + jobs[f'ipa_tests_to_run_{job_id}'] = ' '.join(vm_job['tests']) + jobs[f'ipa_tests_to_ignore_{job_id}'] = ' '.join( + vm_job.get('ignore', '')) + jobs[f'ipa_tests_type_{job_id}'] = vm_job.get( + 'type', 'integration') + + containers = vm_job.get('containers') + replicas = 0 + clients = 0 + if containers: + replicas = containers.get('replicas', 0) + clients = containers.get('clients', 0) + jobs[f'ipa_tests_replicas_{job_id}'] = replicas + jobs[f'ipa_tests_clients_{job_id}'] = clients + + job_name = f'{job_name}_to_{len(vm["vm_jobs"])}' + if job_name in matrix_jobs: + raise ValueError(f"Environment names should be unique:{job_name}") + matrix_jobs[job_name] = jobs + print("##vso[task.setVariable variable=matrix;isOutput=true]" + + json.dumps(matrix_jobs)) diff --git a/ipatests/azure/setup_containers.py b/ipatests/azure/scripts/setup_containers.py similarity index 86% rename from ipatests/azure/setup_containers.py rename to ipatests/azure/scripts/setup_containers.py index adedcc19b..fd42519cc 100644 --- a/ipatests/azure/setup_containers.py +++ b/ipatests/azure/scripts/setup_containers.py @@ -7,16 +7,20 @@ from jinja2 import Template logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') -IPA_CLIENT_NUM = int(os.environ.get('IPA_CLIENT_NUM', 0)) -IPA_REPLICA_NUM = int(os.environ.get('IPA_REPLICA_NUM', 0)) -IPA_CONT_PREFIX = os.environ.get('IPA_CONT_PREFIX', '1') -IPA_TEST_DOMAIN = os.environ.get('IPA_TEST_DOMAIN', 'ipa.test') +IPA_TESTS_ENV_WORKING_DIR = os.environ.get('IPA_TESTS_ENV_WORKING_DIR') +IPA_TESTS_ENV_NAME = os.environ.get('IPA_TESTS_ENV_NAME') +IPA_TESTS_ENV_ID = os.environ.get('IPA_TESTS_ENV_ID', '1') +IPA_TESTS_CLIENTS = int(os.environ.get('IPA_TESTS_CLIENTS', 0)) +IPA_TESTS_REPLICAS = int(os.environ.get('IPA_TESTS_REPLICAS', 0)) +IPA_TESTS_DOMAIN = os.environ.get('IPA_TESTS_DOMAIN', 'ipa.test') IPA_SSH_PRIV_KEY = os.environ.get('IPA_SSH_PRIV_KEY', '/root/.ssh/id_rsa') IPA_DNS_FORWARDER = os.environ.get('IPA_DNS_FORWARDER', '8.8.8.8') IPA_NETWORK = os.environ.get('IPA_NETWORK', 'ipanet') -IPA_CONTROLLER = os.environ.get('IPA_CONTROLLER', 'master') +IPA_CONTROLLER_TYPE = os.environ.get('IPA_CONTROLLER_TYPE', 'master') +IPA_TEST_CONFIG_TEMPLATE = os.environ.get( + 'IPA_TEST_CONFIG_TEMPLATE', './templates/ipa-test-config-template.yaml') -CONTAINER_DIR = "container_{}".format(IPA_CONT_PREFIX) +IPA_TESTS_ENV_DIR = os.path.join(IPA_TESTS_ENV_WORKING_DIR, IPA_TESTS_ENV_NAME) IPA_TEST_CONFIG = "ipa-test-config.yaml" @@ -25,7 +29,7 @@ class Container: Represents group of Docker container """ def __init__(self, role, dns=IPA_DNS_FORWARDER, num=1, - prefix=IPA_CONT_PREFIX, domain=IPA_TEST_DOMAIN): + prefix=IPA_TESTS_ENV_ID, domain=IPA_TESTS_DOMAIN): self.role = role self.num = num self.prefix = prefix @@ -57,7 +61,7 @@ class Container: """ ipv4 address of container """ - ipanet = '{}_{}'.format(IPA_CONT_PREFIX, IPA_NETWORK) + ipanet = '{}_{}'.format(IPA_TESTS_ENV_ID, IPA_NETWORK) dcont = self.dclient.containers.get(name) return dcont.attrs['NetworkSettings']['Networks'][ipanet]['IPAddress'] @@ -177,7 +181,7 @@ class Controller(Container): """ Manages groups of containers """ - def __init__(self, contr_type=IPA_CONTROLLER): + def __init__(self, contr_type=IPA_CONTROLLER_TYPE): self.containers = [] self.contr_type = contr_type if self.contr_type == 'master': @@ -253,20 +257,20 @@ class Controller(Container): container.setup_resolvconf() def generate_ipa_test_config(self, config): - template = 'ipa-test-config-template.yaml' - with open(os.path.join('./templates', template), 'r') as f: + with open(IPA_TEST_CONFIG_TEMPLATE, 'r') as f: + # assert foobar template = Template(f.read(), trim_blocks=True, lstrip_blocks=True) print(template.render(config)) - with open(os.path.join(CONTAINER_DIR, IPA_TEST_CONFIG), 'w') as f: + with open(os.path.join(IPA_TESTS_ENV_DIR, IPA_TEST_CONFIG), 'w') as f: f.write(template.render(config)) controller = Controller() master = Container(role='master') -clients = Container(role='client', num=IPA_CLIENT_NUM, dns=master.ips[0]) -replicas = Container(role='replica', num=IPA_REPLICA_NUM, dns=master.ips[0]) +clients = Container(role='client', num=IPA_TESTS_CLIENTS, dns=master.ips[0]) +replicas = Container(role='replica', num=IPA_TESTS_REPLICAS, dns=master.ips[0]) controller.append(master) controller.append(clients) @@ -280,7 +284,7 @@ controller.setup_resolvconf() config = { 'dns_forwarder': IPA_DNS_FORWARDER, 'ssh_private_key': IPA_SSH_PRIV_KEY, - 'domain_name': IPA_TEST_DOMAIN, + 'domain_name': IPA_TESTS_DOMAIN, 'master': master.ips, 'replicas': replicas.ips, 'clients': clients.ips, diff --git a/ipatests/azure/scripts/variables-fedora.sh b/ipatests/azure/scripts/variables-fedora.sh new file mode 100755 index 000000000..41d7d888f --- /dev/null +++ b/ipatests/azure/scripts/variables-fedora.sh @@ -0,0 +1,7 @@ +#!/bin/bash -eux + +# Put the platform-specific definitions here + +function firewalld_cmd() { + firewall-cmd $@ +} diff --git a/ipatests/azure/scripts/variables.sh b/ipatests/azure/scripts/variables.sh new file mode 100755 index 000000000..21181bf75 --- /dev/null +++ b/ipatests/azure/scripts/variables.sh @@ -0,0 +1,14 @@ +#!/bin/bash -eu + +HTTPD_SYSTEMD_NAME='httpd.service' +HTTPD_LOGDIR='/var/log/httpd' +HTTPD_ERRORLOG="${HTTPD_LOGDIR}/error_log" +HTTPD_BASEDIR='/etc/httpd' +HTTPD_ALIASDIR="${HTTPD_BASEDIR}/alias" +BIND_BASEDIR='/var/named' +BIND_DATADIR="${BIND_BASEDIR}/data" + +function firewalld_cmd() { :; } + +# this should be the last to override base variables with platform specific +source "$IPA_TESTS_SCRIPTS/variables-${IPA_PLATFORM}.sh" diff --git a/ipatests/azure/templates/generate-job-variables.yml b/ipatests/azure/templates/generate-job-variables.yml new file mode 100644 index 000000000..b3579a2d8 --- /dev/null +++ b/ipatests/azure/templates/generate-job-variables.yml @@ -0,0 +1,21 @@ +steps: +- script: | + # don't set 'set -x' here because this breaks variables + # https://github.com/microsoft/azure-pipelines-yaml/blob/master/design/readonly-variables.md + set -eu + total_envs=0 + for project in $(seq $(MAX_CONTAINER_ENVS)); do + # no more configured environments + tests_varname="IPA_TESTS_TO_RUN_${project}" + [ -z "${!tests_varname:-}" ] && break; + let "total_envs=total_envs+1" + + name_varname="IPA_TESTS_ENV_NAME_${project}" + [ -z "${!name_varname:-}" ] && \ + { echo "ipa_tests_env_name_${project} is mandatory."; exit 1; } + done + [ "$total_envs" -eq 0 ] && { echo 'Nothing to test'; env | sort ; exit 1; } + [ "$total_envs" -gt $(MAX_CONTAINER_ENVS) ] && \ + { echo "The number of defined jobs:${total_envs} cannot be greater than $(MAX_CONTAINER_ENVS)"; exit 1; } + echo "##vso[task.setvariable variable=ipa_tests_total_envs]$total_envs" + displayName: Generate environment variables diff --git a/ipatests/azure/templates/generate-matrix.yml b/ipatests/azure/templates/generate-matrix.yml new file mode 100644 index 000000000..7087078ae --- /dev/null +++ b/ipatests/azure/templates/generate-matrix.yml @@ -0,0 +1,9 @@ +parameters: + definition: '' + displayName: '' + name: '' + +steps: + - script: python3 $(IPA_TESTS_SCRIPTS)/generate-matrix.py ${{ parameters.definition }} + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} diff --git a/ipatests/azure/templates/publish-build.yml b/ipatests/azure/templates/publish-build.yml index bf573e78b..94b034a2d 100644 --- a/ipatests/azure/templates/publish-build.yml +++ b/ipatests/azure/templates/publish-build.yml @@ -1,12 +1,11 @@ -steps: -- task: PublishPipelineArtifact@0 - displayName: Publish packages - inputs: - artifactName: 'packages-$(Build.BuildId)-$(Agent.OS)-$(Agent.OSArchitecture)' - targetPath: $(Agent.BuildDirectory)/s/dist -- task: PublishPipelineArtifact@0 - displayName: Publish container image - inputs: - artifactName: 'image-$(Build.BuildId)-$(Agent.OS)-$(Agent.OSArchitecture)' - targetPath: $(Agent.BuildDirectory)/s/freeipa-azure-builder-container.tar.gz +parameters: + artifactName: '' + targetPath: '' + displayName: '' +steps: +- task: PublishPipelineArtifact@1 + inputs: + artifactName: ${{ parameters.artifactName }} + targetPath: ${{ parameters.targetPath }} + displayName: ${{ parameters.displayName }} diff --git a/ipatests/azure/templates/run-test.yml b/ipatests/azure/templates/run-test.yml index 7e0991f8b..a98d1dc7d 100644 --- a/ipatests/azure/templates/run-test.yml +++ b/ipatests/azure/templates/run-test.yml @@ -1,39 +1,58 @@ -parameters: - logsPath: 'logs' - taskToRun: 'run-tests' - testsToRun: '' - testsToIgnore: '' - testsToDedicate: '' - steps: - script: | - set -e - case "$SYSTEM_PARALLELEXECUTIONTYPE" in - # slicing - "MultiMachine" ) + set -eux - TOTALJOBSINPHASE="$(System.TotalJobsInPhase)" - JOBPOSITIONINPHASE="$(System.JobPositionInPhase)" - ;; + workdir="$IPA_TESTS_ENV_WORKING_DIR" + rm -rf "$workdir" + mkdir "$workdir" - * ) - # matrix job's strategy also sets these variables - # MultiConfiguration - TOTALJOBSINPHASE= - JOBPOSITIONINPHASE= - ;; - esac + ln -sfr \ + ${BUILD_REPOSITORY_LOCALPATH}/${IPA_TESTS_SCRIPTS}/{azure-run-tests.sh,setup_containers.py} \ + ./ - project="1" - docker exec \ - --env TESTS_TO_RUN="${{ parameters.testsToRun }}" \ - --env TESTS_TO_IGNORE="${{ parameters.testsToIgnore }}" \ - --env TESTS_TO_DEDICATE="${{ parameters.testsToDedicate }}" \ - --env CI_RUNNER_LOGS_DIR="${{ parameters.logsPath }}" \ - --env SYSTEM_TOTALJOBSINPHASE="$TOTALJOBSINPHASE" \ - --env SYSTEM_JOBPOSITIONINPHASE="$JOBPOSITIONINPHASE" \ - -t \ - "${project}"_master_1 \ - /bin/bash --noprofile --norc \ - -x /freeipa/ipatests/azure/azure-${{parameters.taskToRun}}.sh - displayName: Run test + function runner() { + set -o pipefail + local project_id="$1" + local project_name_varname="IPA_TESTS_ENV_NAME_${project_id}" + local project_name="${!project_name_varname}" + [ -z "$project_name" ] && \ + { echo "Project name is not set for project:${project_id}"; exit 1 ;} + local workdir="$IPA_TESTS_ENV_WORKING_DIR" + local logfile="runner_${project_name}.log" + local project_dir="${workdir}/${project_name}" + rm -rf "$project_dir" + mkdir "$project_dir" + # live-logging of tests within environment: '1' + if [ "$project_id" == "1" ]; then + ./azure-run-tests.sh "$project_id" 2>&1 | \ + ts '[%Y-%m-%d %H:%M:%S]' 2>&1 | tee "${project_dir}/${logfile}" + result=$? + else + ./azure-run-tests.sh "$project_id" 2>&1 | \ + ts '[%Y-%m-%d %H:%M:%S]' 2>&1 > "${project_dir}/${logfile}" + result=$? + fi + + result_file="result_${project_id}" + if [ "$result" -eq 0 ]; then + result_msg="${project_name}: PASSED" + else + result_msg="${project_name}: FAILED" + fi + echo "$result_msg" > "$result_file" + exit $result + } + export -f runner + + result=1 + rm -f result_* + { parallel \ + --tag \ + --jobs $(MAX_CONTAINER_ENVS) \ + --linebuffer \ + 'runner {}' ::: "$(seq $(ipa_tests_total_envs))" && result=0 ; } || \ + result=$? + echo "Results:" + cat $(eval echo result_{1..$(ipa_tests_total_envs)}) + exit $result + displayName: Run tests diff --git a/ipatests/azure/templates/save-test-artifacts.yml b/ipatests/azure/templates/save-test-artifacts.yml index c624d1ea7..5c3bfd61a 100644 --- a/ipatests/azure/templates/save-test-artifacts.yml +++ b/ipatests/azure/templates/save-test-artifacts.yml @@ -1,11 +1,12 @@ parameters: - logsPath: 'logs' + logsPath: $(IPA_TESTS_ENV_WORKING_DIR) logsArtifact: '' steps: -- task: PublishPipelineArtifact@0 +- task: PublishPipelineArtifact@1 displayName: Publish logs inputs: artifactName: ${{parameters.logsArtifact}} + # globbing is adjusted in .artifactignore on IPA_TESTS_ENV_WORKING_DIR targetPath: ${{parameters.logsPath}} condition: always() diff --git a/ipatests/azure/templates/setup-test-environment.yml b/ipatests/azure/templates/setup-test-environment.yml index 28993106f..9edefa17c 100644 --- a/ipatests/azure/templates/setup-test-environment.yml +++ b/ipatests/azure/templates/setup-test-environment.yml @@ -1,8 +1,5 @@ parameters: imageName: 'freeipa-azure-builder:latest' - topology: - replicas: 0 - clients: 0 steps: - script: | @@ -13,49 +10,28 @@ steps: sudo chown root:root /etc/docker/daemon.json sudo systemctl restart docker sudo modprobe ip6_tables - sudo modprobe {nfs,nfsd} - python3 -m pip install docker --user displayName: Configure containerization to allow IPv6 network + +- script: | + set -e + sudo modprobe {nfs,nfsd} + displayName: Configure NFS to allow NFS server/client within containers + - task: DownloadPipelineArtifact@0 displayName: Download prebuilt packages inputs: artifactName: 'packages-$(Build.BuildId)-$(Agent.OS)-$(Agent.OSArchitecture)' - targetPath: $(Agent.BuildDirectory)/s/dist + targetPath: $(Build.Repository.LocalPath)/dist + - task: DownloadPipelineArtifact@0 displayName: Download pre-built container inputs: artifactName: 'image-$(Build.BuildId)-$(Agent.OS)-$(Agent.OSArchitecture)' - targetPath: $(Agent.BuildDirectory)/s + targetPath: $(Build.Repository.LocalPath) + - script: | set -e - docker load --input $(Agent.BuildDirectory)/s/freeipa-azure-builder-container.tar.gz + docker load --input $(Build.Repository.LocalPath)/freeipa-azure-builder-container.tar.gz docker images docker inspect freeipa-azure-builder:latest displayName: Import pre-built container to the engine -- script: | - set -ex - project="1" - cont_dirname="container_$project" - rm -rf "$cont_dirname" - mkdir -p "$cont_dirname"/exports - ln -sfr ipatests/azure/docker-compose.yml "$cont_dirname"/ - touch "$cont_dirname"/ipa-test-config.yaml - cd "$cont_dirname" - export IPA_DOCKER_IMAGE="${{ parameters.imageName }}" - export IPA_NETWORK=ipanet - export IPA_IPV6_SUBNET="2001:db8:1:$project::/64" - docker-compose -p "$project" up -d \ - --scale replica="${{ parameters.topology.replicas }}" \ - --scale client="${{ parameters.topology.clients }}" \ - --force-recreate --remove-orphans - displayName: Create and start container(s) for running tests -- script: | - set -ex - ln -sfr ipatests/azure/templates ./ - ln -sfr ipatests/azure/setup_containers.py ./ - project="1" - export IPA_CONT_PREFIX="$project" - export IPA_CLIENT_NUM="${{ parameters.topology.clients }}" - export IPA_REPLICA_NUM="${{ parameters.topology.replicas }}" - python3 setup_containers.py - displayName: Setup container(s) for running tests diff --git a/ipatests/azure/templates/test-jobs.yml b/ipatests/azure/templates/test-jobs.yml index 964af7d6a..d7231db6c 100644 --- a/ipatests/azure/templates/test-jobs.yml +++ b/ipatests/azure/templates/test-jobs.yml @@ -1,40 +1,53 @@ -parameters: - testsToIgnore: [] - testsToRun: [] - testsToDedicate: [] - taskToRun: '' - topology: - replicas: 0 - clients: 0 - steps: - script: | set -e - env + env | sort + displayName: Print Host Enviroment + +- script: | + set -e printf "Available entropy: %s\n" $(cat /proc/sys/kernel/random/entropy_avail) sudo apt-get install -y rng-tools sudo service rng-tools start sleep 3 printf "Available entropy: %s\n" $(cat /proc/sys/kernel/random/entropy_avail) displayName: Increase entropy level + +- script: | + set -e + sudo apt-get install -y \ + parallel \ + moreutils \ + python3-docker + # ubuntu's one is too old: different API + python3 -m pip install docker --user + displayName: Install Host's tests requirements + - template: setup-test-environment.yml - parameters: - containerName: 'freeipa-azure-builder:latest' - topology: - replicas: ${{ parameters.topology.replicas }} - clients: ${{ parameters.topology.clients }} + - template: run-test.yml - parameters: - logsPath: $(CI_RUNNER_LOGS_DIR) - taskToRun: ${{ parameters.taskToRun }} - testsToRun: ${{ join(' ', parameters.testsToRun ) }} - testsToIgnore: ${{ join(' ', parameters.testsToIgnore ) }} - testsToDedicate: ${{ join(' ', parameters.testsToDedicate ) }} + - task: PublishTestResults@2 inputs: - testResultsFiles: $(CI_RUNNER_LOGS_DIR)/nosetests.xml + testResultsFiles: 'ipa_envs/*/$(CI_RUNNER_LOGS_DIR)/nosetests.xml' testRunTitle: $(System.JobIdentifier) results condition: succeededOrFailed() + +- script: | + set -e + + artifacts_ignore_path="${IPA_TESTS_ENV_WORKING_DIR}/.artifactignore" + cat > "$artifacts_ignore_path" <