From 3049b9587f0ded768ba3a61ec0e98368a6316a0a Mon Sep 17 00:00:00 2001 From: Stanislav Levin Date: Wed, 19 May 2021 18:47:16 +0300 Subject: [PATCH] azure: Collect installed packages The list of installed packages may be useful for checking the versions of packages for analysis. Previously, only the newly installed packages can be observed on Build phase. This is convenient for experienced users of PR-CI. Note: the read-only access provided for non-master containers to be able to execute Azure scripts. The logs are still collected only on controller. Only RPM-based collection is implemented for Fedora. By default nothing is collected. Users may want to override `installed_packages` function in the corresponding `ipatests/azure/scripts/variables-DISTRO.sh`. Reviewed-By: Alexander Bokovoy --- ipatests/azure/Dockerfiles/docker-compose.yml | 2 + ipatests/azure/scripts/azure-run-tests.sh | 44 ++++++++++++++++++- ipatests/azure/scripts/variables-fedora.sh | 4 ++ ipatests/azure/scripts/variables.sh | 2 + ipatests/azure/templates/test-jobs.yml | 1 + 5 files changed, 51 insertions(+), 2 deletions(-) diff --git a/ipatests/azure/Dockerfiles/docker-compose.yml b/ipatests/azure/Dockerfiles/docker-compose.yml index a6fc4e13c..5a8c613ab 100644 --- a/ipatests/azure/Dockerfiles/docker-compose.yml +++ b/ipatests/azure/Dockerfiles/docker-compose.yml @@ -28,6 +28,7 @@ services: mem_limit: "1900m" volumes: - /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd + - ${BUILD_REPOSITORY_LOCALPATH}:${IPA_TESTS_REPO_PATH}:ro networks: - ${IPA_NETWORK} @@ -42,6 +43,7 @@ services: mem_limit: "536870912" volumes: - /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd + - ${BUILD_REPOSITORY_LOCALPATH}:${IPA_TESTS_REPO_PATH}:ro # nfs server - ./exports:/exports - /lib/modules:/lib/modules:ro diff --git a/ipatests/azure/scripts/azure-run-tests.sh b/ipatests/azure/scripts/azure-run-tests.sh index 661185aa0..567d8996c 100755 --- a/ipatests/azure/scripts/azure-run-tests.sh +++ b/ipatests/azure/scripts/azure-run-tests.sh @@ -44,6 +44,11 @@ IPA_TESTS_REPLICAS="${!IPA_TESTS_REPLICAS_VARNAME:-0}" IPA_TESTS_CONTROLLER="${PROJECT_ID}_master_1" IPA_TESTS_LOGSDIR="${IPA_TESTS_REPO_PATH}/ipa_envs/${IPA_TESTS_ENV_NAME}/${CI_RUNNER_LOGS_DIR}" +# path to azure scripts inside container +IPA_TESTS_SCRIPTS_IN="${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}" +# path to azure scripts outside of container +IPA_TESTS_SCRIPTS_OUT="${BUILD_REPOSITORY_LOCALPATH}/${IPA_TESTS_SCRIPTS}" + IPA_TESTS_NETWORK_INTERNAL_VARNAME="IPA_TESTS_NETWORK_INTERNAL_${PROJECT_ID}" IPA_NETWORK_INTERNAL="${!IPA_TESTS_NETWORK_INTERNAL_VARNAME:-false}" @@ -51,6 +56,7 @@ IPA_TESTS_DOMAIN="${IPA_TESTS_DOMAIN:-ipa.test}" # bash4 IPA_TESTS_REALM="${IPA_TESTS_DOMAIN^^}" + # for base tests only 1 master is needed even if another was specified if [ "$IPA_TESTS_TYPE" == "base" ]; then IPA_TESTS_CLIENTS="0" @@ -63,6 +69,9 @@ project_dir="${IPA_TESTS_ENV_WORKING_DIR}/${IPA_TESTS_ENV_NAME}" # path for journal if containers setup fails SYSTEMD_BOOT_LOG="${project_dir}/systemd_boot_logs" +# path to directory where to dump list of packages outside of container +IPA_INSTALLED_PKGS_DIR="${project_dir}/installed_packages" + BASH_CMD="/bin/bash --noprofile --norc" function containers() { @@ -132,12 +141,43 @@ python3 setup_containers.py || \ exit 1; } +# collect list of all the installed packages +mkdir -p "$IPA_INSTALLED_PKGS_DIR" + +# controller +docker exec -t \ + --env IPA_TESTS_SCRIPTS="${IPA_TESTS_SCRIPTS_IN}" \ + --env IPA_PLATFORM="$IPA_PLATFORM" \ + "$IPA_TESTS_CONTROLLER" \ + $BASH_CMD -eu \ + -c \ + "source '${IPA_TESTS_SCRIPTS_IN}/variables.sh' && \ + echo '# Controller container: $IPA_TESTS_CONTROLLER' && \ + echo '# IPA platform: '\$IPA_PLATFORM && \ + installed_packages \ + " > "${IPA_INSTALLED_PKGS_DIR}/packages_controller_${IPA_TESTS_CONTROLLER}.log" + +# workers +for container in $(containers); do + docker exec -t \ + --env IPA_TESTS_SCRIPTS="${IPA_TESTS_SCRIPTS_IN}" \ + --env IPA_PLATFORM="$IPA_PLATFORM" \ + "$container" \ + $BASH_CMD -eu \ + -c \ + "source '${IPA_TESTS_SCRIPTS_IN}/variables.sh' && \ + echo '# Container: $container' && \ + echo '# IPA platform: '\$IPA_PLATFORM && \ + installed_packages \ + " > "${IPA_INSTALLED_PKGS_DIR}/packages_${container}.log" +done + # path to runner within container -tests_runner="${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}/azure-run-${IPA_TESTS_TYPE}-tests.sh" +tests_runner="${IPA_TESTS_SCRIPTS_IN}/azure-run-${IPA_TESTS_TYPE}-tests.sh" tests_result=1 { docker exec -t \ - --env IPA_TESTS_SCRIPTS="${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}" \ + --env IPA_TESTS_SCRIPTS="${IPA_TESTS_SCRIPTS_IN}" \ --env IPA_PLATFORM="$IPA_PLATFORM" \ --env IPA_TESTS_DOMAIN="$IPA_TESTS_DOMAIN" \ --env IPA_TESTS_REALM="$IPA_TESTS_REALM" \ diff --git a/ipatests/azure/scripts/variables-fedora.sh b/ipatests/azure/scripts/variables-fedora.sh index 41d7d888f..62600c9b0 100755 --- a/ipatests/azure/scripts/variables-fedora.sh +++ b/ipatests/azure/scripts/variables-fedora.sh @@ -5,3 +5,7 @@ function firewalld_cmd() { firewall-cmd $@ } + +function installed_packages() { + rpm -qa | sort +} diff --git a/ipatests/azure/scripts/variables.sh b/ipatests/azure/scripts/variables.sh index 6b855def8..1efb2adea 100755 --- a/ipatests/azure/scripts/variables.sh +++ b/ipatests/azure/scripts/variables.sh @@ -12,5 +12,7 @@ BIND_LOGGING_OPTIONS_CONF='/etc/named/ipa-logging-ext.conf' function firewalld_cmd() { :; } +function installed_packages() { :; } + # this should be the last to override base variables with platform specific source "$IPA_TESTS_SCRIPTS/variables-${IPA_PLATFORM}.sh" diff --git a/ipatests/azure/templates/test-jobs.yml b/ipatests/azure/templates/test-jobs.yml index e0e25c633..b8c5bcb93 100644 --- a/ipatests/azure/templates/test-jobs.yml +++ b/ipatests/azure/templates/test-jobs.yml @@ -161,6 +161,7 @@ steps: !*/*.yaml !*/*.log !*/systemd_boot_logs/*.log + !*/installed_packages/*.log !*/memory.stats !*.tar.gz EOF