diff --git a/ipatests/azure/scripts/dump_cores.sh b/ipatests/azure/scripts/dump_cores.sh new file mode 100755 index 000000000..03e932805 --- /dev/null +++ b/ipatests/azure/scripts/dump_cores.sh @@ -0,0 +1,54 @@ +#!/bin/bash -eu + +IPA_TESTS_ENV_WORKING_DIR="${IPA_TESTS_REPO_PATH}/ipa_envs" +COREDUMPS_DIR="${IPA_TESTS_ENV_WORKING_DIR}/${COREDUMPS_SUBDIR}" + +since_time="$(cat '/coredumpctl.time.mark' || echo '-1h')" +debugger="/debugger.sh" + +cat > "$debugger" < "\${CORE_PID}.stacktrace" 2>&1 +EOF +chmod +x "$debugger" + +# make sure coredumpctl installed +which coredumpctl +coredumpctl \ + --no-pager --directory="$HOST_JOURNAL" --since="$since_time" list ||: + +rm -rvf "$COREDUMPS_DIR" ||: +mkdir "$COREDUMPS_DIR" +cd "$COREDUMPS_DIR" + +pids="$(coredumpctl --no-pager --directory="$HOST_JOURNAL" --since="$since_time" -F COREDUMP_PID || echo '')" +for pid in $pids; do + # core dump + { coredumpctl \ + --no-pager \ + --since="$since_time" \ + --directory="$HOST_JOURNAL" \ + -o "${pid}.core" dump "$pid" && \ + tar -czf "${pid}.core.tar.gz" --remove-files "${pid}.core" ; } ||: + + # stacktrace + { CORE_PID="$pid" \ + coredumpctl \ + --no-pager \ + --since="$since_time" \ + --directory="$HOST_JOURNAL" \ + --debugger="$debugger" \ + debug "$pid" && \ + tar \ + -czf "${pid}.stacktrace.tar.gz" \ + --remove-files "${pid}.stacktrace" ; } ||: +done + +chmod a+rw -R "$COREDUMPS_DIR" diff --git a/ipatests/azure/scripts/install-debuginfo-fedora.sh b/ipatests/azure/scripts/install-debuginfo-fedora.sh new file mode 100755 index 000000000..bddeef332 --- /dev/null +++ b/ipatests/azure/scripts/install-debuginfo-fedora.sh @@ -0,0 +1,20 @@ +#!/bin/bash -eu + +function install_debuginfo() { + dnf makecache ||: + dnf install -y \ + ${IPA_TESTS_REPO_PATH}/dist/rpms_debuginfo/*.rpm \ + gdb + + dnf debuginfo-install -y \ + 389-ds-base \ + bind \ + bind-dyndb-ldap \ + certmonger \ + gssproxy \ + httpd \ + krb5-server \ + krb5-workstation \ + samba \ + sssd +} diff --git a/ipatests/azure/scripts/install-debuginfo.sh b/ipatests/azure/scripts/install-debuginfo.sh new file mode 100755 index 000000000..9513b073f --- /dev/null +++ b/ipatests/azure/scripts/install-debuginfo.sh @@ -0,0 +1,8 @@ +#!/bin/bash -eu + +function install_debuginfo() { :; } + +# override install_debuginfo for the platform specifics +source "${IPA_TESTS_SCRIPTS}/install-debuginfo-${IPA_PLATFORM}.sh" + +install_debuginfo diff --git a/ipatests/azure/templates/build-fedora.yml b/ipatests/azure/templates/build-fedora.yml index 310f39d7e..60a3baa34 100644 --- a/ipatests/azure/templates/build-fedora.yml +++ b/ipatests/azure/templates/build-fedora.yml @@ -3,4 +3,8 @@ steps: set -e echo "Running make target 'rpms'" make V=0 rpms LOG_COMPILE='gdb.minimal -return-child-result -ex run -ex "thread apply all bt" -ex "quit" --args' + mkdir -p $(builddir)/dist/rpms_debuginfo + find $(builddir)/dist/rpms/ -type f \ + \( -name "*-debuginfo-*.rpm" -o -name '*-debugsource-*.rpm' \) \ + -exec mv {} $(builddir)/dist/rpms_debuginfo/ \; displayName: Build packages diff --git a/ipatests/azure/templates/test-jobs.yml b/ipatests/azure/templates/test-jobs.yml index d7231db6c..e0ab236bd 100644 --- a/ipatests/azure/templates/test-jobs.yml +++ b/ipatests/azure/templates/test-jobs.yml @@ -18,11 +18,28 @@ steps: sudo apt-get install -y \ parallel \ moreutils \ + systemd-coredump \ python3-docker # ubuntu's one is too old: different API python3 -m pip install docker --user displayName: Install Host's tests requirements +- script: | + set -eu + date +'%Y-%m-%d %H:%M:%S' > coredumpctl.time.mark + systemd_conf="/etc/systemd/system.conf" + sudo sed -i 's/^DumpCore=.*/#&/g' "$systemd_conf" + sudo sed -i 's/^DefaultLimitCORE=.*/#&/g' "$systemd_conf" + echo -e 'DumpCore=yes\nDefaultLimitCORE=infinity' | \ + sudo tee -a "$systemd_conf" >/dev/null + cat "$systemd_conf" + coredump_conf="/etc/systemd/coredump.conf" + cat "$coredump_conf" + sudo systemctl daemon-reexec + # for ns-slapd debugging + sudo sysctl -w fs.suid_dumpable=1 + displayName: Allow coredumps + - template: setup-test-environment.yml - template: run-test.yml @@ -33,12 +50,59 @@ steps: testRunTitle: $(System.JobIdentifier) results condition: succeededOrFailed() +- script: | + set -eu + # check the host first, containers cores were dumped here + COREDUMPS_SUBDIR="coredumps" + COREDUMPS_DIR="${IPA_TESTS_ENV_WORKING_DIR}/${COREDUMPS_SUBDIR}" + rm -rfv "$COREDUMPS_DIR" ||: + mkdir "$COREDUMPS_DIR" + since_time="$(cat coredumpctl.time.mark || echo '-1h')" + sudo coredumpctl --no-pager --since="$since_time" list ||: + + pids="$(sudo coredumpctl --no-pager --since="$since_time" -F COREDUMP_PID || echo '')" + # nothing to dump + [ -z "$pids" ] && exit 0 + + # continue in container + HOST_JOURNAL="/var/log/host_journal" + CONTAINER_COREDUMP="dump_cores" + docker create --privileged \ + -v "$(realpath coredumpctl.time.mark)":/coredumpctl.time.mark:ro \ + -v /var/lib/systemd/coredump:/var/lib/systemd/coredump:ro \ + -v /var/log/journal:"$HOST_JOURNAL":ro \ + -v "${BUILD_REPOSITORY_LOCALPATH}":"${IPA_TESTS_REPO_PATH}" \ + --name "$CONTAINER_COREDUMP" freeipa-azure-builder + docker start "$CONTAINER_COREDUMP" + + docker exec -t \ + --env IPA_TESTS_REPO_PATH="${IPA_TESTS_REPO_PATH}" \ + --env IPA_TESTS_SCRIPTS="${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}" \ + --env IPA_PLATFORM="${IPA_PLATFORM}" \ + "$CONTAINER_COREDUMP" \ + /bin/bash --noprofile --norc -eux \ + "${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}/install-debuginfo.sh" + + docker exec -t \ + --env IPA_TESTS_REPO_PATH="${IPA_TESTS_REPO_PATH}" \ + --env COREDUMPS_SUBDIR="$COREDUMPS_SUBDIR" \ + --env HOST_JOURNAL="$HOST_JOURNAL" \ + "$CONTAINER_COREDUMP" \ + /bin/bash --noprofile --norc -eux \ + "${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}/dump_cores.sh" + # there should be no crashes + exit 1 + condition: succeededOrFailed() + displayName: Check for coredumps + - script: | set -e artifacts_ignore_path="${IPA_TESTS_ENV_WORKING_DIR}/.artifactignore" cat > "$artifacts_ignore_path" <