From d1b53ded8b566f8ca4ed191a76efe7076263ed5f Mon Sep 17 00:00:00 2001 From: Stanislav Levin Date: Tue, 24 Mar 2020 11:01:16 +0300 Subject: [PATCH] Azure: Gather coredumps Applications may crash. If a crash happens on a remote system during CI run it's sometimes very hard to understand the reason. The most important means to analyze such is a stack trace. It's also very important to check whether there was a core dump or not, even a test passed. For Docker environment, the core dumps are collected by the host's systemd-coredump, which knows nothing about such containers (for now). To build an informative thread stack trace debuginfo packages should be installed. But they can't be installed on the host OS (ubuntu), That's why after all the tests completed an additional container should be up and the host's core dumps and host's journal should be passed into it. Even if there weren't enough debuginfo packages at CI-runtime, the core dump could be analyzed locally later. Fixes: https://pagure.io/freeipa/issue/8251 Signed-off-by: Stanislav Levin Reviewed-By: Alexander Bokovoy Reviewed-By: Rob Crittenden --- ipatests/azure/scripts/dump_cores.sh | 54 ++++++++++++++++ .../azure/scripts/install-debuginfo-fedora.sh | 20 ++++++ ipatests/azure/scripts/install-debuginfo.sh | 8 +++ ipatests/azure/templates/build-fedora.yml | 4 ++ ipatests/azure/templates/test-jobs.yml | 64 +++++++++++++++++++ 5 files changed, 150 insertions(+) create mode 100755 ipatests/azure/scripts/dump_cores.sh create mode 100755 ipatests/azure/scripts/install-debuginfo-fedora.sh create mode 100755 ipatests/azure/scripts/install-debuginfo.sh diff --git a/ipatests/azure/scripts/dump_cores.sh b/ipatests/azure/scripts/dump_cores.sh new file mode 100755 index 000000000..03e932805 --- /dev/null +++ b/ipatests/azure/scripts/dump_cores.sh @@ -0,0 +1,54 @@ +#!/bin/bash -eu + +IPA_TESTS_ENV_WORKING_DIR="${IPA_TESTS_REPO_PATH}/ipa_envs" +COREDUMPS_DIR="${IPA_TESTS_ENV_WORKING_DIR}/${COREDUMPS_SUBDIR}" + +since_time="$(cat '/coredumpctl.time.mark' || echo '-1h')" +debugger="/debugger.sh" + +cat > "$debugger" < "\${CORE_PID}.stacktrace" 2>&1 +EOF +chmod +x "$debugger" + +# make sure coredumpctl installed +which coredumpctl +coredumpctl \ + --no-pager --directory="$HOST_JOURNAL" --since="$since_time" list ||: + +rm -rvf "$COREDUMPS_DIR" ||: +mkdir "$COREDUMPS_DIR" +cd "$COREDUMPS_DIR" + +pids="$(coredumpctl --no-pager --directory="$HOST_JOURNAL" --since="$since_time" -F COREDUMP_PID || echo '')" +for pid in $pids; do + # core dump + { coredumpctl \ + --no-pager \ + --since="$since_time" \ + --directory="$HOST_JOURNAL" \ + -o "${pid}.core" dump "$pid" && \ + tar -czf "${pid}.core.tar.gz" --remove-files "${pid}.core" ; } ||: + + # stacktrace + { CORE_PID="$pid" \ + coredumpctl \ + --no-pager \ + --since="$since_time" \ + --directory="$HOST_JOURNAL" \ + --debugger="$debugger" \ + debug "$pid" && \ + tar \ + -czf "${pid}.stacktrace.tar.gz" \ + --remove-files "${pid}.stacktrace" ; } ||: +done + +chmod a+rw -R "$COREDUMPS_DIR" diff --git a/ipatests/azure/scripts/install-debuginfo-fedora.sh b/ipatests/azure/scripts/install-debuginfo-fedora.sh new file mode 100755 index 000000000..bddeef332 --- /dev/null +++ b/ipatests/azure/scripts/install-debuginfo-fedora.sh @@ -0,0 +1,20 @@ +#!/bin/bash -eu + +function install_debuginfo() { + dnf makecache ||: + dnf install -y \ + ${IPA_TESTS_REPO_PATH}/dist/rpms_debuginfo/*.rpm \ + gdb + + dnf debuginfo-install -y \ + 389-ds-base \ + bind \ + bind-dyndb-ldap \ + certmonger \ + gssproxy \ + httpd \ + krb5-server \ + krb5-workstation \ + samba \ + sssd +} diff --git a/ipatests/azure/scripts/install-debuginfo.sh b/ipatests/azure/scripts/install-debuginfo.sh new file mode 100755 index 000000000..9513b073f --- /dev/null +++ b/ipatests/azure/scripts/install-debuginfo.sh @@ -0,0 +1,8 @@ +#!/bin/bash -eu + +function install_debuginfo() { :; } + +# override install_debuginfo for the platform specifics +source "${IPA_TESTS_SCRIPTS}/install-debuginfo-${IPA_PLATFORM}.sh" + +install_debuginfo diff --git a/ipatests/azure/templates/build-fedora.yml b/ipatests/azure/templates/build-fedora.yml index 310f39d7e..60a3baa34 100644 --- a/ipatests/azure/templates/build-fedora.yml +++ b/ipatests/azure/templates/build-fedora.yml @@ -3,4 +3,8 @@ steps: set -e echo "Running make target 'rpms'" make V=0 rpms LOG_COMPILE='gdb.minimal -return-child-result -ex run -ex "thread apply all bt" -ex "quit" --args' + mkdir -p $(builddir)/dist/rpms_debuginfo + find $(builddir)/dist/rpms/ -type f \ + \( -name "*-debuginfo-*.rpm" -o -name '*-debugsource-*.rpm' \) \ + -exec mv {} $(builddir)/dist/rpms_debuginfo/ \; displayName: Build packages diff --git a/ipatests/azure/templates/test-jobs.yml b/ipatests/azure/templates/test-jobs.yml index d7231db6c..e0ab236bd 100644 --- a/ipatests/azure/templates/test-jobs.yml +++ b/ipatests/azure/templates/test-jobs.yml @@ -18,11 +18,28 @@ steps: sudo apt-get install -y \ parallel \ moreutils \ + systemd-coredump \ python3-docker # ubuntu's one is too old: different API python3 -m pip install docker --user displayName: Install Host's tests requirements +- script: | + set -eu + date +'%Y-%m-%d %H:%M:%S' > coredumpctl.time.mark + systemd_conf="/etc/systemd/system.conf" + sudo sed -i 's/^DumpCore=.*/#&/g' "$systemd_conf" + sudo sed -i 's/^DefaultLimitCORE=.*/#&/g' "$systemd_conf" + echo -e 'DumpCore=yes\nDefaultLimitCORE=infinity' | \ + sudo tee -a "$systemd_conf" >/dev/null + cat "$systemd_conf" + coredump_conf="/etc/systemd/coredump.conf" + cat "$coredump_conf" + sudo systemctl daemon-reexec + # for ns-slapd debugging + sudo sysctl -w fs.suid_dumpable=1 + displayName: Allow coredumps + - template: setup-test-environment.yml - template: run-test.yml @@ -33,12 +50,59 @@ steps: testRunTitle: $(System.JobIdentifier) results condition: succeededOrFailed() +- script: | + set -eu + # check the host first, containers cores were dumped here + COREDUMPS_SUBDIR="coredumps" + COREDUMPS_DIR="${IPA_TESTS_ENV_WORKING_DIR}/${COREDUMPS_SUBDIR}" + rm -rfv "$COREDUMPS_DIR" ||: + mkdir "$COREDUMPS_DIR" + since_time="$(cat coredumpctl.time.mark || echo '-1h')" + sudo coredumpctl --no-pager --since="$since_time" list ||: + + pids="$(sudo coredumpctl --no-pager --since="$since_time" -F COREDUMP_PID || echo '')" + # nothing to dump + [ -z "$pids" ] && exit 0 + + # continue in container + HOST_JOURNAL="/var/log/host_journal" + CONTAINER_COREDUMP="dump_cores" + docker create --privileged \ + -v "$(realpath coredumpctl.time.mark)":/coredumpctl.time.mark:ro \ + -v /var/lib/systemd/coredump:/var/lib/systemd/coredump:ro \ + -v /var/log/journal:"$HOST_JOURNAL":ro \ + -v "${BUILD_REPOSITORY_LOCALPATH}":"${IPA_TESTS_REPO_PATH}" \ + --name "$CONTAINER_COREDUMP" freeipa-azure-builder + docker start "$CONTAINER_COREDUMP" + + docker exec -t \ + --env IPA_TESTS_REPO_PATH="${IPA_TESTS_REPO_PATH}" \ + --env IPA_TESTS_SCRIPTS="${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}" \ + --env IPA_PLATFORM="${IPA_PLATFORM}" \ + "$CONTAINER_COREDUMP" \ + /bin/bash --noprofile --norc -eux \ + "${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}/install-debuginfo.sh" + + docker exec -t \ + --env IPA_TESTS_REPO_PATH="${IPA_TESTS_REPO_PATH}" \ + --env COREDUMPS_SUBDIR="$COREDUMPS_SUBDIR" \ + --env HOST_JOURNAL="$HOST_JOURNAL" \ + "$CONTAINER_COREDUMP" \ + /bin/bash --noprofile --norc -eux \ + "${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}/dump_cores.sh" + # there should be no crashes + exit 1 + condition: succeededOrFailed() + displayName: Check for coredumps + - script: | set -e artifacts_ignore_path="${IPA_TESTS_ENV_WORKING_DIR}/.artifactignore" cat > "$artifacts_ignore_path" <