diff --git a/ipaserver/install/dnskeysyncinstance.py b/ipaserver/install/dnskeysyncinstance.py
index 0091250f5..d7f052319 100644
--- a/ipaserver/install/dnskeysyncinstance.py
+++ b/ipaserver/install/dnskeysyncinstance.py
@@ -5,6 +5,7 @@
 from __future__ import print_function
 
 import logging
+import errno
 import os
 import pwd
 import grp
@@ -463,9 +464,15 @@ class DNSKeySyncInstance(service.Service):
         # remove softhsm pin, to make sure new installation will generate
         # new token database
         # do not delete *so pin*, user can need it to get token data
+        installutils.remove_file(paths.DNSSEC_SOFTHSM_PIN)
+        installutils.remove_file(paths.DNSSEC_SOFTHSM2_CONF)
+
         try:
-            os.remove(paths.DNSSEC_SOFTHSM_PIN)
-        except Exception:
-            pass
+            shutil.rmtree(paths.DNSSEC_TOKENS_DIR)
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                logger.exception(
+                    "Failed to remove %s", paths.DNSSEC_TOKENS_DIR
+                )
 
         installutils.remove_keytab(self.keytab)
diff --git a/ipaserver/install/httpinstance.py b/ipaserver/install/httpinstance.py
index 6acb9126b..e33d943f8 100644
--- a/ipaserver/install/httpinstance.py
+++ b/ipaserver/install/httpinstance.py
@@ -22,13 +22,14 @@ from __future__ import absolute_import
 
 import logging
 import os
-import os.path
-import dbus
+import glob
+import errno
 import shlex
 import pipes
 import tempfile
 
 from augeas import Augeas
+import dbus
 
 from ipalib.install import certmonger
 from ipapython import ipaldap
@@ -493,19 +494,40 @@ class HTTPInstance(service.Service):
             except ValueError as error:
                 logger.debug("%s", error)
 
-        installutils.remove_keytab(self.keytab)
-        installutils.remove_file(paths.HTTP_CCACHE)
-
         # Remove the configuration files we create
-        installutils.remove_file(paths.HTTPD_CERT_FILE)
-        installutils.remove_file(paths.HTTPD_KEY_FILE)
-        installutils.remove_file(paths.HTTPD_IPA_REWRITE_CONF)
-        installutils.remove_file(paths.HTTPD_IPA_CONF)
-        installutils.remove_file(paths.HTTPD_IPA_PKI_PROXY_CONF)
-        installutils.remove_file(paths.HTTPD_IPA_KDCPROXY_CONF_SYMLINK)
-        installutils.remove_file(paths.HTTPD_IPA_KDCPROXY_CONF)
+        installutils.remove_keytab(self.keytab)
+        remove_files = [
+            paths.HTTP_CCACHE,
+            paths.HTTPD_CERT_FILE,
+            paths.HTTPD_KEY_FILE,
+            paths.HTTPD_IPA_REWRITE_CONF,
+            paths.HTTPD_IPA_CONF,
+            paths.HTTPD_IPA_PKI_PROXY_CONF,
+            paths.HTTPD_IPA_KDCPROXY_CONF_SYMLINK,
+            paths.HTTPD_IPA_KDCPROXY_CONF,
+            paths.GSSPROXY_CONF,
+            paths.GSSAPI_SESSION_KEY,
+            paths.HTTPD_PASSWORD_CONF,
+            paths.SYSTEMD_SYSTEM_HTTPD_IPA_CONF,
+        ]
+        # NSS DB backups
+        remove_files.extend(
+            glob.glob(os.path.join(paths.HTTPD_ALIAS_DIR, '*.ipasave'))
+        )
         if paths.HTTPD_IPA_WSGI_MODULES_CONF is not None:
-            installutils.remove_file(paths.HTTPD_IPA_WSGI_MODULES_CONF)
+            remove_files.append(paths.HTTPD_IPA_WSGI_MODULES_CONF)
+
+        for filename in remove_files:
+            installutils.remove_file(filename)
+
+        try:
+            os.rmdir(paths.SYSTEMD_SYSTEM_HTTPD_D_DIR)
+        except OSError as e:
+            if e.errno not in {errno.ENOENT, errno.ENOTEMPTY}:
+                logger.error(
+                    "Failed to remove directory %s",
+                    paths.SYSTEMD_SYSTEM_HTTPD_D_DIR
+                )
 
         # Restore SELinux boolean states
         boolean_states = {name: self.restore_state(name)
diff --git a/ipaserver/install/installutils.py b/ipaserver/install/installutils.py
index 885549a80..0a21be4d5 100644
--- a/ipaserver/install/installutils.py
+++ b/ipaserver/install/installutils.py
@@ -964,14 +964,14 @@ def check_server_configuration():
 
 
 def remove_file(filename):
-    """
-    Remove a file and log any exceptions raised.
+    """Remove a file and log any exceptions raised.
     """
     try:
-        if os.path.lexists(filename):
-            os.unlink(filename)
+        os.unlink(filename)
     except Exception as e:
-        logger.error('Error removing %s: %s', filename, str(e))
+        # ignore missing file
+        if getattr(e, 'errno', None) != errno.ENOENT:
+            logger.error('Error removing %s: %s', filename, str(e))
 
 
 def rmtree(path):