diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py
index 3176742e0..68cc77b9f 100644
--- a/ipaserver/install/cainstance.py
+++ b/ipaserver/install/cainstance.py
@@ -1304,7 +1304,16 @@ class CAInstance(DogtagInstance):
         keyfile = os.path.join(paths.PKI_TOMCAT,
                                self.service_prefix + '.keys')
         keystore = IPAKEMKeys({'server_keys': keyfile})
-        keystore.remove_keys(self.service_prefix)
+        # Call remove_server_keys_file explicitly to ensure that the key
+        # file is always removed.
+        keystore.remove_server_keys_file()
+        try:
+            keystore.remove_keys(self.service_prefix)
+        except (ldap.CONNECT_ERROR, ldap.SERVER_DOWN):
+            logger.debug(
+                "Cannot remove custodia keys now, server_del takes care of "
+                "them later."
+            )
 
     def add_lightweight_ca_tracking_requests(self):
         try:
diff --git a/ipaserver/install/custodiainstance.py b/ipaserver/install/custodiainstance.py
index 46998164a..748f84763 100644
--- a/ipaserver/install/custodiainstance.py
+++ b/ipaserver/install/custodiainstance.py
@@ -16,6 +16,7 @@ from ipaserver.install import ldapupdate
 from ipaserver.install import sysupgrade
 from base64 import b64decode
 from jwcrypto.common import json_decode
+import ldap
 import shutil
 import os
 import stat
@@ -70,7 +71,16 @@ class CustodiaInstance(SimpleServiceInstance):
             'server_keys': self.server_keys,
             'ldap_uri': self.ldap_uri
         })
-        keystore.remove_server_keys()
+        # Call remove_server_keys_file explicitly to ensure that the key
+        # file is always removed.
+        keystore.remove_server_keys_file()
+        try:
+            keystore.remove_server_keys()
+        except (ldap.CONNECT_ERROR, ldap.SERVER_DOWN):
+            logger.debug(
+                "Cannot remove custodia keys now, server_del takes care of "
+                "them later."
+            )
         installutils.remove_file(self.config_file)
         sysupgrade.set_upgrade_state('custodia', 'installed', False)
 
diff --git a/ipaserver/secrets/kem.py b/ipaserver/secrets/kem.py
index 266d975d5..5521c4772 100644
--- a/ipaserver/secrets/kem.py
+++ b/ipaserver/secrets/kem.py
@@ -235,6 +235,20 @@ class IPAKEMKeys(KEMKeysStore):
         ldapconn.set_key(KEY_USAGE_SIG, principal, pubkeys[0])
         ldapconn.set_key(KEY_USAGE_ENC, principal, pubkeys[1])
 
+    def remove_server_keys_file(self):
+        """Remove keys from disk
+
+        The method does not fail when the file is missing.
+        """
+        try:
+            os.unlink(self.config['server_keys'])
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                raise
+            return False
+        else:
+            return True
+
     def remove_server_keys(self):
         """Remove keys from LDAP and disk
         """
@@ -243,15 +257,11 @@ class IPAKEMKeys(KEMKeysStore):
     def remove_keys(self, servicename):
         """Remove keys from LDAP and disk
         """
+        self.remove_server_keys_file()
         principal = '%s/%s@%s' % (servicename, self.host, self.realm)
         ldapconn = KEMLdap(self.ldap_uri)
         ldapconn.del_key(KEY_USAGE_SIG, principal)
         ldapconn.del_key(KEY_USAGE_ENC, principal)
-        try:
-            os.unlink(self.config['server_keys'])
-        except OSError as e:
-            if e.errno != errno.ENOENT:
-                raise
 
     @property
     def server_keys(self):