diff --git a/ipaclient/install/client.py b/ipaclient/install/client.py
index 8731afd69..6622959ef 100644
--- a/ipaclient/install/client.py
+++ b/ipaclient/install/client.py
@@ -3329,6 +3329,10 @@ def uninstall(options):
     # Remove the IPA configuration file
     remove_file(paths.IPA_DEFAULT_CONF)
 
+    # Remove misc backups
+    remove_file(paths.OPENLDAP_LDAP_CONF + '.ipabkp')
+    remove_file(paths.NSSWITCH_CONF + '.ipabkp')
+
     # Remove the CA cert from the systemwide certificate store
     tasks.remove_ca_certs_from_systemwide_ca_store()
 
diff --git a/ipaserver/install/bindinstance.py b/ipaserver/install/bindinstance.py
index cd70cc983..82e443039 100644
--- a/ipaserver/install/bindinstance.py
+++ b/ipaserver/install/bindinstance.py
@@ -1204,6 +1204,8 @@ class BindInstance(service.Service):
             except ValueError as error:
                 logger.debug('%s', error)
 
+        installutils.rmtree(paths.BIND_LDAP_DNS_IPA_WORKDIR)
+
         # disabled by default, by ldap_enable()
         if enabled:
             self.enable()
diff --git a/ipaserver/install/dsinstance.py b/ipaserver/install/dsinstance.py
index 982634b7c..9daee0a68 100644
--- a/ipaserver/install/dsinstance.py
+++ b/ipaserver/install/dsinstance.py
@@ -1055,12 +1055,18 @@ class DsInstance(service.Service):
             logger.debug("Removing DS instance %s", serverid)
             try:
                 remove_ds_instance(serverid)
-                installutils.remove_keytab(paths.DS_KEYTAB)
-                installutils.remove_ccache(run_as=DS_USER)
             except ipautil.CalledProcessError:
                 logger.error("Failed to remove DS instance. You may "
                              "need to remove instance data manually")
 
+            installutils.remove_keytab(paths.DS_KEYTAB)
+            installutils.remove_ccache(run_as=DS_USER)
+
+            # Remove scripts dir
+            scripts = paths.VAR_LIB_DIRSRV_INSTANCE_SCRIPTS_TEMPLATE % (
+                serverid)
+            installutils.rmtree(scripts)
+
         # Just eat this state
         self.restore_state("user_exists")
 
diff --git a/ipaserver/install/server/install.py b/ipaserver/install/server/install.py
index b35b9bc85..8a009db7d 100644
--- a/ipaserver/install/server/install.py
+++ b/ipaserver/install/server/install.py
@@ -1150,6 +1150,10 @@ def uninstall(installer):
                      'but it may\n'
                      'mean your system hasn\'t be restored to its '
                      'pre-installation state.', SYSRESTORE_DIR_PATH)
+    else:
+        # sysrestore.state has no state left, remove it
+        sysrestore = os.path.join(SYSRESTORE_DIR_PATH, 'sysrestore.state')
+        installutils.remove_file(sysrestore)
 
     # Note that this name will be wrong after the first uninstall.
     dirname = dsinstance.config_dirname(