diff --git a/src/conf/virsecretobj.c b/src/conf/virsecretobj.c
index e5dafa47a1..7ad77c7413 100644
--- a/src/conf/virsecretobj.c
+++ b/src/conf/virsecretobj.c
@@ -648,6 +648,29 @@ virSecretObjListGetUUIDs(virSecretObjListPtr secrets,
 }
 
 
+int
+virSecretObjDeleteConfig(virSecretObjPtr secret)
+{
+    if (!secret->def->ephemeral &&
+        unlink(secret->configFile) < 0 && errno != ENOENT) {
+        virReportSystemError(errno, _("cannot unlink '%s'"),
+                             secret->configFile);
+        return -1;
+    }
+
+    return 0;
+}
+
+
+void
+virSecretObjDeleteData(virSecretObjPtr secret)
+{
+    /* The configFile will already be removed, so secret won't be
+     * loaded again if this fails */
+    (void)unlink(secret->base64File);
+}
+
+
 static int
 virSecretLoadValidateUUID(virSecretDefPtr def,
                           const char *file)
diff --git a/src/conf/virsecretobj.h b/src/conf/virsecretobj.h
index 2e8dcf69eb..8f1247a1ff 100644
--- a/src/conf/virsecretobj.h
+++ b/src/conf/virsecretobj.h
@@ -93,6 +93,10 @@ int virSecretObjListGetUUIDs(virSecretObjListPtr secrets,
                              virSecretObjListACLFilter filter,
                              virConnectPtr conn);
 
+int virSecretObjDeleteConfig(virSecretObjPtr secret);
+
+void virSecretObjDeleteData(virSecretObjPtr secret);
+
 int virSecretLoadAllConfigs(virSecretObjListPtr secrets,
                             const char *configDir);
 #endif /* __VIRSECRETOBJ_H__ */
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index 5a6265fe96..6134ac2b84 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -896,6 +896,8 @@ virDomainObjListRename;
 
 # conf/virsecretobj.h
 virSecretLoadAllConfigs;
+virSecretObjDeleteConfig;
+virSecretObjDeleteData;
 virSecretObjEndAPI;
 virSecretObjListAdd;
 virSecretObjListExport;
diff --git a/src/secret/secret_driver.c b/src/secret/secret_driver.c
index c8b4163f9a..0767424448 100644
--- a/src/secret/secret_driver.c
+++ b/src/secret/secret_driver.c
@@ -176,19 +176,6 @@ secretSaveValue(const virSecretObj *secret)
     return ret;
 }
 
-static int
-secretDeleteSaved(const virSecretObj *secret)
-{
-    if (unlink(secret->configFile) < 0 && errno != ENOENT)
-        return -1;
-
-    /* When the XML is missing, the rest may waste disk space, but the secret
-       won't be loaded again, so we have succeeded already. */
-    (void)unlink(secret->base64File);
-
-    return 0;
-}
-
 /* Driver functions */
 
 static int
@@ -326,8 +313,10 @@ secretDefineXML(virConnectPtr conn,
             goto restore_backup;
         }
     } else if (backup && !backup->ephemeral) {
-        if (secretDeleteSaved(secret) < 0)
+        if (virSecretObjDeleteConfig(secret) < 0)
             goto restore_backup;
+
+        virSecretObjDeleteData(secret);
     }
     /* Saved successfully - drop old values */
     new_attrs = NULL;
@@ -490,10 +479,11 @@ secretUndefine(virSecretPtr obj)
     if (virSecretUndefineEnsureACL(obj->conn, secret->def) < 0)
         goto cleanup;
 
-    if (!secret->def->ephemeral &&
-        secretDeleteSaved(secret) < 0)
+    if (virSecretObjDeleteConfig(secret) < 0)
         goto cleanup;
 
+    virSecretObjDeleteData(secret);
+
     virSecretObjListRemove(driver->secrets, secret);
 
     ret = 0;