diff --git a/src/security/security_dac.c b/src/security/security_dac.c
index 95dbe4636f..c327e4c9e0 100644
--- a/src/security/security_dac.c
+++ b/src/security/security_dac.c
@@ -1022,6 +1022,9 @@ virSecurityDACRestoreImageLabelInt(virSecurityManager *mgr,
         if (rc == 1) {
             VIR_DEBUG("Skipping image label restore on %s because FS is shared",
                       src->path);
+
+            ignore_value(virSecurityDACRecallLabel(priv, src->path, NULL, NULL));
+
             return 0;
         }
     }
diff --git a/src/security/security_selinux.c b/src/security/security_selinux.c
index 453ac67d25..779a52ac11 100644
--- a/src/security/security_selinux.c
+++ b/src/security/security_selinux.c
@@ -1837,8 +1837,15 @@ virSecuritySELinuxRestoreImageLabelInt(virSecurityManager *mgr,
         }
 
         if (rc == 1) {
+            g_autofree char *oldlabel = NULL;
+
             VIR_DEBUG("Skipping image label restore on %s because FS is shared",
                       src->path);
+
+            /* We still want to remove the local reference of the remembered
+             * seclabel. The destination will take its own reference when
+             * starting the migrated VM */
+            ignore_value(virSecuritySELinuxRecallLabel(src->path, &oldlabel));
             return 0;
         }
     }