diff --git a/src/qemu/qemu_domain_address.c b/src/qemu/qemu_domain_address.c
index 1171dc5b14..68dbf9e95b 100644
--- a/src/qemu/qemu_domain_address.c
+++ b/src/qemu/qemu_domain_address.c
@@ -3122,6 +3122,29 @@ qemuDomainAssignMemoryDeviceSlot(virQEMUDriverPtr driver,
 }
 
 
+void
+qemuDomainReleaseMemoryDeviceSlot(virDomainObjPtr vm,
+                                  virDomainMemoryDefPtr mem)
+{
+    switch (mem->model) {
+    case VIR_DOMAIN_MEMORY_MODEL_DIMM:
+    case VIR_DOMAIN_MEMORY_MODEL_NVDIMM:
+        /* We don't need to release anything. Slot map is not
+         * kept around. It's constructed every time when
+         * assigning new slot. */
+        break;
+
+    case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
+        qemuDomainReleaseDeviceAddress(vm, &mem->info);
+        break;
+
+    case VIR_DOMAIN_MEMORY_MODEL_NONE:
+    case VIR_DOMAIN_MEMORY_MODEL_LAST:
+        break;
+    }
+}
+
+
 static int
 qemuDomainAssignMemorySlots(virDomainDefPtr def)
 {
diff --git a/src/qemu/qemu_domain_address.h b/src/qemu/qemu_domain_address.h
index 20a46160d5..bfd56e163e 100644
--- a/src/qemu/qemu_domain_address.h
+++ b/src/qemu/qemu_domain_address.h
@@ -60,6 +60,9 @@ int qemuDomainAssignMemoryDeviceSlot(virQEMUDriverPtr driver,
                                      virDomainObjPtr vm,
                                      virDomainMemoryDefPtr mem);
 
+void qemuDomainReleaseMemoryDeviceSlot(virDomainObjPtr vm,
+                                       virDomainMemoryDefPtr mem);
+
 int qemuDomainEnsureVirtioAddress(bool *releaseAddr,
                                   virDomainObjPtr vm,
                                   virDomainDeviceDefPtr dev,
diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c
index 94c3df6368..dc2b46057c 100644
--- a/src/qemu/qemu_hotplug.c
+++ b/src/qemu/qemu_hotplug.c
@@ -2400,6 +2400,7 @@ qemuDomainAttachMemory(virQEMUDriverPtr driver,
     g_autofree char *devstr = NULL;
     g_autofree char *objalias = NULL;
     bool objAdded = false;
+    bool releaseaddr = false;
     bool teardownlabel = false;
     bool teardowncgroup = false;
     bool teardowndevice = false;
@@ -2416,6 +2417,7 @@ qemuDomainAttachMemory(virQEMUDriverPtr driver,
 
     if (qemuDomainAssignMemoryDeviceSlot(driver, vm, mem) < 0)
         goto cleanup;
+    releaseaddr = true;
 
     /* in cases where we are using a VM with aliases generated according to the
      * index of the memory device we need to keep continue using that scheme */
@@ -2492,6 +2494,8 @@ qemuDomainAttachMemory(virQEMUDriverPtr driver,
         if (teardowndevice &&
             qemuDomainNamespaceTeardownMemory(vm, mem) <  0)
             VIR_WARN("Unable to remove memory device from /dev");
+        if (releaseaddr)
+            qemuDomainReleaseMemoryDeviceSlot(vm, mem);
     }
 
     virJSONValueFree(props);
@@ -4457,6 +4461,8 @@ qemuDomainRemoveMemoryDevice(virQEMUDriverPtr driver,
     if (qemuProcessDestroyMemoryBackingPath(driver, vm, mem) < 0)
         VIR_WARN("Unable to destroy memory backing path");
 
+    qemuDomainReleaseMemoryDeviceSlot(vm, mem);
+
     virDomainMemoryDefFree(mem);
 
     /* fix the balloon size */