virtio-devices: properly join all threads on Drop

This change is important to do a proper resource cleanup. We decided
to do this repetitive approach as VirtioCommon can't implement Drop
without major changes to the corresponding code. Also, devices such as
Net can't easily use the epoll_threads-abstraction from VirtioCommon as
it has multiple threads with different semantics.

Signed-off-by: Philipp Schuster <philipp.schuster@cyberus-technology.de>

virtio-devices/src/balloon.rs

@@ -450,6 +450,7 @@ impl Drop for Balloon {
             // Ignore the result because there is nothing we can do about it.
             let _ = kill_evt.write(1);
         }
+        self.common.wait_for_epoll_threads();
     }
 }
 

virtio-devices/src/block.rs

@@ -658,6 +658,7 @@ impl Drop for Block {
             // Ignore the result because there is nothing we can do about it.
             let _ = kill_evt.write(1);
         }
+        self.common.wait_for_epoll_threads();
     }
 }
 

virtio-devices/src/console.rs

@@ -710,6 +710,7 @@ impl Drop for Console {
             // Ignore the result because there is nothing we can do about it.
             let _ = kill_evt.write(1);
         }
+        self.common.wait_for_epoll_threads();
     }
 }
 

virtio-devices/src/iommu.rs

@@ -1000,6 +1000,7 @@ impl Drop for Iommu {
             // Ignore the result because there is nothing we can do about it.
             let _ = kill_evt.write(1);
         }
+        self.common.wait_for_epoll_threads();
     }
 }
 

virtio-devices/src/mem.rs

@@ -902,6 +902,7 @@ impl Drop for Mem {
             // Ignore the result because there is nothing we can do about it.
             let _ = kill_evt.write(1);
         }
+        self.common.wait_for_epoll_threads();
     }
 }
 

virtio-devices/src/net.rs

@@ -670,6 +670,11 @@ impl Drop for Net {
         }
         // Needed to ensure all references to tap FDs are dropped (#4868)
         self.common.wait_for_epoll_threads();
+        if let Some(thread) = self.ctrl_queue_epoll_thread.take() {
+            if let Err(e) = thread.join() {
+                error!("Error joining thread: {:?}", e);
+            }
+        }
     }
 }
 

virtio-devices/src/pmem.rs

@@ -364,6 +364,7 @@ impl Drop for Pmem {
             // Ignore the result because there is nothing we can do about it.
             let _ = kill_evt.write(1);
         }
+        self.common.wait_for_epoll_threads();
     }
 }
 

virtio-devices/src/rng.rs

@@ -228,6 +228,7 @@ impl Drop for Rng {
             // Ignore the result because there is nothing we can do about it.
             let _ = kill_evt.write(1);
         }
+        self.common.wait_for_epoll_threads();
     }
 }
 

virtio-devices/src/vhost_user/blk.rs

@@ -223,6 +223,12 @@ impl Drop for Blk {
                 error!("failed to kill vhost-user-blk: {:?}", e);
             }
         }
+        self.common.wait_for_epoll_threads();
+        if let Some(thread) = self.epoll_thread.take() {
+            if let Err(e) = thread.join() {
+                error!("Error joining thread: {:?}", e);
+            }
+        }
     }
 }
 

virtio-devices/src/vhost_user/fs.rs

@@ -462,6 +462,12 @@ impl Drop for Fs {
             // Ignore the result because there is nothing we can do about it.
             let _ = kill_evt.write(1);
         }
+        self.common.wait_for_epoll_threads();
+        if let Some(thread) = self.epoll_thread.take() {
+            if let Err(e) = thread.join() {
+                error!("Error joining thread: {:?}", e);
+            }
+        }
     }
 }
 

virtio-devices/src/vhost_user/net.rs

@@ -251,6 +251,19 @@ impl Drop for Net {
                 error!("failed to kill vhost-user-net: {:?}", e);
             }
         }
+
+        self.common.wait_for_epoll_threads();
+
+        if let Some(thread) = self.epoll_thread.take() {
+            if let Err(e) = thread.join() {
+                error!("Error joining thread: {:?}", e);
+            }
+        }
+        if let Some(thread) = self.ctrl_queue_epoll_thread.take() {
+            if let Err(e) = thread.join() {
+                error!("Error joining thread: {:?}", e);
+            }
+        }
     }
 }
 

virtio-devices/src/vsock/device.rs

@@ -397,6 +397,7 @@ where
             // Ignore the result because there is nothing we can do about it.
             let _ = kill_evt.write(1);
         }
+        self.common.wait_for_epoll_threads();
     }
 }
 

virtio-devices/src/watchdog.rs

@@ -278,6 +278,7 @@ impl Drop for Watchdog {
             // Ignore the result because there is nothing we can do about it.
             let _ = kill_evt.write(1);
         }
+        self.common.wait_for_epoll_threads();
     }
 }