aarch64: Enable IRQ routing for legacy devices

On AArch64, interrupt controller (GIC) is emulated by KVM. VMM need to
set IRQ routing for devices, including legacy ones.

Before this commit, IRQ routing was only set for MSI. Legacy routing
entries of type KVM_IRQ_ROUTING_IRQCHIP were missing. That is way legacy
devices (like serial device ttyS0) does not work.

The setting of X86 IRQ routing entries are not impacted.

Signed-off-by: Michael Zhao <michael.zhao@arm.com>

arch/src/aarch64/layout.rs

@@ -78,9 +78,9 @@ pub const FDT_MAX_SIZE: usize = 0x20_0000;
 // * bigger than 32
 // * less than 1023 and
 // * a multiple of 32.
-// We are setting up our interrupt controller to support a maximum of 128 interrupts.
+// We are setting up our interrupt controller to support a maximum of 256 interrupts.
 /// First usable interrupt on aarch64.
-pub const IRQ_BASE: u32 = 32;
+pub const IRQ_BASE: u32 = 0;
 
 /// Last usable interrupt on aarch64.
-pub const IRQ_MAX: u32 = 159;
+pub const IRQ_MAX: u32 = 255;

devices/src/gic.rs

@@ -6,16 +6,16 @@ use super::interrupt_controller::{Error, InterruptController};
 use std::result;
 use std::sync::Arc;
 use vm_device::interrupt::{
-    InterruptIndex, InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig,
+    InterruptIndex, InterruptManager, InterruptSourceConfig, InterruptSourceGroup,
+    LegacyIrqSourceConfig, MsiIrqGroupConfig,
 };
 use vmm_sys_util::eventfd::EventFd;
 
 type Result<T> = result::Result<T, Error>;
 
-// Reserve 32 IRQs (GSI 32 ~ 64) for legacy device.
-// GsiAllocator should allocate beyond this: from 64 on
+// Reserve 32 IRQs for legacy device.
+pub const IRQ_LEGACY_BASE: usize = 0;
 pub const IRQ_LEGACY_COUNT: usize = 32;
-pub const IRQ_SPI_OFFSET: usize = 32;
 
 // This Gic struct implements InterruptController to provide interrupt delivery service.
 // The Gic source files in arch/ folder maintain the Aarch64 specific Gic device.
@@ -34,7 +34,7 @@ impl Gic {
     ) -> Result<Gic> {
         let interrupt_source_group = interrupt_manager
             .create_group(MsiIrqGroupConfig {
-                base: IRQ_SPI_OFFSET as InterruptIndex,
+                base: IRQ_LEGACY_BASE as InterruptIndex,
                 count: IRQ_LEGACY_COUNT as InterruptIndex,
             })
             .map_err(Error::CreateInterruptSourceGroup)?;
@@ -47,9 +47,26 @@ impl Gic {
 
 impl InterruptController for Gic {
     fn enable(&self) -> Result<()> {
+        // Set irqfd for legacy interrupts
         self.interrupt_source_group
             .enable()
             .map_err(Error::EnableInterrupt)?;
+
+        // Set irq_routing for legacy interrupts.
+        //   irqchip: Hardcode to 0 as we support only 1 GIC
+        //   pin: Use irq number as pin
+        for i in IRQ_LEGACY_BASE..(IRQ_LEGACY_BASE + IRQ_LEGACY_COUNT) {
+            let config = LegacyIrqSourceConfig {
+                irqchip: 0,
+                pin: i as u32,
+            };
+            self.interrupt_source_group
+                .update(
+                    i as InterruptIndex,
+                    InterruptSourceConfig::LegacyIrq(config),
+                )
+                .map_err(Error::EnableInterrupt)?;
+        }
         Ok(())
     }
 

hypervisor/src/kvm/mod.rs

@@ -57,8 +57,8 @@ pub use kvm_bindings;
 use kvm_bindings::KVMIO;
 pub use kvm_bindings::{
     kvm_create_device, kvm_device_type_KVM_DEV_TYPE_VFIO, kvm_irq_routing, kvm_irq_routing_entry,
-    kvm_userspace_memory_region, KVM_IRQ_ROUTING_MSI, KVM_MEM_LOG_DIRTY_PAGES, KVM_MEM_READONLY,
-    KVM_MSI_VALID_DEVID,
+    kvm_userspace_memory_region, KVM_IRQ_ROUTING_IRQCHIP, KVM_IRQ_ROUTING_MSI,
+    KVM_MEM_LOG_DIRTY_PAGES, KVM_MEM_READONLY, KVM_MSI_VALID_DEVID,
 };
 #[cfg(target_arch = "aarch64")]
 use kvm_bindings::{

vm-allocator/src/system.rs

@@ -42,11 +42,11 @@ fn pagesize() -> usize {
 ///   #[cfg(target_arch = "x86_64")]
 ///   assert_eq!(allocator.allocate_irq(), Some(5));
 ///   #[cfg(target_arch = "aarch64")]
-///   assert_eq!(allocator.allocate_irq(), Some(32));
+///   assert_eq!(allocator.allocate_irq(), Some(0));
 ///   #[cfg(target_arch = "x86_64")]
 ///   assert_eq!(allocator.allocate_irq(), Some(6));
 ///   #[cfg(target_arch = "aarch64")]
-///   assert_eq!(allocator.allocate_irq(), Some(33));
+///   assert_eq!(allocator.allocate_irq(), Some(1));
 ///   assert_eq!(allocator.allocate_mmio_addresses(None, 0x1000, Some(0x1000)), Some(GuestAddress(0x1fff_f000)));
 ///
 /// ```

vm-device/src/interrupt/mod.rs

@@ -70,7 +70,10 @@ pub type InterruptIndex = u32;
 ///
 /// On x86 platforms, legacy interrupts means those interrupts routed through PICs or IOAPICs.
 #[derive(Copy, Clone, Debug)]
-pub struct LegacyIrqSourceConfig {}
+pub struct LegacyIrqSourceConfig {
+    pub irqchip: u32,
+    pub pin: u32,
+}
 
 /// Configuration data for MSI/MSI-X interrupts.
 ///

vmm/src/interrupt.rs

@@ -337,7 +337,7 @@ where
 pub mod kvm {
     use super::*;
     use hypervisor::kvm::KVM_MSI_VALID_DEVID;
-    use hypervisor::kvm::{kvm_irq_routing_entry, KVM_IRQ_ROUTING_MSI};
+    use hypervisor::kvm::{kvm_irq_routing_entry, KVM_IRQ_ROUTING_IRQCHIP, KVM_IRQ_ROUTING_MSI};
 
     type KvmMsiInterruptGroup = MsiInterruptGroup<kvm_irq_routing_entry>;
     type KvmRoutingEntry = RoutingEntry<kvm_irq_routing_entry>;
@@ -370,6 +370,20 @@ pub mod kvm {
                     masked: false,
                 };
 
+                return Ok(Box::new(kvm_entry));
+            } else if let InterruptSourceConfig::LegacyIrq(cfg) = &config {
+                let mut kvm_route = kvm_irq_routing_entry {
+                    gsi,
+                    type_: KVM_IRQ_ROUTING_IRQCHIP,
+                    ..Default::default()
+                };
+                kvm_route.u.irqchip.irqchip = cfg.irqchip;
+                kvm_route.u.irqchip.pin = cfg.pin;
+                let kvm_entry = KvmRoutingEntry {
+                    route: kvm_route,
+                    masked: false,
+                };
+
                 return Ok(Box::new(kvm_entry));
             }
 
@@ -493,7 +507,7 @@ mod tests {
         let res = get_dist_regs(gic.device());
         assert!(res.is_ok());
         let state = res.unwrap();
-        assert_eq!(state.len(), 244);
+        assert_eq!(state.len(), 649);
 
         let res = set_dist_regs(gic.device(), &state);
         assert!(res.is_ok());