support booting with more than 255 CPUs

SDM[*1] says that if there are CPUs with APIC ID
greater than 254, BIOS is to pass control to OS
in x2APIC mode. Use the fact that QEMU passes in
"etc/max-cpus" max possible "APIC ID + 1" to
detect need for x2APIC mode. Also instead of
CMOS_BIOS_SMP_COUNT which is limited to 256 CPUs
use a new rom file "etc/boot-cpus" that QEMU
supporting more than 256 CPUs will provide.

*1) SDM: Volume 3: EXTENDED XAPIC (X2APIC):
     Initialization by System Software

Signed-off-by: Igor Mammedov <imammedo@redhat.com>

2 files changed, 21 insertions, 4 deletions

diff --git a/src/fw/smp.c b/src/fw/smp.c
index 9c404b9..bcbad69 100644
--- a/src/fw/smp.c
+++ b/src/fw/smp.c
@@ -20,6 +20,9 @@
 #define APIC_LINT1   ((u8*)BUILD_APIC_ADDR + 0x360)
 
 #define APIC_ENABLED 0x0100
+#define MSR_IA32_APIC_BASE 0x01B
+#define MSR_LOCAL_APIC_ID 0x802
+#define MSR_IA32_APICBASE_EXTD (1ULL << 10) /* Enable x2APIC mode */
 
 static struct { u32 index; u64 val; } smp_msr[32];
 static u32 smp_msr_count;
@@ -59,11 +62,21 @@ int apic_id_is_present(u8 apic_id)
 static int
 apic_id_init(void)
 {
-    // Track found apic id for use in legacy internal bios tables
     u32 eax, ebx, ecx, cpuid_features;
     cpuid(1, &eax, &ebx, &ecx, &cpuid_features);
-    u8 apic_id = ebx>>24;
-    FoundAPICIDs[apic_id/32] |= 1 << (apic_id % 32);
+    u32 apic_id = ebx>>24;
+    if (MaxCountCPUs < 256) { // xAPIC mode
+        // Track found apic id for use in legacy internal bios tables
+        FoundAPICIDs[apic_id/32] |= 1 << (apic_id % 32);
+    } else if (ecx & CPUID_X2APIC) {
+        // switch to x2APIC mode
+        u64 apic_base = rdmsr(MSR_IA32_APIC_BASE);
+        wrmsr(MSR_IA32_APIC_BASE, apic_base | MSR_IA32_APICBASE_EXTD);
+        apic_id = rdmsr(MSR_LOCAL_APIC_ID);
+    } else {
+        // x2APIC is masked by CPUID
+        apic_id = -1;
+    }
     return apic_id;
 }
 
@@ -101,7 +114,6 @@ smp_scan(void)
     }
 
     // mark the BSP initial APIC ID as found, too:
-    apic_id_init();
     CountCPUs = 1;
 
     // Setup jump trampoline to counter code.
@@ -131,6 +143,10 @@ smp_scan(void)
     u32 sipi_vector = BUILD_AP_BOOT_ADDR >> 12;
     writel(APIC_ICR_LOW, 0x000C4600 | sipi_vector);
 
+    // switch to x2APIC mode after sending SIPI so that
+    // x2APIC and xAPIC mode could share AP wake up code
+    apic_id_init();
+
     // Wait for other CPUs to process the SIPI.
     u16 expected_cpus_count = qemu_get_present_cpus_count();
     while (expected_cpus_count != CountCPUs)
diff --git a/src/x86.h b/src/x86.h
index 53378e9..a770e6f 100644
--- a/src/x86.h
+++ b/src/x86.h
@@ -68,6 +68,7 @@ static inline void wbinvd(void)
 #define CPUID_MSR (1 << 5)
 #define CPUID_APIC (1 << 9)
 #define CPUID_MTRR (1 << 12)
+#define CPUID_X2APIC (1 << 21)
 static inline void __cpuid(u32 index, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx)
 {
     asm("cpuid"