UefiCpuPkg: Has APs in 64 bit long-mode before booting to OS.

During the finalization of Mp initialization before booting into the OS,
 depending on whether Mwait is supported or not, AsmRelocateApLoop
 places Aps in MWAIT-loop or HLT-loop.

Since paging is necessary for long mode, the original implementation of
moving APs to 32-bit was to disable paging to ensure that the booting
does not crash.

The current modification creates a page table in reserved memory,
avoiding switching modes and reclaiming memory by OS. This modification
is only for 64 bit mode.

More specifically, we keep the AMD logic as the original code flow,
extract and update the Intel-related code, where the APs would stay
in 64-bit, and run in a Mwait or Hlt loop until the OS wake them up.

Signed-off-by: Ray Ni <ray.ni@intel.com>
Signed-off-by: Yuanhao Xie <yuanhao.xie@intel.com>
Reviewed-by: Ray Ni <ray.ni@intel.com>

7 files changed, 184 insertions, 200 deletions

diff --git a/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf b/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf
index cd07de3..8c8b81d 100644
--- a/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf
+++ b/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf
@@ -1,7 +1,7 @@
 ## @file

 #  MP Initialize Library instance for DXE driver.

 #

-#  Copyright (c) 2016 - 2021, Intel Corporation. All rights reserved.<BR>

+#  Copyright (c) 2016 - 2022, Intel Corporation. All rights reserved.<BR>

 #  SPDX-License-Identifier: BSD-2-Clause-Patent

 #

 ##

@@ -24,9 +24,11 @@
 [Sources.IA32]

   Ia32/AmdSev.c

   Ia32/MpFuncs.nasm

+  Ia32/CreatePageTable.c

 

 [Sources.X64]

   X64/AmdSev.c

+  X64/CreatePageTable.c

   X64/MpFuncs.nasm

 

 [Sources.common]

@@ -57,6 +59,9 @@
   CcExitLib

   MicrocodeLib

 

+[LibraryClasses.X64]

+  CpuPageTableLib

+

 [Protocols]

   gEfiTimerArchProtocolGuid                     ## SOMETIMES_CONSUMES

 

diff --git a/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c b/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c
index 445e085..beab06a 100644
--- a/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c
+++ b/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c
@@ -28,6 +28,7 @@ volatile BOOLEAN  mStopCheckAllApsStatus       = TRUE;
 VOID              *mReservedApLoopFunc         = NULL;

 UINTN             mReservedTopOfApStack;

 volatile UINT32   mNumberToFinish = 0;

+UINTN             mApPageTable;

 

 //

 // Begin wakeup buffer allocation below 0x88000

@@ -407,12 +408,9 @@ RelocateApLoop (
     AsmRelocateApLoopFunc (

       MwaitSupport,

       CpuMpData->ApTargetCState,

-      CpuMpData->PmCodeSegment,

       StackStart - ProcessorNumber * AP_SAFE_STACK_SIZE,

       (UINTN)&mNumberToFinish,

-      CpuMpData->Pm16CodeSegment,

-      CpuMpData->SevEsAPBuffer,

-      CpuMpData->WakeupBuffer

+      mApPageTable

       );

   }

 

@@ -477,7 +475,6 @@ InitMpGlobalData (
   )

 {

   EFI_STATUS                       Status;

-  EFI_PHYSICAL_ADDRESS             Address;

   UINTN                            ApSafeBufferSize;

   UINTN                            Index;

   EFI_GCD_MEMORY_SPACE_DESCRIPTOR  MemDesc;

@@ -545,60 +542,45 @@ InitMpGlobalData (
   // Allocating it in advance since memory services are not available in

   // Exit Boot Services callback function.

   //

-  ApSafeBufferSize = EFI_PAGES_TO_SIZE (

-                       EFI_SIZE_TO_PAGES (

-                         CpuMpData->AddressMap.RelocateApLoopFuncSize

-                         )

-                       );

-  Address = BASE_4GB - 1;

-  Status  = gBS->AllocatePages (

-                   AllocateMaxAddress,

-                   EfiReservedMemoryType,

-                   EFI_SIZE_TO_PAGES (ApSafeBufferSize),

-                   &Address

-                   );

-  ASSERT_EFI_ERROR (Status);

-

-  mReservedApLoopFunc = (VOID *)(UINTN)Address;

-  ASSERT (mReservedApLoopFunc != NULL);

-

-  //

-  // Make sure that the buffer memory is executable if NX protection is enabled

-  // for EfiReservedMemoryType.

-  //

-  // TODO: Check EFI_MEMORY_XP bit set or not once it's available in DXE GCD

-  //       service.

+  // +------------+

+  // | Ap Loop    |

+  // +------------+

+  // | Stack * N  |

+  // +------------+ (low address)

   //

-  Status = gDS->GetMemorySpaceDescriptor (Address, &MemDesc);

-  if (!EFI_ERROR (Status)) {

-    gDS->SetMemorySpaceAttributes (

-           Address,

-           ApSafeBufferSize,

-           MemDesc.Attributes & (~EFI_MEMORY_XP)

-           );

-  }

-

   ApSafeBufferSize = EFI_PAGES_TO_SIZE (

                        EFI_SIZE_TO_PAGES (

                          CpuMpData->CpuCount * AP_SAFE_STACK_SIZE

+                         + CpuMpData->AddressMap.RelocateApLoopFuncSize

                          )

                        );

-  Address = BASE_4GB - 1;

-  Status  = gBS->AllocatePages (

-                   AllocateMaxAddress,

-                   EfiReservedMemoryType,

-                   EFI_SIZE_TO_PAGES (ApSafeBufferSize),

-                   &Address

-                   );

-  ASSERT_EFI_ERROR (Status);

 

-  mReservedTopOfApStack = (UINTN)Address + ApSafeBufferSize;

+  mReservedTopOfApStack = (UINTN)AllocateReservedPages (EFI_SIZE_TO_PAGES (ApSafeBufferSize));

+  ASSERT (mReservedTopOfApStack != 0);

   ASSERT ((mReservedTopOfApStack & (UINTN)(CPU_STACK_ALIGNMENT - 1)) == 0);

-  CopyMem (

-    mReservedApLoopFunc,

-    CpuMpData->AddressMap.RelocateApLoopFuncAddress,

-    CpuMpData->AddressMap.RelocateApLoopFuncSize

-    );

+  ASSERT ((AP_SAFE_STACK_SIZE & (CPU_STACK_ALIGNMENT - 1)) == 0);

+

+  mReservedApLoopFunc = (VOID *)(mReservedTopOfApStack + CpuMpData->CpuCount * AP_SAFE_STACK_SIZE);

+  if (StandardSignatureIsAuthenticAMD ()) {

+    CopyMem (

+      mReservedApLoopFunc,

+      CpuMpData->AddressMap.RelocateApLoopFuncAddressAmd,

+      CpuMpData->AddressMap.RelocateApLoopFuncSizeAmd

+      );

+  } else {

+    CopyMem (

+      mReservedApLoopFunc,

+      CpuMpData->AddressMap.RelocateApLoopFuncAddress,

+      CpuMpData->AddressMap.RelocateApLoopFuncSize

+      );

+

+    mApPageTable = CreatePageTable (

+                     mReservedTopOfApStack,

+                     ApSafeBufferSize

+                     );

+  }

+

+  mReservedTopOfApStack += CpuMpData->CpuCount * AP_SAFE_STACK_SIZE;

 

   Status = gBS->CreateEvent (

                   EVT_TIMER | EVT_NOTIFY_SIGNAL,

diff --git a/UefiCpuPkg/Library/MpInitLib/Ia32/CreatePageTable.c b/UefiCpuPkg/Library/MpInitLib/Ia32/CreatePageTable.c
new file mode 100644
index 0000000..525885b
--- /dev/null
+++ b/UefiCpuPkg/Library/MpInitLib/Ia32/CreatePageTable.c
@@ -0,0 +1,27 @@
+/** @file

+  Function to create page talbe.

+  Only create page table for x64, and leave the CreatePageTable empty for Ia32.

+

+  Copyright (c) 2022, Intel Corporation. All rights reserved.<BR>

+  SPDX-License-Identifier: BSD-2-Clause-Patent

+

+**/

+

+#include <Base.h>

+

+/**

+  Only create page table for x64, and leave the CreatePageTable empty for Ia32.

+

+  @param[in]      LinearAddress  The start of the linear address range.

+  @param[in]      Length         The length of the linear address range.

+

+  @return The page table to be created.

+**/

+UINTN

+CreatePageTable (

+  IN UINTN  Address,

+  IN UINTN  Length

+  )

+{

+  return 0;

+}

diff --git a/UefiCpuPkg/Library/MpInitLib/MpLib.h b/UefiCpuPkg/Library/MpInitLib/MpLib.h
index 1102003..13d515c 100644
--- a/UefiCpuPkg/Library/MpInitLib/MpLib.h
+++ b/UefiCpuPkg/Library/MpInitLib/MpLib.h
@@ -392,12 +392,9 @@ typedef
 (EFIAPI *ASM_RELOCATE_AP_LOOP)(

   IN BOOLEAN                 MwaitSupport,

   IN UINTN                   ApTargetCState,

-  IN UINTN                   PmCodeSegment,

   IN UINTN                   TopOfApStack,

   IN UINTN                   NumberToFinish,

-  IN UINTN                   Pm16CodeSegment,

-  IN UINTN                   SevEsAPJumpTable,

-  IN UINTN                   WakeupBuffer

+  IN UINTN                   Cr3

   );

 

 /**

@@ -513,6 +510,20 @@ WakeUpAP (
   );

 

 /**

+  Create 1:1 mapping page table in reserved memory to map the specified address range.

+

+  @param[in]      LinearAddress  The start of the linear address range.

+  @param[in]      Length         The length of the linear address range.

+

+  @return The page table to be created.

+**/

+UINTN

+CreatePageTable (

+  IN UINTN  Address,

+  IN UINTN  Length

+  );

+

+/**

   Initialize global data for MP support.

 

   @param[in] CpuMpData  The pointer to CPU MP Data structure.

diff --git a/UefiCpuPkg/Library/MpInitLib/X64/CreatePageTable.c b/UefiCpuPkg/Library/MpInitLib/X64/CreatePageTable.c
new file mode 100644
index 0000000..548ef3f
--- /dev/null
+++ b/UefiCpuPkg/Library/MpInitLib/X64/CreatePageTable.c
@@ -0,0 +1,75 @@
+/** @file

+  Function to create page talbe.

+  Only create page table for x64, and leave the CreatePageTable empty for Ia32.

+

+  Copyright (c) 2022, Intel Corporation. All rights reserved.<BR>

+  SPDX-License-Identifier: BSD-2-Clause-Patent

+

+**/

+#include <Library/CpuPageTableLib.h>

+#include <Library/MemoryAllocationLib.h>

+#include <Base.h>

+#include <Library/BaseMemoryLib.h>

+#include <Library/DebugLib.h>

+

+/**

+  Create 1:1 mapping page table in reserved memory to map the specified address range.

+

+  @param[in]      LinearAddress  The start of the linear address range.

+  @param[in]      Length         The length of the linear address range.

+

+  @return The page table to be created.

+**/

+UINTN

+CreatePageTable (

+  IN UINTN  Address,

+  IN UINTN  Length

+  )

+{

+  EFI_STATUS  Status;

+  VOID        *PageTableBuffer;

+  UINTN       PageTableBufferSize;

+  UINTN       PageTable;

+

+  IA32_MAP_ATTRIBUTE  MapAttribute;

+  IA32_MAP_ATTRIBUTE  MapMask;

+

+  MapAttribute.Uint64         = Address;

+  MapAttribute.Bits.Present   = 1;

+  MapAttribute.Bits.ReadWrite = 1;

+

+  MapMask.Bits.PageTableBaseAddress = 1;

+  MapMask.Bits.Present              = 1;

+  MapMask.Bits.ReadWrite            = 1;

+

+  PageTable           = 0;

+  PageTableBufferSize = 0;

+

+  Status = PageTableMap (

+             &PageTable,

+             Paging4Level,

+             NULL,

+             &PageTableBufferSize,

+             Address,

+             Length,

+             &MapAttribute,

+             &MapMask

+             );

+  ASSERT (Status == EFI_BUFFER_TOO_SMALL);

+  DEBUG ((DEBUG_INFO, "AP Page Table Buffer Size = %x\n", PageTableBufferSize));

+

+  PageTableBuffer = AllocateReservedPages (EFI_SIZE_TO_PAGES (PageTableBufferSize));

+  ASSERT (PageTableBuffer != NULL);

+  Status = PageTableMap (

+             &PageTable,

+             Paging4Level,

+             PageTableBuffer,

+             &PageTableBufferSize,

+             Address,

+             Length,

+             &MapAttribute,

+             &MapMask

+             );

+  ASSERT_EFI_ERROR (Status);

+  return PageTable;

+}

diff --git a/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm b/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm
index 39c3e86..8ae287d 100644
--- a/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm
+++ b/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm
@@ -279,120 +279,42 @@ CProcedureInvoke:
 RendezvousFunnelProcEnd:

 

 ;-------------------------------------------------------------------------------------

-;  AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);

+;  AsmRelocateApLoop (MwaitSupport, ApTargetCState, TopOfApStack, CountTofinish, Cr3);

+;  This function is called during the finalizaiton of Mp initialization before booting

+;  to OS, and aim to put Aps either in Mwait or HLT.

 ;-------------------------------------------------------------------------------------

-AsmRelocateApLoopStart:

-BITS 64

-    cmp        qword [rsp + 56], 0  ; SevEsAPJumpTable

-    je         NoSevEs

-

-    ;

-    ; Perform some SEV-ES related setup before leaving 64-bit mode

-    ;

-    push       rcx

-    push       rdx

-

-    ;

-    ; Get the RDX reset value using CPUID

-    ;

-    mov        rax, 1

-    cpuid

-    mov        rsi, rax          ; Save off the reset value for RDX

-

-    ;

-    ; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call

-    ;   - Must be done while in 64-bit long mode so that writes to

-    ;     the GHCB memory will be unencrypted.

-    ;   - No NAE events can be generated once this is set otherwise

-    ;     the AP_RESET_HOLD SW_EXITCODE will be overwritten.

-    ;

-    mov        rcx, 0xc0010130

-    rdmsr                        ; Retrieve current GHCB address

-    shl        rdx, 32

-    or         rdx, rax

-

-    mov        rdi, rdx

-    xor        rax, rax

-    mov        rcx, 0x800

-    shr        rcx, 3

-    rep stosq                    ; Clear the GHCB

-

-    mov        rax, 0x80000004   ; VMGEXIT AP_RESET_HOLD

-    mov        [rdx + 0x390], rax

-    mov        rax, 114          ; Set SwExitCode valid bit

-    bts        [rdx + 0x3f0], rax

-    inc        rax               ; Set SwExitInfo1 valid bit

-    bts        [rdx + 0x3f0], rax

-    inc        rax               ; Set SwExitInfo2 valid bit

-    bts        [rdx + 0x3f0], rax

+; +----------------+

+; | Cr3            |  rsp+40

+; +----------------+

+; | CountTofinish  |  r9

+; +----------------+

+; | TopOfApStack   |  r8

+; +----------------+

+; | ApTargetCState |  rdx

+; +----------------+

+; | MwaitSupport   |  rcx

+; +----------------+

+; | the return     |

+; +----------------+ low address

 

-    pop        rdx

-    pop        rcx

-

-NoSevEs:

-    cli                          ; Disable interrupt before switching to 32-bit mode

-    mov        rax, [rsp + 40]   ; CountTofinish

+AsmRelocateApLoopStart:

+    mov        rax, r9           ; CountTofinish

     lock dec   dword [rax]       ; (*CountTofinish)--

 

-    mov        r10, [rsp + 48]   ; Pm16CodeSegment

-    mov        rax, [rsp + 56]   ; SevEsAPJumpTable

-    mov        rbx, [rsp + 64]   ; WakeupBuffer

-    mov        rsp, r9           ; TopOfApStack

-

-    push       rax               ; Save SevEsAPJumpTable

-    push       rbx               ; Save WakeupBuffer

-    push       r10               ; Save Pm16CodeSegment

-    push       rcx               ; Save MwaitSupport

-    push       rdx               ; Save ApTargetCState

-

-    lea        rax, [PmEntry]    ; rax <- The start address of transition code

-

-    push       r8

-    push       rax

-

-    ;

-    ; Clear R8 - R15, for reset, before going into 32-bit mode

-    ;

-    xor        r8, r8

-    xor        r9, r9

-    xor        r10, r10

-    xor        r11, r11

-    xor        r12, r12

-    xor        r13, r13

-    xor        r14, r14

-    xor        r15, r15

-

-    ;

-    ; Far return into 32-bit mode

-    ;

-    retfq

-

-BITS 32

-PmEntry:

-    mov        eax, cr0

-    btr        eax, 31           ; Clear CR0.PG

-    mov        cr0, eax          ; Disable paging and caches

-

-    mov        ecx, 0xc0000080

-    rdmsr

-    and        ah, ~ 1           ; Clear LME

-    wrmsr

-    mov        eax, cr4

-    and        al, ~ (1 << 5)    ; Clear PAE

-    mov        cr4, eax

-

-    pop        edx

-    add        esp, 4

-    pop        ecx,

-    add        esp, 4

+    mov        rax, [rsp + 40]    ; Cr3

+    ; Do not push on old stack, since old stack is not mapped

+    ; in the page table pointed by cr3

+    mov        cr3, rax

+    mov        rsp, r8            ; TopOfApStack

 

 MwaitCheck:

     cmp        cl, 1              ; Check mwait-monitor support

     jnz        HltLoop

-    mov        ebx, edx           ; Save C-State to ebx

+    mov        rbx, rdx           ; Save C-State to ebx

+

 MwaitLoop:

     cli

-    mov        eax, esp           ; Set Monitor Address

+    mov        rax, rsp           ; Set Monitor Address

     xor        ecx, ecx           ; ecx = 0

     xor        edx, edx           ; edx = 0

     monitor

@@ -402,49 +324,10 @@ MwaitLoop:
     jmp        MwaitLoop

 

 HltLoop:

-    pop        edx                ; PM16CodeSegment

-    add        esp, 4

-    pop        ebx                ; WakeupBuffer

-    add        esp, 4

-    pop        eax                ; SevEsAPJumpTable

-    add        esp, 4

-    cmp        eax, 0             ; Check for SEV-ES

-    je         DoHlt

-

-    cli

-    ;

-    ; SEV-ES is enabled, use VMGEXIT (GHCB information already

-    ; set by caller)

-    ;

-BITS 64

-    rep        vmmcall

-BITS 32

-

-    ;

-    ; Back from VMGEXIT AP_HLT_LOOP

-    ;   Push the FLAGS/CS/IP values to use

-    ;

-    push       word 0x0002        ; EFLAGS

-    xor        ecx, ecx

-    mov        cx, [eax + 2]      ; CS

-    push       cx

-    mov        cx, [eax]          ; IP

-    push       cx

-    push       word 0x0000        ; For alignment, will be discarded

-

-    push       edx

-    push       ebx

-

-    mov        edx, esi           ; Restore RDX reset value

-

-    retf

-

-DoHlt:

     cli

     hlt

-    jmp        DoHlt

+    jmp        HltLoop

 

-BITS 64

 AsmRelocateApLoopEnd:

 

 ;-------------------------------------------------------------------------------------

diff --git a/UefiCpuPkg/UefiCpuPkg.dsc b/UefiCpuPkg/UefiCpuPkg.dsc
index f9a4608..781aced 100644
--- a/UefiCpuPkg/UefiCpuPkg.dsc
+++ b/UefiCpuPkg/UefiCpuPkg.dsc
@@ -94,6 +94,7 @@
   MemoryAllocationLib|MdePkg/Library/UefiMemoryAllocationLib/UefiMemoryAllocationLib.inf

   HobLib|MdePkg/Library/DxeHobLib/DxeHobLib.inf

   CpuExceptionHandlerLib|UefiCpuPkg/Library/CpuExceptionHandlerLib/DxeCpuExceptionHandlerLib.inf

+  CpuPageTableLib|UefiCpuPkg/Library/CpuPageTableLib/CpuPageTableLib.inf

   MpInitLib|UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf

   RegisterCpuFeaturesLib|UefiCpuPkg/Library/RegisterCpuFeaturesLib/DxeRegisterCpuFeaturesLib.inf

   CpuCacheInfoLib|UefiCpuPkg/Library/CpuCacheInfoLib/DxeCpuCacheInfoLib.inf