qemu

whpx-all.c (87034B)

---

 /*
  * QEMU Windows Hypervisor Platform accelerator (WHPX)
  *
  * Copyright Microsoft Corp. 2017
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  *
  */
 
 #include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/address-spaces.h"
 #include "exec/ioport.h"
 #include "exec/gdbstub.h"
 #include "qemu/accel.h"
 #include "sysemu/whpx.h"
 #include "sysemu/cpus.h"
 #include "sysemu/runstate.h"
 #include "qemu/main-loop.h"
 #include "hw/boards.h"
 #include "hw/i386/ioapic.h"
 #include "hw/i386/apic_internal.h"
 #include "qemu/error-report.h"
 #include "qapi/error.h"
 #include "qapi/qapi-types-common.h"
 #include "qapi/qapi-visit-common.h"
 #include "migration/blocker.h"
 #include <winerror.h>
 
 #include "whpx-internal.h"
 #include "whpx-accel-ops.h"
 
 #include <WinHvPlatform.h>
 #include <WinHvEmulation.h>
 
 #define HYPERV_APIC_BUS_FREQUENCY      (200000000ULL)
 
 static const WHV_REGISTER_NAME whpx_register_names[] = {
 
     /* X64 General purpose registers */
     WHvX64RegisterRax,
     WHvX64RegisterRcx,
     WHvX64RegisterRdx,
     WHvX64RegisterRbx,
     WHvX64RegisterRsp,
     WHvX64RegisterRbp,
     WHvX64RegisterRsi,
     WHvX64RegisterRdi,
     WHvX64RegisterR8,
     WHvX64RegisterR9,
     WHvX64RegisterR10,
     WHvX64RegisterR11,
     WHvX64RegisterR12,
     WHvX64RegisterR13,
     WHvX64RegisterR14,
     WHvX64RegisterR15,
     WHvX64RegisterRip,
     WHvX64RegisterRflags,
 
     /* X64 Segment registers */
     WHvX64RegisterEs,
     WHvX64RegisterCs,
     WHvX64RegisterSs,
     WHvX64RegisterDs,
     WHvX64RegisterFs,
     WHvX64RegisterGs,
     WHvX64RegisterLdtr,
     WHvX64RegisterTr,
 
     /* X64 Table registers */
     WHvX64RegisterIdtr,
     WHvX64RegisterGdtr,
 
     /* X64 Control Registers */
     WHvX64RegisterCr0,
     WHvX64RegisterCr2,
     WHvX64RegisterCr3,
     WHvX64RegisterCr4,
     WHvX64RegisterCr8,
 
     /* X64 Debug Registers */
     /*
      * WHvX64RegisterDr0,
      * WHvX64RegisterDr1,
      * WHvX64RegisterDr2,
      * WHvX64RegisterDr3,
      * WHvX64RegisterDr6,
      * WHvX64RegisterDr7,
      */
 
     /* X64 Floating Point and Vector Registers */
     WHvX64RegisterXmm0,
     WHvX64RegisterXmm1,
     WHvX64RegisterXmm2,
     WHvX64RegisterXmm3,
     WHvX64RegisterXmm4,
     WHvX64RegisterXmm5,
     WHvX64RegisterXmm6,
     WHvX64RegisterXmm7,
     WHvX64RegisterXmm8,
     WHvX64RegisterXmm9,
     WHvX64RegisterXmm10,
     WHvX64RegisterXmm11,
     WHvX64RegisterXmm12,
     WHvX64RegisterXmm13,
     WHvX64RegisterXmm14,
     WHvX64RegisterXmm15,
     WHvX64RegisterFpMmx0,
     WHvX64RegisterFpMmx1,
     WHvX64RegisterFpMmx2,
     WHvX64RegisterFpMmx3,
     WHvX64RegisterFpMmx4,
     WHvX64RegisterFpMmx5,
     WHvX64RegisterFpMmx6,
     WHvX64RegisterFpMmx7,
     WHvX64RegisterFpControlStatus,
     WHvX64RegisterXmmControlStatus,
 
     /* X64 MSRs */
     WHvX64RegisterEfer,
 #ifdef TARGET_X86_64
     WHvX64RegisterKernelGsBase,
 #endif
     WHvX64RegisterApicBase,
     /* WHvX64RegisterPat, */
     WHvX64RegisterSysenterCs,
     WHvX64RegisterSysenterEip,
     WHvX64RegisterSysenterEsp,
     WHvX64RegisterStar,
 #ifdef TARGET_X86_64
     WHvX64RegisterLstar,
     WHvX64RegisterCstar,
     WHvX64RegisterSfmask,
 #endif
 
     /* Interrupt / Event Registers */
     /*
      * WHvRegisterPendingInterruption,
      * WHvRegisterInterruptState,
      * WHvRegisterPendingEvent0,
      * WHvRegisterPendingEvent1
      * WHvX64RegisterDeliverabilityNotifications,
      */
 };
 
 struct whpx_register_set {
     WHV_REGISTER_VALUE values[RTL_NUMBER_OF(whpx_register_names)];
 };
 
 /*
  * The current implementation of instruction stepping sets the TF flag
  * in RFLAGS, causing the CPU to raise an INT1 after each instruction.
  * This corresponds to the WHvX64ExceptionTypeDebugTrapOrFault exception.
  *
  * This approach has a few limitations:
  *     1. Stepping over a PUSHF/SAHF instruction will save the TF flag
  *        along with the other flags, possibly restoring it later. It would
  *        result in another INT1 when the flags are restored, triggering
  *        a stop in gdb that could be cleared by doing another step.
  *
  *        Stepping over a POPF/LAHF instruction will let it overwrite the
  *        TF flags, ending the stepping mode.
  *
  *     2. Stepping over an instruction raising an exception (e.g. INT, DIV,
  *        or anything that could result in a page fault) will save the flags
  *        to the stack, clear the TF flag, and let the guest execute the
  *        handler. Normally, the guest will restore the original flags,
  *        that will continue single-stepping.
  *
  *     3. Debuggers running on the guest may wish to set TF to do instruction
  *        stepping. INT1 events generated by it would be intercepted by us,
  *        as long as the gdb is connected to QEMU.
  *
  * In practice this means that:
  *     1. Stepping through flags-modifying instructions may cause gdb to
  *        continue or stop in unexpected places. This will be fully recoverable
  *        and will not crash the target.
  *
  *     2. Stepping over an instruction that triggers an exception will step
  *        over the exception handler, not into it.
  *
  *     3. Debugging the guest via gdb, while running debugger on the guest
  *        at the same time may lead to unexpected effects. Removing all
  *        breakpoints set via QEMU will prevent any further interference
  *        with the guest-level debuggers.
  *
  * The limitations can be addressed as shown below:
  *     1. PUSHF/SAHF/POPF/LAHF/IRET instructions can be emulated instead of
  *        stepping through them. The exact semantics of the instructions is
  *        defined in the "Combined Volume Set of Intel 64 and IA-32
  *        Architectures Software Developer's Manuals", however it involves a
  *        fair amount of corner cases due to compatibility with real mode,
  *        virtual 8086 mode, and differences between 64-bit and 32-bit modes.
  *
  *     2. We could step into the guest's exception handlers using the following
  *        sequence:
  *          a. Temporarily enable catching of all exception types via
  *             whpx_set_exception_exit_bitmap().
  *          b. Once an exception is intercepted, read the IDT/GDT and locate
  *             the original handler.
  *          c. Patch the original handler, injecting an INT3 at the beginning.
  *          d. Update the exception exit bitmap to only catch the
  *             WHvX64ExceptionTypeBreakpointTrap exception.
  *          e. Let the affected CPU run in the exclusive mode.
  *          f. Restore the original handler and the exception exit bitmap.
  *        Note that handling all corner cases related to IDT/GDT is harder
  *        than it may seem. See x86_cpu_get_phys_page_attrs_debug() for a
  *        rough idea.
  *
  *     3. In order to properly support guest-level debugging in parallel with
  *        the QEMU-level debugging, we would need to be able to pass some INT1
  *        events to the guest. This could be done via the following methods:
  *          a. Using the WHvRegisterPendingEvent register. As of Windows 21H1,
  *             it seems to only work for interrupts and not software
  *             exceptions.
  *          b. Locating and patching the original handler by parsing IDT/GDT.
  *             This involves relatively complex logic outlined in the previous
  *             paragraph.
  *          c. Emulating the exception invocation (i.e. manually updating RIP,
  *             RFLAGS, and pushing the old values to stack). This is even more
  *             complicated than the previous option, since it involves checking
  *             CPL, gate attributes, and doing various adjustments depending
  *             on the current CPU mode, whether the CPL is changing, etc.
  */
 typedef enum WhpxStepMode {
     WHPX_STEP_NONE = 0,
     /* Halt other VCPUs */
     WHPX_STEP_EXCLUSIVE,
 } WhpxStepMode;
 
 struct whpx_vcpu {
     WHV_EMULATOR_HANDLE emulator;
     bool window_registered;
     bool interruptable;
     bool ready_for_pic_interrupt;
     uint64_t tpr;
     uint64_t apic_base;
     bool interruption_pending;
 
     /* Must be the last field as it may have a tail */
     WHV_RUN_VP_EXIT_CONTEXT exit_ctx;
 };
 
 static bool whpx_allowed;
 static bool whp_dispatch_initialized;
 static HMODULE hWinHvPlatform, hWinHvEmulation;
 static uint32_t max_vcpu_index;
 static WHV_PROCESSOR_XSAVE_FEATURES whpx_xsave_cap;
 
 struct whpx_state whpx_global;
 struct WHPDispatch whp_dispatch;
 
 static bool whpx_has_xsave(void)
 {
     return whpx_xsave_cap.XsaveSupport;
 }
 
 /*
  * VP support
  */
 
 static struct whpx_vcpu *get_whpx_vcpu(CPUState *cpu)
 {
     return (struct whpx_vcpu *)cpu->hax_vcpu;
 }
 
 static WHV_X64_SEGMENT_REGISTER whpx_seg_q2h(const SegmentCache *qs, int v86,
                                              int r86)
 {
     WHV_X64_SEGMENT_REGISTER hs;
     unsigned flags = qs->flags;
 
     hs.Base = qs->base;
     hs.Limit = qs->limit;
     hs.Selector = qs->selector;
 
     if (v86) {
         hs.Attributes = 0;
         hs.SegmentType = 3;
         hs.Present = 1;
         hs.DescriptorPrivilegeLevel = 3;
         hs.NonSystemSegment = 1;
 
     } else {
         hs.Attributes = (flags >> DESC_TYPE_SHIFT);
 
         if (r86) {
             /* hs.Base &= 0xfffff; */
         }
     }
 
     return hs;
 }
 
 static SegmentCache whpx_seg_h2q(const WHV_X64_SEGMENT_REGISTER *hs)
 {
     SegmentCache qs;
 
     qs.base = hs->Base;
     qs.limit = hs->Limit;
     qs.selector = hs->Selector;
 
     qs.flags = ((uint32_t)hs->Attributes) << DESC_TYPE_SHIFT;
 
     return qs;
 }
 
 /* X64 Extended Control Registers */
 static void whpx_set_xcrs(CPUState *cpu)
 {
     CPUX86State *env = cpu->env_ptr;
     HRESULT hr;
     struct whpx_state *whpx = &whpx_global;
     WHV_REGISTER_VALUE xcr0;
     WHV_REGISTER_NAME xcr0_name = WHvX64RegisterXCr0;
 
     if (!whpx_has_xsave()) {
         return;
     }
 
     /* Only xcr0 is supported by the hypervisor currently */
     xcr0.Reg64 = env->xcr0;
     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
         whpx->partition, cpu->cpu_index, &xcr0_name, 1, &xcr0);
     if (FAILED(hr)) {
         error_report("WHPX: Failed to set register xcr0, hr=%08lx", hr);
     }
 }
 
 static int whpx_set_tsc(CPUState *cpu)
 {
     CPUX86State *env = cpu->env_ptr;
     WHV_REGISTER_NAME tsc_reg = WHvX64RegisterTsc;
     WHV_REGISTER_VALUE tsc_val;
     HRESULT hr;
     struct whpx_state *whpx = &whpx_global;
 
     /*
      * Suspend the partition prior to setting the TSC to reduce the variance
      * in TSC across vCPUs. When the first vCPU runs post suspend, the
      * partition is automatically resumed.
      */
     if (whp_dispatch.WHvSuspendPartitionTime) {
 
         /*
          * Unable to suspend partition while setting TSC is not a fatal
          * error. It just increases the likelihood of TSC variance between
          * vCPUs and some guest OS are able to handle that just fine.
          */
         hr = whp_dispatch.WHvSuspendPartitionTime(whpx->partition);
         if (FAILED(hr)) {
             warn_report("WHPX: Failed to suspend partition, hr=%08lx", hr);
         }
     }
 
     tsc_val.Reg64 = env->tsc;
     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
         whpx->partition, cpu->cpu_index, &tsc_reg, 1, &tsc_val);
     if (FAILED(hr)) {
         error_report("WHPX: Failed to set TSC, hr=%08lx", hr);
         return -1;
     }
 
     return 0;
 }
 
 /*
  * The CR8 register in the CPU is mapped to the TPR register of the APIC,
  * however, they use a slightly different encoding. Specifically:
  *
  *     APIC.TPR[bits 7:4] = CR8[bits 3:0]
  *
  * This mechanism is described in section 10.8.6.1 of Volume 3 of Intel 64
  * and IA-32 Architectures Software Developer's Manual.
  *
  * The functions below translate the value of CR8 to TPR and vice versa.
  */
 
 static uint64_t whpx_apic_tpr_to_cr8(uint64_t tpr)
 {
     return tpr >> 4;
 }
 
 static uint64_t whpx_cr8_to_apic_tpr(uint64_t cr8)
 {
     return cr8 << 4;
 }
 
 static void whpx_set_registers(CPUState *cpu, int level)
 {
     struct whpx_state *whpx = &whpx_global;
     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
     CPUX86State *env = cpu->env_ptr;
     X86CPU *x86_cpu = X86_CPU(cpu);
     struct whpx_register_set vcxt;
     HRESULT hr;
     int idx;
     int idx_next;
     int i;
     int v86, r86;
 
     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
 
     /*
      * Following MSRs have side effects on the guest or are too heavy for
      * runtime. Limit them to full state update.
      */
     if (level >= WHPX_SET_RESET_STATE) {
         whpx_set_tsc(cpu);
     }
 
     memset(&vcxt, 0, sizeof(struct whpx_register_set));
 
     v86 = (env->eflags & VM_MASK);
     r86 = !(env->cr[0] & CR0_PE_MASK);
 
     vcpu->tpr = whpx_apic_tpr_to_cr8(cpu_get_apic_tpr(x86_cpu->apic_state));
     vcpu->apic_base = cpu_get_apic_base(x86_cpu->apic_state);
 
     idx = 0;
 
     /* Indexes for first 16 registers match between HV and QEMU definitions */
     idx_next = 16;
     for (idx = 0; idx < CPU_NB_REGS; idx += 1) {
         vcxt.values[idx].Reg64 = (uint64_t)env->regs[idx];
     }
     idx = idx_next;
 
     /* Same goes for RIP and RFLAGS */
     assert(whpx_register_names[idx] == WHvX64RegisterRip);
     vcxt.values[idx++].Reg64 = env->eip;
 
     assert(whpx_register_names[idx] == WHvX64RegisterRflags);
     vcxt.values[idx++].Reg64 = env->eflags;
 
     /* Translate 6+4 segment registers. HV and QEMU order matches  */
     assert(idx == WHvX64RegisterEs);
     for (i = 0; i < 6; i += 1, idx += 1) {
         vcxt.values[idx].Segment = whpx_seg_q2h(&env->segs[i], v86, r86);
     }
 
     assert(idx == WHvX64RegisterLdtr);
     vcxt.values[idx++].Segment = whpx_seg_q2h(&env->ldt, 0, 0);
 
     assert(idx == WHvX64RegisterTr);
     vcxt.values[idx++].Segment = whpx_seg_q2h(&env->tr, 0, 0);
 
     assert(idx == WHvX64RegisterIdtr);
     vcxt.values[idx].Table.Base = env->idt.base;
     vcxt.values[idx].Table.Limit = env->idt.limit;
     idx += 1;
 
     assert(idx == WHvX64RegisterGdtr);
     vcxt.values[idx].Table.Base = env->gdt.base;
     vcxt.values[idx].Table.Limit = env->gdt.limit;
     idx += 1;
 
     /* CR0, 2, 3, 4, 8 */
     assert(whpx_register_names[idx] == WHvX64RegisterCr0);
     vcxt.values[idx++].Reg64 = env->cr[0];
     assert(whpx_register_names[idx] == WHvX64RegisterCr2);
     vcxt.values[idx++].Reg64 = env->cr[2];
     assert(whpx_register_names[idx] == WHvX64RegisterCr3);
     vcxt.values[idx++].Reg64 = env->cr[3];
     assert(whpx_register_names[idx] == WHvX64RegisterCr4);
     vcxt.values[idx++].Reg64 = env->cr[4];
     assert(whpx_register_names[idx] == WHvX64RegisterCr8);
     vcxt.values[idx++].Reg64 = vcpu->tpr;
 
     /* 8 Debug Registers - Skipped */
 
     /*
      * Extended control registers needs to be handled separately depending
      * on whether xsave is supported/enabled or not.
      */
     whpx_set_xcrs(cpu);
 
     /* 16 XMM registers */
     assert(whpx_register_names[idx] == WHvX64RegisterXmm0);
     idx_next = idx + 16;
     for (i = 0; i < sizeof(env->xmm_regs) / sizeof(ZMMReg); i += 1, idx += 1) {
         vcxt.values[idx].Reg128.Low64 = env->xmm_regs[i].ZMM_Q(0);
         vcxt.values[idx].Reg128.High64 = env->xmm_regs[i].ZMM_Q(1);
     }
     idx = idx_next;
 
     /* 8 FP registers */
     assert(whpx_register_names[idx] == WHvX64RegisterFpMmx0);
     for (i = 0; i < 8; i += 1, idx += 1) {
         vcxt.values[idx].Fp.AsUINT128.Low64 = env->fpregs[i].mmx.MMX_Q(0);
         /* vcxt.values[idx].Fp.AsUINT128.High64 =
                env->fpregs[i].mmx.MMX_Q(1);
         */
     }
 
     /* FP control status register */
     assert(whpx_register_names[idx] == WHvX64RegisterFpControlStatus);
     vcxt.values[idx].FpControlStatus.FpControl = env->fpuc;
     vcxt.values[idx].FpControlStatus.FpStatus =
         (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
     vcxt.values[idx].FpControlStatus.FpTag = 0;
     for (i = 0; i < 8; ++i) {
         vcxt.values[idx].FpControlStatus.FpTag |= (!env->fptags[i]) << i;
     }
     vcxt.values[idx].FpControlStatus.Reserved = 0;
     vcxt.values[idx].FpControlStatus.LastFpOp = env->fpop;
     vcxt.values[idx].FpControlStatus.LastFpRip = env->fpip;
     idx += 1;
 
     /* XMM control status register */
     assert(whpx_register_names[idx] == WHvX64RegisterXmmControlStatus);
     vcxt.values[idx].XmmControlStatus.LastFpRdp = 0;
     vcxt.values[idx].XmmControlStatus.XmmStatusControl = env->mxcsr;
     vcxt.values[idx].XmmControlStatus.XmmStatusControlMask = 0x0000ffff;
     idx += 1;
 
     /* MSRs */
     assert(whpx_register_names[idx] == WHvX64RegisterEfer);
     vcxt.values[idx++].Reg64 = env->efer;
 #ifdef TARGET_X86_64
     assert(whpx_register_names[idx] == WHvX64RegisterKernelGsBase);
     vcxt.values[idx++].Reg64 = env->kernelgsbase;
 #endif
 
     assert(whpx_register_names[idx] == WHvX64RegisterApicBase);
     vcxt.values[idx++].Reg64 = vcpu->apic_base;
 
     /* WHvX64RegisterPat - Skipped */
 
     assert(whpx_register_names[idx] == WHvX64RegisterSysenterCs);
     vcxt.values[idx++].Reg64 = env->sysenter_cs;
     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEip);
     vcxt.values[idx++].Reg64 = env->sysenter_eip;
     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEsp);
     vcxt.values[idx++].Reg64 = env->sysenter_esp;
     assert(whpx_register_names[idx] == WHvX64RegisterStar);
     vcxt.values[idx++].Reg64 = env->star;
 #ifdef TARGET_X86_64
     assert(whpx_register_names[idx] == WHvX64RegisterLstar);
     vcxt.values[idx++].Reg64 = env->lstar;
     assert(whpx_register_names[idx] == WHvX64RegisterCstar);
     vcxt.values[idx++].Reg64 = env->cstar;
     assert(whpx_register_names[idx] == WHvX64RegisterSfmask);
     vcxt.values[idx++].Reg64 = env->fmask;
 #endif
 
     /* Interrupt / Event Registers - Skipped */
 
     assert(idx == RTL_NUMBER_OF(whpx_register_names));
 
     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
         whpx->partition, cpu->cpu_index,
         whpx_register_names,
         RTL_NUMBER_OF(whpx_register_names),
         &vcxt.values[0]);
 
     if (FAILED(hr)) {
         error_report("WHPX: Failed to set virtual processor context, hr=%08lx",
                      hr);
     }
 
     return;
 }
 
 static int whpx_get_tsc(CPUState *cpu)
 {
     CPUX86State *env = cpu->env_ptr;
     WHV_REGISTER_NAME tsc_reg = WHvX64RegisterTsc;
     WHV_REGISTER_VALUE tsc_val;
     HRESULT hr;
     struct whpx_state *whpx = &whpx_global;
 
     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
         whpx->partition, cpu->cpu_index, &tsc_reg, 1, &tsc_val);
     if (FAILED(hr)) {
         error_report("WHPX: Failed to get TSC, hr=%08lx", hr);
         return -1;
     }
 
     env->tsc = tsc_val.Reg64;
     return 0;
 }
 
 /* X64 Extended Control Registers */
 static void whpx_get_xcrs(CPUState *cpu)
 {
     CPUX86State *env = cpu->env_ptr;
     HRESULT hr;
     struct whpx_state *whpx = &whpx_global;
     WHV_REGISTER_VALUE xcr0;
     WHV_REGISTER_NAME xcr0_name = WHvX64RegisterXCr0;
 
     if (!whpx_has_xsave()) {
         return;
     }
 
     /* Only xcr0 is supported by the hypervisor currently */
     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
         whpx->partition, cpu->cpu_index, &xcr0_name, 1, &xcr0);
     if (FAILED(hr)) {
         error_report("WHPX: Failed to get register xcr0, hr=%08lx", hr);
         return;
     }
 
     env->xcr0 = xcr0.Reg64;
 }
 
 static void whpx_get_registers(CPUState *cpu)
 {
     struct whpx_state *whpx = &whpx_global;
     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
     CPUX86State *env = cpu->env_ptr;
     X86CPU *x86_cpu = X86_CPU(cpu);
     struct whpx_register_set vcxt;
     uint64_t tpr, apic_base;
     HRESULT hr;
     int idx;
     int idx_next;
     int i;
 
     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
 
     if (!env->tsc_valid) {
         whpx_get_tsc(cpu);
         env->tsc_valid = !runstate_is_running();
     }
 
     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
         whpx->partition, cpu->cpu_index,
         whpx_register_names,
         RTL_NUMBER_OF(whpx_register_names),
         &vcxt.values[0]);
     if (FAILED(hr)) {
         error_report("WHPX: Failed to get virtual processor context, hr=%08lx",
                      hr);
     }
 
     if (whpx_apic_in_platform()) {
         /*
          * Fetch the TPR value from the emulated APIC. It may get overwritten
          * below with the value from CR8 returned by
          * WHvGetVirtualProcessorRegisters().
          */
         whpx_apic_get(x86_cpu->apic_state);
         vcpu->tpr = whpx_apic_tpr_to_cr8(
             cpu_get_apic_tpr(x86_cpu->apic_state));
     }
 
     idx = 0;
 
     /* Indexes for first 16 registers match between HV and QEMU definitions */
     idx_next = 16;
     for (idx = 0; idx < CPU_NB_REGS; idx += 1) {
         env->regs[idx] = vcxt.values[idx].Reg64;
     }
     idx = idx_next;
 
     /* Same goes for RIP and RFLAGS */
     assert(whpx_register_names[idx] == WHvX64RegisterRip);
     env->eip = vcxt.values[idx++].Reg64;
     assert(whpx_register_names[idx] == WHvX64RegisterRflags);
     env->eflags = vcxt.values[idx++].Reg64;
 
     /* Translate 6+4 segment registers. HV and QEMU order matches  */
     assert(idx == WHvX64RegisterEs);
     for (i = 0; i < 6; i += 1, idx += 1) {
         env->segs[i] = whpx_seg_h2q(&vcxt.values[idx].Segment);
     }
 
     assert(idx == WHvX64RegisterLdtr);
     env->ldt = whpx_seg_h2q(&vcxt.values[idx++].Segment);
     assert(idx == WHvX64RegisterTr);
     env->tr = whpx_seg_h2q(&vcxt.values[idx++].Segment);
     assert(idx == WHvX64RegisterIdtr);
     env->idt.base = vcxt.values[idx].Table.Base;
     env->idt.limit = vcxt.values[idx].Table.Limit;
     idx += 1;
     assert(idx == WHvX64RegisterGdtr);
     env->gdt.base = vcxt.values[idx].Table.Base;
     env->gdt.limit = vcxt.values[idx].Table.Limit;
     idx += 1;
 
     /* CR0, 2, 3, 4, 8 */
     assert(whpx_register_names[idx] == WHvX64RegisterCr0);
     env->cr[0] = vcxt.values[idx++].Reg64;
     assert(whpx_register_names[idx] == WHvX64RegisterCr2);
     env->cr[2] = vcxt.values[idx++].Reg64;
     assert(whpx_register_names[idx] == WHvX64RegisterCr3);
     env->cr[3] = vcxt.values[idx++].Reg64;
     assert(whpx_register_names[idx] == WHvX64RegisterCr4);
     env->cr[4] = vcxt.values[idx++].Reg64;
     assert(whpx_register_names[idx] == WHvX64RegisterCr8);
     tpr = vcxt.values[idx++].Reg64;
     if (tpr != vcpu->tpr) {
         vcpu->tpr = tpr;
         cpu_set_apic_tpr(x86_cpu->apic_state, whpx_cr8_to_apic_tpr(tpr));
     }
 
     /* 8 Debug Registers - Skipped */
 
     /*
      * Extended control registers needs to be handled separately depending
      * on whether xsave is supported/enabled or not.
      */
     whpx_get_xcrs(cpu);
 
     /* 16 XMM registers */
     assert(whpx_register_names[idx] == WHvX64RegisterXmm0);
     idx_next = idx + 16;
     for (i = 0; i < sizeof(env->xmm_regs) / sizeof(ZMMReg); i += 1, idx += 1) {
         env->xmm_regs[i].ZMM_Q(0) = vcxt.values[idx].Reg128.Low64;
         env->xmm_regs[i].ZMM_Q(1) = vcxt.values[idx].Reg128.High64;
     }
     idx = idx_next;
 
     /* 8 FP registers */
     assert(whpx_register_names[idx] == WHvX64RegisterFpMmx0);
     for (i = 0; i < 8; i += 1, idx += 1) {
         env->fpregs[i].mmx.MMX_Q(0) = vcxt.values[idx].Fp.AsUINT128.Low64;
         /* env->fpregs[i].mmx.MMX_Q(1) =
                vcxt.values[idx].Fp.AsUINT128.High64;
         */
     }
 
     /* FP control status register */
     assert(whpx_register_names[idx] == WHvX64RegisterFpControlStatus);
     env->fpuc = vcxt.values[idx].FpControlStatus.FpControl;
     env->fpstt = (vcxt.values[idx].FpControlStatus.FpStatus >> 11) & 0x7;
     env->fpus = vcxt.values[idx].FpControlStatus.FpStatus & ~0x3800;
     for (i = 0; i < 8; ++i) {
         env->fptags[i] = !((vcxt.values[idx].FpControlStatus.FpTag >> i) & 1);
     }
     env->fpop = vcxt.values[idx].FpControlStatus.LastFpOp;
     env->fpip = vcxt.values[idx].FpControlStatus.LastFpRip;
     idx += 1;
 
     /* XMM control status register */
     assert(whpx_register_names[idx] == WHvX64RegisterXmmControlStatus);
     env->mxcsr = vcxt.values[idx].XmmControlStatus.XmmStatusControl;
     idx += 1;
 
     /* MSRs */
     assert(whpx_register_names[idx] == WHvX64RegisterEfer);
     env->efer = vcxt.values[idx++].Reg64;
 #ifdef TARGET_X86_64
     assert(whpx_register_names[idx] == WHvX64RegisterKernelGsBase);
     env->kernelgsbase = vcxt.values[idx++].Reg64;
 #endif
 
     assert(whpx_register_names[idx] == WHvX64RegisterApicBase);
     apic_base = vcxt.values[idx++].Reg64;
     if (apic_base != vcpu->apic_base) {
         vcpu->apic_base = apic_base;
         cpu_set_apic_base(x86_cpu->apic_state, vcpu->apic_base);
     }
 
     /* WHvX64RegisterPat - Skipped */
 
     assert(whpx_register_names[idx] == WHvX64RegisterSysenterCs);
     env->sysenter_cs = vcxt.values[idx++].Reg64;
     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEip);
     env->sysenter_eip = vcxt.values[idx++].Reg64;
     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEsp);
     env->sysenter_esp = vcxt.values[idx++].Reg64;
     assert(whpx_register_names[idx] == WHvX64RegisterStar);
     env->star = vcxt.values[idx++].Reg64;
 #ifdef TARGET_X86_64
     assert(whpx_register_names[idx] == WHvX64RegisterLstar);
     env->lstar = vcxt.values[idx++].Reg64;
     assert(whpx_register_names[idx] == WHvX64RegisterCstar);
     env->cstar = vcxt.values[idx++].Reg64;
     assert(whpx_register_names[idx] == WHvX64RegisterSfmask);
     env->fmask = vcxt.values[idx++].Reg64;
 #endif
 
     /* Interrupt / Event Registers - Skipped */
 
     assert(idx == RTL_NUMBER_OF(whpx_register_names));
 
     if (whpx_apic_in_platform()) {
         whpx_apic_get(x86_cpu->apic_state);
     }
 
     x86_update_hflags(env);
 
     return;
 }
 
 static HRESULT CALLBACK whpx_emu_ioport_callback(
     void *ctx,
     WHV_EMULATOR_IO_ACCESS_INFO *IoAccess)
 {
     MemTxAttrs attrs = { 0 };
     address_space_rw(&address_space_io, IoAccess->Port, attrs,
                      &IoAccess->Data, IoAccess->AccessSize,
                      IoAccess->Direction);
     return S_OK;
 }
 
 static HRESULT CALLBACK whpx_emu_mmio_callback(
     void *ctx,
     WHV_EMULATOR_MEMORY_ACCESS_INFO *ma)
 {
     cpu_physical_memory_rw(ma->GpaAddress, ma->Data, ma->AccessSize,
                            ma->Direction);
     return S_OK;
 }
 
 static HRESULT CALLBACK whpx_emu_getreg_callback(
     void *ctx,
     const WHV_REGISTER_NAME *RegisterNames,
     UINT32 RegisterCount,
     WHV_REGISTER_VALUE *RegisterValues)
 {
     HRESULT hr;
     struct whpx_state *whpx = &whpx_global;
     CPUState *cpu = (CPUState *)ctx;
 
     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
         whpx->partition, cpu->cpu_index,
         RegisterNames, RegisterCount,
         RegisterValues);
     if (FAILED(hr)) {
         error_report("WHPX: Failed to get virtual processor registers,"
                      " hr=%08lx", hr);
     }
 
     return hr;
 }
 
 static HRESULT CALLBACK whpx_emu_setreg_callback(
     void *ctx,
     const WHV_REGISTER_NAME *RegisterNames,
     UINT32 RegisterCount,
     const WHV_REGISTER_VALUE *RegisterValues)
 {
     HRESULT hr;
     struct whpx_state *whpx = &whpx_global;
     CPUState *cpu = (CPUState *)ctx;
 
     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
         whpx->partition, cpu->cpu_index,
         RegisterNames, RegisterCount,
         RegisterValues);
     if (FAILED(hr)) {
         error_report("WHPX: Failed to set virtual processor registers,"
                      " hr=%08lx", hr);
     }
 
     /*
      * The emulator just successfully wrote the register state. We clear the
      * dirty state so we avoid the double write on resume of the VP.
      */
     cpu->vcpu_dirty = false;
 
     return hr;
 }
 
 static HRESULT CALLBACK whpx_emu_translate_callback(
     void *ctx,
     WHV_GUEST_VIRTUAL_ADDRESS Gva,
     WHV_TRANSLATE_GVA_FLAGS TranslateFlags,
     WHV_TRANSLATE_GVA_RESULT_CODE *TranslationResult,
     WHV_GUEST_PHYSICAL_ADDRESS *Gpa)
 {
     HRESULT hr;
     struct whpx_state *whpx = &whpx_global;
     CPUState *cpu = (CPUState *)ctx;
     WHV_TRANSLATE_GVA_RESULT res;
 
     hr = whp_dispatch.WHvTranslateGva(whpx->partition, cpu->cpu_index,
                                       Gva, TranslateFlags, &res, Gpa);
     if (FAILED(hr)) {
         error_report("WHPX: Failed to translate GVA, hr=%08lx", hr);
     } else {
         *TranslationResult = res.ResultCode;
     }
 
     return hr;
 }
 
 static const WHV_EMULATOR_CALLBACKS whpx_emu_callbacks = {
     .Size = sizeof(WHV_EMULATOR_CALLBACKS),
     .WHvEmulatorIoPortCallback = whpx_emu_ioport_callback,
     .WHvEmulatorMemoryCallback = whpx_emu_mmio_callback,
     .WHvEmulatorGetVirtualProcessorRegisters = whpx_emu_getreg_callback,
     .WHvEmulatorSetVirtualProcessorRegisters = whpx_emu_setreg_callback,
     .WHvEmulatorTranslateGvaPage = whpx_emu_translate_callback,
 };
 
 static int whpx_handle_mmio(CPUState *cpu, WHV_MEMORY_ACCESS_CONTEXT *ctx)
 {
     HRESULT hr;
     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
     WHV_EMULATOR_STATUS emu_status;
 
     hr = whp_dispatch.WHvEmulatorTryMmioEmulation(
         vcpu->emulator, cpu,
         &vcpu->exit_ctx.VpContext, ctx,
         &emu_status);
     if (FAILED(hr)) {
         error_report("WHPX: Failed to parse MMIO access, hr=%08lx", hr);
         return -1;
     }
 
     if (!emu_status.EmulationSuccessful) {
         error_report("WHPX: Failed to emulate MMIO access with"
                      " EmulatorReturnStatus: %u", emu_status.AsUINT32);
         return -1;
     }
 
     return 0;
 }
 
 static int whpx_handle_portio(CPUState *cpu,
                               WHV_X64_IO_PORT_ACCESS_CONTEXT *ctx)
 {
     HRESULT hr;
     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
     WHV_EMULATOR_STATUS emu_status;
 
     hr = whp_dispatch.WHvEmulatorTryIoEmulation(
         vcpu->emulator, cpu,
         &vcpu->exit_ctx.VpContext, ctx,
         &emu_status);
     if (FAILED(hr)) {
         error_report("WHPX: Failed to parse PortIO access, hr=%08lx", hr);
         return -1;
     }
 
     if (!emu_status.EmulationSuccessful) {
         error_report("WHPX: Failed to emulate PortIO access with"
                      " EmulatorReturnStatus: %u", emu_status.AsUINT32);
         return -1;
     }
 
     return 0;
 }
 
 /*
  * Controls whether we should intercept various exceptions on the guest,
  * namely breakpoint/single-step events.
  *
  * The 'exceptions' argument accepts a bitmask, e.g:
  * (1 << WHvX64ExceptionTypeDebugTrapOrFault) | (...)
  */
 static HRESULT whpx_set_exception_exit_bitmap(UINT64 exceptions)
 {
     struct whpx_state *whpx = &whpx_global;
     WHV_PARTITION_PROPERTY prop = { 0, };
     HRESULT hr;
 
     if (exceptions == whpx->exception_exit_bitmap) {
         return S_OK;
     }
 
     prop.ExceptionExitBitmap = exceptions;
 
     hr = whp_dispatch.WHvSetPartitionProperty(
         whpx->partition,
         WHvPartitionPropertyCodeExceptionExitBitmap,
         &prop,
         sizeof(WHV_PARTITION_PROPERTY));
 
     if (SUCCEEDED(hr)) {
         whpx->exception_exit_bitmap = exceptions;
     }
 
     return hr;
 }
 
 
 /*
  * This function is called before/after stepping over a single instruction.
  * It will update the CPU registers to arm/disarm the instruction stepping
  * accordingly.
  */
 static HRESULT whpx_vcpu_configure_single_stepping(CPUState *cpu,
     bool set,
     uint64_t *exit_context_rflags)
 {
     WHV_REGISTER_NAME reg_name;
     WHV_REGISTER_VALUE reg_value;
     HRESULT hr;
     struct whpx_state *whpx = &whpx_global;
 
     /*
      * If we are trying to step over a single instruction, we need to set the
      * TF bit in rflags. Otherwise, clear it.
      */
     reg_name = WHvX64RegisterRflags;
     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
         whpx->partition,
         cpu->cpu_index,
         &reg_name,
         1,
         &reg_value);
 
     if (FAILED(hr)) {
         error_report("WHPX: Failed to get rflags, hr=%08lx", hr);
         return hr;
     }
 
     if (exit_context_rflags) {
         assert(*exit_context_rflags == reg_value.Reg64);
     }
 
     if (set) {
         /* Raise WHvX64ExceptionTypeDebugTrapOrFault after each instruction */
         reg_value.Reg64 |= TF_MASK;
     } else {
         reg_value.Reg64 &= ~TF_MASK;
     }
 
     if (exit_context_rflags) {
         *exit_context_rflags = reg_value.Reg64;
     }
 
     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
         whpx->partition,
         cpu->cpu_index,
         &reg_name,
         1,
         &reg_value);
 
     if (FAILED(hr)) {
         error_report("WHPX: Failed to set rflags,"
             " hr=%08lx",
             hr);
         return hr;
     }
 
     reg_name = WHvRegisterInterruptState;
     reg_value.Reg64 = 0;
 
     /* Suspend delivery of hardware interrupts during single-stepping. */
     reg_value.InterruptState.InterruptShadow = set != 0;
 
     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
     whpx->partition,
         cpu->cpu_index,
         &reg_name,
         1,
         &reg_value);
 
     if (FAILED(hr)) {
         error_report("WHPX: Failed to set InterruptState,"
             " hr=%08lx",
             hr);
         return hr;
     }
 
     if (!set) {
         /*
          * We have just finished stepping over a single instruction,
          * and intercepted the INT1 generated by it.
          * We need to now hide the INT1 from the guest,
          * as it would not be expecting it.
          */
 
         reg_name = WHvX64RegisterPendingDebugException;
         hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
         whpx->partition,
             cpu->cpu_index,
             &reg_name,
             1,
             &reg_value);
 
         if (FAILED(hr)) {
             error_report("WHPX: Failed to get pending debug exceptions,"
                          "hr=%08lx", hr);
             return hr;
         }
 
         if (reg_value.PendingDebugException.SingleStep) {
             reg_value.PendingDebugException.SingleStep = 0;
 
             hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
                 whpx->partition,
                 cpu->cpu_index,
                 &reg_name,
                 1,
                 &reg_value);
 
             if (FAILED(hr)) {
                 error_report("WHPX: Failed to clear pending debug exceptions,"
                              "hr=%08lx", hr);
              return hr;
             }
         }
 
     }
 
     return S_OK;
 }
 
 /* Tries to find a breakpoint at the specified address. */
 static struct whpx_breakpoint *whpx_lookup_breakpoint_by_addr(uint64_t address)
 {
     struct whpx_state *whpx = &whpx_global;
     int i;
 
     if (whpx->breakpoints.breakpoints) {
         for (i = 0; i < whpx->breakpoints.breakpoints->used; i++) {
             if (address == whpx->breakpoints.breakpoints->data[i].address) {
                 return &whpx->breakpoints.breakpoints->data[i];
             }
         }
     }
 
     return NULL;
 }
 
 /*
  * Linux uses int3 (0xCC) during startup (see int3_selftest()) and for
  * debugging user-mode applications. Since the WHPX API does not offer
  * an easy way to pass the intercepted exception back to the guest, we
  * resort to using INT1 instead, and let the guest always handle INT3.
  */
 static const uint8_t whpx_breakpoint_instruction = 0xF1;
 
 /*
  * The WHPX QEMU backend implements breakpoints by writing the INT1
  * instruction into memory (ignoring the DRx registers). This raises a few
  * issues that need to be carefully handled:
  *
  * 1. Although unlikely, other parts of QEMU may set multiple breakpoints
  *    at the same location, and later remove them in arbitrary order.
  *    This should not cause memory corruption, and should only remove the
  *    physical breakpoint instruction when the last QEMU breakpoint is gone.
  *
  * 2. Writing arbitrary virtual memory may fail if it's not mapped to a valid
  *    physical location. Hence, physically adding/removing a breakpoint can
  *    theoretically fail at any time. We need to keep track of it.
  *
  * The function below rebuilds a list of low-level breakpoints (one per
  * address, tracking the original instruction and any errors) from the list of
  * high-level breakpoints (set via cpu_breakpoint_insert()).
  *
  * In order to optimize performance, this function stores the list of
  * high-level breakpoints (a.k.a. CPU breakpoints) used to compute the
  * low-level ones, so that it won't be re-invoked until these breakpoints
  * change.
  *
  * Note that this function decides which breakpoints should be inserted into,
  * memory, but doesn't actually do it. The memory accessing is done in
  * whpx_apply_breakpoints().
  */
 static void whpx_translate_cpu_breakpoints(
     struct whpx_breakpoints *breakpoints,
     CPUState *cpu,
     int cpu_breakpoint_count)
 {
     CPUBreakpoint *bp;
     int cpu_bp_index = 0;
 
     breakpoints->original_addresses =
         g_renew(vaddr, breakpoints->original_addresses, cpu_breakpoint_count);
 
     breakpoints->original_address_count = cpu_breakpoint_count;
 
     int max_breakpoints = cpu_breakpoint_count +
         (breakpoints->breakpoints ? breakpoints->breakpoints->used : 0);
 
     struct whpx_breakpoint_collection *new_breakpoints =
         g_malloc0(sizeof(struct whpx_breakpoint_collection)
                   + max_breakpoints * sizeof(struct whpx_breakpoint));
 
     new_breakpoints->allocated = max_breakpoints;
     new_breakpoints->used = 0;
 
     /*
      * 1. Preserve all old breakpoints that could not be automatically
      * cleared when the CPU got stopped.
      */
     if (breakpoints->breakpoints) {
         int i;
         for (i = 0; i < breakpoints->breakpoints->used; i++) {
             if (breakpoints->breakpoints->data[i].state != WHPX_BP_CLEARED) {
                 new_breakpoints->data[new_breakpoints->used++] =
                     breakpoints->breakpoints->data[i];
             }
         }
     }
 
     /* 2. Map all CPU breakpoints to WHPX breakpoints */
     QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
         int i;
         bool found = false;
 
         /* This will be used to detect changed CPU breakpoints later. */
         breakpoints->original_addresses[cpu_bp_index++] = bp->pc;
 
         for (i = 0; i < new_breakpoints->used; i++) {
             /*
              * WARNING: This loop has O(N^2) complexity, where N is the
              * number of breakpoints. It should not be a bottleneck in
              * real-world scenarios, since it only needs to run once after
              * the breakpoints have been modified.
              * If this ever becomes a concern, it can be optimized by storing
              * high-level breakpoint objects in a tree or hash map.
              */
 
             if (new_breakpoints->data[i].address == bp->pc) {
                 /* There was already a breakpoint at this address. */
                 if (new_breakpoints->data[i].state == WHPX_BP_CLEAR_PENDING) {
                     new_breakpoints->data[i].state = WHPX_BP_SET;
                 } else if (new_breakpoints->data[i].state == WHPX_BP_SET) {
                     new_breakpoints->data[i].state = WHPX_BP_SET_PENDING;
                 }
 
                 found = true;
                 break;
             }
         }
 
         if (!found && new_breakpoints->used < new_breakpoints->allocated) {
             /* No WHPX breakpoint at this address. Create one. */
             new_breakpoints->data[new_breakpoints->used].address = bp->pc;
             new_breakpoints->data[new_breakpoints->used].state =
                 WHPX_BP_SET_PENDING;
             new_breakpoints->used++;
         }
     }
 
     /*
      * Free the previous breakpoint list. This can be optimized by keeping
      * it as shadow buffer for the next computation instead of freeing
      * it immediately.
      */
     g_free(breakpoints->breakpoints);
 
     breakpoints->breakpoints = new_breakpoints;
 }
 
 /*
  * Physically inserts/removes the breakpoints by reading and writing the
  * physical memory, keeping a track of the failed attempts.
  *
  * Passing resuming=true  will try to set all previously unset breakpoints.
  * Passing resuming=false will remove all inserted ones.
  */
 static void whpx_apply_breakpoints(
     struct whpx_breakpoint_collection *breakpoints,
     CPUState *cpu,
     bool resuming)
 {
     int i, rc;
     if (!breakpoints) {
         return;
     }
 
     for (i = 0; i < breakpoints->used; i++) {
         /* Decide what to do right now based on the last known state. */
         WhpxBreakpointState state = breakpoints->data[i].state;
         switch (state) {
         case WHPX_BP_CLEARED:
             if (resuming) {
                 state = WHPX_BP_SET_PENDING;
             }
             break;
         case WHPX_BP_SET_PENDING:
             if (!resuming) {
                 state = WHPX_BP_CLEARED;
             }
             break;
         case WHPX_BP_SET:
             if (!resuming) {
                 state = WHPX_BP_CLEAR_PENDING;
             }
             break;
         case WHPX_BP_CLEAR_PENDING:
             if (resuming) {
                 state = WHPX_BP_SET;
             }
             break;
         }
 
         if (state == WHPX_BP_SET_PENDING) {
             /* Remember the original instruction. */
             rc = cpu_memory_rw_debug(cpu,
                 breakpoints->data[i].address,
                 &breakpoints->data[i].original_instruction,
                 1,
                 false);
 
             if (!rc) {
                 /* Write the breakpoint instruction. */
                 rc = cpu_memory_rw_debug(cpu,
                     breakpoints->data[i].address,
                     (void *)&whpx_breakpoint_instruction,
                     1,
                     true);
             }
 
             if (!rc) {
                 state = WHPX_BP_SET;
             }
 
         }
 
         if (state == WHPX_BP_CLEAR_PENDING) {
             /* Restore the original instruction. */
             rc = cpu_memory_rw_debug(cpu,
                 breakpoints->data[i].address,
                 &breakpoints->data[i].original_instruction,
                 1,
                 true);
 
             if (!rc) {
                 state = WHPX_BP_CLEARED;
             }
         }
 
         breakpoints->data[i].state = state;
     }
 }
 
 /*
  * This function is called when the a VCPU is about to start and no other
  * VCPUs have been started so far. Since the VCPU start order could be
  * arbitrary, it doesn't have to be VCPU#0.
  *
  * It is used to commit the breakpoints into memory, and configure WHPX
  * to intercept debug exceptions.
  *
  * Note that whpx_set_exception_exit_bitmap() cannot be called if one or
  * more VCPUs are already running, so this is the best place to do it.
  */
 static int whpx_first_vcpu_starting(CPUState *cpu)
 {
     struct whpx_state *whpx = &whpx_global;
     HRESULT hr;
 
     g_assert(qemu_mutex_iothread_locked());
 
     if (!QTAILQ_EMPTY(&cpu->breakpoints) ||
             (whpx->breakpoints.breakpoints &&
              whpx->breakpoints.breakpoints->used)) {
         CPUBreakpoint *bp;
         int i = 0;
         bool update_pending = false;
 
         QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
             if (i >= whpx->breakpoints.original_address_count ||
                 bp->pc != whpx->breakpoints.original_addresses[i]) {
                 update_pending = true;
             }
 
             i++;
         }
 
         if (i != whpx->breakpoints.original_address_count) {
             update_pending = true;
         }
 
         if (update_pending) {
             /*
              * The CPU breakpoints have changed since the last call to
              * whpx_translate_cpu_breakpoints(). WHPX breakpoints must
              * now be recomputed.
              */
             whpx_translate_cpu_breakpoints(&whpx->breakpoints, cpu, i);
         }
 
         /* Actually insert the breakpoints into the memory. */
         whpx_apply_breakpoints(whpx->breakpoints.breakpoints, cpu, true);
     }
 
     uint64_t exception_mask;
     if (whpx->step_pending ||
         (whpx->breakpoints.breakpoints &&
          whpx->breakpoints.breakpoints->used)) {
         /*
          * We are either attempting to single-step one or more CPUs, or
          * have one or more breakpoints enabled. Both require intercepting
          * the WHvX64ExceptionTypeBreakpointTrap exception.
          */
 
         exception_mask = 1UL << WHvX64ExceptionTypeDebugTrapOrFault;
     } else {
         /* Let the guest handle all exceptions. */
         exception_mask = 0;
     }
 
     hr = whpx_set_exception_exit_bitmap(exception_mask);
     if (!SUCCEEDED(hr)) {
         error_report("WHPX: Failed to update exception exit mask,"
                      "hr=%08lx.", hr);
         return 1;
     }
 
     return 0;
 }
 
 /*
  * This function is called when the last VCPU has finished running.
  * It is used to remove any previously set breakpoints from memory.
  */
 static int whpx_last_vcpu_stopping(CPUState *cpu)
 {
     whpx_apply_breakpoints(whpx_global.breakpoints.breakpoints, cpu, false);
     return 0;
 }
 
 /* Returns the address of the next instruction that is about to be executed. */
 static vaddr whpx_vcpu_get_pc(CPUState *cpu, bool exit_context_valid)
 {
     if (cpu->vcpu_dirty) {
         /* The CPU registers have been modified by other parts of QEMU. */
         CPUArchState *env = (CPUArchState *)(cpu->env_ptr);
         return env->eip;
     } else if (exit_context_valid) {
         /*
          * The CPU registers have not been modified by neither other parts
          * of QEMU, nor this port by calling WHvSetVirtualProcessorRegisters().
          * This is the most common case.
          */
         struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
         return vcpu->exit_ctx.VpContext.Rip;
     } else {
         /*
          * The CPU registers have been modified by a call to
          * WHvSetVirtualProcessorRegisters() and must be re-queried from
          * the target.
          */
         WHV_REGISTER_VALUE reg_value;
         WHV_REGISTER_NAME reg_name = WHvX64RegisterRip;
         HRESULT hr;
         struct whpx_state *whpx = &whpx_global;
 
         hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
             whpx->partition,
             cpu->cpu_index,
             &reg_name,
             1,
             &reg_value);
 
         if (FAILED(hr)) {
             error_report("WHPX: Failed to get PC, hr=%08lx", hr);
             return 0;
         }
 
         return reg_value.Reg64;
     }
 }
 
 static int whpx_handle_halt(CPUState *cpu)
 {
     CPUX86State *env = cpu->env_ptr;
     int ret = 0;
 
     qemu_mutex_lock_iothread();
     if (!((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
           (env->eflags & IF_MASK)) &&
         !(cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
         cpu->exception_index = EXCP_HLT;
         cpu->halted = true;
         ret = 1;
     }
     qemu_mutex_unlock_iothread();
 
     return ret;
 }
 
 static void whpx_vcpu_pre_run(CPUState *cpu)
 {
     HRESULT hr;
     struct whpx_state *whpx = &whpx_global;
     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
     CPUX86State *env = cpu->env_ptr;
     X86CPU *x86_cpu = X86_CPU(cpu);
     int irq;
     uint8_t tpr;
     WHV_X64_PENDING_INTERRUPTION_REGISTER new_int;
     UINT32 reg_count = 0;
     WHV_REGISTER_VALUE reg_values[3];
     WHV_REGISTER_NAME reg_names[3];
 
     memset(&new_int, 0, sizeof(new_int));
     memset(reg_values, 0, sizeof(reg_values));
 
     qemu_mutex_lock_iothread();
 
     /* Inject NMI */
     if (!vcpu->interruption_pending &&
         cpu->interrupt_request & (CPU_INTERRUPT_NMI | CPU_INTERRUPT_SMI)) {
         if (cpu->interrupt_request & CPU_INTERRUPT_NMI) {
             cpu->interrupt_request &= ~CPU_INTERRUPT_NMI;
             vcpu->interruptable = false;
             new_int.InterruptionType = WHvX64PendingNmi;
             new_int.InterruptionPending = 1;
             new_int.InterruptionVector = 2;
         }
         if (cpu->interrupt_request & CPU_INTERRUPT_SMI) {
             cpu->interrupt_request &= ~CPU_INTERRUPT_SMI;
         }
     }
 
     /*
      * Force the VCPU out of its inner loop to process any INIT requests or
      * commit pending TPR access.
      */
     if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
         if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) &&
             !(env->hflags & HF_SMM_MASK)) {
             cpu->exit_request = 1;
         }
         if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
             cpu->exit_request = 1;
         }
     }
 
     /* Get pending hard interruption or replay one that was overwritten */
     if (!whpx_apic_in_platform()) {
         if (!vcpu->interruption_pending &&
             vcpu->interruptable && (env->eflags & IF_MASK)) {
             assert(!new_int.InterruptionPending);
             if (cpu->interrupt_request & CPU_INTERRUPT_HARD) {
                 cpu->interrupt_request &= ~CPU_INTERRUPT_HARD;
                 irq = cpu_get_pic_interrupt(env);
                 if (irq >= 0) {
                     new_int.InterruptionType = WHvX64PendingInterrupt;
                     new_int.InterruptionPending = 1;
                     new_int.InterruptionVector = irq;
                 }
             }
         }
 
         /* Setup interrupt state if new one was prepared */
         if (new_int.InterruptionPending) {
             reg_values[reg_count].PendingInterruption = new_int;
             reg_names[reg_count] = WHvRegisterPendingInterruption;
             reg_count += 1;
         }
     } else if (vcpu->ready_for_pic_interrupt &&
                (cpu->interrupt_request & CPU_INTERRUPT_HARD)) {
         cpu->interrupt_request &= ~CPU_INTERRUPT_HARD;
         irq = cpu_get_pic_interrupt(env);
         if (irq >= 0) {
             reg_names[reg_count] = WHvRegisterPendingEvent;
             reg_values[reg_count].ExtIntEvent = (WHV_X64_PENDING_EXT_INT_EVENT)
             {
                 .EventPending = 1,
                 .EventType = WHvX64PendingEventExtInt,
                 .Vector = irq,
             };
             reg_count += 1;
         }
      }
 
     /* Sync the TPR to the CR8 if was modified during the intercept */
     tpr = whpx_apic_tpr_to_cr8(cpu_get_apic_tpr(x86_cpu->apic_state));
     if (tpr != vcpu->tpr) {
         vcpu->tpr = tpr;
         reg_values[reg_count].Reg64 = tpr;
         cpu->exit_request = 1;
         reg_names[reg_count] = WHvX64RegisterCr8;
         reg_count += 1;
     }
 
     /* Update the state of the interrupt delivery notification */
     if (!vcpu->window_registered &&
         cpu->interrupt_request & CPU_INTERRUPT_HARD) {
         reg_values[reg_count].DeliverabilityNotifications =
             (WHV_X64_DELIVERABILITY_NOTIFICATIONS_REGISTER) {
                 .InterruptNotification = 1
             };
         vcpu->window_registered = 1;
         reg_names[reg_count] = WHvX64RegisterDeliverabilityNotifications;
         reg_count += 1;
     }
 
     qemu_mutex_unlock_iothread();
     vcpu->ready_for_pic_interrupt = false;
 
     if (reg_count) {
         hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
             whpx->partition, cpu->cpu_index,
             reg_names, reg_count, reg_values);
         if (FAILED(hr)) {
             error_report("WHPX: Failed to set interrupt state registers,"
                          " hr=%08lx", hr);
         }
     }
 
     return;
 }
 
 static void whpx_vcpu_post_run(CPUState *cpu)
 {
     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
     CPUX86State *env = cpu->env_ptr;
     X86CPU *x86_cpu = X86_CPU(cpu);
 
     env->eflags = vcpu->exit_ctx.VpContext.Rflags;
 
     uint64_t tpr = vcpu->exit_ctx.VpContext.Cr8;
     if (vcpu->tpr != tpr) {
         vcpu->tpr = tpr;
         qemu_mutex_lock_iothread();
         cpu_set_apic_tpr(x86_cpu->apic_state, whpx_cr8_to_apic_tpr(vcpu->tpr));
         qemu_mutex_unlock_iothread();
     }
 
     vcpu->interruption_pending =
         vcpu->exit_ctx.VpContext.ExecutionState.InterruptionPending;
 
     vcpu->interruptable =
         !vcpu->exit_ctx.VpContext.ExecutionState.InterruptShadow;
 
     return;
 }
 
 static void whpx_vcpu_process_async_events(CPUState *cpu)
 {
     CPUX86State *env = cpu->env_ptr;
     X86CPU *x86_cpu = X86_CPU(cpu);
     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 
     if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) &&
         !(env->hflags & HF_SMM_MASK)) {
         whpx_cpu_synchronize_state(cpu);
         do_cpu_init(x86_cpu);
         vcpu->interruptable = true;
     }
 
     if (cpu->interrupt_request & CPU_INTERRUPT_POLL) {
         cpu->interrupt_request &= ~CPU_INTERRUPT_POLL;
         apic_poll_irq(x86_cpu->apic_state);
     }
 
     if (((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
          (env->eflags & IF_MASK)) ||
         (cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
         cpu->halted = false;
     }
 
     if (cpu->interrupt_request & CPU_INTERRUPT_SIPI) {
         whpx_cpu_synchronize_state(cpu);
         do_cpu_sipi(x86_cpu);
     }
 
     if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
         cpu->interrupt_request &= ~CPU_INTERRUPT_TPR;
         whpx_cpu_synchronize_state(cpu);
         apic_handle_tpr_access_report(x86_cpu->apic_state, env->eip,
                                       env->tpr_access_type);
     }
 
     return;
 }
 
 static int whpx_vcpu_run(CPUState *cpu)
 {
     HRESULT hr;
     struct whpx_state *whpx = &whpx_global;
     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
     struct whpx_breakpoint *stepped_over_bp = NULL;
     WhpxStepMode exclusive_step_mode = WHPX_STEP_NONE;
     int ret;
 
     g_assert(qemu_mutex_iothread_locked());
 
     if (whpx->running_cpus++ == 0) {
         /* Insert breakpoints into memory, update exception exit bitmap. */
         ret = whpx_first_vcpu_starting(cpu);
         if (ret != 0) {
             return ret;
         }
     }
 
     if (whpx->breakpoints.breakpoints &&
         whpx->breakpoints.breakpoints->used > 0)
     {
         uint64_t pc = whpx_vcpu_get_pc(cpu, true);
         stepped_over_bp = whpx_lookup_breakpoint_by_addr(pc);
         if (stepped_over_bp && stepped_over_bp->state != WHPX_BP_SET) {
             stepped_over_bp = NULL;
         }
 
         if (stepped_over_bp) {
             /*
              * We are trying to run the instruction overwritten by an active
              * breakpoint. We will temporarily disable the breakpoint, suspend
              * other CPUs, and step over the instruction.
              */
             exclusive_step_mode = WHPX_STEP_EXCLUSIVE;
         }
     }
 
     if (exclusive_step_mode == WHPX_STEP_NONE) {
         whpx_vcpu_process_async_events(cpu);
         if (cpu->halted && !whpx_apic_in_platform()) {
             cpu->exception_index = EXCP_HLT;
             qatomic_set(&cpu->exit_request, false);
             return 0;
         }
     }
 
     qemu_mutex_unlock_iothread();
 
     if (exclusive_step_mode != WHPX_STEP_NONE) {
         start_exclusive();
         g_assert(cpu == current_cpu);
         g_assert(!cpu->running);
         cpu->running = true;
 
         hr = whpx_set_exception_exit_bitmap(
             1UL << WHvX64ExceptionTypeDebugTrapOrFault);
         if (!SUCCEEDED(hr)) {
             error_report("WHPX: Failed to update exception exit mask, "
                          "hr=%08lx.", hr);
             return 1;
         }
 
         if (stepped_over_bp) {
             /* Temporarily disable the triggered breakpoint. */
             cpu_memory_rw_debug(cpu,
                 stepped_over_bp->address,
                 &stepped_over_bp->original_instruction,
                 1,
                 true);
         }
     } else {
         cpu_exec_start(cpu);
     }
 
     do {
         if (cpu->vcpu_dirty) {
             whpx_set_registers(cpu, WHPX_SET_RUNTIME_STATE);
             cpu->vcpu_dirty = false;
         }
 
         if (exclusive_step_mode == WHPX_STEP_NONE) {
             whpx_vcpu_pre_run(cpu);
 
             if (qatomic_read(&cpu->exit_request)) {
                 whpx_vcpu_kick(cpu);
             }
         }
 
         if (exclusive_step_mode != WHPX_STEP_NONE || cpu->singlestep_enabled) {
             whpx_vcpu_configure_single_stepping(cpu, true, NULL);
         }
 
         hr = whp_dispatch.WHvRunVirtualProcessor(
             whpx->partition, cpu->cpu_index,
             &vcpu->exit_ctx, sizeof(vcpu->exit_ctx));
 
         if (FAILED(hr)) {
             error_report("WHPX: Failed to exec a virtual processor,"
                          " hr=%08lx", hr);
             ret = -1;
             break;
         }
 
         if (exclusive_step_mode != WHPX_STEP_NONE || cpu->singlestep_enabled) {
             whpx_vcpu_configure_single_stepping(cpu,
                 false,
                 &vcpu->exit_ctx.VpContext.Rflags);
         }
 
         whpx_vcpu_post_run(cpu);
 
         switch (vcpu->exit_ctx.ExitReason) {
         case WHvRunVpExitReasonMemoryAccess:
             ret = whpx_handle_mmio(cpu, &vcpu->exit_ctx.MemoryAccess);
             break;
 
         case WHvRunVpExitReasonX64IoPortAccess:
             ret = whpx_handle_portio(cpu, &vcpu->exit_ctx.IoPortAccess);
             break;
 
         case WHvRunVpExitReasonX64InterruptWindow:
             vcpu->ready_for_pic_interrupt = 1;
             vcpu->window_registered = 0;
             ret = 0;
             break;
 
         case WHvRunVpExitReasonX64ApicEoi:
             assert(whpx_apic_in_platform());
             ioapic_eoi_broadcast(vcpu->exit_ctx.ApicEoi.InterruptVector);
             break;
 
         case WHvRunVpExitReasonX64Halt:
             /*
              * WARNING: as of build 19043.1526 (21H1), this exit reason is no
              * longer used.
              */
             ret = whpx_handle_halt(cpu);
             break;
 
         case WHvRunVpExitReasonX64ApicInitSipiTrap: {
             WHV_INTERRUPT_CONTROL ipi = {0};
             uint64_t icr = vcpu->exit_ctx.ApicInitSipi.ApicIcr;
             uint32_t delivery_mode =
                 (icr & APIC_ICR_DELIV_MOD) >> APIC_ICR_DELIV_MOD_SHIFT;
             int dest_shorthand =
                 (icr & APIC_ICR_DEST_SHORT) >> APIC_ICR_DEST_SHORT_SHIFT;
             bool broadcast = false;
             bool include_self = false;
             uint32_t i;
 
             /* We only registered for INIT and SIPI exits. */
             if ((delivery_mode != APIC_DM_INIT) &&
                 (delivery_mode != APIC_DM_SIPI)) {
                 error_report(
                     "WHPX: Unexpected APIC exit that is not a INIT or SIPI");
                 break;
             }
 
             if (delivery_mode == APIC_DM_INIT) {
                 ipi.Type = WHvX64InterruptTypeInit;
             } else {
                 ipi.Type = WHvX64InterruptTypeSipi;
             }
 
             ipi.DestinationMode =
                 ((icr & APIC_ICR_DEST_MOD) >> APIC_ICR_DEST_MOD_SHIFT) ?
                     WHvX64InterruptDestinationModeLogical :
                     WHvX64InterruptDestinationModePhysical;
 
             ipi.TriggerMode =
                 ((icr & APIC_ICR_TRIGGER_MOD) >> APIC_ICR_TRIGGER_MOD_SHIFT) ?
                     WHvX64InterruptTriggerModeLevel :
                     WHvX64InterruptTriggerModeEdge;
 
             ipi.Vector = icr & APIC_VECTOR_MASK;
             switch (dest_shorthand) {
             /* no shorthand. Bits 56-63 contain the destination. */
             case 0:
                 ipi.Destination = (icr >> 56) & APIC_VECTOR_MASK;
                 hr = whp_dispatch.WHvRequestInterrupt(whpx->partition,
                         &ipi, sizeof(ipi));
                 if (FAILED(hr)) {
                     error_report("WHPX: Failed to request interrupt  hr=%08lx",
                         hr);
                 }
 
                 break;
 
             /* self */
             case 1:
                 include_self = true;
                 break;
 
             /* broadcast, including self */
             case 2:
                 broadcast = true;
                 include_self = true;
                 break;
 
             /* broadcast, excluding self */
             case 3:
                 broadcast = true;
                 break;
             }
 
             if (!broadcast && !include_self) {
                 break;
             }
 
             for (i = 0; i <= max_vcpu_index; i++) {
                 if (i == cpu->cpu_index && !include_self) {
                     continue;
                 }
 
                 /*
                  * Assuming that APIC Ids are identity mapped since
                  * WHvX64RegisterApicId & WHvX64RegisterInitialApicId registers
                  * are not handled yet and the hypervisor doesn't allow the
                  * guest to modify the APIC ID.
                  */
                 ipi.Destination = i;
                 hr = whp_dispatch.WHvRequestInterrupt(whpx->partition,
                         &ipi, sizeof(ipi));
                 if (FAILED(hr)) {
                     error_report(
                         "WHPX: Failed to request SIPI for %d,  hr=%08lx",
                         i, hr);
                 }
             }
 
             break;
         }
 
         case WHvRunVpExitReasonCanceled:
             if (exclusive_step_mode != WHPX_STEP_NONE) {
                 /*
                  * We are trying to step over a single instruction, and
                  * likely got a request to stop from another thread.
                  * Delay it until we are done stepping
                  * over.
                  */
                 ret = 0;
             } else {
                 cpu->exception_index = EXCP_INTERRUPT;
                 ret = 1;
             }
             break;
         case WHvRunVpExitReasonX64MsrAccess: {
             WHV_REGISTER_VALUE reg_values[3] = {0};
             WHV_REGISTER_NAME reg_names[3];
             UINT32 reg_count;
 
             reg_names[0] = WHvX64RegisterRip;
             reg_names[1] = WHvX64RegisterRax;
             reg_names[2] = WHvX64RegisterRdx;
 
             reg_values[0].Reg64 =
                 vcpu->exit_ctx.VpContext.Rip +
                 vcpu->exit_ctx.VpContext.InstructionLength;
 
             /*
              * For all unsupported MSR access we:
              *     ignore writes
              *     return 0 on read.
              */
             reg_count = vcpu->exit_ctx.MsrAccess.AccessInfo.IsWrite ?
                         1 : 3;
 
             hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
                 whpx->partition,
                 cpu->cpu_index,
                 reg_names, reg_count,
                 reg_values);
 
             if (FAILED(hr)) {
                 error_report("WHPX: Failed to set MsrAccess state "
                              " registers, hr=%08lx", hr);
             }
             ret = 0;
             break;
         }
         case WHvRunVpExitReasonX64Cpuid: {
             WHV_REGISTER_VALUE reg_values[5];
             WHV_REGISTER_NAME reg_names[5];
             UINT32 reg_count = 5;
             UINT64 cpuid_fn, rip = 0, rax = 0, rcx = 0, rdx = 0, rbx = 0;
             X86CPU *x86_cpu = X86_CPU(cpu);
             CPUX86State *env = &x86_cpu->env;
 
             memset(reg_values, 0, sizeof(reg_values));
 
             rip = vcpu->exit_ctx.VpContext.Rip +
                   vcpu->exit_ctx.VpContext.InstructionLength;
             cpuid_fn = vcpu->exit_ctx.CpuidAccess.Rax;
 
             /*
              * Ideally, these should be supplied to the hypervisor during VCPU
              * initialization and it should be able to satisfy this request.
              * But, currently, WHPX doesn't support setting CPUID values in the
              * hypervisor once the partition has been setup, which is too late
              * since VCPUs are realized later. For now, use the values from
              * QEMU to satisfy these requests, until WHPX adds support for
              * being able to set these values in the hypervisor at runtime.
              */
             cpu_x86_cpuid(env, cpuid_fn, 0, (UINT32 *)&rax, (UINT32 *)&rbx,
                 (UINT32 *)&rcx, (UINT32 *)&rdx);
             switch (cpuid_fn) {
             case 0x40000000:
                 /* Expose the vmware cpu frequency cpuid leaf */
                 rax = 0x40000010;
                 rbx = rcx = rdx = 0;
                 break;
 
             case 0x40000010:
                 rax = env->tsc_khz;
                 rbx = env->apic_bus_freq / 1000; /* Hz to KHz */
                 rcx = rdx = 0;
                 break;
 
             case 0x80000001:
                 /* Remove any support of OSVW */
                 rcx &= ~CPUID_EXT3_OSVW;
                 break;
             }
 
             reg_names[0] = WHvX64RegisterRip;
             reg_names[1] = WHvX64RegisterRax;
             reg_names[2] = WHvX64RegisterRcx;
             reg_names[3] = WHvX64RegisterRdx;
             reg_names[4] = WHvX64RegisterRbx;
 
             reg_values[0].Reg64 = rip;
             reg_values[1].Reg64 = rax;
             reg_values[2].Reg64 = rcx;
             reg_values[3].Reg64 = rdx;
             reg_values[4].Reg64 = rbx;
 
             hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
                 whpx->partition, cpu->cpu_index,
                 reg_names,
                 reg_count,
                 reg_values);
 
             if (FAILED(hr)) {
                 error_report("WHPX: Failed to set CpuidAccess state registers,"
                              " hr=%08lx", hr);
             }
             ret = 0;
             break;
         }
         case WHvRunVpExitReasonException:
             whpx_get_registers(cpu);
 
             if ((vcpu->exit_ctx.VpException.ExceptionType ==
                  WHvX64ExceptionTypeDebugTrapOrFault) &&
                 (vcpu->exit_ctx.VpException.InstructionByteCount >= 1) &&
                 (vcpu->exit_ctx.VpException.InstructionBytes[0] ==
                  whpx_breakpoint_instruction)) {
                 /* Stopped at a software breakpoint. */
                 cpu->exception_index = EXCP_DEBUG;
             } else if ((vcpu->exit_ctx.VpException.ExceptionType ==
                         WHvX64ExceptionTypeDebugTrapOrFault) &&
                        !cpu->singlestep_enabled) {
                 /*
                  * Just finished stepping over a breakpoint, but the
                  * gdb does not expect us to do single-stepping.
                  * Don't do anything special.
                  */
                 cpu->exception_index = EXCP_INTERRUPT;
             } else {
                 /* Another exception or debug event. Report it to GDB. */
                 cpu->exception_index = EXCP_DEBUG;
             }
 
             ret = 1;
             break;
         case WHvRunVpExitReasonNone:
         case WHvRunVpExitReasonUnrecoverableException:
         case WHvRunVpExitReasonInvalidVpRegisterValue:
         case WHvRunVpExitReasonUnsupportedFeature:
         default:
             error_report("WHPX: Unexpected VP exit code %d",
                          vcpu->exit_ctx.ExitReason);
             whpx_get_registers(cpu);
             qemu_mutex_lock_iothread();
             qemu_system_guest_panicked(cpu_get_crash_info(cpu));
             qemu_mutex_unlock_iothread();
             break;
         }
 
     } while (!ret);
 
     if (stepped_over_bp) {
         /* Restore the breakpoint we stepped over */
         cpu_memory_rw_debug(cpu,
             stepped_over_bp->address,
             (void *)&whpx_breakpoint_instruction,
             1,
             true);
     }
 
     if (exclusive_step_mode != WHPX_STEP_NONE) {
         g_assert(cpu_in_exclusive_context(cpu));
         cpu->running = false;
         end_exclusive();
 
         exclusive_step_mode = WHPX_STEP_NONE;
     } else {
         cpu_exec_end(cpu);
     }
 
     qemu_mutex_lock_iothread();
     current_cpu = cpu;
 
     if (--whpx->running_cpus == 0) {
         whpx_last_vcpu_stopping(cpu);
     }
 
     qatomic_set(&cpu->exit_request, false);
 
     return ret < 0;
 }
 
 static void do_whpx_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
 {
     if (!cpu->vcpu_dirty) {
         whpx_get_registers(cpu);
         cpu->vcpu_dirty = true;
     }
 }
 
 static void do_whpx_cpu_synchronize_post_reset(CPUState *cpu,
                                                run_on_cpu_data arg)
 {
     whpx_set_registers(cpu, WHPX_SET_RESET_STATE);
     cpu->vcpu_dirty = false;
 }
 
 static void do_whpx_cpu_synchronize_post_init(CPUState *cpu,
                                               run_on_cpu_data arg)
 {
     whpx_set_registers(cpu, WHPX_SET_FULL_STATE);
     cpu->vcpu_dirty = false;
 }
 
 static void do_whpx_cpu_synchronize_pre_loadvm(CPUState *cpu,
                                                run_on_cpu_data arg)
 {
     cpu->vcpu_dirty = true;
 }
 
 /*
  * CPU support.
  */
 
 void whpx_cpu_synchronize_state(CPUState *cpu)
 {
     if (!cpu->vcpu_dirty) {
         run_on_cpu(cpu, do_whpx_cpu_synchronize_state, RUN_ON_CPU_NULL);
     }
 }
 
 void whpx_cpu_synchronize_post_reset(CPUState *cpu)
 {
     run_on_cpu(cpu, do_whpx_cpu_synchronize_post_reset, RUN_ON_CPU_NULL);
 }
 
 void whpx_cpu_synchronize_post_init(CPUState *cpu)
 {
     run_on_cpu(cpu, do_whpx_cpu_synchronize_post_init, RUN_ON_CPU_NULL);
 }
 
 void whpx_cpu_synchronize_pre_loadvm(CPUState *cpu)
 {
     run_on_cpu(cpu, do_whpx_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL);
 }
 
 void whpx_cpu_synchronize_pre_resume(bool step_pending)
 {
     whpx_global.step_pending = step_pending;
 }
 
 /*
  * Vcpu support.
  */
 
 static Error *whpx_migration_blocker;
 
 static void whpx_cpu_update_state(void *opaque, bool running, RunState state)
 {
     CPUX86State *env = opaque;
 
     if (running) {
         env->tsc_valid = false;
     }
 }
 
 int whpx_init_vcpu(CPUState *cpu)
 {
     HRESULT hr;
     struct whpx_state *whpx = &whpx_global;
     struct whpx_vcpu *vcpu = NULL;
     Error *local_error = NULL;
     CPUX86State *env = cpu->env_ptr;
     X86CPU *x86_cpu = X86_CPU(cpu);
     UINT64 freq = 0;
     int ret;
 
     /* Add migration blockers for all unsupported features of the
      * Windows Hypervisor Platform
      */
     if (whpx_migration_blocker == NULL) {
         error_setg(&whpx_migration_blocker,
                "State blocked due to non-migratable CPUID feature support,"
                "dirty memory tracking support, and XSAVE/XRSTOR support");
 
         if (migrate_add_blocker(whpx_migration_blocker, &local_error) < 0) {
             error_report_err(local_error);
             error_free(whpx_migration_blocker);
             ret = -EINVAL;
             goto error;
         }
     }
 
     vcpu = g_new0(struct whpx_vcpu, 1);
 
     if (!vcpu) {
         error_report("WHPX: Failed to allocte VCPU context.");
         ret = -ENOMEM;
         goto error;
     }
 
     hr = whp_dispatch.WHvEmulatorCreateEmulator(
         &whpx_emu_callbacks,
         &vcpu->emulator);
     if (FAILED(hr)) {
         error_report("WHPX: Failed to setup instruction completion support,"
                      " hr=%08lx", hr);
         ret = -EINVAL;
         goto error;
     }
 
     hr = whp_dispatch.WHvCreateVirtualProcessor(
         whpx->partition, cpu->cpu_index, 0);
     if (FAILED(hr)) {
         error_report("WHPX: Failed to create a virtual processor,"
                      " hr=%08lx", hr);
         whp_dispatch.WHvEmulatorDestroyEmulator(vcpu->emulator);
         ret = -EINVAL;
         goto error;
     }
 
     /*
      * vcpu's TSC frequency is either specified by user, or use the value
      * provided by Hyper-V if the former is not present. In the latter case, we
      * query it from Hyper-V and record in env->tsc_khz, so that vcpu's TSC
      * frequency can be migrated later via this field.
      */
     if (!env->tsc_khz) {
         hr = whp_dispatch.WHvGetCapability(
             WHvCapabilityCodeProcessorClockFrequency, &freq, sizeof(freq),
                 NULL);
         if (hr != WHV_E_UNKNOWN_CAPABILITY) {
             if (FAILED(hr)) {
                 printf("WHPX: Failed to query tsc frequency, hr=0x%08lx\n", hr);
             } else {
                 env->tsc_khz = freq / 1000; /* Hz to KHz */
             }
         }
     }
 
     env->apic_bus_freq = HYPERV_APIC_BUS_FREQUENCY;
     hr = whp_dispatch.WHvGetCapability(
         WHvCapabilityCodeInterruptClockFrequency, &freq, sizeof(freq), NULL);
     if (hr != WHV_E_UNKNOWN_CAPABILITY) {
         if (FAILED(hr)) {
             printf("WHPX: Failed to query apic bus frequency hr=0x%08lx\n", hr);
         } else {
             env->apic_bus_freq = freq;
         }
     }
 
     /*
      * If the vmware cpuid frequency leaf option is set, and we have a valid
      * tsc value, trap the corresponding cpuid's.
      */
     if (x86_cpu->vmware_cpuid_freq && env->tsc_khz) {
         UINT32 cpuidExitList[] = {1, 0x80000001, 0x40000000, 0x40000010};
 
         hr = whp_dispatch.WHvSetPartitionProperty(
                 whpx->partition,
                 WHvPartitionPropertyCodeCpuidExitList,
                 cpuidExitList,
                 RTL_NUMBER_OF(cpuidExitList) * sizeof(UINT32));
 
         if (FAILED(hr)) {
             error_report("WHPX: Failed to set partition CpuidExitList hr=%08lx",
                         hr);
             ret = -EINVAL;
             goto error;
         }
     }
 
     vcpu->interruptable = true;
     cpu->vcpu_dirty = true;
     cpu->hax_vcpu = (struct hax_vcpu_state *)vcpu;
     max_vcpu_index = max(max_vcpu_index, cpu->cpu_index);
     qemu_add_vm_change_state_handler(whpx_cpu_update_state, cpu->env_ptr);
 
     return 0;
 
 error:
     g_free(vcpu);
 
     return ret;
 }
 
 int whpx_vcpu_exec(CPUState *cpu)
 {
     int ret;
     int fatal;
 
     for (;;) {
         if (cpu->exception_index >= EXCP_INTERRUPT) {
             ret = cpu->exception_index;
             cpu->exception_index = -1;
             break;
         }
 
         fatal = whpx_vcpu_run(cpu);
 
         if (fatal) {
             error_report("WHPX: Failed to exec a virtual processor");
             abort();
         }
     }
 
     return ret;
 }
 
 void whpx_destroy_vcpu(CPUState *cpu)
 {
     struct whpx_state *whpx = &whpx_global;
     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 
     whp_dispatch.WHvDeleteVirtualProcessor(whpx->partition, cpu->cpu_index);
     whp_dispatch.WHvEmulatorDestroyEmulator(vcpu->emulator);
     g_free(cpu->hax_vcpu);
     return;
 }
 
 void whpx_vcpu_kick(CPUState *cpu)
 {
     struct whpx_state *whpx = &whpx_global;
     whp_dispatch.WHvCancelRunVirtualProcessor(
         whpx->partition, cpu->cpu_index, 0);
 }
 
 /*
  * Memory support.
  */
 
 static void whpx_update_mapping(hwaddr start_pa, ram_addr_t size,
                                 void *host_va, int add, int rom,
                                 const char *name)
 {
     struct whpx_state *whpx = &whpx_global;
     HRESULT hr;
 
     /*
     if (add) {
         printf("WHPX: ADD PA:%p Size:%p, Host:%p, %s, '%s'\n",
                (void*)start_pa, (void*)size, host_va,
                (rom ? "ROM" : "RAM"), name);
     } else {
         printf("WHPX: DEL PA:%p Size:%p, Host:%p,      '%s'\n",
                (void*)start_pa, (void*)size, host_va, name);
     }
     */
 
     if (add) {
         hr = whp_dispatch.WHvMapGpaRange(whpx->partition,
                                          host_va,
                                          start_pa,
                                          size,
                                          (WHvMapGpaRangeFlagRead |
                                           WHvMapGpaRangeFlagExecute |
                                           (rom ? 0 : WHvMapGpaRangeFlagWrite)));
     } else {
         hr = whp_dispatch.WHvUnmapGpaRange(whpx->partition,
                                            start_pa,
                                            size);
     }
 
     if (FAILED(hr)) {
         error_report("WHPX: Failed to %s GPA range '%s' PA:%p, Size:%p bytes,"
                      " Host:%p, hr=%08lx",
                      (add ? "MAP" : "UNMAP"), name,
                      (void *)(uintptr_t)start_pa, (void *)size, host_va, hr);
     }
 }
 
 static void whpx_process_section(MemoryRegionSection *section, int add)
 {
     MemoryRegion *mr = section->mr;
     hwaddr start_pa = section->offset_within_address_space;
     ram_addr_t size = int128_get64(section->size);
     unsigned int delta;
     uint64_t host_va;
 
     if (!memory_region_is_ram(mr)) {
         return;
     }
 
     delta = qemu_real_host_page_size() - (start_pa & ~qemu_real_host_page_mask());
     delta &= ~qemu_real_host_page_mask();
     if (delta > size) {
         return;
     }
     start_pa += delta;
     size -= delta;
     size &= qemu_real_host_page_mask();
     if (!size || (start_pa & ~qemu_real_host_page_mask())) {
         return;
     }
 
     host_va = (uintptr_t)memory_region_get_ram_ptr(mr)
             + section->offset_within_region + delta;
 
     whpx_update_mapping(start_pa, size, (void *)(uintptr_t)host_va, add,
                         memory_region_is_rom(mr), mr->name);
 }
 
 static void whpx_region_add(MemoryListener *listener,
                            MemoryRegionSection *section)
 {
     memory_region_ref(section->mr);
     whpx_process_section(section, 1);
 }
 
 static void whpx_region_del(MemoryListener *listener,
                            MemoryRegionSection *section)
 {
     whpx_process_section(section, 0);
     memory_region_unref(section->mr);
 }
 
 static void whpx_transaction_begin(MemoryListener *listener)
 {
 }
 
 static void whpx_transaction_commit(MemoryListener *listener)
 {
 }
 
 static void whpx_log_sync(MemoryListener *listener,
                          MemoryRegionSection *section)
 {
     MemoryRegion *mr = section->mr;
 
     if (!memory_region_is_ram(mr)) {
         return;
     }
 
     memory_region_set_dirty(mr, 0, int128_get64(section->size));
 }
 
 static MemoryListener whpx_memory_listener = {
     .name = "whpx",
     .begin = whpx_transaction_begin,
     .commit = whpx_transaction_commit,
     .region_add = whpx_region_add,
     .region_del = whpx_region_del,
     .log_sync = whpx_log_sync,
     .priority = 10,
 };
 
 static void whpx_memory_init(void)
 {
     memory_listener_register(&whpx_memory_listener, &address_space_memory);
 }
 
 /*
  * Load the functions from the given library, using the given handle. If a
  * handle is provided, it is used, otherwise the library is opened. The
  * handle will be updated on return with the opened one.
  */
 static bool load_whp_dispatch_fns(HMODULE *handle,
     WHPFunctionList function_list)
 {
     HMODULE hLib = *handle;
 
     #define WINHV_PLATFORM_DLL "WinHvPlatform.dll"
     #define WINHV_EMULATION_DLL "WinHvEmulation.dll"
     #define WHP_LOAD_FIELD_OPTIONAL(return_type, function_name, signature) \
         whp_dispatch.function_name = \
             (function_name ## _t)GetProcAddress(hLib, #function_name); \
 
     #define WHP_LOAD_FIELD(return_type, function_name, signature) \
         whp_dispatch.function_name = \
             (function_name ## _t)GetProcAddress(hLib, #function_name); \
         if (!whp_dispatch.function_name) { \
             error_report("Could not load function %s", #function_name); \
             goto error; \
         } \
 
     #define WHP_LOAD_LIB(lib_name, handle_lib) \
     if (!handle_lib) { \
         handle_lib = LoadLibrary(lib_name); \
         if (!handle_lib) { \
             error_report("Could not load library %s.", lib_name); \
             goto error; \
         } \
     } \
 
     switch (function_list) {
     case WINHV_PLATFORM_FNS_DEFAULT:
         WHP_LOAD_LIB(WINHV_PLATFORM_DLL, hLib)
         LIST_WINHVPLATFORM_FUNCTIONS(WHP_LOAD_FIELD)
         break;
 
     case WINHV_EMULATION_FNS_DEFAULT:
         WHP_LOAD_LIB(WINHV_EMULATION_DLL, hLib)
         LIST_WINHVEMULATION_FUNCTIONS(WHP_LOAD_FIELD)
         break;
 
     case WINHV_PLATFORM_FNS_SUPPLEMENTAL:
         WHP_LOAD_LIB(WINHV_PLATFORM_DLL, hLib)
         LIST_WINHVPLATFORM_FUNCTIONS_SUPPLEMENTAL(WHP_LOAD_FIELD_OPTIONAL)
         break;
     }
 
     *handle = hLib;
     return true;
 
 error:
     if (hLib) {
         FreeLibrary(hLib);
     }
 
     return false;
 }
 
 static void whpx_set_kernel_irqchip(Object *obj, Visitor *v,
                                    const char *name, void *opaque,
                                    Error **errp)
 {
     struct whpx_state *whpx = &whpx_global;
     OnOffSplit mode;
 
     if (!visit_type_OnOffSplit(v, name, &mode, errp)) {
         return;
     }
 
     switch (mode) {
     case ON_OFF_SPLIT_ON:
         whpx->kernel_irqchip_allowed = true;
         whpx->kernel_irqchip_required = true;
         break;
 
     case ON_OFF_SPLIT_OFF:
         whpx->kernel_irqchip_allowed = false;
         whpx->kernel_irqchip_required = false;
         break;
 
     case ON_OFF_SPLIT_SPLIT:
         error_setg(errp, "WHPX: split irqchip currently not supported");
         error_append_hint(errp,
             "Try without kernel-irqchip or with kernel-irqchip=on|off");
         break;
 
     default:
         /*
          * The value was checked in visit_type_OnOffSplit() above. If
          * we get here, then something is wrong in QEMU.
          */
         abort();
     }
 }
 
 /*
  * Partition support
  */
 
 static int whpx_accel_init(MachineState *ms)
 {
     struct whpx_state *whpx;
     int ret;
     HRESULT hr;
     WHV_CAPABILITY whpx_cap;
     UINT32 whpx_cap_size;
     WHV_PARTITION_PROPERTY prop;
     UINT32 cpuidExitList[] = {1, 0x80000001};
     WHV_CAPABILITY_FEATURES features = {0};
 
     whpx = &whpx_global;
 
     if (!init_whp_dispatch()) {
         ret = -ENOSYS;
         goto error;
     }
 
     whpx->mem_quota = ms->ram_size;
 
     hr = whp_dispatch.WHvGetCapability(
         WHvCapabilityCodeHypervisorPresent, &whpx_cap,
         sizeof(whpx_cap), &whpx_cap_size);
     if (FAILED(hr) || !whpx_cap.HypervisorPresent) {
         error_report("WHPX: No accelerator found, hr=%08lx", hr);
         ret = -ENOSPC;
         goto error;
     }
 
     hr = whp_dispatch.WHvGetCapability(
         WHvCapabilityCodeFeatures, &features, sizeof(features), NULL);
     if (FAILED(hr)) {
         error_report("WHPX: Failed to query capabilities, hr=%08lx", hr);
         ret = -EINVAL;
         goto error;
     }
 
     hr = whp_dispatch.WHvCreatePartition(&whpx->partition);
     if (FAILED(hr)) {
         error_report("WHPX: Failed to create partition, hr=%08lx", hr);
         ret = -EINVAL;
         goto error;
     }
 
     /*
      * Query the XSAVE capability of the partition. Any error here is not
      * considered fatal.
      */
     hr = whp_dispatch.WHvGetPartitionProperty(
         whpx->partition,
         WHvPartitionPropertyCodeProcessorXsaveFeatures,
         &whpx_xsave_cap,
         sizeof(whpx_xsave_cap),
         &whpx_cap_size);
 
     /*
      * Windows version which don't support this property will return with the
      * specific error code.
      */
     if (FAILED(hr) && hr != WHV_E_UNKNOWN_PROPERTY) {
         error_report("WHPX: Failed to query XSAVE capability, hr=%08lx", hr);
     }
 
     if (!whpx_has_xsave()) {
         printf("WHPX: Partition is not XSAVE capable\n");
     }
 
     memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
     prop.ProcessorCount = ms->smp.cpus;
     hr = whp_dispatch.WHvSetPartitionProperty(
         whpx->partition,
         WHvPartitionPropertyCodeProcessorCount,
         &prop,
         sizeof(WHV_PARTITION_PROPERTY));
 
     if (FAILED(hr)) {
         error_report("WHPX: Failed to set partition core count to %d,"
                      " hr=%08lx", ms->smp.cores, hr);
         ret = -EINVAL;
         goto error;
     }
 
     /*
      * Error out if WHP doesn't support apic emulation and user is requiring
      * it.
      */
     if (whpx->kernel_irqchip_required && (!features.LocalApicEmulation ||
             !whp_dispatch.WHvSetVirtualProcessorInterruptControllerState2)) {
         error_report("WHPX: kernel irqchip requested, but unavailable. "
             "Try without kernel-irqchip or with kernel-irqchip=off");
         ret = -EINVAL;
         goto error;
     }
 
     if (whpx->kernel_irqchip_allowed && features.LocalApicEmulation &&
         whp_dispatch.WHvSetVirtualProcessorInterruptControllerState2) {
         WHV_X64_LOCAL_APIC_EMULATION_MODE mode =
             WHvX64LocalApicEmulationModeXApic;
         printf("WHPX: setting APIC emulation mode in the hypervisor\n");
         hr = whp_dispatch.WHvSetPartitionProperty(
             whpx->partition,
             WHvPartitionPropertyCodeLocalApicEmulationMode,
             &mode,
             sizeof(mode));
         if (FAILED(hr)) {
             error_report("WHPX: Failed to enable kernel irqchip hr=%08lx", hr);
             if (whpx->kernel_irqchip_required) {
                 error_report("WHPX: kernel irqchip requested, but unavailable");
                 ret = -EINVAL;
                 goto error;
             }
         } else {
             whpx->apic_in_platform = true;
         }
     }
 
     /* Register for MSR and CPUID exits */
     memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
     prop.ExtendedVmExits.X64MsrExit = 1;
     prop.ExtendedVmExits.X64CpuidExit = 1;
     prop.ExtendedVmExits.ExceptionExit = 1;
     if (whpx_apic_in_platform()) {
         prop.ExtendedVmExits.X64ApicInitSipiExitTrap = 1;
     }
 
     hr = whp_dispatch.WHvSetPartitionProperty(
             whpx->partition,
             WHvPartitionPropertyCodeExtendedVmExits,
             &prop,
             sizeof(WHV_PARTITION_PROPERTY));
     if (FAILED(hr)) {
         error_report("WHPX: Failed to enable MSR & CPUIDexit, hr=%08lx", hr);
         ret = -EINVAL;
         goto error;
     }
 
     hr = whp_dispatch.WHvSetPartitionProperty(
         whpx->partition,
         WHvPartitionPropertyCodeCpuidExitList,
         cpuidExitList,
         RTL_NUMBER_OF(cpuidExitList) * sizeof(UINT32));
 
     if (FAILED(hr)) {
         error_report("WHPX: Failed to set partition CpuidExitList hr=%08lx",
                      hr);
         ret = -EINVAL;
         goto error;
     }
 
     /*
      * We do not want to intercept any exceptions from the guest,
      * until we actually start debugging with gdb.
      */
     whpx->exception_exit_bitmap = -1;
     hr = whpx_set_exception_exit_bitmap(0);
 
     if (FAILED(hr)) {
         error_report("WHPX: Failed to set exception exit bitmap, hr=%08lx", hr);
         ret = -EINVAL;
         goto error;
     }
 
     hr = whp_dispatch.WHvSetupPartition(whpx->partition);
     if (FAILED(hr)) {
         error_report("WHPX: Failed to setup partition, hr=%08lx", hr);
         ret = -EINVAL;
         goto error;
     }
 
     whpx_memory_init();
 
     printf("Windows Hypervisor Platform accelerator is operational\n");
     return 0;
 
 error:
 
     if (NULL != whpx->partition) {
         whp_dispatch.WHvDeletePartition(whpx->partition);
         whpx->partition = NULL;
     }
 
     return ret;
 }
 
 int whpx_enabled(void)
 {
     return whpx_allowed;
 }
 
 bool whpx_apic_in_platform(void) {
     return whpx_global.apic_in_platform;
 }
 
 static void whpx_accel_class_init(ObjectClass *oc, void *data)
 {
     AccelClass *ac = ACCEL_CLASS(oc);
     ac->name = "WHPX";
     ac->init_machine = whpx_accel_init;
     ac->allowed = &whpx_allowed;
 
     object_class_property_add(oc, "kernel-irqchip", "on|off|split",
         NULL, whpx_set_kernel_irqchip,
         NULL, NULL);
     object_class_property_set_description(oc, "kernel-irqchip",
         "Configure WHPX in-kernel irqchip");
 }
 
 static void whpx_accel_instance_init(Object *obj)
 {
     struct whpx_state *whpx = &whpx_global;
 
     memset(whpx, 0, sizeof(struct whpx_state));
     /* Turn on kernel-irqchip, by default */
     whpx->kernel_irqchip_allowed = true;
 }
 
 static const TypeInfo whpx_accel_type = {
     .name = ACCEL_CLASS_NAME("whpx"),
     .parent = TYPE_ACCEL,
     .instance_init = whpx_accel_instance_init,
     .class_init = whpx_accel_class_init,
 };
 
 static void whpx_type_init(void)
 {
     type_register_static(&whpx_accel_type);
 }
 
 bool init_whp_dispatch(void)
 {
     if (whp_dispatch_initialized) {
         return true;
     }
 
     if (!load_whp_dispatch_fns(&hWinHvPlatform, WINHV_PLATFORM_FNS_DEFAULT)) {
         goto error;
     }
 
     if (!load_whp_dispatch_fns(&hWinHvEmulation, WINHV_EMULATION_FNS_DEFAULT)) {
         goto error;
     }
 
     assert(load_whp_dispatch_fns(&hWinHvPlatform,
         WINHV_PLATFORM_FNS_SUPPLEMENTAL));
     whp_dispatch_initialized = true;
 
     return true;
 error:
     if (hWinHvPlatform) {
         FreeLibrary(hWinHvPlatform);
     }
 
     if (hWinHvEmulation) {
         FreeLibrary(hWinHvEmulation);
     }
 
     return false;
 }
 
 type_init(whpx_type_init);