cpu_recompiler_register_cache.h (15368B)
1 // SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com> 2 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) 3 4 #pragma once 5 #include "common/assert.h" 6 #include "cpu_recompiler_types.h" 7 #include "cpu_types.h" 8 9 #if defined(CPU_ARCH_ARM32) 10 #include "vixl/aarch32/macro-assembler-aarch32.h" 11 #elif defined(CPU_ARCH_ARM64) 12 #include "vixl/aarch64/macro-assembler-aarch64.h" 13 #endif 14 15 #include <array> 16 #include <optional> 17 #include <stack> 18 #include <tuple> 19 20 namespace CPU::Recompiler { 21 22 enum RegSize : u8 23 { 24 RegSize_8, 25 RegSize_16, 26 RegSize_32, 27 RegSize_64, 28 }; 29 30 #if defined(CPU_ARCH_X64) 31 32 using HostReg = unsigned; 33 using CodeEmitter = Xbyak::CodeGenerator; 34 using LabelType = Xbyak::Label; 35 enum : u32 36 { 37 HostReg_Count = 16 38 }; 39 constexpr HostReg HostReg_Invalid = static_cast<HostReg>(HostReg_Count); 40 constexpr RegSize HostPointerSize = RegSize_64; 41 42 #elif defined(CPU_ARCH_ARM32) 43 44 using HostReg = unsigned; 45 using CodeEmitter = vixl::aarch32::MacroAssembler; 46 using LabelType = vixl::aarch32::Label; 47 enum : u32 48 { 49 HostReg_Count = vixl::aarch32::kNumberOfRegisters 50 }; 51 constexpr HostReg HostReg_Invalid = static_cast<HostReg>(HostReg_Count); 52 constexpr RegSize HostPointerSize = RegSize_32; 53 54 #elif defined(CPU_ARCH_ARM64) 55 56 using HostReg = unsigned; 57 using CodeEmitter = vixl::aarch64::MacroAssembler; 58 using LabelType = vixl::aarch64::Label; 59 enum : u32 60 { 61 HostReg_Count = vixl::aarch64::kNumberOfRegisters 62 }; 63 constexpr HostReg HostReg_Invalid = static_cast<HostReg>(HostReg_Count); 64 constexpr RegSize HostPointerSize = RegSize_64; 65 66 #else 67 68 #error Unknown architecture. 69 70 #endif 71 72 class CodeGenerator; 73 class RegisterCache; 74 75 enum class HostRegState : u8 76 { 77 None = 0, 78 Usable = (1 << 1), // Can be allocated 79 CallerSaved = (1 << 2), // Register is caller-saved, and should be saved/restored after calling a function. 80 CalleeSaved = (1 << 3), // Register is callee-saved, and should be restored after leaving the block. 81 InUse = (1 << 4), // In-use, must be saved/restored across function call. 82 CalleeSavedAllocated = (1 << 5), // Register was callee-saved and allocated, so should be restored before returning. 83 Discarded = (1 << 6), // Register contents is not used, so do not preserve across function calls. 84 }; 85 IMPLEMENT_ENUM_CLASS_BITWISE_OPERATORS(HostRegState); 86 87 enum class ValueFlags : u8 88 { 89 None = 0, 90 Valid = (1 << 0), 91 Constant = (1 << 1), // The value itself is constant, and not in a register. 92 InHostRegister = (1 << 2), // The value itself is located in a host register. 93 Scratch = (1 << 3), // The value is temporary, and will be released after the Value is destroyed. 94 Dirty = (1 << 4), // For register cache values, the value needs to be written back to the CPU struct. 95 }; 96 IMPLEMENT_ENUM_CLASS_BITWISE_OPERATORS(ValueFlags); 97 98 struct Value 99 { 100 RegisterCache* regcache = nullptr; 101 u64 constant_value = 0; 102 HostReg host_reg = {}; 103 104 RegSize size = RegSize_8; 105 ValueFlags flags = ValueFlags::None; 106 107 Value(); 108 Value(RegisterCache* regcache_, u64 constant_, RegSize size_, ValueFlags flags_); 109 Value(RegisterCache* regcache_, HostReg reg_, RegSize size_, ValueFlags flags_); 110 Value(const Value& other); 111 Value(Value&& other); 112 ~Value(); 113 114 Value& operator=(const Value& other); 115 Value& operator=(Value&& other); 116 117 bool IsConstant() const { return (flags & ValueFlags::Constant) != ValueFlags::None; } 118 bool IsValid() const { return (flags & ValueFlags::Valid) != ValueFlags::None; } 119 bool IsInHostRegister() const { return (flags & ValueFlags::InHostRegister) != ValueFlags::None; } 120 bool IsScratch() const { return (flags & ValueFlags::Scratch) != ValueFlags::None; } 121 122 /// Returns the host register this value is bound to. 123 HostReg GetHostRegister() const 124 { 125 DebugAssert(IsInHostRegister()); 126 return host_reg; 127 } 128 129 /// Returns true if this value is constant and has the specified value. 130 bool HasConstantValue(u64 cv) const 131 { 132 return (((flags & ValueFlags::Constant) != ValueFlags::None) && constant_value == cv); 133 } 134 135 /// Removes the contents of this value. Use with care, as scratch/temporaries are not released. 136 void Clear(); 137 138 /// Releases the host register if needed, and clears the contents. 139 void ReleaseAndClear(); 140 141 /// Flags the value is being discarded. Call Undiscard() to track again. 142 void Discard(); 143 void Undiscard(); 144 145 void AddHostReg(RegisterCache* regcache_, HostReg hr) 146 { 147 DebugAssert(IsValid()); 148 regcache = regcache_; 149 host_reg = hr; 150 flags |= ValueFlags::InHostRegister; 151 } 152 153 void SetHostReg(RegisterCache* regcache_, HostReg hr, RegSize size_) 154 { 155 regcache = regcache_; 156 constant_value = 0; 157 host_reg = hr; 158 size = size_; 159 flags = ValueFlags::Valid | ValueFlags::InHostRegister; 160 } 161 162 void ClearConstant() 163 { 164 // By clearing the constant bit, we should already be in a host register. 165 DebugAssert(IsInHostRegister()); 166 flags &= ~ValueFlags::Constant; 167 } 168 169 bool IsDirty() const { return (flags & ValueFlags::Dirty) != ValueFlags::None; } 170 void SetDirty() { flags |= ValueFlags::Dirty; } 171 void ClearDirty() { flags &= ~ValueFlags::Dirty; } 172 173 /// Returns the same register viewed as a different size. 174 Value ViewAsSize(RegSize view_size) const 175 { 176 if (view_size == size) 177 return *this; 178 179 if (IsConstant()) 180 { 181 // truncate to size 182 switch (view_size) 183 { 184 case RegSize_8: 185 return Value::FromConstant(constant_value & UINT64_C(0xFF), RegSize_8); 186 case RegSize_16: 187 return Value::FromConstant(constant_value & UINT64_C(0xFFFF), RegSize_16); 188 case RegSize_32: 189 return Value::FromConstant(constant_value & UINT64_C(0xFFFFFFFF), RegSize_32); 190 case RegSize_64: 191 default: 192 return Value::FromConstant(constant_value, view_size); 193 } 194 } 195 196 if (IsInHostRegister()) 197 return Value::FromHostReg(regcache, host_reg, view_size); 198 199 // invalid? 200 return Value(); 201 } 202 203 /// Returns the constant value as a signed 32-bit integer, suitable as an immediate. 204 s32 GetS32ConstantValue() const 205 { 206 switch (size) 207 { 208 case RegSize_8: 209 return static_cast<s32>(SignExtend32(Truncate8(constant_value))); 210 211 case RegSize_16: 212 return static_cast<s32>(SignExtend32(Truncate16(constant_value))); 213 214 case RegSize_32: 215 case RegSize_64: 216 default: 217 return static_cast<s32>(constant_value); 218 } 219 } 220 221 /// Returns the constant value as a signed 64-bit integer, suitable as an immediate. 222 s64 GetS64ConstantValue() const 223 { 224 switch (size) 225 { 226 case RegSize_8: 227 return static_cast<s64>(SignExtend64(Truncate8(constant_value))); 228 229 case RegSize_16: 230 return static_cast<s64>(SignExtend64(Truncate16(constant_value))); 231 232 case RegSize_32: 233 return static_cast<s64>(SignExtend64(Truncate32(constant_value))); 234 235 case RegSize_64: 236 default: 237 return static_cast<s64>(constant_value); 238 } 239 } 240 241 static Value FromHostReg(RegisterCache* regcache, HostReg reg, RegSize size) 242 { 243 return Value(regcache, reg, size, ValueFlags::Valid | ValueFlags::InHostRegister); 244 } 245 static Value FromScratch(RegisterCache* regcache, HostReg reg, RegSize size) 246 { 247 return Value(regcache, reg, size, ValueFlags::Valid | ValueFlags::InHostRegister | ValueFlags::Scratch); 248 } 249 static Value FromConstant(u64 cv, RegSize size) 250 { 251 return Value(nullptr, cv, size, ValueFlags::Valid | ValueFlags::Constant); 252 } 253 static Value FromConstantU8(u8 value) { return FromConstant(ZeroExtend64(value), RegSize_8); } 254 static Value FromConstantU16(u16 value) { return FromConstant(ZeroExtend64(value), RegSize_16); } 255 static Value FromConstantU32(u32 value) { return FromConstant(ZeroExtend64(value), RegSize_32); } 256 static Value FromConstantS32(s32 value) { return FromConstant(ZeroExtend64(static_cast<u32>(value)), RegSize_32); } 257 static Value FromConstantU64(u64 value) { return FromConstant(value, RegSize_64); } 258 static Value FromConstantPtr(const void* pointer) 259 { 260 #if defined(CPU_ARCH_ARM64) || defined(CPU_ARCH_X64) 261 return FromConstant(static_cast<u64>(reinterpret_cast<uintptr_t>(pointer)), RegSize_64); 262 #elif defined(CPU_ARCH_ARM32) 263 return FromConstant(static_cast<u32>(reinterpret_cast<uintptr_t>(pointer)), RegSize_32); 264 #else 265 return FromConstant(0, RegSize_32); 266 #endif 267 } 268 269 private: 270 void Release(); 271 }; 272 273 class RegisterCache 274 { 275 public: 276 RegisterCache(CodeGenerator& code_generator); 277 ~RegisterCache(); 278 279 u32 GetActiveCalleeSavedRegisterCount() const { return m_state.callee_saved_order_count; } 280 281 ////////////////////////////////////////////////////////////////////////// 282 // Register Allocation 283 ////////////////////////////////////////////////////////////////////////// 284 void SetHostRegAllocationOrder(std::initializer_list<HostReg> regs); 285 void SetCallerSavedHostRegs(std::initializer_list<HostReg> regs); 286 void SetCalleeSavedHostRegs(std::initializer_list<HostReg> regs); 287 void SetCPUPtrHostReg(HostReg reg); 288 289 /// Returns true if the register is permitted to be used in the register cache. 290 bool IsUsableHostReg(HostReg reg) const; 291 bool IsHostRegInUse(HostReg reg) const; 292 bool HasFreeHostRegister() const; 293 u32 GetUsedHostRegisters() const; 294 u32 GetFreeHostRegisters() const; 295 296 /// Allocates a new host register. If there are no free registers, the guest register which was accessed the longest 297 /// time ago will be evicted. 298 HostReg AllocateHostReg(HostRegState state = HostRegState::InUse); 299 300 /// Allocates a specific host register. If this register is not free, returns false. 301 bool AllocateHostReg(HostReg reg, HostRegState state = HostRegState::InUse); 302 303 /// Flags the host register as discard-able. This means that the contents is no longer required, and will not be 304 /// pushed when saving caller-saved registers. 305 void DiscardHostReg(HostReg reg); 306 307 /// Clears the discard-able flag on a host register, so that the contents will be preserved across function calls. 308 void UndiscardHostReg(HostReg reg); 309 310 /// Frees a host register, making it usable in future allocations. 311 void FreeHostReg(HostReg reg); 312 313 /// Ensures a host register is free, removing any value cached. 314 void EnsureHostRegFree(HostReg reg); 315 316 /// Preallocates caller saved registers, enabling later use without stack pushes. 317 void ReserveCallerSavedRegisters(); 318 319 /// Push/pop volatile host registers. Returns the number of registers pushed/popped. 320 u32 PushCallerSavedRegisters() const; 321 u32 PopCallerSavedRegisters() const; 322 323 /// Restore callee-saved registers. Call at the end of the function. 324 u32 PopCalleeSavedRegisters(bool commit); 325 326 /// Preallocates caller saved registers, enabling later use without stack pushes. 327 void ReserveCalleeSavedRegisters(); 328 329 /// Removes the callee-saved register flag from all registers. Call when compiling code blocks. 330 void AssumeCalleeSavedRegistersAreSaved(); 331 332 /// Pushes the register allocator state, use when entering branched code. 333 void PushState(); 334 335 /// Pops the register allocator state, use when leaving branched code. 336 void PopState(); 337 338 ////////////////////////////////////////////////////////////////////////// 339 // Scratch Register Allocation 340 ////////////////////////////////////////////////////////////////////////// 341 Value GetCPUPtr(); 342 Value AllocateScratch(RegSize size, HostReg reg = HostReg_Invalid); 343 344 ////////////////////////////////////////////////////////////////////////// 345 // Guest Register Caching 346 ////////////////////////////////////////////////////////////////////////// 347 348 /// Returns true if the specified guest register is cached. 349 bool IsGuestRegisterCached(Reg guest_reg) const 350 { 351 const Value& cache_value = m_state.guest_reg_state[static_cast<u8>(guest_reg)]; 352 return cache_value.IsConstant() || cache_value.IsInHostRegister(); 353 } 354 355 /// Returns true if the specified guest register is cached and in a host register. 356 bool IsGuestRegisterInHostRegister(Reg guest_reg) const 357 { 358 const Value& cache_value = m_state.guest_reg_state[static_cast<u8>(guest_reg)]; 359 return cache_value.IsInHostRegister(); 360 } 361 362 /// Returns the host register if the guest register is cached. 363 std::optional<HostReg> GetHostRegisterForGuestRegister(Reg guest_reg) const 364 { 365 if (!m_state.guest_reg_state[static_cast<u8>(guest_reg)].IsInHostRegister()) 366 return std::nullopt; 367 return m_state.guest_reg_state[static_cast<u8>(guest_reg)].GetHostRegister(); 368 } 369 370 /// Returns true if there is a load delay which will be stored at the end of the instruction. 371 bool HasLoadDelay() const { return m_state.load_delay_register != Reg::count; } 372 373 Value ReadGuestRegister(Reg guest_reg, bool cache = true, bool force_host_register = false, 374 HostReg forced_host_reg = HostReg_Invalid); 375 376 /// Reads the guest register to a caller-owned scratch register. This will ensure the cache won't invalidate the value 377 /// from some other write. 378 Value ReadGuestRegisterToScratch(Reg guest_reg); 379 380 /// Creates a copy of value, and stores it to guest_reg. 381 Value WriteGuestRegister(Reg guest_reg, Value&& value); 382 383 /// Stores the specified value to the guest register after the next instruction (load delay). 384 void WriteGuestRegisterDelayed(Reg guest_reg, Value&& value); 385 386 /// Returns the current target for a load delay, or Reg::count. 387 Reg GetLoadDelayRegister() const { return m_state.load_delay_register; } 388 const Value& GetLoadDelayValue() const { return m_state.load_delay_value; } 389 390 /// Moves load delay to the next load delay, and writes any previous load delay to the destination register. 391 void UpdateLoadDelay(); 392 393 /// Cancels any present load delay. 394 void CancelLoadDelay(); 395 396 /// Writes the load delay to the CPU structure, so it is synced up with the interpreter. 397 void WriteLoadDelayToCPU(bool clear); 398 399 /// Flushes the load delay, i.e. writes it to the destination register. 400 void FlushLoadDelay(bool clear); 401 402 void FlushGuestRegister(Reg guest_reg, bool invalidate, bool clear_dirty); 403 void InvalidateGuestRegister(Reg guest_reg); 404 405 void InvalidateAllNonDirtyGuestRegisters(); 406 void FlushAllGuestRegisters(bool invalidate, bool clear_dirty); 407 void FlushCallerSavedGuestRegisters(bool invalidate, bool clear_dirty); 408 bool EvictOneGuestRegister(); 409 410 /// Temporarily prevents register allocation. 411 void InhibitAllocation(); 412 void UninhibitAllocation(); 413 414 private: 415 void ClearRegisterFromOrder(Reg reg); 416 void PushRegisterToOrder(Reg reg); 417 void AppendRegisterToOrder(Reg reg); 418 419 CodeGenerator& m_code_generator; 420 421 std::array<HostReg, HostReg_Count> m_host_register_allocation_order{}; 422 423 HostReg m_cpu_ptr_host_register = {}; 424 425 struct RegAllocState 426 { 427 std::array<HostRegState, HostReg_Count> host_reg_state{}; 428 std::array<HostReg, HostReg_Count> callee_saved_order{}; 429 std::array<Value, static_cast<u8>(Reg::count)> guest_reg_state{}; 430 std::array<Reg, HostReg_Count> guest_reg_order{}; 431 432 u32 available_count = 0; 433 u32 callee_saved_order_count = 0; 434 u32 guest_reg_order_count = 0; 435 u32 allocator_inhibit_count = 0; 436 437 Reg load_delay_register = Reg::count; 438 Value load_delay_value{}; 439 440 Reg next_load_delay_register = Reg::count; 441 Value next_load_delay_value{}; 442 } m_state; 443 444 std::stack<RegAllocState> m_state_stack; 445 }; 446 447 } // namespace CPU::Recompiler