cacheflush.c (9763B)
1 /* 2 * Info about, and flushing the host cpu caches. 3 * 4 * This work is licensed under the terms of the GNU GPL, version 2 or later. 5 * See the COPYING file in the top-level directory. 6 */ 7 8 #include "qemu/osdep.h" 9 #include "qemu/cacheflush.h" 10 #include "qemu/cacheinfo.h" 11 #include "qemu/bitops.h" 12 #include "qemu/host-utils.h" 13 #include "qemu/atomic.h" 14 15 16 int qemu_icache_linesize = 0; 17 int qemu_icache_linesize_log; 18 int qemu_dcache_linesize = 0; 19 int qemu_dcache_linesize_log; 20 21 /* 22 * Operating system specific cache detection mechanisms. 23 */ 24 25 #if defined(_WIN32) 26 27 static void sys_cache_info(int *isize, int *dsize) 28 { 29 SYSTEM_LOGICAL_PROCESSOR_INFORMATION *buf; 30 DWORD size = 0; 31 BOOL success; 32 size_t i, n; 33 34 /* 35 * Check for the required buffer size first. Note that if the zero 36 * size we use for the probe results in success, then there is no 37 * data available; fail in that case. 38 */ 39 success = GetLogicalProcessorInformation(0, &size); 40 if (success || GetLastError() != ERROR_INSUFFICIENT_BUFFER) { 41 return; 42 } 43 44 n = size / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); 45 size = n * sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); 46 buf = g_new0(SYSTEM_LOGICAL_PROCESSOR_INFORMATION, n); 47 if (!GetLogicalProcessorInformation(buf, &size)) { 48 goto fail; 49 } 50 51 for (i = 0; i < n; i++) { 52 if (buf[i].Relationship == RelationCache 53 && buf[i].Cache.Level == 1) { 54 switch (buf[i].Cache.Type) { 55 case CacheUnified: 56 *isize = *dsize = buf[i].Cache.LineSize; 57 break; 58 case CacheInstruction: 59 *isize = buf[i].Cache.LineSize; 60 break; 61 case CacheData: 62 *dsize = buf[i].Cache.LineSize; 63 break; 64 default: 65 break; 66 } 67 } 68 } 69 fail: 70 g_free(buf); 71 } 72 73 #elif defined(CONFIG_DARWIN) 74 # include <sys/sysctl.h> 75 static void sys_cache_info(int *isize, int *dsize) 76 { 77 /* There's only a single sysctl for both I/D cache line sizes. */ 78 long size; 79 size_t len = sizeof(size); 80 if (!sysctlbyname("hw.cachelinesize", &size, &len, NULL, 0)) { 81 *isize = *dsize = size; 82 } 83 } 84 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) 85 # include <sys/sysctl.h> 86 static void sys_cache_info(int *isize, int *dsize) 87 { 88 /* There's only a single sysctl for both I/D cache line sizes. */ 89 int size; 90 size_t len = sizeof(size); 91 if (!sysctlbyname("machdep.cacheline_size", &size, &len, NULL, 0)) { 92 *isize = *dsize = size; 93 } 94 } 95 #else 96 /* POSIX */ 97 98 static void sys_cache_info(int *isize, int *dsize) 99 { 100 # ifdef _SC_LEVEL1_ICACHE_LINESIZE 101 int tmp_isize = (int) sysconf(_SC_LEVEL1_ICACHE_LINESIZE); 102 if (tmp_isize > 0) { 103 *isize = tmp_isize; 104 } 105 # endif 106 # ifdef _SC_LEVEL1_DCACHE_LINESIZE 107 int tmp_dsize = (int) sysconf(_SC_LEVEL1_DCACHE_LINESIZE); 108 if (tmp_dsize > 0) { 109 *dsize = tmp_dsize; 110 } 111 # endif 112 } 113 #endif /* sys_cache_info */ 114 115 116 /* 117 * Architecture (+ OS) specific cache detection mechanisms. 118 */ 119 120 #if defined(__powerpc__) 121 static bool have_coherent_icache; 122 #endif 123 124 #if defined(__aarch64__) && !defined(CONFIG_DARWIN) 125 /* Apple does not expose CTR_EL0, so we must use system interfaces. */ 126 static uint64_t save_ctr_el0; 127 static void arch_cache_info(int *isize, int *dsize) 128 { 129 uint64_t ctr; 130 131 /* 132 * The real cache geometry is in CCSIDR_EL1/CLIDR_EL1/CSSELR_EL1, 133 * but (at least under Linux) these are marked protected by the 134 * kernel. However, CTR_EL0 contains the minimum linesize in the 135 * entire hierarchy, and is used by userspace cache flushing. 136 * 137 * We will also use this value in flush_idcache_range. 138 */ 139 asm volatile("mrs\t%0, ctr_el0" : "=r"(ctr)); 140 save_ctr_el0 = ctr; 141 142 if (*isize == 0 || *dsize == 0) { 143 if (*isize == 0) { 144 *isize = 4 << (ctr & 0xf); 145 } 146 if (*dsize == 0) { 147 *dsize = 4 << ((ctr >> 16) & 0xf); 148 } 149 } 150 } 151 152 #elif defined(_ARCH_PPC) && defined(__linux__) 153 # include "elf.h" 154 155 static void arch_cache_info(int *isize, int *dsize) 156 { 157 if (*isize == 0) { 158 *isize = qemu_getauxval(AT_ICACHEBSIZE); 159 } 160 if (*dsize == 0) { 161 *dsize = qemu_getauxval(AT_DCACHEBSIZE); 162 } 163 have_coherent_icache = qemu_getauxval(AT_HWCAP) & PPC_FEATURE_ICACHE_SNOOP; 164 } 165 166 #else 167 static void arch_cache_info(int *isize, int *dsize) { } 168 #endif /* arch_cache_info */ 169 170 /* 171 * ... and if all else fails ... 172 */ 173 174 static void fallback_cache_info(int *isize, int *dsize) 175 { 176 /* If we can only find one of the two, assume they're the same. */ 177 if (*isize) { 178 if (*dsize) { 179 /* Success! */ 180 } else { 181 *dsize = *isize; 182 } 183 } else if (*dsize) { 184 *isize = *dsize; 185 } else { 186 #if defined(_ARCH_PPC) 187 /* 188 * For PPC, we're going to use the cache sizes computed for 189 * flush_idcache_range. Which means that we must use the 190 * architecture minimum. 191 */ 192 *isize = *dsize = 16; 193 #else 194 /* Otherwise, 64 bytes is not uncommon. */ 195 *isize = *dsize = 64; 196 #endif 197 } 198 } 199 200 static void __attribute__((constructor)) init_cache_info(void) 201 { 202 int isize = 0, dsize = 0; 203 204 sys_cache_info(&isize, &dsize); 205 arch_cache_info(&isize, &dsize); 206 fallback_cache_info(&isize, &dsize); 207 208 assert((isize & (isize - 1)) == 0); 209 assert((dsize & (dsize - 1)) == 0); 210 211 qemu_icache_linesize = isize; 212 qemu_icache_linesize_log = ctz32(isize); 213 qemu_dcache_linesize = dsize; 214 qemu_dcache_linesize_log = ctz32(dsize); 215 216 qatomic64_init(); 217 } 218 219 220 /* 221 * Architecture (+ OS) specific cache flushing mechanisms. 222 */ 223 224 #if defined(__i386__) || defined(__x86_64__) || defined(__s390__) 225 226 /* Caches are coherent and do not require flushing; symbol inline. */ 227 228 #elif defined(__aarch64__) 229 230 #ifdef CONFIG_DARWIN 231 /* Apple does not expose CTR_EL0, so we must use system interfaces. */ 232 extern void sys_icache_invalidate(void *start, size_t len); 233 extern void sys_dcache_flush(void *start, size_t len); 234 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 235 { 236 sys_dcache_flush((void *)rw, len); 237 sys_icache_invalidate((void *)rx, len); 238 } 239 #else 240 241 /* 242 * This is a copy of gcc's __aarch64_sync_cache_range, modified 243 * to fit this three-operand interface. 244 */ 245 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 246 { 247 const unsigned CTR_IDC = 1u << 28; 248 const unsigned CTR_DIC = 1u << 29; 249 const uint64_t ctr_el0 = save_ctr_el0; 250 const uintptr_t icache_lsize = qemu_icache_linesize; 251 const uintptr_t dcache_lsize = qemu_dcache_linesize; 252 uintptr_t p; 253 254 /* 255 * If CTR_EL0.IDC is enabled, Data cache clean to the Point of Unification 256 * is not required for instruction to data coherence. 257 */ 258 if (!(ctr_el0 & CTR_IDC)) { 259 /* 260 * Loop over the address range, clearing one cache line at once. 261 * Data cache must be flushed to unification first to make sure 262 * the instruction cache fetches the updated data. 263 */ 264 for (p = rw & -dcache_lsize; p < rw + len; p += dcache_lsize) { 265 asm volatile("dc\tcvau, %0" : : "r" (p) : "memory"); 266 } 267 asm volatile("dsb\tish" : : : "memory"); 268 } 269 270 /* 271 * If CTR_EL0.DIC is enabled, Instruction cache cleaning to the Point 272 * of Unification is not required for instruction to data coherence. 273 */ 274 if (!(ctr_el0 & CTR_DIC)) { 275 for (p = rx & -icache_lsize; p < rx + len; p += icache_lsize) { 276 asm volatile("ic\tivau, %0" : : "r"(p) : "memory"); 277 } 278 asm volatile ("dsb\tish" : : : "memory"); 279 } 280 281 asm volatile("isb" : : : "memory"); 282 } 283 #endif /* CONFIG_DARWIN */ 284 285 #elif defined(__mips__) 286 287 #ifdef __OpenBSD__ 288 #include <machine/sysarch.h> 289 #else 290 #include <sys/cachectl.h> 291 #endif 292 293 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 294 { 295 if (rx != rw) { 296 cacheflush((void *)rw, len, DCACHE); 297 } 298 cacheflush((void *)rx, len, ICACHE); 299 } 300 301 #elif defined(__powerpc__) 302 303 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 304 { 305 uintptr_t p, b, e; 306 size_t dsize, isize; 307 308 /* 309 * Some processors have coherent caches and support a simplified 310 * flushing procedure. See 311 * POWER9 UM, 4.6.2.2 Instruction Cache Block Invalidate (icbi) 312 * https://ibm.ent.box.com/s/tmklq90ze7aj8f4n32er1mu3sy9u8k3k 313 */ 314 if (have_coherent_icache) { 315 asm volatile ("sync\n\t" 316 "icbi 0,%0\n\t" 317 "isync" 318 : : "r"(rx) : "memory"); 319 return; 320 } 321 322 dsize = qemu_dcache_linesize; 323 isize = qemu_icache_linesize; 324 325 b = rw & ~(dsize - 1); 326 e = (rw + len + dsize - 1) & ~(dsize - 1); 327 for (p = b; p < e; p += dsize) { 328 asm volatile ("dcbst 0,%0" : : "r"(p) : "memory"); 329 } 330 asm volatile ("sync" : : : "memory"); 331 332 b = rx & ~(isize - 1); 333 e = (rx + len + isize - 1) & ~(isize - 1); 334 for (p = b; p < e; p += isize) { 335 asm volatile ("icbi 0,%0" : : "r"(p) : "memory"); 336 } 337 asm volatile ("sync" : : : "memory"); 338 asm volatile ("isync" : : : "memory"); 339 } 340 341 #elif defined(__sparc__) 342 343 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 344 { 345 /* No additional data flush to the RW virtual address required. */ 346 uintptr_t p, end = (rx + len + 7) & -8; 347 for (p = rx & -8; p < end; p += 8) { 348 __asm__ __volatile__("flush\t%0" : : "r" (p)); 349 } 350 } 351 352 #else 353 354 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len) 355 { 356 if (rw != rx) { 357 __builtin___clear_cache((char *)rw, (char *)rw + len); 358 } 359 __builtin___clear_cache((char *)rx, (char *)rx + len); 360 } 361 362 #endif