#pragma once /* * Most of this header is a giant workaround for MSVC to make atomics into a * somewhat unified interface with how GCC and Clang handle them. * * We don't use the absolutely disgusting C11 stdatomic.h header because it is * unable to operate on non _Atomic types, which enforce implicit sequential * consistency and alter the behavior of the standard C binary/unary operators. * * The strictness of the atomic helpers defined here is assumed to be at least * acquire for loads and release for stores. Cmpxchg uses the standard acq/rel * for success, acq for failure, and is assumed to be strong. */ #ifdef UACPI_OVERRIDE_ATOMIC #include "uacpi_atomic.h" #else #include #if defined(_MSC_VER) && !defined(__clang__) #include // mimic __atomic_compare_exchange_n that doesn't exist on MSVC #define UACPI_MAKE_MSVC_CMPXCHG(width, type, suffix) \ static inline int uacpi_do_atomic_cmpxchg##width( \ type volatile *ptr, type volatile *expected, type desired \ ) \ { \ type current; \ \ current = _InterlockedCompareExchange##suffix(ptr, *expected, desired); \ if (current != *expected) { \ *expected = current; \ return 0; \ } \ return 1; \ } #define UACPI_MSVC_CMPXCHG_INVOKE(ptr, expected, desired, width, type) \ uacpi_do_atomic_cmpxchg##width( \ (type volatile*)ptr, (type volatile*)expected, desired \ ) #define UACPI_MSVC_ATOMIC_STORE(ptr, value, type, width) \ _InterlockedExchange##width((type volatile*)(ptr), (type)(value)) #define UACPI_MSVC_ATOMIC_LOAD(ptr, type, width) \ _InterlockedOr##width((type volatile*)(ptr), 0) #define UACPI_MSVC_ATOMIC_INC(ptr, type, width) \ _InterlockedIncrement##width((type volatile*)(ptr)) #define UACPI_MSVC_ATOMIC_DEC(ptr, type, width) \ _InterlockedDecrement##width((type volatile*)(ptr)) UACPI_MAKE_MSVC_CMPXCHG(64, __int64, 64) UACPI_MAKE_MSVC_CMPXCHG(32, long,) UACPI_MAKE_MSVC_CMPXCHG(16, short, 16) #define uacpi_atomic_cmpxchg16(ptr, expected, desired) \ UACPI_MSVC_CMPXCHG_INVOKE(ptr, expected, desired, 16, short) #define uacpi_atomic_cmpxchg32(ptr, expected, desired) \ UACPI_MSVC_CMPXCHG_INVOKE(ptr, expected, desired, 32, long) #define uacpi_atomic_cmpxchg64(ptr, expected, desired) \ UACPI_MSVC_CMPXCHG_INVOKE(ptr, expected, desired, 64, __int64) #define uacpi_atomic_load8(ptr) UACPI_MSVC_ATOMIC_LOAD(ptr, char, 8) #define uacpi_atomic_load16(ptr) UACPI_MSVC_ATOMIC_LOAD(ptr, short, 16) #define uacpi_atomic_load32(ptr) UACPI_MSVC_ATOMIC_LOAD(ptr, long,) #define uacpi_atomic_load64(ptr) UACPI_MSVC_ATOMIC_LOAD(ptr, __int64, 64) #define uacpi_atomic_store8(ptr, value) UACPI_MSVC_ATOMIC_STORE(ptr, value, char, 8) #define uacpi_atomic_store16(ptr, value) UACPI_MSVC_ATOMIC_STORE(ptr, value, short, 16) #define uacpi_atomic_store32(ptr, value) UACPI_MSVC_ATOMIC_STORE(ptr, value, long,) #define uacpi_atomic_store64(ptr, value) UACPI_MSVC_ATOMIC_STORE(ptr, value, __int64, 64) #define uacpi_atomic_inc16(ptr) UACPI_MSVC_ATOMIC_INC(ptr, short, 16) #define uacpi_atomic_inc32(ptr) UACPI_MSVC_ATOMIC_INC(ptr, long,) #define uacpi_atomic_inc64(ptr) UACPI_MSVC_ATOMIC_INC(ptr, __int64, 64) #define uacpi_atomic_dec16(ptr) UACPI_MSVC_ATOMIC_DEC(ptr, short, 16) #define uacpi_atomic_dec32(ptr) UACPI_MSVC_ATOMIC_DEC(ptr, long,) #define uacpi_atomic_dec64(ptr) UACPI_MSVC_ATOMIC_DEC(ptr, __int64, 64) #elif defined(__WATCOMC__) #include static int uacpi_do_atomic_cmpxchg16(volatile uint16_t *ptr, volatile uint16_t *expected, uint16_t desired); #pragma aux uacpi_do_atomic_cmpxchg16 = \ ".486" \ "mov ax, [esi]" \ "lock cmpxchg [edi], bx" \ "mov [esi], ax" \ "setz al" \ "movzx eax, al" \ parm [ edi ] [ esi ] [ ebx ] \ value [ eax ] static int uacpi_do_atomic_cmpxchg32(volatile uint32_t *ptr, volatile uint32_t *expected, uint32_t desired); #pragma aux uacpi_do_atomic_cmpxchg32 = \ ".486" \ "mov eax, [esi]" \ "lock cmpxchg [edi], ebx" \ "mov [esi], eax" \ "setz al" \ "movzx eax, al" \ parm [ edi ] [ esi ] [ ebx ] \ value [ eax ] static int uacpi_do_atomic_cmpxchg64_asm(volatile uint64_t *ptr, volatile uint64_t *expected, uint32_t low, uint32_t high); #pragma aux uacpi_do_atomic_cmpxchg64_asm = \ ".586" \ "mov eax, [esi]" \ "mov edx, [esi + 4]" \ "lock cmpxchg8b [edi]" \ "mov [esi], eax" \ "mov [esi + 4], edx" \ "setz al" \ "movzx eax, al" \ modify [ edx ] \ parm [ edi ] [ esi ] [ ebx ] [ ecx ] \ value [ eax ] static inline int uacpi_do_atomic_cmpxchg64(volatile uint64_t *ptr, volatile uint64_t *expected, uint64_t desired) { return uacpi_do_atomic_cmpxchg64_asm(ptr, expected, desired, desired >> 32); } #define uacpi_atomic_cmpxchg16(ptr, expected, desired) \ uacpi_do_atomic_cmpxchg16((volatile uint16_t*)ptr, (volatile uint16_t*)expected, (uint16_t)desired) #define uacpi_atomic_cmpxchg32(ptr, expected, desired) \ uacpi_do_atomic_cmpxchg32((volatile uint32_t*)ptr, (volatile uint32_t*)expected, (uint32_t)desired) #define uacpi_atomic_cmpxchg64(ptr, expected, desired) \ uacpi_do_atomic_cmpxchg64((volatile uint64_t*)ptr, (volatile uint64_t*)expected, (uint64_t)desired) static uint8_t uacpi_do_atomic_load8(volatile uint8_t *ptr); #pragma aux uacpi_do_atomic_load8 = \ "mov al, [esi]" \ parm [ esi ] \ value [ al ] static uint16_t uacpi_do_atomic_load16(volatile uint16_t *ptr); #pragma aux uacpi_do_atomic_load16 = \ "mov ax, [esi]" \ parm [ esi ] \ value [ ax ] static uint32_t uacpi_do_atomic_load32(volatile uint32_t *ptr); #pragma aux uacpi_do_atomic_load32 = \ "mov eax, [esi]" \ parm [ esi ] \ value [ eax ] static void uacpi_do_atomic_load64_asm(volatile uint64_t *ptr, uint64_t *out); #pragma aux uacpi_do_atomic_load64_asm = \ ".586" \ "xor eax, eax" \ "xor ebx, ebx" \ "xor ecx, ecx" \ "xor edx, edx" \ "lock cmpxchg8b [esi]" \ "mov [edi], eax" \ "mov [edi + 4], edx" \ modify [ eax ebx ecx edx ] \ parm [ esi ] [ edi ] static inline uint64_t uacpi_do_atomic_load64(volatile uint64_t *ptr) { uint64_t value; uacpi_do_atomic_load64_asm(ptr, &value); return value; } #define uacpi_atomic_load8(ptr) uacpi_do_atomic_load8((volatile uint8_t*)ptr) #define uacpi_atomic_load16(ptr) uacpi_do_atomic_load16((volatile uint16_t*)ptr) #define uacpi_atomic_load32(ptr) uacpi_do_atomic_load32((volatile uint32_t*)ptr) #define uacpi_atomic_load64(ptr) uacpi_do_atomic_load64((volatile uint64_t*)ptr) static void uacpi_do_atomic_store8(volatile uint8_t *ptr, uint8_t value); #pragma aux uacpi_do_atomic_store8 = \ "mov [edi], al" \ parm [ edi ] [ eax ] static void uacpi_do_atomic_store16(volatile uint16_t *ptr, uint16_t value); #pragma aux uacpi_do_atomic_store16 = \ "mov [edi], ax" \ parm [ edi ] [ eax ] static void uacpi_do_atomic_store32(volatile uint32_t *ptr, uint32_t value); #pragma aux uacpi_do_atomic_store32 = \ "mov [edi], eax" \ parm [ edi ] [ eax ] static void uacpi_do_atomic_store64_asm(volatile uint64_t *ptr, uint32_t low, uint32_t high); #pragma aux uacpi_do_atomic_store64_asm = \ ".586" \ "xor eax, eax" \ "xor edx, edx" \ "retry: lock cmpxchg8b [edi]" \ "jnz retry" \ modify [ eax edx ] \ parm [ edi ] [ ebx ] [ ecx ] static inline void uacpi_do_atomic_store64(volatile uint64_t *ptr, uint64_t value) { uacpi_do_atomic_store64_asm(ptr, value, value >> 32); } #define uacpi_atomic_store8(ptr, value) uacpi_do_atomic_store8((volatile uint8_t*)ptr, (uint8_t)value) #define uacpi_atomic_store16(ptr, value) uacpi_do_atomic_store16((volatile uint16_t*)ptr, (uint16_t)value) #define uacpi_atomic_store32(ptr, value) uacpi_do_atomic_store32((volatile uint32_t*)ptr, (uint32_t)value) #define uacpi_atomic_store64(ptr, value) uacpi_do_atomic_store64((volatile uint64_t*)ptr, (uint64_t)value) static uint16_t uacpi_do_atomic_inc16(volatile uint16_t *ptr); #pragma aux uacpi_do_atomic_inc16 = \ ".486" \ "mov ax, 1" \ "lock xadd [edi], ax" \ "add ax, 1" \ parm [ edi ] \ value [ ax ] static uint32_t uacpi_do_atomic_inc32(volatile uint32_t *ptr); #pragma aux uacpi_do_atomic_inc32 = \ ".486" \ "mov eax, 1" \ "lock xadd [edi], eax" \ "add eax, 1" \ parm [ edi ] \ value [ eax ] static void uacpi_do_atomic_inc64_asm(volatile uint64_t *ptr, uint64_t *out); #pragma aux uacpi_do_atomic_inc64_asm = \ ".586" \ "xor eax, eax" \ "xor edx, edx" \ "mov ebx, 1" \ "mov ecx, 1" \ "retry: lock cmpxchg8b [esi]" \ "mov ebx, eax" \ "mov ecx, edx" \ "add ebx, 1" \ "adc ecx, 0" \ "jnz retry" \ "mov [edi], ebx" \ "mov [edi + 4], ecx" \ modify [ eax ebx ecx edx ] \ parm [ esi ] [ edi ] static inline uint64_t uacpi_do_atomic_inc64(volatile uint64_t *ptr) { uint64_t value; uacpi_do_atomic_inc64_asm(ptr, &value); return value; } #define uacpi_atomic_inc16(ptr) uacpi_do_atomic_inc16((volatile uint16_t*)ptr) #define uacpi_atomic_inc32(ptr) uacpi_do_atomic_inc32((volatile uint32_t*)ptr) #define uacpi_atomic_inc64(ptr) uacpi_do_atomic_inc64((volatile uint64_t*)ptr) static uint16_t uacpi_do_atomic_dec16(volatile uint16_t *ptr); #pragma aux uacpi_do_atomic_dec16 = \ ".486" \ "mov ax, -1" \ "lock xadd [edi], ax" \ "add ax, -1" \ parm [ edi ] \ value [ ax ] static uint32_t uacpi_do_atomic_dec32(volatile uint32_t *ptr); #pragma aux uacpi_do_atomic_dec32 = \ ".486" \ "mov eax, -1" \ "lock xadd [edi], eax" \ "add eax, -1" \ parm [ edi ] \ value [ eax ] static void uacpi_do_atomic_dec64_asm(volatile uint64_t *ptr, uint64_t *out); #pragma aux uacpi_do_atomic_dec64_asm = \ ".586" \ "xor eax, eax" \ "xor edx, edx" \ "mov ebx, -1" \ "mov ecx, -1" \ "retry: lock cmpxchg8b [esi]" \ "mov ebx, eax" \ "mov ecx, edx" \ "sub ebx, 1" \ "sbb ecx, 0" \ "jnz retry" \ "mov [edi], ebx" \ "mov [edi + 4], ecx" \ modify [ eax ebx ecx edx ] \ parm [ esi ] [ edi ] static inline uint64_t uacpi_do_atomic_dec64(volatile uint64_t *ptr) { uint64_t value; uacpi_do_atomic_dec64_asm(ptr, &value); return value; } #define uacpi_atomic_dec16(ptr) uacpi_do_atomic_dec16((volatile uint16_t*)ptr) #define uacpi_atomic_dec32(ptr) uacpi_do_atomic_dec32((volatile uint32_t*)ptr) #define uacpi_atomic_dec64(ptr) uacpi_do_atomic_dec64((volatile uint64_t*)ptr) #else #define UACPI_DO_CMPXCHG(ptr, expected, desired) \ __atomic_compare_exchange_n(ptr, expected, desired, 0, \ __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE) #define uacpi_atomic_cmpxchg16(ptr, expected, desired) \ UACPI_DO_CMPXCHG(ptr, expected, desired) #define uacpi_atomic_cmpxchg32(ptr, expected, desired) \ UACPI_DO_CMPXCHG(ptr, expected, desired) #define uacpi_atomic_cmpxchg64(ptr, expected, desired) \ UACPI_DO_CMPXCHG(ptr, expected, desired) #define uacpi_atomic_load8(ptr) __atomic_load_n(ptr, __ATOMIC_ACQUIRE) #define uacpi_atomic_load16(ptr) __atomic_load_n(ptr, __ATOMIC_ACQUIRE) #define uacpi_atomic_load32(ptr) __atomic_load_n(ptr, __ATOMIC_ACQUIRE) #define uacpi_atomic_load64(ptr) __atomic_load_n(ptr, __ATOMIC_ACQUIRE) #define uacpi_atomic_store8(ptr, value) __atomic_store_n(ptr, value, __ATOMIC_RELEASE) #define uacpi_atomic_store16(ptr, value) __atomic_store_n(ptr, value, __ATOMIC_RELEASE) #define uacpi_atomic_store32(ptr, value) __atomic_store_n(ptr, value, __ATOMIC_RELEASE) #define uacpi_atomic_store64(ptr, value) __atomic_store_n(ptr, value, __ATOMIC_RELEASE) #define uacpi_atomic_inc16(ptr) __atomic_add_fetch(ptr, 1, __ATOMIC_ACQ_REL) #define uacpi_atomic_inc32(ptr) __atomic_add_fetch(ptr, 1, __ATOMIC_ACQ_REL) #define uacpi_atomic_inc64(ptr) __atomic_add_fetch(ptr, 1, __ATOMIC_ACQ_REL) #define uacpi_atomic_dec16(ptr) __atomic_sub_fetch(ptr, 1, __ATOMIC_ACQ_REL) #define uacpi_atomic_dec32(ptr) __atomic_sub_fetch(ptr, 1, __ATOMIC_ACQ_REL) #define uacpi_atomic_dec64(ptr) __atomic_sub_fetch(ptr, 1, __ATOMIC_ACQ_REL) #endif #if UACPI_POINTER_SIZE == 4 #define uacpi_atomic_load_ptr(ptr_to_ptr) uacpi_atomic_load32(ptr_to_ptr) #define uacpi_atomic_store_ptr(ptr_to_ptr, value) uacpi_atomic_store32(ptr_to_ptr, value) #else #define uacpi_atomic_load_ptr(ptr_to_ptr) uacpi_atomic_load64(ptr_to_ptr) #define uacpi_atomic_store_ptr(ptr_to_ptr, value) uacpi_atomic_store64(ptr_to_ptr, value) #endif #endif