Integrate uACPI
This commit is contained in:
347
kernel/uACPI/include/uacpi/platform/atomic.h
Normal file
347
kernel/uACPI/include/uacpi/platform/atomic.h
Normal file
@@ -0,0 +1,347 @@
|
||||
#pragma once
|
||||
|
||||
/*
|
||||
* Most of this header is a giant workaround for MSVC to make atomics into a
|
||||
* somewhat unified interface with how GCC and Clang handle them.
|
||||
*
|
||||
* We don't use the absolutely disgusting C11 stdatomic.h header because it is
|
||||
* unable to operate on non _Atomic types, which enforce implicit sequential
|
||||
* consistency and alter the behavior of the standard C binary/unary operators.
|
||||
*
|
||||
* The strictness of the atomic helpers defined here is assumed to be at least
|
||||
* acquire for loads and release for stores. Cmpxchg uses the standard acq/rel
|
||||
* for success, acq for failure, and is assumed to be strong.
|
||||
*/
|
||||
|
||||
#ifdef UACPI_OVERRIDE_ATOMIC
|
||||
#include "uacpi_atomic.h"
|
||||
#else
|
||||
|
||||
#include <uacpi/platform/compiler.h>
|
||||
|
||||
#if defined(_MSC_VER) && !defined(__clang__)
|
||||
|
||||
#include <intrin.h>
|
||||
|
||||
// mimic __atomic_compare_exchange_n that doesn't exist on MSVC
|
||||
#define UACPI_MAKE_MSVC_CMPXCHG(width, type, suffix) \
|
||||
static inline int uacpi_do_atomic_cmpxchg##width( \
|
||||
type volatile *ptr, type volatile *expected, type desired \
|
||||
) \
|
||||
{ \
|
||||
type current; \
|
||||
\
|
||||
current = _InterlockedCompareExchange##suffix(ptr, *expected, desired); \
|
||||
if (current != *expected) { \
|
||||
*expected = current; \
|
||||
return 0; \
|
||||
} \
|
||||
return 1; \
|
||||
}
|
||||
|
||||
#define UACPI_MSVC_CMPXCHG_INVOKE(ptr, expected, desired, width, type) \
|
||||
uacpi_do_atomic_cmpxchg##width( \
|
||||
(type volatile*)ptr, (type volatile*)expected, desired \
|
||||
)
|
||||
|
||||
#define UACPI_MSVC_ATOMIC_STORE(ptr, value, type, width) \
|
||||
_InterlockedExchange##width((type volatile*)(ptr), (type)(value))
|
||||
|
||||
#define UACPI_MSVC_ATOMIC_LOAD(ptr, type, width) \
|
||||
_InterlockedOr##width((type volatile*)(ptr), 0)
|
||||
|
||||
#define UACPI_MSVC_ATOMIC_INC(ptr, type, width) \
|
||||
_InterlockedIncrement##width((type volatile*)(ptr))
|
||||
|
||||
#define UACPI_MSVC_ATOMIC_DEC(ptr, type, width) \
|
||||
_InterlockedDecrement##width((type volatile*)(ptr))
|
||||
|
||||
UACPI_MAKE_MSVC_CMPXCHG(64, __int64, 64)
|
||||
UACPI_MAKE_MSVC_CMPXCHG(32, long,)
|
||||
UACPI_MAKE_MSVC_CMPXCHG(16, short, 16)
|
||||
|
||||
#define uacpi_atomic_cmpxchg16(ptr, expected, desired) \
|
||||
UACPI_MSVC_CMPXCHG_INVOKE(ptr, expected, desired, 16, short)
|
||||
|
||||
#define uacpi_atomic_cmpxchg32(ptr, expected, desired) \
|
||||
UACPI_MSVC_CMPXCHG_INVOKE(ptr, expected, desired, 32, long)
|
||||
|
||||
#define uacpi_atomic_cmpxchg64(ptr, expected, desired) \
|
||||
UACPI_MSVC_CMPXCHG_INVOKE(ptr, expected, desired, 64, __int64)
|
||||
|
||||
#define uacpi_atomic_load8(ptr) UACPI_MSVC_ATOMIC_LOAD(ptr, char, 8)
|
||||
#define uacpi_atomic_load16(ptr) UACPI_MSVC_ATOMIC_LOAD(ptr, short, 16)
|
||||
#define uacpi_atomic_load32(ptr) UACPI_MSVC_ATOMIC_LOAD(ptr, long,)
|
||||
#define uacpi_atomic_load64(ptr) UACPI_MSVC_ATOMIC_LOAD(ptr, __int64, 64)
|
||||
|
||||
#define uacpi_atomic_store8(ptr, value) UACPI_MSVC_ATOMIC_STORE(ptr, value, char, 8)
|
||||
#define uacpi_atomic_store16(ptr, value) UACPI_MSVC_ATOMIC_STORE(ptr, value, short, 16)
|
||||
#define uacpi_atomic_store32(ptr, value) UACPI_MSVC_ATOMIC_STORE(ptr, value, long,)
|
||||
#define uacpi_atomic_store64(ptr, value) UACPI_MSVC_ATOMIC_STORE(ptr, value, __int64, 64)
|
||||
|
||||
#define uacpi_atomic_inc16(ptr) UACPI_MSVC_ATOMIC_INC(ptr, short, 16)
|
||||
#define uacpi_atomic_inc32(ptr) UACPI_MSVC_ATOMIC_INC(ptr, long,)
|
||||
#define uacpi_atomic_inc64(ptr) UACPI_MSVC_ATOMIC_INC(ptr, __int64, 64)
|
||||
|
||||
#define uacpi_atomic_dec16(ptr) UACPI_MSVC_ATOMIC_DEC(ptr, short, 16)
|
||||
#define uacpi_atomic_dec32(ptr) UACPI_MSVC_ATOMIC_DEC(ptr, long,)
|
||||
#define uacpi_atomic_dec64(ptr) UACPI_MSVC_ATOMIC_DEC(ptr, __int64, 64)
|
||||
#elif defined(__WATCOMC__)
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
static int uacpi_do_atomic_cmpxchg16(volatile uint16_t *ptr, volatile uint16_t *expected, uint16_t desired);
|
||||
#pragma aux uacpi_do_atomic_cmpxchg16 = \
|
||||
".486" \
|
||||
"mov ax, [esi]" \
|
||||
"lock cmpxchg [edi], bx" \
|
||||
"mov [esi], ax" \
|
||||
"setz al" \
|
||||
"movzx eax, al" \
|
||||
parm [ edi ] [ esi ] [ ebx ] \
|
||||
value [ eax ]
|
||||
|
||||
static int uacpi_do_atomic_cmpxchg32(volatile uint32_t *ptr, volatile uint32_t *expected, uint32_t desired);
|
||||
#pragma aux uacpi_do_atomic_cmpxchg32 = \
|
||||
".486" \
|
||||
"mov eax, [esi]" \
|
||||
"lock cmpxchg [edi], ebx" \
|
||||
"mov [esi], eax" \
|
||||
"setz al" \
|
||||
"movzx eax, al" \
|
||||
parm [ edi ] [ esi ] [ ebx ] \
|
||||
value [ eax ]
|
||||
|
||||
static int uacpi_do_atomic_cmpxchg64_asm(volatile uint64_t *ptr, volatile uint64_t *expected, uint32_t low, uint32_t high);
|
||||
#pragma aux uacpi_do_atomic_cmpxchg64_asm = \
|
||||
".586" \
|
||||
"mov eax, [esi]" \
|
||||
"mov edx, [esi + 4]" \
|
||||
"lock cmpxchg8b [edi]" \
|
||||
"mov [esi], eax" \
|
||||
"mov [esi + 4], edx" \
|
||||
"setz al" \
|
||||
"movzx eax, al" \
|
||||
modify [ edx ] \
|
||||
parm [ edi ] [ esi ] [ ebx ] [ ecx ] \
|
||||
value [ eax ]
|
||||
|
||||
static inline int uacpi_do_atomic_cmpxchg64(volatile uint64_t *ptr, volatile uint64_t *expected, uint64_t desired) {
|
||||
return uacpi_do_atomic_cmpxchg64_asm(ptr, expected, desired, desired >> 32);
|
||||
}
|
||||
|
||||
#define uacpi_atomic_cmpxchg16(ptr, expected, desired) \
|
||||
uacpi_do_atomic_cmpxchg16((volatile uint16_t*)ptr, (volatile uint16_t*)expected, (uint16_t)desired)
|
||||
#define uacpi_atomic_cmpxchg32(ptr, expected, desired) \
|
||||
uacpi_do_atomic_cmpxchg32((volatile uint32_t*)ptr, (volatile uint32_t*)expected, (uint32_t)desired)
|
||||
#define uacpi_atomic_cmpxchg64(ptr, expected, desired) \
|
||||
uacpi_do_atomic_cmpxchg64((volatile uint64_t*)ptr, (volatile uint64_t*)expected, (uint64_t)desired)
|
||||
|
||||
static uint8_t uacpi_do_atomic_load8(volatile uint8_t *ptr);
|
||||
#pragma aux uacpi_do_atomic_load8 = \
|
||||
"mov al, [esi]" \
|
||||
parm [ esi ] \
|
||||
value [ al ]
|
||||
|
||||
static uint16_t uacpi_do_atomic_load16(volatile uint16_t *ptr);
|
||||
#pragma aux uacpi_do_atomic_load16 = \
|
||||
"mov ax, [esi]" \
|
||||
parm [ esi ] \
|
||||
value [ ax ]
|
||||
|
||||
static uint32_t uacpi_do_atomic_load32(volatile uint32_t *ptr);
|
||||
#pragma aux uacpi_do_atomic_load32 = \
|
||||
"mov eax, [esi]" \
|
||||
parm [ esi ] \
|
||||
value [ eax ]
|
||||
|
||||
static void uacpi_do_atomic_load64_asm(volatile uint64_t *ptr, uint64_t *out);
|
||||
#pragma aux uacpi_do_atomic_load64_asm = \
|
||||
".586" \
|
||||
"xor eax, eax" \
|
||||
"xor ebx, ebx" \
|
||||
"xor ecx, ecx" \
|
||||
"xor edx, edx" \
|
||||
"lock cmpxchg8b [esi]" \
|
||||
"mov [edi], eax" \
|
||||
"mov [edi + 4], edx" \
|
||||
modify [ eax ebx ecx edx ] \
|
||||
parm [ esi ] [ edi ]
|
||||
|
||||
static inline uint64_t uacpi_do_atomic_load64(volatile uint64_t *ptr) {
|
||||
uint64_t value;
|
||||
uacpi_do_atomic_load64_asm(ptr, &value);
|
||||
return value;
|
||||
}
|
||||
|
||||
#define uacpi_atomic_load8(ptr) uacpi_do_atomic_load8((volatile uint8_t*)ptr)
|
||||
#define uacpi_atomic_load16(ptr) uacpi_do_atomic_load16((volatile uint16_t*)ptr)
|
||||
#define uacpi_atomic_load32(ptr) uacpi_do_atomic_load32((volatile uint32_t*)ptr)
|
||||
#define uacpi_atomic_load64(ptr) uacpi_do_atomic_load64((volatile uint64_t*)ptr)
|
||||
|
||||
static void uacpi_do_atomic_store8(volatile uint8_t *ptr, uint8_t value);
|
||||
#pragma aux uacpi_do_atomic_store8 = \
|
||||
"mov [edi], al" \
|
||||
parm [ edi ] [ eax ]
|
||||
|
||||
static void uacpi_do_atomic_store16(volatile uint16_t *ptr, uint16_t value);
|
||||
#pragma aux uacpi_do_atomic_store16 = \
|
||||
"mov [edi], ax" \
|
||||
parm [ edi ] [ eax ]
|
||||
|
||||
static void uacpi_do_atomic_store32(volatile uint32_t *ptr, uint32_t value);
|
||||
#pragma aux uacpi_do_atomic_store32 = \
|
||||
"mov [edi], eax" \
|
||||
parm [ edi ] [ eax ]
|
||||
|
||||
static void uacpi_do_atomic_store64_asm(volatile uint64_t *ptr, uint32_t low, uint32_t high);
|
||||
#pragma aux uacpi_do_atomic_store64_asm = \
|
||||
".586" \
|
||||
"xor eax, eax" \
|
||||
"xor edx, edx" \
|
||||
"retry: lock cmpxchg8b [edi]" \
|
||||
"jnz retry" \
|
||||
modify [ eax edx ] \
|
||||
parm [ edi ] [ ebx ] [ ecx ]
|
||||
|
||||
static inline void uacpi_do_atomic_store64(volatile uint64_t *ptr, uint64_t value) {
|
||||
uacpi_do_atomic_store64_asm(ptr, value, value >> 32);
|
||||
}
|
||||
|
||||
#define uacpi_atomic_store8(ptr, value) uacpi_do_atomic_store8((volatile uint8_t*)ptr, (uint8_t)value)
|
||||
#define uacpi_atomic_store16(ptr, value) uacpi_do_atomic_store16((volatile uint16_t*)ptr, (uint16_t)value)
|
||||
#define uacpi_atomic_store32(ptr, value) uacpi_do_atomic_store32((volatile uint32_t*)ptr, (uint32_t)value)
|
||||
#define uacpi_atomic_store64(ptr, value) uacpi_do_atomic_store64((volatile uint64_t*)ptr, (uint64_t)value)
|
||||
|
||||
static uint16_t uacpi_do_atomic_inc16(volatile uint16_t *ptr);
|
||||
#pragma aux uacpi_do_atomic_inc16 = \
|
||||
".486" \
|
||||
"mov ax, 1" \
|
||||
"lock xadd [edi], ax" \
|
||||
"add ax, 1" \
|
||||
parm [ edi ] \
|
||||
value [ ax ]
|
||||
|
||||
static uint32_t uacpi_do_atomic_inc32(volatile uint32_t *ptr);
|
||||
#pragma aux uacpi_do_atomic_inc32 = \
|
||||
".486" \
|
||||
"mov eax, 1" \
|
||||
"lock xadd [edi], eax" \
|
||||
"add eax, 1" \
|
||||
parm [ edi ] \
|
||||
value [ eax ]
|
||||
|
||||
static void uacpi_do_atomic_inc64_asm(volatile uint64_t *ptr, uint64_t *out);
|
||||
#pragma aux uacpi_do_atomic_inc64_asm = \
|
||||
".586" \
|
||||
"xor eax, eax" \
|
||||
"xor edx, edx" \
|
||||
"mov ebx, 1" \
|
||||
"mov ecx, 1" \
|
||||
"retry: lock cmpxchg8b [esi]" \
|
||||
"mov ebx, eax" \
|
||||
"mov ecx, edx" \
|
||||
"add ebx, 1" \
|
||||
"adc ecx, 0" \
|
||||
"jnz retry" \
|
||||
"mov [edi], ebx" \
|
||||
"mov [edi + 4], ecx" \
|
||||
modify [ eax ebx ecx edx ] \
|
||||
parm [ esi ] [ edi ]
|
||||
|
||||
static inline uint64_t uacpi_do_atomic_inc64(volatile uint64_t *ptr) {
|
||||
uint64_t value;
|
||||
uacpi_do_atomic_inc64_asm(ptr, &value);
|
||||
return value;
|
||||
}
|
||||
|
||||
#define uacpi_atomic_inc16(ptr) uacpi_do_atomic_inc16((volatile uint16_t*)ptr)
|
||||
#define uacpi_atomic_inc32(ptr) uacpi_do_atomic_inc32((volatile uint32_t*)ptr)
|
||||
#define uacpi_atomic_inc64(ptr) uacpi_do_atomic_inc64((volatile uint64_t*)ptr)
|
||||
|
||||
static uint16_t uacpi_do_atomic_dec16(volatile uint16_t *ptr);
|
||||
#pragma aux uacpi_do_atomic_dec16 = \
|
||||
".486" \
|
||||
"mov ax, -1" \
|
||||
"lock xadd [edi], ax" \
|
||||
"add ax, -1" \
|
||||
parm [ edi ] \
|
||||
value [ ax ]
|
||||
|
||||
static uint32_t uacpi_do_atomic_dec32(volatile uint32_t *ptr);
|
||||
#pragma aux uacpi_do_atomic_dec32 = \
|
||||
".486" \
|
||||
"mov eax, -1" \
|
||||
"lock xadd [edi], eax" \
|
||||
"add eax, -1" \
|
||||
parm [ edi ] \
|
||||
value [ eax ]
|
||||
|
||||
static void uacpi_do_atomic_dec64_asm(volatile uint64_t *ptr, uint64_t *out);
|
||||
#pragma aux uacpi_do_atomic_dec64_asm = \
|
||||
".586" \
|
||||
"xor eax, eax" \
|
||||
"xor edx, edx" \
|
||||
"mov ebx, -1" \
|
||||
"mov ecx, -1" \
|
||||
"retry: lock cmpxchg8b [esi]" \
|
||||
"mov ebx, eax" \
|
||||
"mov ecx, edx" \
|
||||
"sub ebx, 1" \
|
||||
"sbb ecx, 0" \
|
||||
"jnz retry" \
|
||||
"mov [edi], ebx" \
|
||||
"mov [edi + 4], ecx" \
|
||||
modify [ eax ebx ecx edx ] \
|
||||
parm [ esi ] [ edi ]
|
||||
|
||||
static inline uint64_t uacpi_do_atomic_dec64(volatile uint64_t *ptr) {
|
||||
uint64_t value;
|
||||
uacpi_do_atomic_dec64_asm(ptr, &value);
|
||||
return value;
|
||||
}
|
||||
|
||||
#define uacpi_atomic_dec16(ptr) uacpi_do_atomic_dec16((volatile uint16_t*)ptr)
|
||||
#define uacpi_atomic_dec32(ptr) uacpi_do_atomic_dec32((volatile uint32_t*)ptr)
|
||||
#define uacpi_atomic_dec64(ptr) uacpi_do_atomic_dec64((volatile uint64_t*)ptr)
|
||||
#else
|
||||
|
||||
#define UACPI_DO_CMPXCHG(ptr, expected, desired) \
|
||||
__atomic_compare_exchange_n(ptr, expected, desired, 0, \
|
||||
__ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)
|
||||
|
||||
#define uacpi_atomic_cmpxchg16(ptr, expected, desired) \
|
||||
UACPI_DO_CMPXCHG(ptr, expected, desired)
|
||||
#define uacpi_atomic_cmpxchg32(ptr, expected, desired) \
|
||||
UACPI_DO_CMPXCHG(ptr, expected, desired)
|
||||
#define uacpi_atomic_cmpxchg64(ptr, expected, desired) \
|
||||
UACPI_DO_CMPXCHG(ptr, expected, desired)
|
||||
|
||||
#define uacpi_atomic_load8(ptr) __atomic_load_n(ptr, __ATOMIC_ACQUIRE)
|
||||
#define uacpi_atomic_load16(ptr) __atomic_load_n(ptr, __ATOMIC_ACQUIRE)
|
||||
#define uacpi_atomic_load32(ptr) __atomic_load_n(ptr, __ATOMIC_ACQUIRE)
|
||||
#define uacpi_atomic_load64(ptr) __atomic_load_n(ptr, __ATOMIC_ACQUIRE)
|
||||
|
||||
#define uacpi_atomic_store8(ptr, value) __atomic_store_n(ptr, value, __ATOMIC_RELEASE)
|
||||
#define uacpi_atomic_store16(ptr, value) __atomic_store_n(ptr, value, __ATOMIC_RELEASE)
|
||||
#define uacpi_atomic_store32(ptr, value) __atomic_store_n(ptr, value, __ATOMIC_RELEASE)
|
||||
#define uacpi_atomic_store64(ptr, value) __atomic_store_n(ptr, value, __ATOMIC_RELEASE)
|
||||
|
||||
#define uacpi_atomic_inc16(ptr) __atomic_add_fetch(ptr, 1, __ATOMIC_ACQ_REL)
|
||||
#define uacpi_atomic_inc32(ptr) __atomic_add_fetch(ptr, 1, __ATOMIC_ACQ_REL)
|
||||
#define uacpi_atomic_inc64(ptr) __atomic_add_fetch(ptr, 1, __ATOMIC_ACQ_REL)
|
||||
|
||||
#define uacpi_atomic_dec16(ptr) __atomic_sub_fetch(ptr, 1, __ATOMIC_ACQ_REL)
|
||||
#define uacpi_atomic_dec32(ptr) __atomic_sub_fetch(ptr, 1, __ATOMIC_ACQ_REL)
|
||||
#define uacpi_atomic_dec64(ptr) __atomic_sub_fetch(ptr, 1, __ATOMIC_ACQ_REL)
|
||||
#endif
|
||||
|
||||
#if UACPI_POINTER_SIZE == 4
|
||||
#define uacpi_atomic_load_ptr(ptr_to_ptr) uacpi_atomic_load32(ptr_to_ptr)
|
||||
#define uacpi_atomic_store_ptr(ptr_to_ptr, value) uacpi_atomic_store32(ptr_to_ptr, value)
|
||||
#else
|
||||
#define uacpi_atomic_load_ptr(ptr_to_ptr) uacpi_atomic_load64(ptr_to_ptr)
|
||||
#define uacpi_atomic_store_ptr(ptr_to_ptr, value) uacpi_atomic_store64(ptr_to_ptr, value)
|
||||
#endif
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user