| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /* | ||
| 2 | * Copyright (c) 2026 Tiger Data, Inc. | ||
| 3 | * Licensed under the PostgreSQL License. See LICENSE for details. | ||
| 4 | * | ||
| 5 | * platform.c - Platform abstraction implementation | ||
| 6 | */ | ||
| 7 | |||
| 8 | #include <stdbool.h> | ||
| 9 | |||
| 10 | #include "platform.h" | ||
| 11 | |||
| 12 | /* Cached SIMD capabilities (computed once) */ | ||
| 13 | static SimdCapability g_simd_caps = SIMD_NONE; | ||
| 14 | static bool g_simd_detected = false; | ||
| 15 | |||
| 16 | /* Override mask for testing/benchmarking (0xFFFFFFFF = no override) */ | ||
| 17 | static uint32_t g_simd_override = 0xFFFFFFFF; | ||
| 18 | |||
| 19 | #if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \ | ||
| 20 | defined(_M_IX86) | ||
| 21 | |||
| 22 | #define VS_X86 1 | ||
| 23 | |||
| 24 | /* | ||
| 25 | * CPUID wrapper for x86/x64. | ||
| 26 | * | ||
| 27 | * GCC/Clang provide __get_cpuid, but we use inline asm for portability | ||
| 28 | * and to handle extended leaves properly. | ||
| 29 | */ | ||
| 30 | static void | ||
| 31 | 3252 | cpuid(uint32_t leaf, | |
| 32 | uint32_t subleaf, | ||
| 33 | uint32_t *eax, | ||
| 34 | uint32_t *ebx, | ||
| 35 | uint32_t *ecx, | ||
| 36 | uint32_t *edx) | ||
| 37 | { | ||
| 38 | #if defined(__GNUC__) || defined(__clang__) | ||
| 39 | 4272 | __asm__ __volatile__("cpuid" | |
| 40 | : "=a"(*eax), "=b"(*ebx), "=c"(*ecx), "=d"(*edx) | ||
| 41 | : "a"(leaf), "c"(subleaf)); | ||
| 42 | #elif defined(_MSC_VER) | ||
| 43 | int regs[4]; | ||
| 44 | __cpuidex(regs, (int)leaf, (int)subleaf); | ||
| 45 | *eax = regs[0]; | ||
| 46 | *ebx = regs[1]; | ||
| 47 | *ecx = regs[2]; | ||
| 48 | *edx = regs[3]; | ||
| 49 | #else | ||
| 50 | *eax = *ebx = *ecx = *edx = 0; | ||
| 51 | #endif | ||
| 52 | 2232 | } | |
| 53 | |||
| 54 | /* | ||
| 55 | * Check if OS supports AVX state saving (required for AVX/AVX2/AVX-512). | ||
| 56 | * | ||
| 57 | * Even if CPU supports AVX, the OS must save/restore the wider registers | ||
| 58 | * on context switch. We check this via XGETBV. | ||
| 59 | */ | ||
| 60 | static bool | ||
| 61 | 813 | os_supports_avx(void) | |
| 62 | { | ||
| 63 | 255 | uint32_t eax, ebx, ecx, edx; | |
| 64 | |||
| 65 | /* Check OSXSAVE bit in CPUID.1:ECX */ | ||
| 66 | 813 | cpuid(1, 0, &eax, &ebx, &ecx, &edx); | |
| 67 |
2/4✓ Branch 0 taken 255 times.
✓ Branch 1 taken 558 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
813 | if (!(ecx & (1 << 27))) |
| 68 | ✗ | return false; | |
| 69 | |||
| 70 | /* Check XCR0 via XGETBV - bits 1 and 2 must be set for AVX */ | ||
| 71 | #if defined(__GNUC__) || defined(__clang__) | ||
| 72 | 255 | uint32_t xcr0_lo, xcr0_hi; | |
| 73 | 813 | __asm__ __volatile__("xgetbv" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); | |
| 74 | 813 | return (xcr0_lo & 0x6) == 0x6; | |
| 75 | #elif defined(_MSC_VER) | ||
| 76 | uint64_t xcr0 = _xgetbv(0); | ||
| 77 | return (xcr0 & 0x6) == 0x6; | ||
| 78 | #else | ||
| 79 | return false; | ||
| 80 | #endif | ||
| 81 | } | ||
| 82 | |||
| 83 | /* | ||
| 84 | * Check if OS supports AVX-512 state saving. | ||
| 85 | * | ||
| 86 | * AVX-512 requires additional state beyond AVX (opmask, ZMM upper bits). | ||
| 87 | */ | ||
| 88 | static bool | ||
| 89 | ✗ | os_supports_avx512(void) | |
| 90 | { | ||
| 91 | ✗ | if (!os_supports_avx()) | |
| 92 | ✗ | return false; | |
| 93 | |||
| 94 | #if defined(__GNUC__) || defined(__clang__) | ||
| 95 | ✗ | uint32_t xcr0_lo, xcr0_hi; | |
| 96 | ✗ | __asm__ __volatile__("xgetbv" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); | |
| 97 | /* Bits 5, 6, 7 for opmask, ZMM_Hi256, Hi16_ZMM */ | ||
| 98 | ✗ | return (xcr0_lo & 0xE6) == 0xE6; | |
| 99 | #elif defined(_MSC_VER) | ||
| 100 | uint64_t xcr0 = _xgetbv(0); | ||
| 101 | return (xcr0 & 0xE6) == 0xE6; | ||
| 102 | #else | ||
| 103 | return false; | ||
| 104 | #endif | ||
| 105 | } | ||
| 106 | |||
| 107 | static SimdCapability | ||
| 108 | 813 | detect_simd_x86(void) | |
| 109 | { | ||
| 110 | 813 | SimdCapability caps = SIMD_NONE; | |
| 111 | 255 | uint32_t eax, ebx, ecx, edx; | |
| 112 | |||
| 113 | /* Get max supported CPUID leaf */ | ||
| 114 | 813 | cpuid(0, 0, &eax, &ebx, &ecx, &edx); | |
| 115 | 813 | uint32_t max_leaf = eax; | |
| 116 | |||
| 117 |
2/2✓ Branch 0 taken 255 times.
✓ Branch 1 taken 558 times.
|
813 | if (max_leaf < 1) |
| 118 | ✗ | return caps; | |
| 119 | |||
| 120 | /* CPUID leaf 1: basic feature flags */ | ||
| 121 | 813 | cpuid(1, 0, &eax, &ebx, &ecx, &edx); | |
| 122 | |||
| 123 | /* SSE2: EDX bit 26 */ | ||
| 124 |
1/2✓ Branch 0 taken 813 times.
✗ Branch 1 not taken.
|
813 | if (edx & (1 << 26)) |
| 125 | 813 | caps |= SIMD_SSE2; | |
| 126 | |||
| 127 | /* SSE4.1: ECX bit 19 */ | ||
| 128 |
1/2✓ Branch 0 taken 813 times.
✗ Branch 1 not taken.
|
813 | if (ecx & (1 << 19)) |
| 129 | 813 | caps |= SIMD_SSE4_1; | |
| 130 | |||
| 131 | /* AVX requires OS support */ | ||
| 132 |
3/5✓ Branch 0 taken 813 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 255 times.
✓ Branch 3 taken 558 times.
✗ Branch 4 not taken.
|
1068 | if ((ecx & (1 << 28)) && os_supports_avx()) |
| 133 | { | ||
| 134 | /* Check AVX2 in extended features (leaf 7) */ | ||
| 135 |
1/2✓ Branch 0 taken 813 times.
✗ Branch 1 not taken.
|
813 | if (max_leaf >= 7) |
| 136 | { | ||
| 137 | 813 | cpuid(7, 0, &eax, &ebx, &ecx, &edx); | |
| 138 | |||
| 139 | /* AVX2: EBX bit 5 */ | ||
| 140 |
1/2✓ Branch 0 taken 813 times.
✗ Branch 1 not taken.
|
813 | if (ebx & (1 << 5)) |
| 141 | 813 | caps |= SIMD_AVX2; | |
| 142 | |||
| 143 | /* AVX-512F: EBX bit 16 (requires OS support) */ | ||
| 144 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 813 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
|
813 | if ((ebx & (1 << 16)) && os_supports_avx512()) |
| 145 | { | ||
| 146 | ✗ | caps |= SIMD_AVX512F; | |
| 147 | |||
| 148 | /* AVX512DQ: EBX bit 17 */ | ||
| 149 | ✗ | if (ebx & (1 << 17)) | |
| 150 | ✗ | caps |= SIMD_AVX512DQ; | |
| 151 | |||
| 152 | /* AVX512BW: EBX bit 30 */ | ||
| 153 | ✗ | if (ebx & (1 << 30)) | |
| 154 | ✗ | caps |= SIMD_AVX512BW; | |
| 155 | |||
| 156 | /* VPOPCNTDQ: ECX bit 14 (Ice Lake 2019+) */ | ||
| 157 | ✗ | if (ecx & (1 << 14)) | |
| 158 | ✗ | caps |= SIMD_AVX512_VPOPCNTDQ; | |
| 159 | } | ||
| 160 | } | ||
| 161 | } | ||
| 162 | |||
| 163 | 558 | return caps; | |
| 164 | } | ||
| 165 | |||
| 166 | #elif defined(__aarch64__) || defined(_M_ARM64) | ||
| 167 | |||
| 168 | #define VS_ARM64 1 | ||
| 169 | |||
| 170 | static SimdCapability | ||
| 171 | detect_simd_arm64(void) | ||
| 172 | { | ||
| 173 | /* NEON is mandatory on AArch64 */ | ||
| 174 | return SIMD_NEON; | ||
| 175 | } | ||
| 176 | |||
| 177 | #elif defined(__arm__) || defined(_M_ARM) | ||
| 178 | |||
| 179 | #define VS_ARM32 1 | ||
| 180 | |||
| 181 | #if defined(__ARM_NEON) || defined(__ARM_NEON__) | ||
| 182 | static SimdCapability | ||
| 183 | detect_simd_arm32(void) | ||
| 184 | { | ||
| 185 | return SIMD_NEON; | ||
| 186 | } | ||
| 187 | #else | ||
| 188 | static SimdCapability | ||
| 189 | detect_simd_arm32(void) | ||
| 190 | { | ||
| 191 | return SIMD_NONE; | ||
| 192 | } | ||
| 193 | #endif | ||
| 194 | |||
| 195 | #endif /* architecture detection */ | ||
| 196 | |||
| 197 | SimdCapability | ||
| 198 | 1712 | vs_detect_simd(void) | |
| 199 | { | ||
| 200 |
2/2✓ Branch 0 taken 899 times.
✓ Branch 1 taken 813 times.
|
1712 | if (g_simd_detected) |
| 201 | 899 | return g_simd_caps; | |
| 202 | |||
| 203 | 255 | SimdCapability detected; | |
| 204 | |||
| 205 | #if defined(VS_X86) | ||
| 206 | 813 | detected = detect_simd_x86(); | |
| 207 | #elif defined(VS_ARM64) | ||
| 208 | detected = detect_simd_arm64(); | ||
| 209 | #elif defined(VS_ARM32) | ||
| 210 | detected = detect_simd_arm32(); | ||
| 211 | #else | ||
| 212 | detected = SIMD_NONE; | ||
| 213 | #endif | ||
| 214 | |||
| 215 | /* Apply override mask if set */ | ||
| 216 |
2/2✓ Branch 0 taken 476 times.
✓ Branch 1 taken 337 times.
|
813 | if (g_simd_override != 0xFFFFFFFF) |
| 217 | 476 | g_simd_caps = detected & g_simd_override; | |
| 218 | else | ||
| 219 | 337 | g_simd_caps = detected; | |
| 220 | |||
| 221 | 813 | g_simd_detected = true; | |
| 222 | 813 | return g_simd_caps; | |
| 223 | } | ||
| 224 | |||
| 225 | void | ||
| 226 | 546 | vs_simd_set_override(uint32_t mask) | |
| 227 | { | ||
| 228 | 546 | g_simd_override = mask; | |
| 229 | 546 | g_simd_detected = false; /* Force re-detection */ | |
| 230 | 546 | } | |
| 231 | |||
| 232 | void | ||
| 233 | 546 | vs_simd_reset_cache(void) | |
| 234 | { | ||
| 235 | 546 | g_simd_detected = false; | |
| 236 | 546 | g_simd_caps = SIMD_NONE; | |
| 237 | 546 | } | |
| 238 |