| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /* | ||
| 2 | * Copyright (c) 2026 Tiger Data, Inc. | ||
| 3 | * Licensed under the PostgreSQL License. See LICENSE for details. | ||
| 4 | * | ||
| 5 | * platform.c - Platform abstraction implementation | ||
| 6 | */ | ||
| 7 | |||
| 8 | #include <stdbool.h> | ||
| 9 | |||
| 10 | #include "platform.h" | ||
| 11 | |||
| 12 | #ifdef VS_STANDALONE | ||
| 13 | int vs_standalone_quiet = 0; | ||
| 14 | #endif | ||
| 15 | |||
| 16 | /* Cached SIMD capabilities (computed once) */ | ||
| 17 | static SimdCapability g_simd_caps = SIMD_NONE; | ||
| 18 | static bool g_simd_detected = false; | ||
| 19 | |||
| 20 | /* Override mask for testing/benchmarking (0xFFFFFFFF = no override) */ | ||
| 21 | static uint32_t g_simd_override = 0xFFFFFFFF; | ||
| 22 | |||
| 23 | #if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \ | ||
| 24 | defined(_M_IX86) | ||
| 25 | |||
| 26 | #define VS_X86 1 | ||
| 27 | |||
| 28 | /* | ||
| 29 | * CPUID wrapper for x86/x64. | ||
| 30 | * | ||
| 31 | * GCC/Clang provide __get_cpuid, but we use inline asm for portability | ||
| 32 | * and to handle extended leaves properly. | ||
| 33 | */ | ||
| 34 | static void | ||
| 35 | 5285 | cpuid(uint32_t leaf, | |
| 36 | uint32_t subleaf, | ||
| 37 | uint32_t *eax, | ||
| 38 | uint32_t *ebx, | ||
| 39 | uint32_t *ecx, | ||
| 40 | uint32_t *edx) | ||
| 41 | { | ||
| 42 | #if defined(__GNUC__) || defined(__clang__) | ||
| 43 | 6560 | __asm__ __volatile__("cpuid" | |
| 44 | : "=a"(*eax), "=b"(*ebx), "=c"(*ecx), "=d"(*edx) | ||
| 45 | : "a"(leaf), "c"(subleaf)); | ||
| 46 | #elif defined(_MSC_VER) | ||
| 47 | int regs[4]; | ||
| 48 | __cpuidex(regs, (int)leaf, (int)subleaf); | ||
| 49 | *eax = regs[0]; | ||
| 50 | *ebx = regs[1]; | ||
| 51 | *ecx = regs[2]; | ||
| 52 | *edx = regs[3]; | ||
| 53 | #else | ||
| 54 | *eax = *ebx = *ecx = *edx = 0; | ||
| 55 | #endif | ||
| 56 | 4010 | } | |
| 57 | |||
| 58 | /* | ||
| 59 | * Check if OS supports AVX state saving (required for AVX/AVX2/AVX-512). | ||
| 60 | * | ||
| 61 | * Even if CPU supports AVX, the OS must save/restore the wider registers | ||
| 62 | * on context switch. We check this via XGETBV. | ||
| 63 | */ | ||
| 64 | static bool | ||
| 65 | 2114 | os_supports_avx(void) | |
| 66 | { | ||
| 67 | 510 | uint32_t eax, ebx, ecx, edx; | |
| 68 | |||
| 69 | /* Check OSXSAVE bit in CPUID.1:ECX */ | ||
| 70 | 1859 | cpuid(1, 0, &eax, &ebx, &ecx, &edx); | |
| 71 |
3/4✓ Branch 0 taken 255 times.
✓ Branch 1 taken 1604 times.
✓ Branch 2 taken 255 times.
✗ Branch 3 not taken.
|
2114 | if (!(ecx & (1 << 27))) |
| 72 | ✗ | return false; | |
| 73 | |||
| 74 | /* Check XCR0 via XGETBV - bits 1 and 2 must be set for AVX */ | ||
| 75 | #if defined(__GNUC__) || defined(__clang__) | ||
| 76 | 510 | uint32_t xcr0_lo, xcr0_hi; | |
| 77 | 2114 | __asm__ __volatile__("xgetbv" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); | |
| 78 | 2114 | return (xcr0_lo & 0x6) == 0x6; | |
| 79 | #elif defined(_MSC_VER) | ||
| 80 | uint64_t xcr0 = _xgetbv(0); | ||
| 81 | return (xcr0 & 0x6) == 0x6; | ||
| 82 | #else | ||
| 83 | return false; | ||
| 84 | #endif | ||
| 85 | } | ||
| 86 | |||
| 87 | /* | ||
| 88 | * Check if OS supports AVX-512 state saving. | ||
| 89 | * | ||
| 90 | * AVX-512 requires additional state beyond AVX (opmask, ZMM upper bits). | ||
| 91 | */ | ||
| 92 | static bool | ||
| 93 | 1057 | os_supports_avx512(void) | |
| 94 | { | ||
| 95 |
2/2✓ Branch 1 taken 255 times.
✓ Branch 2 taken 802 times.
|
1312 | if (!os_supports_avx()) |
| 96 | ✗ | return false; | |
| 97 | |||
| 98 | #if defined(__GNUC__) || defined(__clang__) | ||
| 99 | 255 | uint32_t xcr0_lo, xcr0_hi; | |
| 100 | 1057 | __asm__ __volatile__("xgetbv" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); | |
| 101 | /* Bits 5, 6, 7 for opmask, ZMM_Hi256, Hi16_ZMM */ | ||
| 102 | 1057 | return (xcr0_lo & 0xE6) == 0xE6; | |
| 103 | #elif defined(_MSC_VER) | ||
| 104 | uint64_t xcr0 = _xgetbv(0); | ||
| 105 | return (xcr0 & 0xE6) == 0xE6; | ||
| 106 | #else | ||
| 107 | return false; | ||
| 108 | #endif | ||
| 109 | } | ||
| 110 | |||
| 111 | static SimdCapability | ||
| 112 | 1057 | detect_simd_x86(void) | |
| 113 | { | ||
| 114 | 1057 | SimdCapability caps = SIMD_NONE; | |
| 115 | 255 | uint32_t eax, ebx, ecx, edx; | |
| 116 | |||
| 117 | /* Get max supported CPUID leaf */ | ||
| 118 | 1057 | cpuid(0, 0, &eax, &ebx, &ecx, &edx); | |
| 119 | 1057 | uint32_t max_leaf = eax; | |
| 120 | |||
| 121 |
2/2✓ Branch 0 taken 255 times.
✓ Branch 1 taken 802 times.
|
1057 | if (max_leaf < 1) |
| 122 | ✗ | return caps; | |
| 123 | |||
| 124 | /* CPUID leaf 1: basic feature flags */ | ||
| 125 | 1057 | cpuid(1, 0, &eax, &ebx, &ecx, &edx); | |
| 126 | |||
| 127 | /* SSE2: EDX bit 26 */ | ||
| 128 |
1/2✓ Branch 0 taken 1057 times.
✗ Branch 1 not taken.
|
1057 | if (edx & (1 << 26)) |
| 129 | 1057 | caps |= SIMD_SSE2; | |
| 130 | |||
| 131 | /* SSE4.1: ECX bit 19 */ | ||
| 132 |
1/2✓ Branch 0 taken 1057 times.
✗ Branch 1 not taken.
|
1057 | if (ecx & (1 << 19)) |
| 133 | 1057 | caps |= SIMD_SSE4_1; | |
| 134 | |||
| 135 | /* AVX requires OS support */ | ||
| 136 |
3/5✓ Branch 0 taken 1057 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 255 times.
✓ Branch 3 taken 802 times.
✗ Branch 4 not taken.
|
1312 | if ((ecx & (1 << 28)) && os_supports_avx()) |
| 137 | { | ||
| 138 | /* Check AVX2 in extended features (leaf 7) */ | ||
| 139 |
1/2✓ Branch 0 taken 1057 times.
✗ Branch 1 not taken.
|
1057 | if (max_leaf >= 7) |
| 140 | { | ||
| 141 | 1057 | cpuid(7, 0, &eax, &ebx, &ecx, &edx); | |
| 142 | |||
| 143 | /* AVX2: EBX bit 5 */ | ||
| 144 |
1/2✓ Branch 0 taken 1057 times.
✗ Branch 1 not taken.
|
1057 | if (ebx & (1 << 5)) |
| 145 | 1057 | caps |= SIMD_AVX2; | |
| 146 | |||
| 147 | /* AVX-512F: EBX bit 16 (requires OS support) */ | ||
| 148 |
3/4✓ Branch 0 taken 1057 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 802 times.
✓ Branch 4 taken 255 times.
|
1057 | if ((ebx & (1 << 16)) && os_supports_avx512()) |
| 149 | { | ||
| 150 | 1057 | caps |= SIMD_AVX512F; | |
| 151 | |||
| 152 | /* AVX512DQ: EBX bit 17 */ | ||
| 153 |
1/2✓ Branch 0 taken 1057 times.
✗ Branch 1 not taken.
|
1057 | if (ebx & (1 << 17)) |
| 154 | 1057 | caps |= SIMD_AVX512DQ; | |
| 155 | |||
| 156 | /* AVX512BW: EBX bit 30 */ | ||
| 157 |
1/2✓ Branch 0 taken 1057 times.
✗ Branch 1 not taken.
|
1057 | if (ebx & (1 << 30)) |
| 158 | 1057 | caps |= SIMD_AVX512BW; | |
| 159 | |||
| 160 | /* VPOPCNTDQ: ECX bit 14 (Ice Lake 2019+) */ | ||
| 161 |
1/2✓ Branch 0 taken 1057 times.
✗ Branch 1 not taken.
|
1057 | if (ecx & (1 << 14)) |
| 162 | 1057 | caps |= SIMD_AVX512_VPOPCNTDQ; | |
| 163 | } | ||
| 164 | } | ||
| 165 | } | ||
| 166 | |||
| 167 | 802 | return caps; | |
| 168 | } | ||
| 169 | |||
| 170 | #elif defined(__aarch64__) || defined(_M_ARM64) | ||
| 171 | |||
| 172 | #define VS_ARM64 1 | ||
| 173 | |||
| 174 | static SimdCapability | ||
| 175 | detect_simd_arm64(void) | ||
| 176 | { | ||
| 177 | /* NEON is mandatory on AArch64 */ | ||
| 178 | return SIMD_NEON; | ||
| 179 | } | ||
| 180 | |||
| 181 | #elif defined(__arm__) || defined(_M_ARM) | ||
| 182 | |||
| 183 | #define VS_ARM32 1 | ||
| 184 | |||
| 185 | #if defined(__ARM_NEON) || defined(__ARM_NEON__) | ||
| 186 | static SimdCapability | ||
| 187 | detect_simd_arm32(void) | ||
| 188 | { | ||
| 189 | return SIMD_NEON; | ||
| 190 | } | ||
| 191 | #else | ||
| 192 | static SimdCapability | ||
| 193 | detect_simd_arm32(void) | ||
| 194 | { | ||
| 195 | return SIMD_NONE; | ||
| 196 | } | ||
| 197 | #endif | ||
| 198 | |||
| 199 | #endif /* architecture detection */ | ||
| 200 | |||
| 201 | SimdCapability | ||
| 202 | 1876 | vs_detect_simd(void) | |
| 203 | { | ||
| 204 |
2/2✓ Branch 0 taken 819 times.
✓ Branch 1 taken 1057 times.
|
1876 | if (g_simd_detected) |
| 205 | 819 | return g_simd_caps; | |
| 206 | |||
| 207 | 255 | SimdCapability detected; | |
| 208 | |||
| 209 | #if defined(VS_X86) | ||
| 210 | 1057 | detected = detect_simd_x86(); | |
| 211 | #elif defined(VS_ARM64) | ||
| 212 | detected = detect_simd_arm64(); | ||
| 213 | #elif defined(VS_ARM32) | ||
| 214 | detected = detect_simd_arm32(); | ||
| 215 | #else | ||
| 216 | detected = SIMD_NONE; | ||
| 217 | #endif | ||
| 218 | |||
| 219 | /* Apply override mask if set */ | ||
| 220 |
2/2✓ Branch 0 taken 698 times.
✓ Branch 1 taken 359 times.
|
1057 | if (g_simd_override != 0xFFFFFFFF) |
| 221 | 698 | g_simd_caps = detected & g_simd_override; | |
| 222 | else | ||
| 223 | 359 | g_simd_caps = detected; | |
| 224 | |||
| 225 | 1057 | g_simd_detected = true; | |
| 226 | 1057 | return g_simd_caps; | |
| 227 | } | ||
| 228 | |||
| 229 | void | ||
| 230 | 790 | vs_simd_set_override(uint32_t mask) | |
| 231 | { | ||
| 232 | 790 | g_simd_override = mask; | |
| 233 | 790 | g_simd_detected = false; /* Force re-detection */ | |
| 234 | 790 | } | |
| 235 | |||
| 236 | void | ||
| 237 | 790 | vs_simd_reset_cache(void) | |
| 238 | { | ||
| 239 | 790 | g_simd_detected = false; | |
| 240 | 790 | g_simd_caps = SIMD_NONE; | |
| 241 | 790 | } | |
| 242 |