GCC Code Coverage Report

Directory: src/
File: src/algo/vecops.c
Date: 2026-10-23 09:17:32
Exec Total Coverage
Lines: 105 105 100.0%
Functions: 20 20 100.0%
Branches: 36 36 100.0%
Line Branch Exec Source
1 /*
2 * Copyright (c) 2026 Tiger Data, Inc.
3 * Licensed under the PostgreSQL License. See LICENSE for details.
4 *
5 * vecops.c - General vector operations with SIMD dispatch
6 *
7 * Provides compiler-vectorized implementations with target_clones for
8 * automatic ISA selection. Hand-optimized SIMD implementations can be
9 * added in vecops_avx2.c, vecops_avx512.c, vecops_neon.c if needed.
10 */
11
12 #include "vs_config.h"
13
14 #include <math.h>
15 #include <stdatomic.h>
16 #include <stddef.h>
17 #include <string.h>
18
19 #include "algo/simd_utils.h"
20 #include "algo/vecops.h"
21 #include "core/platform.h"
22
23 /*
24 * Compiler-Vectorized Implementations
25 *
26 * These use target_clones to generate multiple versions for different ISAs.
27 * The dynamic linker selects the best version at load time.
28 */
29
30 VS_TARGET_CLONES static float
31 35530142 vecops_dot_product_compiler(const float *a, const float *b, Dimension dim)
32 {
33 35530142 float sum = 0.0f;
34
2/2
✓ Branch 0 taken 1349017793 times.
✓ Branch 1 taken 35530142 times.
 1384547935 for (Dimension i = 0; i < dim; i++)
35 1349017793 sum += a[i] * b[i];
36 35530142 return sum;
37 }
38
39 VS_TARGET_CLONES static float
40 34725640 vecops_l2_norm_squared_compiler(const float *v, Dimension dim)
41 {
42 34725640 float sum = 0.0f;
43
2/2
✓ Branch 0 taken 1368925091 times.
✓ Branch 1 taken 34725640 times.
 1403650731 for (Dimension i = 0; i < dim; i++)
44 1368925091 sum += v[i] * v[i];
45 34725640 return sum;
46 }
47
48 VS_TARGET_CLONES static float
49 623225 vecops_vector_sum_compiler(const float *v, Dimension dim)
50 {
51 623225 float sum = 0.0f;
52
2/2
✓ Branch 0 taken 36869474 times.
✓ Branch 1 taken 623225 times.
 37492699 for (Dimension i = 0; i < dim; i++)
53 36869474 sum += v[i];
54 623225 return sum;
55 }
56
57 VS_TARGET_CLONES static void
58 722880 vecops_vector_sub_compiler(
59 const float *a, const float *b, float *out, Dimension dim)
60 {
61
2/2
✓ Branch 0 taken 43433481 times.
✓ Branch 1 taken 722880 times.
 44156361 for (Dimension i = 0; i < dim; i++)
62 43433481 out[i] = a[i] - b[i];
63 722880 }
64
65 VS_TARGET_CLONES static void
66 715 vecops_vector_add_compiler(
67 const float *a, const float *b, float *out, Dimension dim)
68 {
69
2/2
✓ Branch 0 taken 62803 times.
✓ Branch 1 taken 715 times.
 63518 for (Dimension i = 0; i < dim; i++)
70 62803 out[i] = a[i] + b[i];
71 715 }
72
73 VS_TARGET_CLONES static void
74 37238 vecops_vector_scale_compiler(
75 const float *v, float scalar, float *out, Dimension dim)
76 {
77
2/2
✓ Branch 0 taken 318073 times.
✓ Branch 1 taken 37238 times.
 355311 for (Dimension i = 0; i < dim; i++)
78 318073 out[i] = v[i] * scalar;
79 37238 }
80
81 VS_TARGET_CLONES static float
82 19959289 vecops_l2_distance_squared_compiler(
83 const float *a, const float *b, Dimension dim)
84 {
85 19959289 float sum = 0.0f;
86
2/2
✓ Branch 0 taken 1026943861 times.
✓ Branch 1 taken 19959289 times.
 1046903150 for (Dimension i = 0; i < dim; i++)
87 {
88 1026943861 float diff = a[i] - b[i];
89 1026943861 sum += diff * diff;
90 }
91 19959289 return sum;
92 }
93
94 /*
95 * Function Pointer Dispatch
96 */
97
98 typedef float (*DotProductFn)(const float *, const float *, Dimension);
99 typedef float (*NormFn)(const float *, Dimension);
100 typedef float (*SumFn)(const float *, Dimension);
101 typedef void (*BinaryOpFn)(const float *, const float *, float *, Dimension);
102 typedef void (*ScaleFn)(const float *, float, float *, Dimension);
103 typedef float (*DistanceFn)(const float *, const float *, Dimension);
104
105 /*
106 * Selected implementations, published on first use.
107 *
108 * Atomic because two threads racing the first call can both pass the
109 * g_initialized check and both store here. The stores pick the same
110 * implementation, but concurrent non-atomic stores to one object are
111 * a data race whatever they write. Only standalone runs threads; a
112 * PostgreSQL backend is a process with its own copy.
113 */
114 static _Atomic(DotProductFn) g_dot_product_fn = NULL;
115 static _Atomic(NormFn) g_l2_norm_squared_fn = NULL;
116 static _Atomic(SumFn) g_vector_sum_fn = NULL;
117 static _Atomic(BinaryOpFn) g_vector_sub_fn = NULL;
118 static _Atomic(BinaryOpFn) g_vector_add_fn = NULL;
119 static _Atomic(ScaleFn) g_vector_scale_fn = NULL;
120 static _Atomic(DistanceFn) g_l2_distance_squared_fn = NULL;
121
122 static _Atomic(const char *) g_impl_name = NULL;
123 static _Atomic(bool) g_initialized = false;
124
125 void
126 16 vs_vecops_force_reinit(void)
127 {
128 16 g_initialized = false;
129 16 g_dot_product_fn = NULL;
130 16 g_l2_norm_squared_fn = NULL;
131 16 g_vector_sum_fn = NULL;
132 16 g_vector_sub_fn = NULL;
133 16 g_vector_add_fn = NULL;
134 16 g_vector_scale_fn = NULL;
135 16 g_l2_distance_squared_fn = NULL;
136 16 g_impl_name = NULL;
137 16 }
138
139 int
140 215 vs_vecops_init(void)
141 {
142
2/2
✓ Branch 0 taken 185 times.
✓ Branch 1 taken 30 times.
 215 if (g_initialized)
143 6 return 0;
144
145 /*
146 * For now, use compiler-vectorized implementations for all operations.
147 * Hand-optimized SIMD can be added later if profiling shows benefit.
148 *
149 * The compiler does a good job with these simple loops, especially
150 * with target_clones generating AVX2/AVX-512 versions automatically.
151 */
152 209 g_dot_product_fn = vecops_dot_product_compiler;
153 209 g_l2_norm_squared_fn = vecops_l2_norm_squared_compiler;
154 209 g_vector_sum_fn = vecops_vector_sum_compiler;
155 209 g_vector_sub_fn = vecops_vector_sub_compiler;
156 209 g_vector_add_fn = vecops_vector_add_compiler;
157 209 g_vector_scale_fn = vecops_vector_scale_compiler;
158 209 g_l2_distance_squared_fn = vecops_l2_distance_squared_compiler;
159 209 g_impl_name = "compiler";
160
161 209 g_initialized = true;
162 209 return 0;
163 }
164
165 const char *
166 4 vs_vecops_impl_name(void)
167 {
168
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 2 times.
 4 if (vs_unlikely(!g_initialized))
169 2 vs_vecops_init();
170 4 return g_impl_name;
171 }
172
173 /*
174 * Public API
175 */
176
177 float
178 35530142 vs_dot_product(const float *a, const float *b, Dimension dim)
179 {
180
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 35530139 times.
 35530142 if (vs_unlikely(!g_initialized))
181 3 vs_vecops_init();
182 35530142 return g_dot_product_fn(a, b, dim);
183 }
184
185 float
186 34725640 vs_l2_norm_squared(const float *v, Dimension dim)
187 {
188
2/2
✓ Branch 0 taken 122 times.
✓ Branch 1 taken 34725518 times.
 34725640 if (vs_unlikely(!g_initialized))
189 122 vs_vecops_init();
190 34725640 return g_l2_norm_squared_fn(v, dim);
191 }
192
193 float
194 24260 vs_l2_norm(const float *v, Dimension dim)
195 {
196 24260 return sqrtf(vs_l2_norm_squared(v, dim));
197 }
198
199 float
200 623225 vec32_sum(const float *v, Dimension dim)
201 {
202
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 623223 times.
 623225 if (vs_unlikely(!g_initialized))
203 2 vs_vecops_init();
204 623225 return g_vector_sum_fn(v, dim);
205 }
206
207 void
208 722880 vec32_sub(const float *a, const float *b, float *out, Dimension dim)
209 {
210
2/2
✓ Branch 0 taken 63 times.
✓ Branch 1 taken 722817 times.
 722880 if (vs_unlikely(!g_initialized))
211 63 vs_vecops_init();
212 722880 g_vector_sub_fn(a, b, out, dim);
213 722880 }
214
215 void
216 715 vec32_add(const float *a, const float *b, float *out, Dimension dim)
217 {
218
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 713 times.
 715 if (vs_unlikely(!g_initialized))
219 2 vs_vecops_init();
220 715 g_vector_add_fn(a, b, out, dim);
221 715 }
222
223 void
224 37238 vec32_scale(const float *v, float scalar, float *out, Dimension dim)
225 {
226
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 37236 times.
 37238 if (vs_unlikely(!g_initialized))
227 2 vs_vecops_init();
228 37238 g_vector_scale_fn(v, scalar, out, dim);
229 37238 }
230
231 void
232 94 vec32_mean(const float *vectors, uint32_t nvecs, Dimension dim, float *out)
233 {
234 94 memset(out, 0, dim * sizeof(float));
235
2/2
✓ Branch 0 taken 707 times.
✓ Branch 1 taken 94 times.
 801 for (uint32_t i = 0; i < nvecs; i++)
236 707 vec32_add(out, vectors + (size_t)i * dim, out, dim);
237 94 vec32_scale(out, 1.0f / (float)nvecs, out, dim);
238 94 }
239
240 void
241 16 vs_global_mean(
242 const float *centroids,
243 uint32_t ncentroids,
244 Dimension dim,
245 DistanceMetric metric,
246 float *out)
247 {
248 16 vec32_mean(centroids, ncentroids, dim, out);
249
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 14 times.
 16 if (metric == DISTANCE_COSINE)
250 2 vs_l2_normalize(out, dim);
251 16 }
252
253 float
254 19959289 vs_l2_distance_squared(const float *a, const float *b, Dimension dim)
255 {
256
2/2
✓ Branch 0 taken 13 times.
✓ Branch 1 taken 19959276 times.
 19959289 if (vs_unlikely(!g_initialized))
257 13 vs_vecops_init();
258 19959289 return g_l2_distance_squared_fn(a, b, dim);
259 }
260