| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /* | ||
| 2 | * Copyright (c) 2026 Tiger Data, Inc. | ||
| 3 | * Licensed under the PostgreSQL License. See LICENSE for details. | ||
| 4 | * | ||
| 5 | * index.c - Standalone in-memory index build | ||
| 6 | * | ||
| 7 | * Builds centroid tree and per-cluster posting lists from a flat | ||
| 8 | * vector array. Used by bindings and CLI benchmark. | ||
| 9 | * | ||
| 10 | * When RaBitQ encoding is enabled, posting data is written to pages | ||
| 11 | * via PrismPostingBuilder. The full-precision vectors are stored in a | ||
| 12 | * flat array for reranking. | ||
| 13 | */ | ||
| 14 | |||
| 15 | #include <math.h> | ||
| 16 | #include <pthread.h> | ||
| 17 | #include <stdatomic.h> | ||
| 18 | #include <stdio.h> | ||
| 19 | #include <stdlib.h> | ||
| 20 | #include <string.h> | ||
| 21 | #include <time.h> | ||
| 22 | #include <unistd.h> | ||
| 23 | |||
| 24 | #include "algo/distance.h" | ||
| 25 | #include "algo/hkmeans.h" | ||
| 26 | #include "algo/kmeans.h" | ||
| 27 | #include "algo/kmeans_internal.h" | ||
| 28 | #include "algo/topk.h" | ||
| 29 | #include "algo/vecops.h" | ||
| 30 | #include "core/log.h" | ||
| 31 | #include "core/memory.h" | ||
| 32 | #include "index/centroid_build.h" | ||
| 33 | #include "index/centroid_page.h" | ||
| 34 | #include "index/index_build.h" | ||
| 35 | #include "index/parallel_build.h" | ||
| 36 | #include "index/posting_build.h" | ||
| 37 | #include "index/posting_convert.h" | ||
| 38 | #include "index/posting_page.h" | ||
| 39 | #include "quant/rabitq.h" | ||
| 40 | #include "standalone/index.h" | ||
| 41 | |||
| 42 | static uint64_t | ||
| 43 | 590 | now_ns(void) | |
| 44 | { | ||
| 45 | struct timespec ts; | ||
| 46 | 590 | clock_gettime(CLOCK_MONOTONIC, &ts); | |
| 47 | 590 | return (uint64_t)ts.tv_sec * 1000000000ULL + (uint64_t)ts.tv_nsec; | |
| 48 | } | ||
| 49 | |||
| 50 | /* Arena-safe grow: alloc new, copy old, old freed on context delete */ | ||
| 51 | static void * | ||
| 52 | 86 | arena_grow(void *old, size_t old_size, size_t new_size) | |
| 53 | { | ||
| 54 | 86 | void *new_buf = vs_alloc(new_size); | |
| 55 |
2/4✓ Branch 0 taken 86 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 86 times.
✗ Branch 3 not taken.
|
86 | if (old != NULL && old_size > 0) |
| 56 | 86 | memcpy(new_buf, old, old_size); | |
| 57 | 86 | return new_buf; | |
| 58 | } | ||
| 59 | |||
| 60 | /* ---------------------------------------------------------------- | ||
| 61 | * ArrayPageStorage — page array backed by memory context | ||
| 62 | * ---------------------------------------------------------------- */ | ||
| 63 | |||
| 64 | static Page | ||
| 65 | 3815390 | aps_read_page(VsStorage *self, BlockNumber blkno) | |
| 66 | { | ||
| 67 | 3815390 | ArrayPageStorage *s = (ArrayPageStorage *)self; | |
| 68 | 3815390 | return s->pages + (size_t)blkno * BLCKSZ; | |
| 69 | } | ||
| 70 | |||
| 71 | static void | ||
| 72 | 3815390 | aps_release_page(VsStorage *self, BlockNumber blkno) | |
| 73 | { | ||
| 74 | (void)self; | ||
| 75 | (void)blkno; | ||
| 76 | 3815390 | } | |
| 77 | |||
| 78 | static Page | ||
| 79 | 5695 | aps_write_page(VsStorage *self, BlockNumber blkno) | |
| 80 | { | ||
| 81 | 5695 | ArrayPageStorage *s = (ArrayPageStorage *)self; | |
| 82 | 5695 | return s->pages + (size_t)blkno * BLCKSZ; | |
| 83 | } | ||
| 84 | |||
| 85 | static Page | ||
| 86 | 533 | aps_new_page(VsStorage *self, BlockNumber *blkno_out) | |
| 87 | { | ||
| 88 | 533 | ArrayPageStorage *s = (ArrayPageStorage *)self; | |
| 89 | 533 | pthread_mutex_lock(&s->alloc_mutex); | |
| 90 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 533 times.
|
533 | if (s->next_blkno >= s->page_cap) |
| 91 | { | ||
| 92 | ✗ | VsMemCtx prev = vs_memctx_switch(s->memctx); | |
| 93 | ✗ | uint32_t new_cap = s->page_cap * 2; | |
| 94 | ✗ | s->pages = arena_grow( | |
| 95 | ✗ | s->pages, | |
| 96 | ✗ | (size_t)s->page_cap * BLCKSZ, | |
| 97 | ✗ | (size_t)new_cap * BLCKSZ); | |
| 98 | ✗ | s->page_cap = new_cap; | |
| 99 | ✗ | vs_memctx_switch(prev); | |
| 100 | } | ||
| 101 | 533 | *blkno_out = s->next_blkno++; | |
| 102 | 533 | pthread_mutex_unlock(&s->alloc_mutex); | |
| 103 | 533 | return s->pages + (size_t)*blkno_out * BLCKSZ; | |
| 104 | } | ||
| 105 | |||
| 106 | static void | ||
| 107 | 6228 | aps_commit_page(VsStorage *self, BlockNumber blkno) | |
| 108 | { | ||
| 109 | (void)self; | ||
| 110 | (void)blkno; | ||
| 111 | 6228 | } | |
| 112 | |||
| 113 | static BlockNumber | ||
| 114 | 86 | aps_extend(VsStorage *self, uint32_t npages) | |
| 115 | { | ||
| 116 | 86 | ArrayPageStorage *s = (ArrayPageStorage *)self; | |
| 117 | 86 | pthread_mutex_lock(&s->alloc_mutex); | |
| 118 | |||
| 119 | 86 | BlockNumber start = s->next_blkno; | |
| 120 | 86 | uint32_t needed = start + npages; | |
| 121 | |||
| 122 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 86 times.
|
86 | while (needed > s->page_cap) |
| 123 | { | ||
| 124 | ✗ | VsMemCtx prev = vs_memctx_switch(s->memctx); | |
| 125 | ✗ | uint32_t new_cap = s->page_cap * 2; | |
| 126 | ✗ | if (new_cap < needed) | |
| 127 | ✗ | new_cap = needed; | |
| 128 | ✗ | s->pages = arena_grow( | |
| 129 | ✗ | s->pages, | |
| 130 | ✗ | (size_t)s->page_cap * BLCKSZ, | |
| 131 | ✗ | (size_t)new_cap * BLCKSZ); | |
| 132 | ✗ | s->page_cap = new_cap; | |
| 133 | ✗ | vs_memctx_switch(prev); | |
| 134 | } | ||
| 135 | |||
| 136 | 86 | s->next_blkno = needed; | |
| 137 | 86 | pthread_mutex_unlock(&s->alloc_mutex); | |
| 138 | 86 | return start; | |
| 139 | } | ||
| 140 | |||
| 141 | static uint32_t | ||
| 142 | 3706 | aps_rerank( | |
| 143 | VsStorage *self, | ||
| 144 | const float *query, | ||
| 145 | Dimension dim, | ||
| 146 | const VsTopKEntry *candidates, | ||
| 147 | uint32_t count, | ||
| 148 | uint32_t keep, | ||
| 149 | uint32_t *out_indices, | ||
| 150 | Distance *out_distances) | ||
| 151 | { | ||
| 152 | 3706 | ArrayPageStorage *s = (ArrayPageStorage *)self; | |
| 153 | |||
| 154 |
2/4✓ Branch 0 taken 3706 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 3706 times.
|
3706 | if (s->all_vectors == NULL || count == 0) |
| 155 | ✗ | return 0; | |
| 156 | |||
| 157 | 3706 | vs_topk_reset_to_k(&s->rerank_topk, keep); | |
| 158 | |||
| 159 |
2/2✓ Branch 0 taken 68460 times.
✓ Branch 1 taken 3706 times.
|
72166 | for (uint32_t i = 0; i < count; i++) |
| 160 | { | ||
| 161 | Distance d; | ||
| 162 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 68460 times.
|
68460 | if (candidates[i].error == 0.0f) |
| 163 | { | ||
| 164 | ✗ | d = candidates[i].distance; | |
| 165 | } | ||
| 166 | else | ||
| 167 | { | ||
| 168 | 68460 | uint32_t vid = prism_posting_decode_vector_id(candidates[i].id); | |
| 169 |
1/2✓ Branch 0 taken 68460 times.
✗ Branch 1 not taken.
|
68460 | if (vid < s->nvecs) |
| 170 | { | ||
| 171 | 68460 | const float *vec = s->all_vectors + (size_t)vid * dim; | |
| 172 | 68460 | d = vs_l2_distance_squared(query, vec, dim); | |
| 173 | } | ||
| 174 | else | ||
| 175 | { | ||
| 176 | ✗ | d = candidates[i].distance; | |
| 177 | } | ||
| 178 | } | ||
| 179 | |||
| 180 | 68460 | vs_topk_insert(&s->rerank_topk, d, 0.0f, (uint64_t)i); | |
| 181 | } | ||
| 182 | |||
| 183 | uint32_t nresults; | ||
| 184 | 3706 | vs_topk_extract_sorted(&s->rerank_topk, s->rerank_entries, &nresults); | |
| 185 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 3706 times.
|
3706 | if (nresults > s->rerank_cap) |
| 186 | ✗ | nresults = s->rerank_cap; | |
| 187 | |||
| 188 |
2/2✓ Branch 0 taken 5136 times.
✓ Branch 1 taken 3706 times.
|
8842 | for (uint32_t i = 0; i < nresults; i++) |
| 189 | { | ||
| 190 | 5136 | out_indices[i] = (uint32_t)s->rerank_entries[i].id; | |
| 191 | 5136 | out_distances[i] = s->rerank_entries[i].distance; | |
| 192 | } | ||
| 193 | |||
| 194 | 3706 | return nresults; | |
| 195 | } | ||
| 196 | |||
| 197 | static const VsStorageOps array_page_storage_ops = { | ||
| 198 | .read_page = aps_read_page, | ||
| 199 | .release_page = aps_release_page, | ||
| 200 | .write_page = aps_write_page, | ||
| 201 | .new_page = aps_new_page, | ||
| 202 | .commit_page = aps_commit_page, | ||
| 203 | .extend = aps_extend, | ||
| 204 | .rerank = aps_rerank, | ||
| 205 | }; | ||
| 206 | |||
| 207 | /* ---------------------------------------------------------------- | ||
| 208 | * Cluster list helpers (per-cluster vector ID lists) | ||
| 209 | * ---------------------------------------------------------------- */ | ||
| 210 | |||
| 211 | static void | ||
| 212 | 204 | cluster_list_init(PrismClusterList *cl, uint32_t initial_cap) | |
| 213 | { | ||
| 214 | 204 | cl->count = 0; | |
| 215 | 204 | cl->capacity = initial_cap; | |
| 216 | 204 | cl->ids = vs_alloc(initial_cap * sizeof(uint32_t)); | |
| 217 | 204 | } | |
| 218 | |||
| 219 | static void | ||
| 220 | 12400 | cluster_list_append(PrismClusterList *cl, uint32_t id) | |
| 221 | { | ||
| 222 |
2/2✓ Branch 0 taken 86 times.
✓ Branch 1 taken 12314 times.
|
12400 | if (cl->count == cl->capacity) |
| 223 | { | ||
| 224 | 86 | uint32_t old_cap = cl->capacity; | |
| 225 | 86 | uint32_t new_cap = old_cap * 2; | |
| 226 | 172 | cl->ids = arena_grow( | |
| 227 | 86 | cl->ids, | |
| 228 | old_cap * sizeof(uint32_t), | ||
| 229 | new_cap * sizeof(uint32_t)); | ||
| 230 | 86 | cl->capacity = new_cap; | |
| 231 | } | ||
| 232 | 12400 | cl->ids[cl->count++] = id; | |
| 233 | 12400 | } | |
| 234 | |||
| 235 | /* ---------------------------------------------------------------- | ||
| 236 | * ArrayPageStorage initialization helper | ||
| 237 | * ---------------------------------------------------------------- */ | ||
| 238 | |||
| 239 | static ArrayPageStorage | ||
| 240 | 102 | make_array_page_storage(uint32_t est_pages, VsMemCtx memctx) | |
| 241 | { | ||
| 242 | /* | ||
| 243 | * No zero-fill: every page is fully overwritten on its first write | ||
| 244 | * (flush copies the whole BLCKSZ buffer), and reserved/spill pages | ||
| 245 | * that no worker writes are never linked into a chain, so they are | ||
| 246 | * never read. Zeroing the whole pre-reserved array (hundreds of MB | ||
| 247 | * for large builds) would be pure overhead. | ||
| 248 | */ | ||
| 249 | 204 | ArrayPageStorage s = { | |
| 250 | .base = {.ops = &array_page_storage_ops}, | ||
| 251 | 102 | .pages = vs_alloc((size_t)est_pages * BLCKSZ), | |
| 252 | .next_blkno = 0, | ||
| 253 | .page_cap = est_pages, | ||
| 254 | .memctx = memctx, | ||
| 255 | }; | ||
| 256 | 102 | pthread_mutex_init(&s.alloc_mutex, NULL); | |
| 257 | 102 | return s; | |
| 258 | } | ||
| 259 | |||
| 260 | /* ---------------------------------------------------------------- | ||
| 261 | * Normalize helpers | ||
| 262 | * ---------------------------------------------------------------- */ | ||
| 263 | |||
| 264 | static void | ||
| 265 | 2898 | normalize_vector(float *v, Dimension dim) | |
| 266 | { | ||
| 267 | 2898 | float norm = vs_l2_norm(v, dim); | |
| 268 |
2/2✓ Branch 0 taken 2818 times.
✓ Branch 1 taken 80 times.
|
2898 | if (norm > 0.0f) |
| 269 | 2818 | vec32_scale(v, 1.0f / norm, v, dim); | |
| 270 | 2898 | } | |
| 271 | |||
| 272 | static void | ||
| 273 | 6 | normalize_all(float *data, uint32_t nvecs, Dimension dim) | |
| 274 | { | ||
| 275 |
2/2✓ Branch 0 taken 2880 times.
✓ Branch 1 taken 6 times.
|
2886 | for (uint32_t i = 0; i < nvecs; i++) |
| 276 | 2880 | normalize_vector(data + (size_t)i * dim, dim); | |
| 277 | 6 | } | |
| 278 | |||
| 279 | static uint32_t | ||
| 280 | ✗ | sa_detect_nthreads(void) | |
| 281 | { | ||
| 282 | ✗ | long n = sysconf(_SC_NPROCESSORS_ONLN); | |
| 283 | ✗ | if (n < 1) | |
| 284 | ✗ | n = 1; | |
| 285 | ✗ | if (n > 64) | |
| 286 | ✗ | n = 64; | |
| 287 | ✗ | return (uint32_t)n; | |
| 288 | } | ||
| 289 | |||
| 290 | /* ---------------------------------------------------------------- | ||
| 291 | * Assign vectors to cluster lists (serial, inline) | ||
| 292 | * | ||
| 293 | * Used only by flat mode and no-rabitq brute-force path which need | ||
| 294 | * per-cluster vector ID lists. The pages mode parallel path does | ||
| 295 | * assignment inline in each posting worker instead. | ||
| 296 | * ---------------------------------------------------------------- */ | ||
| 297 | |||
| 298 | static void | ||
| 299 | 16 | assign_to_cluster_lists( | |
| 300 | PrismIndex *idx, | ||
| 301 | const HKMeansResult *tree, | ||
| 302 | const PrismAssignParams *bp, | ||
| 303 | Dimension dim, | ||
| 304 | uint32_t nlist) | ||
| 305 | { | ||
| 306 | 16 | PrismBuildWorkerBufs bufs = prism_build_worker_bufs_create(dim); | |
| 307 | |||
| 308 |
2/2✓ Branch 0 taken 12400 times.
✓ Branch 1 taken 16 times.
|
12416 | for (uint32_t i = 0; i < idx->nvecs; i++) |
| 309 | { | ||
| 310 | 12400 | const float *vec = idx->all_vectors + (size_t)i * dim; | |
| 311 | PrismBuildAssignment asgn = | ||
| 312 | 12400 | prism_build_assign_vector(tree, vec, bp, &bufs); | |
| 313 | |||
| 314 | 12400 | cluster_list_append(&idx->clusters[asgn.primary], i); | |
| 315 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 12400 times.
|
12400 | if (asgn.secondary != PRISM_INVALID_CLUSTER) |
| 316 | ✗ | cluster_list_append(&idx->clusters[asgn.secondary], i); | |
| 317 | } | ||
| 318 | |||
| 319 | 16 | prism_build_worker_bufs_free(&bufs); | |
| 320 | |||
| 321 | 16 | idx->max_cluster_size = 0; | |
| 322 |
2/2✓ Branch 0 taken 204 times.
✓ Branch 1 taken 16 times.
|
220 | for (uint32_t c = 0; c < nlist; c++) |
| 323 |
2/2✓ Branch 0 taken 36 times.
✓ Branch 1 taken 168 times.
|
204 | if (idx->clusters[c].count > idx->max_cluster_size) |
| 324 | 36 | idx->max_cluster_size = idx->clusters[c].count; | |
| 325 | 16 | } | |
| 326 | |||
| 327 | /* ---------------------------------------------------------------- | ||
| 328 | * Build | ||
| 329 | * ---------------------------------------------------------------- */ | ||
| 330 | |||
| 331 | PrismIndex * | ||
| 332 | 110 | prism_index_build( | |
| 333 | Vec32Source *src, | ||
| 334 | const PrismIndexConfig *config, | ||
| 335 | PrismBuildStats *stats) | ||
| 336 | { | ||
| 337 | /* Each path below writes only the phases it ran, and ms_refine has no | ||
| 338 | * standalone producer at all, while callers read the struct whole. */ | ||
| 339 |
2/2✓ Branch 0 taken 16 times.
✓ Branch 1 taken 94 times.
|
110 | if (stats != NULL) |
| 340 | 16 | memset(stats, 0, sizeof(*stats)); | |
| 341 | |||
| 342 |
8/8✓ Branch 0 taken 108 times.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 106 times.
✓ Branch 3 taken 2 times.
✓ Branch 4 taken 104 times.
✓ Branch 5 taken 2 times.
✓ Branch 6 taken 2 times.
✓ Branch 7 taken 102 times.
|
110 | if (src == NULL || src->nvecs == 0 || src->dim == 0 || config == NULL) |
| 343 | 8 | return NULL; | |
| 344 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 102 times.
|
102 | if (src->dim > PRISM_INDEX_MAX_DIM) |
| 345 | { | ||
| 346 | ✗ | vs_warn("index build: %u dimensions exceeds the layout ceiling %u", | |
| 347 | (unsigned)src->dim, | ||
| 348 | (unsigned)PRISM_INDEX_MAX_DIM); | ||
| 349 | ✗ | return NULL; | |
| 350 | } | ||
| 351 | |||
| 352 | 102 | uint32_t nvecs = src->nvecs; | |
| 353 | 102 | Dimension dim = src->dim; | |
| 354 | |||
| 355 | 102 | vs_distance_init(); | |
| 356 | |||
| 357 | /* Resolve the worker count for the parallel driver (paged builds). */ | ||
| 358 | uint32_t nworkers; | ||
| 359 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 102 times.
|
102 | if (config->nworkers < 0) |
| 360 | { | ||
| 361 | ✗ | uint32_t ncpu = sa_detect_nthreads(); | |
| 362 | ✗ | nworkers = ncpu > 1 ? ncpu - 1 : 0; | |
| 363 | } | ||
| 364 | else | ||
| 365 | { | ||
| 366 | 102 | nworkers = (uint32_t)config->nworkers; | |
| 367 | } | ||
| 368 | |||
| 369 | /* Long-lived context for index data. Build-phase temporaries | ||
| 370 | * go into a child context that gets deleted after build. */ | ||
| 371 | 102 | VsMemCtx idx_ctx = vs_memctx_create(NULL, "index"); | |
| 372 | 102 | VsMemCtx build_ctx = vs_memctx_create(idx_ctx, "index_build"); | |
| 373 | 102 | VsMemCtx old_ctx = vs_memctx_switch(idx_ctx); | |
| 374 | |||
| 375 | 102 | PrismIndex *idx = vs_alloc0(sizeof(PrismIndex)); | |
| 376 | 102 | idx->memctx = idx_ctx; | |
| 377 | 102 | idx->base.dim = dim; | |
| 378 | 102 | idx->base.metric = config->metric; | |
| 379 | 102 | idx->base.centroid_format = config->centroid_fmt; | |
| 380 | |||
| 381 | /* Resolve nlist */ | ||
| 382 | 102 | uint32_t nlist = config->nlist; | |
| 383 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 102 times.
|
102 | if (nlist == 0) |
| 384 | ✗ | nlist = prism_auto_nlist((double)nvecs, dim, config->target_pages); | |
| 385 | |||
| 386 | /* Resolve fan_out */ | ||
| 387 | 102 | uint32_t fan_out = config->fan_out; | |
| 388 |
2/2✓ Branch 0 taken 92 times.
✓ Branch 1 taken 10 times.
|
102 | if (fan_out == 0) |
| 389 | 92 | fan_out = prism_auto_fan_out(0, nlist, 0); | |
| 390 | 102 | idx->fan_out = fan_out; | |
| 391 | |||
| 392 | /* --- Phase: load vectors --- */ | ||
| 393 | 102 | uint64_t t_phase = now_ns(); | |
| 394 | |||
| 395 | 102 | idx->all_vectors = vs_alloc((size_t)nvecs * dim * sizeof(float)); | |
| 396 | |||
| 397 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 102 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
|
102 | if (src->read_all != NULL && src->read_all(src, idx->all_vectors)) |
| 398 | { | ||
| 399 | ✗ | idx->nvecs = nvecs; | |
| 400 | } | ||
| 401 | else | ||
| 402 | { | ||
| 403 | 102 | idx->nvecs = 0; | |
| 404 | const float *vec; | ||
| 405 | uint32_t id; | ||
| 406 |
2/2✓ Branch 1 taken 189162 times.
✓ Branch 2 taken 102 times.
|
189264 | while (src->next(src, 1, &vec, &id)) |
| 407 | { | ||
| 408 | 189162 | memcpy(idx->all_vectors + (size_t)id * dim, | |
| 409 | vec, | ||
| 410 | dim * sizeof(float)); | ||
| 411 | 189162 | idx->nvecs++; | |
| 412 | } | ||
| 413 | } | ||
| 414 | |||
| 415 |
2/2✓ Branch 0 taken 6 times.
✓ Branch 1 taken 96 times.
|
102 | if (idx->base.metric == DISTANCE_COSINE) |
| 416 | 6 | normalize_all(idx->all_vectors, idx->nvecs, dim); | |
| 417 | |||
| 418 | 102 | double ms_sample = (double)(now_ns() - t_phase) / 1e6; | |
| 419 | |||
| 420 | /* | ||
| 421 | * Paged RaBitQ builds with at least one worker run the shared parallel | ||
| 422 | * build driver (sampling + k-means + bounded streaming posting) — the same | ||
| 423 | * code path the PostgreSQL extension uses. Flat and serial builds keep the | ||
| 424 | * in-memory path below. | ||
| 425 | */ | ||
| 426 |
2/2✓ Branch 0 taken 98 times.
✓ Branch 1 taken 4 times.
|
200 | bool use_driver = config->encode_rabitq && |
| 427 |
2/2✓ Branch 0 taken 86 times.
✓ Branch 1 taken 12 times.
|
98 | config->posting_fmt == PRISM_POSTING_FMT_PAGES; |
| 428 | |||
| 429 | 102 | HKMeansResult *tree = NULL; | |
| 430 | 102 | uint32_t drv_nlist = 0; /* driver path: streamed tree shape */ | |
| 431 | 102 | uint8_t drv_nlevels = 0; | |
| 432 | 102 | float *km_vectors = NULL; | |
| 433 | 102 | uint32_t km_nvecs = 0; | |
| 434 | 102 | double ms_kmeans = 0; | |
| 435 | 102 | float *sa_global_mean = NULL; /* driver's leaf-centroid mean */ | |
| 436 | |||
| 437 |
2/2✓ Branch 0 taken 86 times.
✓ Branch 1 taken 16 times.
|
102 | if (use_driver) |
| 438 | { | ||
| 439 | 86 | t_phase = now_ns(); | |
| 440 | |||
| 441 | /* The driver produces fastscan-packed pages when requested; the query | ||
| 442 | * path must know to read them as fastscan (not AoS). */ | ||
| 443 | 86 | idx->base.fastscan = config->fastscan != 0; | |
| 444 | |||
| 445 | /* Upper bound on leaves (fan_out^nlevels, matching the tree the driver | ||
| 446 | * builds). */ | ||
| 447 | 86 | uint32_t max_nlist = prism_max_nlist(nlist, fan_out); | |
| 448 | |||
| 449 | /* | ||
| 450 | * Long-lived posting storage for the driver's streamed pages. The | ||
| 451 | * driver reserves a centroid region at the front (left unwritten here | ||
| 452 | * — the centroid pages go to centroid_storage below) and writes | ||
| 453 | * posting pages after it. Head blocks are formula-derived | ||
| 454 | * (first_posting + leaf), so no head array is needed. | ||
| 455 | */ | ||
| 456 | 86 | uint32_t est_pages = idx->nvecs / 4 + max_nlist + 256; | |
| 457 | 86 | idx->posting_storage = make_array_page_storage(est_pages, idx_ctx); | |
| 458 | |||
| 459 | 86 | RelationData heap_rel = { | |
| 460 | 86 | .vectors = idx->all_vectors, | |
| 461 | 86 | .nvecs = idx->nvecs, | |
| 462 | .dim = dim, | ||
| 463 | }; | ||
| 464 | 86 | RelationData index_rel = { | |
| 465 | 86 | .page_count = &idx->posting_storage.next_blkno, | |
| 466 | }; | ||
| 467 | /* The driver streams worker→leader, so it needs at least one worker; | ||
| 468 | * a serial (nworkers==0) PAGES build runs through the driver with one. | ||
| 469 | */ | ||
| 470 | 86 | IndexInfo index_info = { | |
| 471 |
2/2✓ Branch 0 taken 16 times.
✓ Branch 1 taken 70 times.
|
86 | .ii_ParallelWorkers = (int)(nworkers > 0 ? nworkers : 1)}; |
| 472 | 86 | PrismBuildConfig cfg = { | |
| 473 | .dim = dim, | ||
| 474 | 86 | .metric = config->metric, | |
| 475 | 86 | .centroid_format = idx->base.centroid_format, | |
| 476 | /* The tree expands to up to fan_out^nlevels leaves; size the | ||
| 477 | * shared regions for that bound (matches the PG caller). */ | ||
| 478 | .nlist = max_nlist, | ||
| 479 | .fan_out = fan_out, | ||
| 480 | 86 | .soar_lambda = config->soar_lambda, | |
| 481 | 86 | .boundary_epsilon = config->boundary_epsilon, | |
| 482 | 86 | .fastscan = config->fastscan != 0, | |
| 483 | }; | ||
| 484 | |||
| 485 | 86 | double heap_tuples = 0, indtuples = 0, soar_dupes = 0; | |
| 486 | |||
| 487 | /* Build temporaries (samples, accumulators, the tree) live in | ||
| 488 | * build_ctx and are freed after the build; page growth uses the | ||
| 489 | * storage's own (idx_ctx) context, so the index pages outlive it. */ | ||
| 490 | 86 | vs_memctx_switch(build_ctx); | |
| 491 | 86 | bool ok = do_parallel_build( | |
| 492 | &heap_rel, | ||
| 493 | &index_rel, | ||
| 494 | &index_info, | ||
| 495 | &cfg, | ||
| 496 | &idx->posting_storage.base, | ||
| 497 | NULL, /* no build-progress seam in standalone (no-op stub) */ | ||
| 498 | &drv_nlist, | ||
| 499 | &drv_nlevels, | ||
| 500 | &heap_tuples, | ||
| 501 | &indtuples, | ||
| 502 | &soar_dupes, | ||
| 503 | /* The driver streams the centroid tree into posting_storage | ||
| 504 | * (no in-RAM tree). Capture the leaf-centroid mean it encoded | ||
| 505 | * against so the page-backed query centering matches. */ | ||
| 506 | &sa_global_mean, | ||
| 507 | &idx->first_posting); | ||
| 508 | 86 | vs_memctx_switch(idx_ctx); | |
| 509 | |||
| 510 | /* The driver always launches at least one worker, so it does not fail | ||
| 511 | * here; on the off chance it does, the empty guard below returns. */ | ||
| 512 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 86 times.
|
86 | if (!ok) |
| 513 | ✗ | drv_nlist = 0; | |
| 514 | |||
| 515 | 86 | ms_kmeans = (double)(now_ns() - t_phase) / 1e6; | |
| 516 | } | ||
| 517 | |||
| 518 |
2/2✓ Branch 0 taken 16 times.
✓ Branch 1 taken 86 times.
|
102 | if (!use_driver) |
| 519 | { | ||
| 520 | /* --- Phase: kmeans (serial; FLAT/no-RaBitQ do not need parallel) | ||
| 521 | * --- */ | ||
| 522 | 16 | t_phase = now_ns(); | |
| 523 | 16 | vs_memctx_switch(build_ctx); | |
| 524 | |||
| 525 | /* Subsample by stride into a contiguous buffer for cache-friendly | ||
| 526 | * k-means iteration. */ | ||
| 527 | 16 | uint32_t max_samples = idx->nvecs < 256000 ? idx->nvecs : 256000; | |
| 528 |
1/2✓ Branch 0 taken 16 times.
✗ Branch 1 not taken.
|
16 | uint32_t stride = max_samples > 0 ? idx->nvecs / max_samples : 1; |
| 529 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 16 times.
|
16 | if (stride < 1) |
| 530 | ✗ | stride = 1; | |
| 531 |
1/2✓ Branch 0 taken 16 times.
✗ Branch 1 not taken.
|
16 | km_nvecs = (stride > 1) ? max_samples : idx->nvecs; |
| 532 | |||
| 533 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 16 times.
|
16 | if (stride > 1) |
| 534 | { | ||
| 535 | ✗ | km_vectors = vs_alloc((size_t)km_nvecs * dim * sizeof(float)); | |
| 536 | ✗ | for (uint32_t i = 0; i < km_nvecs; i++) | |
| 537 | ✗ | memcpy(km_vectors + (size_t)i * dim, | |
| 538 | ✗ | idx->all_vectors + (size_t)i * stride * dim, | |
| 539 | dim * sizeof(float)); | ||
| 540 | } | ||
| 541 | else | ||
| 542 | { | ||
| 543 | 16 | km_vectors = idx->all_vectors; | |
| 544 | } | ||
| 545 | |||
| 546 | 16 | KMeansOptions km_opts = VS_KMEANS_OPTIONS_DEFAULT; | |
| 547 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 16 times.
|
16 | if (config->km_nredo > 0) |
| 548 | ✗ | km_opts.nredo = config->km_nredo; | |
| 549 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 16 times.
|
16 | if (config->km_max_iter > 0) |
| 550 | ✗ | km_opts.max_iterations = config->km_max_iter; | |
| 551 | |||
| 552 | 16 | tree = vs_hkmeans_f32( | |
| 553 | km_vectors, | ||
| 554 | km_nvecs, | ||
| 555 | NULL, | ||
| 556 | dim, | ||
| 557 | nlist, | ||
| 558 | fan_out, | ||
| 559 | idx->base.metric, | ||
| 560 | &km_opts); | ||
| 561 | |||
| 562 | 16 | ms_kmeans = (double)(now_ns() - t_phase) / 1e6; | |
| 563 | } /* end serial in-memory k-means */ | ||
| 564 | |||
| 565 | /* --- Phase: setup --- */ | ||
| 566 | 102 | t_phase = now_ns(); | |
| 567 | |||
| 568 | 102 | vs_memctx_switch(idx_ctx); | |
| 569 | |||
| 570 |
3/4✓ Branch 0 taken 86 times.
✓ Branch 1 taken 16 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 102 times.
|
102 | if (use_driver ? drv_nlist == 0 : tree == NULL) |
| 571 | { | ||
| 572 | ✗ | vs_memctx_switch(old_ctx); | |
| 573 | ✗ | vs_memctx_delete(idx_ctx); /* frees idx, build_ctx, everything */ | |
| 574 | ✗ | return NULL; | |
| 575 | } | ||
| 576 | |||
| 577 |
2/2✓ Branch 0 taken 86 times.
✓ Branch 1 taken 16 times.
|
102 | if (use_driver) |
| 578 | { | ||
| 579 | /* Paged parallel build: do_parallel_build streamed the whole centroid | ||
| 580 | * tree + posting heads into posting_storage (no in-RAM tree). The | ||
| 581 | * query routes page-backed over those pages via idx->base, so skip the | ||
| 582 | * tree-centroid finalize entirely (the returned tree is only a carrier | ||
| 583 | * of leaf/level counts). */ | ||
| 584 | 86 | nlist = drv_nlist; | |
| 585 | 86 | idx->nlist = nlist; | |
| 586 | 86 | idx->base.nlevels = drv_nlevels; | |
| 587 | 86 | idx->base.nlist = nlist; | |
| 588 |
1/2✓ Branch 0 taken 86 times.
✗ Branch 1 not taken.
|
86 | idx->base.fan_out = (uint8_t)(fan_out <= UINT8_MAX ? fan_out |
| 589 | : UINT8_MAX); | ||
| 590 | /* Query knobs: mirror the PG GUC defaults so the paged beam search | ||
| 591 | * behaves identically in both engines (a zero beam scale would | ||
| 592 | * collapse the beam to width 1 and skip the coverage floors). */ | ||
| 593 | 86 | idx->base.centroid_error_scale = 0.0f; | |
| 594 | 86 | idx->base.centroid_beam_scale = 0.5f; | |
| 595 | /* The streaming build reserves block 0 for the meta page and writes | ||
| 596 | * the root centroid page in place at block 1, after its subtrees. */ | ||
| 597 | 86 | idx->base.first_centroid = 1; | |
| 598 | 86 | idx->base.params = vs_rabitq_create(dim, VS_RABITQ_BUILD_SEED); | |
| 599 | /* The paged query needs no per-cluster lists, but the bindings API | ||
| 600 | * reads idx->clusters[c].count for build stats; give it zeroed entries | ||
| 601 | * (count 0 -> the API falls back to size estimates). */ | ||
| 602 | 86 | idx->clusters = vs_alloc0((size_t)nlist * sizeof(PrismClusterList)); | |
| 603 | 86 | idx->base.pt_global_mean = vs_alloc(dim * sizeof(float)); | |
| 604 |
1/2✓ Branch 0 taken 86 times.
✗ Branch 1 not taken.
|
86 | if (sa_global_mean != NULL) |
| 605 | 86 | vs_rabitq_rotate( | |
| 606 | 86 | idx->base.params, | |
| 607 | sa_global_mean, | ||
| 608 | idx->base.pt_global_mean); | ||
| 609 | else | ||
| 610 | ✗ | memset(idx->base.pt_global_mean, 0, (size_t)dim * sizeof(float)); | |
| 611 | 86 | idx->has_posting_data = true; | |
| 612 | 86 | idx->posting_fmt = config->posting_fmt; | |
| 613 | |||
| 614 | 86 | vs_memctx_switch(old_ctx); | |
| 615 | 86 | vs_memctx_delete(build_ctx); | |
| 616 | |||
| 617 |
2/2✓ Branch 0 taken 14 times.
✓ Branch 1 taken 72 times.
|
86 | if (stats != NULL) |
| 618 | { | ||
| 619 | 14 | stats->ms_sample = ms_sample; | |
| 620 | 14 | stats->ms_kmeans = ms_kmeans; | |
| 621 | 14 | stats->ms_total = ms_sample + ms_kmeans; | |
| 622 | 14 | stats->nworkers = nworkers; | |
| 623 | } | ||
| 624 | |||
| 625 | 86 | idx->base.centroid_storage = &idx->posting_storage.base; | |
| 626 | 86 | idx->base.posting_storage = &idx->posting_storage.base; | |
| 627 | 86 | idx->base.page_base = idx->posting_storage.pages; | |
| 628 | |||
| 629 | 86 | idx->posting_storage.all_vectors = idx->all_vectors; | |
| 630 | 86 | idx->posting_storage.nvecs = idx->nvecs; | |
| 631 | 86 | idx->posting_storage.metric = idx->base.metric; | |
| 632 | 86 | uint32_t rerank_cap = 256; | |
| 633 | 86 | vs_topk_init(&idx->posting_storage.rerank_topk, rerank_cap); | |
| 634 | 86 | idx->posting_storage.rerank_entries = vs_alloc( | |
| 635 | rerank_cap * sizeof(VsTopKEntry)); | ||
| 636 | 86 | idx->posting_storage.rerank_cap = rerank_cap; | |
| 637 | |||
| 638 | 86 | return idx; | |
| 639 | } | ||
| 640 | |||
| 641 | 16 | nlist = tree->nleaves; | |
| 642 | 16 | idx->nlist = nlist; | |
| 643 | 16 | idx->base.nlevels = (uint8_t)tree->nlevels; | |
| 644 |
1/2✓ Branch 0 taken 16 times.
✗ Branch 1 not taken.
|
16 | idx->base.fan_out = (uint8_t)(fan_out <= UINT8_MAX ? fan_out : UINT8_MAX); |
| 645 | 16 | idx->base.nlist = nlist; | |
| 646 | |||
| 647 | /* Normalize leaf centroids for cosine */ | ||
| 648 |
2/2✓ Branch 0 taken 2 times.
✓ Branch 1 taken 14 times.
|
16 | if (idx->base.metric == DISTANCE_COSINE) |
| 649 | { | ||
| 650 |
2/2✓ Branch 0 taken 18 times.
✓ Branch 1 taken 2 times.
|
20 | for (uint32_t c = 0; c < nlist; c++) |
| 651 | 18 | normalize_vector(hk_leaf_centroids(tree) + (size_t)c * dim, dim); | |
| 652 | } | ||
| 653 | |||
| 654 | /* Global mean */ | ||
| 655 | 16 | idx->global_mean = vs_alloc(dim * sizeof(float)); | |
| 656 | 16 | vs_global_mean( | |
| 657 | 16 | hk_leaf_centroids(tree), | |
| 658 | nlist, | ||
| 659 | dim, | ||
| 660 | idx->base.metric, | ||
| 661 | idx->global_mean); | ||
| 662 | |||
| 663 | /* Save leaf centroids for per-cluster query preparation */ | ||
| 664 | 16 | idx->leaf_centroids = vs_alloc((size_t)nlist * dim * sizeof(float)); | |
| 665 | 32 | memcpy(idx->leaf_centroids, | |
| 666 | 16 | hk_leaf_centroids(tree), | |
| 667 | 16 | (size_t)nlist * dim * sizeof(float)); | |
| 668 | |||
| 669 | /* RaBitQ params */ | ||
| 670 | 16 | idx->base.params = vs_rabitq_create(dim, VS_RABITQ_BUILD_SEED); | |
| 671 | |||
| 672 | /* Precompute P^T * centroids for zero-alloc query path. */ | ||
| 673 | 16 | idx->pt_centroids = vs_alloc((size_t)nlist * dim * sizeof(float)); | |
| 674 |
2/2✓ Branch 0 taken 204 times.
✓ Branch 1 taken 16 times.
|
220 | for (uint32_t c = 0; c < nlist; c++) |
| 675 | 204 | vs_rabitq_rotate( | |
| 676 | 204 | idx->base.params, | |
| 677 | 204 | idx->leaf_centroids + (size_t)c * dim, | |
| 678 | 204 | idx->pt_centroids + (size_t)c * dim); | |
| 679 | |||
| 680 | 16 | idx->base.pt_global_mean = vs_alloc(dim * sizeof(float)); | |
| 681 | 16 | vs_rabitq_rotate( | |
| 682 | 16 | idx->base.params, idx->global_mean, idx->base.pt_global_mean); | |
| 683 | |||
| 684 | /* Build centroid pages */ | ||
| 685 | 16 | uint32_t est_centroid_pages = nlist + 100; | |
| 686 | idx->centroid_storage = | ||
| 687 | 16 | make_array_page_storage(est_centroid_pages, idx_ctx); | |
| 688 | |||
| 689 | uint32_t max_ent = | ||
| 690 | 16 | prism_centroid_max_entries_fmt(dim, idx->base.centroid_format); | |
| 691 | |||
| 692 | /* Temporary arrays for centroid page layout (build context) */ | ||
| 693 | 16 | vs_memctx_switch(build_ctx); | |
| 694 | 16 | BlockNumber *node_first_blkno = vs_alloc( | |
| 695 | 16 | tree->nnodes * sizeof(BlockNumber)); | |
| 696 | 16 | idx->base.first_centroid = 0; | |
| 697 | 16 | prism_compute_centroid_layout( | |
| 698 | tree, max_ent, idx->base.first_centroid, node_first_blkno); | ||
| 699 | |||
| 700 | /* Switch back to index context for posting data. | ||
| 701 | * Posting lists are built before centroid pages so that | ||
| 702 | * centroid leaf entries store actual posting block numbers. */ | ||
| 703 | 16 | vs_memctx_switch(idx_ctx); | |
| 704 | |||
| 705 | /* Initialize per-cluster ID lists */ | ||
| 706 |
1/2✓ Branch 0 taken 16 times.
✗ Branch 1 not taken.
|
16 | uint32_t est_per_cluster = nlist > 0 ? nvecs / nlist + 1 : 1; |
| 707 | 16 | idx->clusters = vs_alloc0(nlist * sizeof(PrismClusterList)); | |
| 708 |
2/2✓ Branch 0 taken 204 times.
✓ Branch 1 taken 16 times.
|
220 | for (uint32_t c = 0; c < nlist; c++) |
| 709 | 204 | cluster_list_init(&idx->clusters[c], est_per_cluster); | |
| 710 | |||
| 711 | 16 | double ms_setup = (double)(now_ns() - t_phase) / 1e6; | |
| 712 | 16 | const PrismAssignParams bp = { | |
| 713 | .dim = dim, | ||
| 714 | 16 | .metric = config->metric, | |
| 715 | 16 | .soar_lambda = config->soar_lambda, | |
| 716 | 16 | .boundary_epsilon = config->boundary_epsilon, | |
| 717 | }; | ||
| 718 | |||
| 719 | /* --- Phase: posting --- */ | ||
| 720 | 16 | t_phase = now_ns(); | |
| 721 | |||
| 722 | 16 | double ms_parallel = 0, ms_merge = 0; | |
| 723 | 16 | uint32_t stat_nworkers = nworkers, stat_pages = 0; | |
| 724 | 16 | uint32_t stat_merge_in = 0, stat_merge_out = 0; | |
| 725 |
2/2✓ Branch 0 taken 12 times.
✓ Branch 1 taken 4 times.
|
16 | if (config->encode_rabitq) |
| 726 | { | ||
| 727 | 12 | idx->posting_fmt = config->posting_fmt; | |
| 728 | |||
| 729 | /* The driver path returned above, so this is always the flat | ||
| 730 | * mode, which needs per-cluster vector lists. */ | ||
| 731 | 12 | assign_to_cluster_lists(idx, tree, &bp, dim, nlist); | |
| 732 | |||
| 733 | 12 | idx->flat_pages = vs_alloc(nlist * sizeof(char *)); | |
| 734 | |||
| 735 |
2/2✓ Branch 0 taken 168 times.
✓ Branch 1 taken 12 times.
|
180 | for (uint32_t c = 0; c < nlist; c++) |
| 736 | { | ||
| 737 | 168 | PrismClusterList *cl = &idx->clusters[c]; | |
| 738 | 168 | const float *cent = hk_leaf_centroids(tree) + (size_t)c * dim; | |
| 739 | |||
| 740 | PrismFlatPostingBuilder builder; | ||
| 741 | 168 | prism_flat_posting_builder_init( | |
| 742 | 168 | &builder, idx->base.params, dim, c, cent, cl->count); | |
| 743 | |||
| 744 |
2/2✓ Branch 0 taken 11000 times.
✓ Branch 1 taken 168 times.
|
11168 | for (uint32_t i = 0; i < cl->count; i++) |
| 745 | { | ||
| 746 | 11000 | uint32_t vid = cl->ids[i]; | |
| 747 | 11000 | const float *vec = idx->all_vectors + (size_t)vid * dim; | |
| 748 | ItemPointerData tid; | ||
| 749 | 11000 | prism_posting_set_vector_id(&tid, vid); | |
| 750 | 11000 | prism_flat_posting_builder_add(&builder, tid, vec); | |
| 751 | } | ||
| 752 | |||
| 753 | 168 | idx->flat_pages[c] = prism_flat_posting_builder_finish(&builder); | |
| 754 | 168 | prism_flat_posting_builder_cleanup(&builder); | |
| 755 | } | ||
| 756 | |||
| 757 | 12 | idx->has_posting_data = true; | |
| 758 | } | ||
| 759 | else | ||
| 760 | { | ||
| 761 | /* No RaBitQ — still need cluster lists for brute-force */ | ||
| 762 | 4 | assign_to_cluster_lists(idx, tree, &bp, dim, nlist); | |
| 763 | } | ||
| 764 | |||
| 765 |
1/2✓ Branch 0 taken 16 times.
✗ Branch 1 not taken.
|
32 | idx->has_replication = config->soar_lambda > 0.0 || |
| 766 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 16 times.
|
16 | config->boundary_epsilon > 0.0; |
| 767 | |||
| 768 | /* --- Phase: centroid pages --- */ | ||
| 769 | 16 | double ms_posting = (double)(now_ns() - t_phase) / 1e6; | |
| 770 | 16 | t_phase = now_ns(); | |
| 771 | { | ||
| 772 | /* This (non-driver) path stores cluster indices as leaf child blocks; | ||
| 773 | * the inline flat query maps them to per-cluster lists. Leaf j's child | ||
| 774 | * is posting_base + global_leaf_index, so posting_base = 0 yields the | ||
| 775 | * leaf (cluster) index directly. The paged build takes the use_driver | ||
| 776 | * path, which writes the centroid tree itself and returns above. */ | ||
| 777 | 16 | vs_memctx_switch(build_ctx); | |
| 778 | 16 | bool needs_rq_params = | |
| 779 |
2/2✓ Branch 0 taken 4 times.
✓ Branch 1 taken 12 times.
|
20 | (idx->base.centroid_format == PRISM_CENTROID_FMT_RABITQ || |
| 780 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
|
4 | idx->base.centroid_format == PRISM_CENTROID_FMT_FASTSCAN); |
| 781 | 16 | prism_write_centroid_tree( | |
| 782 | &idx->centroid_storage.base, | ||
| 783 | tree, | ||
| 784 | dim, | ||
| 785 | fan_out, | ||
| 786 | 0, /* in-RAM tree levels are absolute */ | ||
| 787 | idx->base.centroid_format, | ||
| 788 | needs_rq_params ? idx->base.params : NULL, | ||
| 789 |
2/2✓ Branch 0 taken 12 times.
✓ Branch 1 taken 4 times.
|
16 | idx->global_mean, |
| 790 | 0, /* posting_base: leaf child = global leaf (cluster) index */ | ||
| 791 | node_first_blkno, | ||
| 792 | NULL, /* pt_centroids on posting pages, not here */ | ||
| 793 | NULL); /* in-RAM assignment descends the exact float tree */ | ||
| 794 | 16 | vs_memctx_switch(idx_ctx); | |
| 795 | } | ||
| 796 | |||
| 797 | 16 | double ms_centroid = (double)(now_ns() - t_phase) / 1e6; | |
| 798 | |||
| 799 | 16 | vs_free(tree); | |
| 800 | |||
| 801 | 16 | vs_memctx_switch(old_ctx); | |
| 802 | 16 | vs_memctx_delete(build_ctx); | |
| 803 | |||
| 804 | /* Fill build stats */ | ||
| 805 |
2/2✓ Branch 0 taken 2 times.
✓ Branch 1 taken 14 times.
|
16 | if (stats != NULL) |
| 806 | { | ||
| 807 | 2 | stats->ms_sample = ms_sample; | |
| 808 | 2 | stats->ms_kmeans = ms_kmeans; | |
| 809 | 2 | stats->ms_setup = ms_setup; | |
| 810 | 2 | stats->ms_posting = ms_posting; | |
| 811 | 2 | stats->ms_parallel = ms_parallel; | |
| 812 | 2 | stats->ms_merge = ms_merge; | |
| 813 | 2 | stats->ms_centroid = ms_centroid; | |
| 814 | 2 | stats->ms_total = ms_sample + ms_kmeans + ms_setup + ms_posting + | |
| 815 | ms_centroid; | ||
| 816 | 2 | stats->nworkers = stat_nworkers; | |
| 817 | 2 | stats->total_pages = stat_pages; | |
| 818 | 2 | stats->merge_input = stat_merge_in; | |
| 819 | 2 | stats->merge_output = stat_merge_out; | |
| 820 | } | ||
| 821 | |||
| 822 | /* Wire storage pointers in base (concrete storage owned above) */ | ||
| 823 | 16 | idx->base.centroid_storage = &idx->centroid_storage.base; | |
| 824 | 16 | idx->base.posting_storage = &idx->posting_storage.base; | |
| 825 | 16 | idx->base.page_base = idx->posting_storage.pages; | |
| 826 | |||
| 827 | /* Wire rerank data on posting storage */ | ||
| 828 | 16 | idx->posting_storage.all_vectors = idx->all_vectors; | |
| 829 | 16 | idx->posting_storage.nvecs = idx->nvecs; | |
| 830 | 16 | idx->posting_storage.metric = idx->base.metric; | |
| 831 | |||
| 832 | /* Pre-allocate rerank buffers */ | ||
| 833 | 16 | uint32_t rerank_cap = 256; | |
| 834 | 16 | vs_topk_init(&idx->posting_storage.rerank_topk, rerank_cap); | |
| 835 | 16 | idx->posting_storage.rerank_entries = vs_alloc( | |
| 836 | rerank_cap * sizeof(VsTopKEntry)); | ||
| 837 | 16 | idx->posting_storage.rerank_cap = rerank_cap; | |
| 838 | |||
| 839 | 16 | return idx; | |
| 840 | } | ||
| 841 | |||
| 842 | void | ||
| 843 | 102 | prism_index_destroy(PrismIndex *idx) | |
| 844 | { | ||
| 845 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 102 times.
|
102 | if (idx == NULL) |
| 846 | ✗ | return; | |
| 847 | |||
| 848 | 102 | vs_topk_cleanup(&idx->posting_storage.rerank_topk); | |
| 849 | |||
| 850 | /* Deleting the memory context frees idx and all owned data. */ | ||
| 851 | 102 | vs_memctx_delete(idx->memctx); | |
| 852 | } | ||
| 853 |