GCC Code Coverage Report


Directory: src/
File: src/pg/bufstorage.c
Date: 2026-11-18 16:12:57
Exec Total Coverage
Lines: 201 248 81.0%
Functions: 16 17 94.1%
Branches: 65 102 63.7%

Line Branch Exec Source
1 /*
2 * Copyright (c) 2026 Tiger Data, Inc.
3 * Licensed under the PostgreSQL License. See LICENSE for details.
4 *
5 * bufstorage.c - PG buffer cache VsStorage implementation
6 *
7 * Read path: ReadBuffer + LockBuffer(SHARE) -> return page
8 * UnlockReleaseBuffer on release
9 *
10 * Write path: ReadBufferExtended(P_NEW) + exclusive lock
11 * GenericXLogStart/RegisterBuffer/Finish for WAL
12 * Commit releases buffer
13 */
14
15 #include <postgres.h>
16
17 #include <access/generic_xlog.h>
18 #include <access/heapam.h>
19 #include <access/htup_details.h>
20 #include <access/tableam.h>
21 #include <catalog/pg_am_d.h>
22 #include <executor/tuptable.h>
23 #include <pgstat.h>
24 #include <storage/buf_internals.h>
25 #include <storage/bufmgr.h>
26 #include <storage/read_stream.h>
27 #include <utils/memutils.h>
28 #include <utils/snapmgr.h>
29
30 #include "algo/distance.h"
31 #include "algo/topk.h"
32 #include "algo/vecops.h"
33 #include "amcache.h"
34 #include "index/posting_page.h"
35 #include "pg/bufstorage.h"
36 #include "support_pg.h"
37 #include "typeinfo.h"
38
39 /* Downcast from base to concrete type */
40 #define PG_STORAGE(self) ((VsPgStorage *)(self))
41
42 /* ----------------------------------------------------------------
43 * Backend-local buffer-id cache (prism.recent_buffers)
44 *
45 * ~22% of warm query CPU is BufTableLookup hash probes inside
46 * ReadBuffer, for index pages that essentially never leave
47 * shared_buffers. Remember the buffer id per block (per backend) and
48 * re-pin it via ReadRecentBuffer, which validates the tag and pins
49 * without touching the buffer mapping table. A stale id (page evicted
50 * or buffer reused) just fails validation and falls back to
51 * ReadBuffer, which refreshes the cached id — correctness never
52 * depends on the cache.
53 *
54 * The cache stores 4-byte buffer ids, not page data, so it does not
55 * duplicate shared_buffers (~11 MB per backend for a 21 GB index).
56 * One relation is cached at a time per backend; switching indexes
57 * swaps the cache.
58 * ---------------------------------------------------------------- */
59 static bool g_recent_buffers = true;
60 static RelFileLocator g_bufcache_locator; /* zeroed = invalid */
61 static Buffer *g_bufcache = NULL;
62 static BlockNumber g_bufcache_len = 0;
63 static MemoryContext g_bufcache_ctx = NULL;
64
65 void
66 293 vs_pg_storage_set_recent_buffers(bool enabled)
67 {
68 293 g_recent_buffers = enabled;
69 293 }
70
71 static inline Buffer *
72 15786956 bufcache_slot(Relation index, BlockNumber blkno)
73 {
74 15786956 const RelFileLocator *loc = &index->rd_locator;
75
76
6/8
✓ Branch 0 taken 15786757 times.
✓ Branch 1 taken 199 times.
✓ Branch 2 taken 15786472 times.
✓ Branch 3 taken 285 times.
✓ Branch 4 taken 15786472 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 15786472 times.
15786956 if (unlikely(
77 g_bufcache == NULL ||
78 !RelFileLocatorEquals(g_bufcache_locator, *loc)))
79 {
80 484 BlockNumber nblocks = RelationGetNumberOfBlocks(index);
81
82 /* Headroom so post-build inserts don't invalidate the cache. */
83 484 nblocks += nblocks / 8 + 1024;
84
85 /* Backend-lifetime cache data belongs under CacheMemoryContext
86 * (as a named child, so it is attributed in
87 * pg_backend_memory_contexts rather than hiding in the top
88 * context). */
89
2/2
✓ Branch 0 taken 199 times.
✓ Branch 1 taken 285 times.
484 if (g_bufcache_ctx == NULL)
90 199 g_bufcache_ctx = AllocSetContextCreate(
91 CacheMemoryContext,
92 "prism recent-buffers cache",
93 ALLOCSET_START_SMALL_SIZES);
94
95
2/2
✓ Branch 0 taken 285 times.
✓ Branch 1 taken 199 times.
484 if (g_bufcache != NULL)
96 285 pfree(g_bufcache);
97 968 g_bufcache = MemoryContextAlloc(
98 484 g_bufcache_ctx, (Size)nblocks * sizeof(Buffer));
99
2/2
✓ Branch 0 taken 519825 times.
✓ Branch 1 taken 484 times.
520309 for (BlockNumber i = 0; i < nblocks; i++)
100 519825 g_bufcache[i] = InvalidBuffer;
101 484 g_bufcache_len = nblocks;
102 484 g_bufcache_locator = *loc;
103
104 /*
105 * Eagerly seed the cache from the buffer descriptors instead of
106 * populating one miss at a time: measured 38% of reads in a
107 * fresh backend (14% even warmed) fall through to the shared
108 * buffer-mapping hash purely because a slot was never
109 * populated. The tags are read WITHOUT the header lock -- a
110 * torn or stale id is harmless because ReadRecentBuffer
111 * re-validates the tag under its own pin, exactly as it does
112 * for ids that went stale after a normal miss fill.
113 */
114
2/2
✓ Branch 0 taken 6856960 times.
✓ Branch 1 taken 484 times.
6857444 for (int b = 0; b < NBuffers; b++)
115 {
116
2/2
✓ Branch 0 taken 902179 times.
✓ Branch 1 taken 5954781 times.
6856960 BufferDesc *hdr = GetBufferDescriptor(b);
117
118
2/2
✓ Branch 0 taken 902179 times.
✓ Branch 1 taken 5954781 times.
7759139 if (BufTagMatchesRelFileLocator(&hdr->tag, loc) &&
119
1/2
✓ Branch 0 taken 16245 times.
✗ Branch 1 not taken.
16245 BufTagGetForkNum(&hdr->tag) == MAIN_FORKNUM &&
120
1/2
✓ Branch 0 taken 16245 times.
✗ Branch 1 not taken.
16245 hdr->tag.blockNum < g_bufcache_len)
121 16245 g_bufcache[hdr->tag.blockNum] = BufferDescriptorGetBuffer(hdr);
122 }
123 }
124
125
1/2
✓ Branch 0 taken 15786956 times.
✗ Branch 1 not taken.
15786956 if (unlikely(blkno >= g_bufcache_len))
126 return NULL;
127 15786956 return &g_bufcache[blkno];
128 }
129
130 /* ----------------------------------------------------------------
131 * Read path
132 * ---------------------------------------------------------------- */
133
134 /* Buffer-id cache effectiveness counters (diagnostic; exposed via
135 * vs_routing_stats). */
136 uint64_t vs_bufcache_hits;
137 uint64_t vs_bufcache_cold;
138 uint64_t vs_bufcache_stale;
139
140 static Page
141 15786956 pg_read_page(VsStorage *self, BlockNumber blkno)
142 {
143 15786956 VsPgStorage *s = PG_STORAGE(self);
144 15786956 Buffer buf;
145
146 /* One pinned buffer at a time: a read before the previous release
147 * would overwrite cur_buf and leak its pin. */
148
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 15786956 times.
15786956 Assert(s->cur_buf == InvalidBuffer);
149
150
1/2
✓ Branch 0 taken 15786956 times.
✗ Branch 1 not taken.
15786956 if (g_recent_buffers)
151 {
152 15786956 Buffer *slot = bufcache_slot(s->index, blkno);
153
154
5/6
✓ Branch 0 taken 15786956 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 15781164 times.
✓ Branch 3 taken 5792 times.
✓ Branch 4 taken 15538829 times.
✓ Branch 5 taken 242335 times.
31568120 if (slot != NULL && *slot != InvalidBuffer &&
155 15781164 ReadRecentBuffer(s->index->rd_locator, MAIN_FORKNUM, blkno, *slot))
156 {
157 15538829 buf = *slot;
158 15538829 vs_bufcache_hits++;
159 /*
160 * ReadRecentBuffer counts the hit in the backend-wide
161 * pgBufferUsage (EXPLAIN BUFFERS) but, taking a locator rather
162 * than a Relation, cannot attribute it to the index the way
163 * ReadBuffer does. Do that here, or pg_statio_*_indexes shows a
164 * warm prism index with almost no block hits.
165 */
166
4/4
✓ Branch 0 taken 3369610 times.
✓ Branch 1 taken 12169219 times.
✓ Branch 2 taken 72 times.
✓ Branch 3 taken 3369538 times.
15538829 pgstat_count_buffer_hit(s->index);
167 }
168 else
169 {
170 /* Diagnostic: distinguish never-populated slots (first
171 * touch) from stale ids (eviction churn). */
172
2/2
✓ Branch 0 taken 5792 times.
✓ Branch 1 taken 242335 times.
248127 if (slot == NULL || *slot == InvalidBuffer)
173 5792 vs_bufcache_cold++;
174 else
175 242335 vs_bufcache_stale++;
176 248127 buf = ReadBuffer(s->index, blkno);
177
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 248127 times.
248127 if (slot != NULL)
178 248127 *slot = buf;
179 }
180 }
181 else
182 ✗ buf = ReadBuffer(s->index, blkno);
183
184 15786956 LockBuffer(buf, BUFFER_LOCK_SHARE);
185 15786956 s->cur_buf = buf;
186 15786956 s->read_count++;
187
188 15786956 return BufferGetPage(buf);
189 }
190
191 static void
192 15814659 pg_release_page(VsStorage *self, BlockNumber blkno)
193 {
194 15814659 VsPgStorage *s = PG_STORAGE(self);
195
196 15814659 (void)blkno;
197 15814659 UnlockReleaseBuffer(s->cur_buf);
198 15814659 s->cur_buf = InvalidBuffer;
199 15814659 }
200
201 /*
202 * Issue an async prefetch (posix_fadvise WILLNEED) for one index page so a
203 * batch of upcoming random reads can overlap instead of stalling serially on
204 * a cold cache. No-op for pages already in shared buffers. Gated on
205 * effective_io_concurrency so an operator can disable prefetching (0).
206 */
207 static void
208 11248 pg_prefetch_page(VsStorage *self, BlockNumber blkno)
209 {
210 11248 VsPgStorage *s = PG_STORAGE(self);
211
212
1/2
✓ Branch 0 taken 11248 times.
✗ Branch 1 not taken.
11248 if (effective_io_concurrency == 0)
213 return;
214
215 11248 PrefetchBuffer(s->index, MAIN_FORKNUM, blkno);
216 }
217
218 /* ----------------------------------------------------------------
219 * Write path
220 * ---------------------------------------------------------------- */
221
222 static Page
223 78271 pg_write_page(VsStorage *self, BlockNumber blkno)
224 {
225 78271 VsPgStorage *s = PG_STORAGE(self);
226
227 78271 Buffer buf = ReadBuffer(s->index, blkno);
228 78271 LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
229 78271 s->cur_buf = buf;
230
231 78271 return BufferGetPage(buf);
232 }
233
234 static Page
235 2324 pg_new_page(VsStorage *self, BlockNumber *blkno_out)
236 {
237 2324 VsPgStorage *s = PG_STORAGE(self);
238
239 2324 Buffer buf = ReadBufferExtended(
240 s->index, MAIN_FORKNUM, P_NEW, RBM_NORMAL, NULL);
241 2324 LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
242
243 2324 *blkno_out = BufferGetBlockNumber(buf);
244 2324 s->cur_buf = buf;
245
246 2324 return BufferGetPage(buf);
247 }
248
249 static void
250 80595 pg_commit_page(VsStorage *self, BlockNumber blkno)
251 {
252 80595 VsPgStorage *s = PG_STORAGE(self);
253
254 80595 (void)blkno;
255
256
2/2
✓ Branch 0 taken 38153 times.
✓ Branch 1 taken 42442 times.
80595 if (s->build_mode)
257 {
258 /* During index build, skip per-page WAL logging.
259 * log_newpage_range() at the end covers all pages. */
260 38153 MarkBufferDirty(s->cur_buf);
261 }
262 else
263 {
264 /*
265 * prism pages store their data in the content area between pd_lower
266 * and pd_upper — the region PostgreSQL treats as the free "hole" and
267 * omits from standard-layout full-page images (GenericXLog assumes the
268 * standard layout). Cover the hole (pd_lower = pd_upper) before
269 * logging so the entire page is preserved; prism never reads
270 * pd_lower back (scans locate data via PageGetContents /
271 * pd_special, and centroid pages derive their metadata cursor from
272 * entry_count -- see prism_centroid_meta_end, which exists because of
273 * this overwrite).
274 */
275 42442 PageHeader ph = (PageHeader)BufferGetPage(s->cur_buf);
276 42442 ph->pd_lower = ph->pd_upper;
277
278 42442 GenericXLogState *state = GenericXLogStart(s->index);
279 42442 GenericXLogRegisterBuffer(state, s->cur_buf, GENERIC_XLOG_FULL_IMAGE);
280 42442 GenericXLogFinish(state);
281 }
282
283 80595 UnlockReleaseBuffer(s->cur_buf);
284 80595 s->cur_buf = InvalidBuffer;
285 80595 }
286
287 /* ----------------------------------------------------------------
288 * Bulk extend: pre-allocate contiguous pages
289 * ---------------------------------------------------------------- */
290
291 static BlockNumber
292 249 pg_extend(VsStorage *self, uint32_t npages)
293 {
294 249 VsPgStorage *s = PG_STORAGE(self);
295
296 249 BlockNumber start = InvalidBlockNumber;
297 249 uint32_t remaining = npages;
298
299
2/2
✓ Branch 0 taken 1127 times.
✓ Branch 1 taken 249 times.
1376 while (remaining > 0)
300 {
301 1127 uint32_t batch = Min(remaining, 512);
302 1127 Buffer *buffers = palloc(batch * sizeof(Buffer));
303 1127 uint32_t got = 0;
304
305 2254 BlockNumber batch_start = ExtendBufferedRelBy(
306 1127 BMR_REL(s->index),
307 MAIN_FORKNUM,
308 NULL,
309 0,
310 batch,
311 buffers,
312 &got);
313
314
2/2
✓ Branch 0 taken 249 times.
✓ Branch 1 taken 878 times.
1127 if (start == InvalidBlockNumber)
315 249 start = batch_start;
316
317
2/2
✓ Branch 0 taken 11903 times.
✓ Branch 1 taken 1127 times.
13030 for (uint32_t i = 0; i < got; i++)
318 11903 ReleaseBuffer(buffers[i]);
319
320 1127 pfree(buffers);
321 1127 remaining -= got;
322 }
323
324 249 return start;
325 }
326
327 /* ----------------------------------------------------------------
328 * Rerank: fetch vectors from heap, compute exact L2
329 * ---------------------------------------------------------------- */
330
331 /* Comparator for sorting candidate indices by TID block order */
332 static int
333 76119 cmp_tid_order(const void *a, const void *b, void *arg)
334 {
335 76119 const VsTopKEntry *cands = arg;
336
337 /* Encoded ids are (block << 16) | offset — strictly monotonic in
338 * (block, offset), so raw id comparison IS TID order; no need to
339 * decode both TIDs on every comparison. */
340 76119 uint64_t id_a = cands[*(const uint32_t *)a].id;
341 76119 uint64_t id_b = cands[*(const uint32_t *)b].id;
342
343
2/2
✓ Branch 0 taken 38582 times.
✓ Branch 1 taken 37537 times.
76119 if (id_a < id_b)
344 return -1;
345
1/2
✓ Branch 0 taken 38582 times.
✗ Branch 1 not taken.
38582 if (id_a > id_b)
346 38582 return 1;
347 return 0;
348 }
349
350 /*
351 * pg_rerank - Rerank candidates with exact L2 distances.
352 *
353 * Fetches full-precision vectors from the heap table and computes
354 * exact L2 distance for candidates with nonzero error. Candidates
355 * are visited in TID block-number order to minimize buffer cache
356 * thrash — sequential block access avoids re-pinning the same
357 * buffer multiple times.
358 *
359 * Rows are fetched under the scan's snapshot, following HOT chains the way
360 * the executor's own heap fetch does. A candidate whose row is not visible is
361 * left out: the executor would discard it anyway, so keeping it would only
362 * take a top-k slot from a visible row. Its vector is never read either,
363 * which matters because an aborted insert or a vacuumed delete may already
364 * have made its TOAST chunks unreadable.
365 */
366 static uint32_t
367 ✗ pg_rerank(
368 VsStorage *self,
369 const float *query,
370 Dimension dim,
371 const VsTopKEntry *candidates,
372 uint32_t count,
373 uint32_t keep,
374 uint32_t *out_indices,
375 Distance *out_distances)
376 {
377 ✗ VsPgStorage *s = PG_STORAGE(self);
378
379 ✗ if (s->rel == NULL || count == 0)
380 return 0;
381
382 ✗ AttrNumber vec_attnum = s->index->rd_index->indkey.values[0];
383
384 /*
385 * Widening buffer for a vec16 heap column, allocated once per rerank
386 * call -- rerank runs once per query over the whole candidate set, so this
387 * is not a per-tuple allocation.
388 */
389 ✗ Vec32Access input = vec32_access(s->type_info, dim, CurrentMemoryContext);
390
391 /* Sort by TID block order for sequential I/O */
392 ✗ uint32_t *order = palloc(count * sizeof(uint32_t));
393 ✗ for (uint32_t i = 0; i < count; i++)
394 ✗ order[i] = i;
395 ✗ qsort_arg(
396 order, count, sizeof(uint32_t), cmp_tid_order, (void *)candidates);
397
398 ✗ VsTopK topk;
399 ✗ vs_topk_init(&topk, keep);
400
401 ✗ Assert(s->snapshot != NULL);
402 ✗ TupleTableSlot *slot = table_slot_create(s->rel, NULL);
403 #if PG_VERSION_NUM >= 190000
404 IndexFetchTableData *fetch = table_index_fetch_begin(s->rel, SO_NONE);
405 #else
406 ✗ IndexFetchTableData *fetch = table_index_fetch_begin(s->rel);
407 #endif
408
409 ✗ for (uint32_t i = 0; i < count; i++)
410 {
411 ✗ uint32_t idx = order[i];
412
413 ✗ Distance d;
414 ✗ if (candidates[idx].error == 0.0f)
415 {
416 ✗ d = candidates[idx].distance;
417 }
418 else
419 {
420 ✗ ItemPointerData tid = prism_posting_decode_tid(candidates[idx].id);
421 ✗ bool call_again = false;
422 ✗ bool all_dead = false;
423 ✗ if (!table_index_fetch_tuple(
424 fetch,
425 &tid,
426 s->snapshot,
427 slot,
428 &call_again,
429 &all_dead))
430 ✗ continue;
431
432 ✗ bool isnull;
433 ✗ Datum val = slot_getattr(slot, vec_attnum, &isnull);
434 ✗ if (!isnull)
435 {
436 ✗ Vec32Ref qref = {.data = query, .dim = dim};
437 ✗ Vec32Ref vref = vec32_read(&input, val);
438 ✗ d = vs_distance(qref, vref, s->metric);
439 }
440 else
441 {
442 ✗ d = candidates[idx].distance;
443 }
444 ✗ ExecClearTuple(slot);
445 }
446
447 ✗ vs_topk_insert_unique(&topk, d, 0.0f, (uint64_t)idx);
448 }
449
450 ✗ table_index_fetch_end(fetch);
451 ✗ ExecDropSingleTupleTableSlot(slot);
452
453 ✗ VsTopKEntry *entries = palloc(topk.cand_count * sizeof(VsTopKEntry));
454 ✗ uint32_t nresults;
455 ✗ vs_topk_extract_sorted_unique(&topk, entries, &nresults);
456
457 ✗ for (uint32_t i = 0; i < nresults; i++)
458 {
459 ✗ out_indices[i] = (uint32_t)entries[i].id;
460 ✗ out_distances[i] = entries[i].distance;
461 }
462
463 ✗ pfree(entries);
464 ✗ vs_topk_cleanup(&topk);
465 ✗ pfree(order);
466
467 ✗ return nresults;
468 }
469
470 /* ----------------------------------------------------------------
471 * Heap AM rerank with read_stream
472 *
473 * Uses PostgreSQL's read_stream API for batched async I/O when
474 * fetching heap tuples for reranking. The stream callback yields
475 * block numbers in candidate order, and each buffer is processed
476 * directly as it arrives.
477 * ---------------------------------------------------------------- */
478
479 typedef struct RerankStreamState
480 {
481 const VsTopKEntry *candidates;
482 const uint32_t *order;
483 VsTopK *topk;
484 VsPgStorage *storage;
485 uint32_t count;
486 uint32_t pos;
487 } RerankStreamState;
488
489 static BlockNumber
490 15571 rerank_stream_cb(
491 ReadStream *stream, void *callback_private_data, void *per_buffer_data)
492 {
493 15571 RerankStreamState *st = callback_private_data;
494
495
2/2
✓ Branch 0 taken 15425 times.
✓ Branch 1 taken 362 times.
15787 while (st->pos < st->count)
496 {
497 15425 uint32_t idx = st->order[st->pos++];
498
499 /* Already exact — insert directly, no heap fetch */
500
2/2
✓ Branch 0 taken 216 times.
✓ Branch 1 taken 15209 times.
15425 if (st->candidates[idx].error == 0.0f)
501 {
502 216 vs_topk_insert_unique(
503 st->topk,
504 216 st->candidates[idx].distance,
505 0.0f,
506 (uint64_t)idx);
507 216 continue;
508 }
509
510 15209 *(uint32_t *)per_buffer_data = idx;
511 15209 ItemPointerData tid = prism_posting_decode_tid(st->candidates[idx].id);
512 15209 return ItemPointerGetBlockNumber(&tid);
513 }
514
515 return InvalidBlockNumber;
516 }
517
518 static uint32_t
519 362 pg_rerank_readstream(
520 VsStorage *self,
521 const float *query,
522 Dimension dim,
523 const VsTopKEntry *candidates,
524 uint32_t count,
525 uint32_t keep,
526 uint32_t *out_indices,
527 Distance *out_distances)
528 {
529 362 VsPgStorage *s = PG_STORAGE(self);
530
531
2/4
✓ Branch 0 taken 362 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 362 times.
✗ Branch 3 not taken.
362 if (s->rel == NULL || count == 0)
532 return 0;
533
534 362 AttrNumber vec_attnum = s->index->rd_index->indkey.values[0];
535
536 /* See pg_rerank: one widening buffer per call, not per candidate. */
537 362 Vec32Access input = vec32_access(s->type_info, dim, CurrentMemoryContext);
538
539 362 uint32_t *order = palloc(count * sizeof(uint32_t));
540
2/2
✓ Branch 1 taken 15425 times.
✓ Branch 2 taken 362 times.
15787 for (uint32_t i = 0; i < count; i++)
541 15425 order[i] = i;
542 362 qsort_arg(
543 order, count, sizeof(uint32_t), cmp_tid_order, (void *)candidates);
544
545 362 VsTopK topk;
546 362 vs_topk_init(&topk, keep);
547
548 362 RerankStreamState state = {
549 .candidates = candidates,
550 .order = order,
551 .topk = &topk,
552 .storage = s,
553 .count = count,
554 .pos = 0,
555 };
556
557 724 TupleTableSlot *slot = MakeSingleTupleTableSlot(
558 362 RelationGetDescr(s->rel), &TTSOpsBufferHeapTuple);
559
560 362 ReadStream *stream = read_stream_begin_relation(
561 READ_STREAM_DEFAULT,
562 NULL,
563 s->rel,
564 MAIN_FORKNUM,
565 rerank_stream_cb,
566 &state,
567 sizeof(uint32_t));
568
569 362 void *per_buffer_data;
570 362 Buffer buf;
571
572
2/2
✓ Branch 1 taken 15209 times.
✓ Branch 2 taken 362 times.
15933 while (BufferIsValid(
573 15571 buf = read_stream_next_buffer(stream, &per_buffer_data)))
574 {
575 15209 uint32_t idx = *(uint32_t *)per_buffer_data;
576 15209 ItemPointerData tid = prism_posting_decode_tid(candidates[idx].id);
577
578 /* The executor's heap fetch, on the buffer the stream pinned: follow
579 * the HOT chain under the scan's snapshot, holding the page lock for
580 * the walk only. An invisible row is left out (see pg_rerank). */
581 15209 HeapTupleData tuple;
582 15209 bool all_dead;
583 15209 LockBuffer(buf, BUFFER_LOCK_SHARE);
584 15209 bool found = heap_hot_search_buffer(
585 &tid, s->rel, buf, s->snapshot, &tuple, &all_dead, true);
586 15209 LockBuffer(buf, BUFFER_LOCK_UNLOCK);
587
588
2/2
✓ Branch 0 taken 72 times.
✓ Branch 1 taken 15137 times.
15209 if (!found)
589 {
590 72 ReleaseBuffer(buf);
591 72 continue;
592 }
593
594 15137 Distance d = candidates[idx].distance;
595 15137 ExecStoreBufferHeapTuple(&tuple, slot, buf);
596
597 15137 bool isnull;
598 15137 Datum val = slot_getattr(slot, vec_attnum, &isnull);
599
1/2
✓ Branch 0 taken 15137 times.
✗ Branch 1 not taken.
15137 if (!isnull)
600 {
601 15137 Vec32Ref qref = {.data = query, .dim = dim};
602 15137 Vec32Ref vref = vec32_read(&input, val);
603 15137 d = vs_distance(qref, vref, s->metric);
604 }
605 15137 ExecClearTuple(slot);
606
607 15137 vs_topk_insert_unique(&topk, d, 0.0f, (uint64_t)idx);
608 15137 ReleaseBuffer(buf);
609 }
610
611 362 read_stream_end(stream);
612 362 ExecDropSingleTupleTableSlot(slot);
613
614 362 VsTopKEntry *entries = palloc(topk.cand_count * sizeof(VsTopKEntry));
615 362 uint32_t nresults;
616 362 vs_topk_extract_sorted_unique(&topk, entries, &nresults);
617
618
2/2
✓ Branch 1 taken 11027 times.
✓ Branch 2 taken 362 times.
11389 for (uint32_t i = 0; i < nresults; i++)
619 {
620 11027 out_indices[i] = (uint32_t)entries[i].id;
621 11027 out_distances[i] = entries[i].distance;
622 }
623
624 362 pfree(entries);
625 362 vs_topk_cleanup(&topk);
626 362 pfree(order);
627
628 362 return nresults;
629 }
630
631 /* ----------------------------------------------------------------
632 * Static vtables
633 * ---------------------------------------------------------------- */
634
635 static const VsStorageOps pg_storage_ops = {
636 .read_page = pg_read_page,
637 .release_page = pg_release_page,
638 .prefetch = pg_prefetch_page,
639 .write_page = pg_write_page,
640 .new_page = pg_new_page,
641 .commit_page = pg_commit_page,
642 .extend = pg_extend,
643 .rerank = pg_rerank,
644 };
645
646 /*
647 * Read for a caller that sweeps the index rather than following a query's
648 * access pattern -- inspection. Identical to pg_read_page minus the
649 * recent-buffer cache, which for a sweep is worse than useless: its slots
650 * would fill with blocks no query asks for again, evicting the ones the
651 * query path reuses, and its hit/cold/stale counters would report the sweep
652 * instead of the workload they exist to measure.
653 *
654 * A separate entry in the ops table rather than a flag inside pg_read_page:
655 * the query path reads a page for every posting page it scans, and should
656 * not test a condition that only inspection can change.
657 */
658 static Page
659 27703 pg_read_page_nocache(VsStorage *self, BlockNumber blkno)
660 {
661 27703 VsPgStorage *s = PG_STORAGE(self);
662 27703 Buffer buf = ReadBuffer(s->index, blkno);
663
664 27703 LockBuffer(buf, BUFFER_LOCK_SHARE);
665 27703 s->cur_buf = buf;
666 27703 s->read_count++;
667
668 27703 return BufferGetPage(buf);
669 }
670
671 static const VsStorageOps pg_storage_readstream_ops = {
672 .read_page = pg_read_page,
673 .release_page = pg_release_page,
674 .prefetch = pg_prefetch_page,
675 .write_page = pg_write_page,
676 .new_page = pg_new_page,
677 .commit_page = pg_commit_page,
678 .extend = pg_extend,
679 .rerank = pg_rerank_readstream,
680 };
681
682 /* Inspection: the standard ops with the cache-bypassing read. */
683 static const VsStorageOps pg_storage_inspect_ops = {
684 .read_page = pg_read_page_nocache,
685 .release_page = pg_release_page,
686 .prefetch = pg_prefetch_page,
687 .write_page = pg_write_page,
688 .new_page = pg_new_page,
689 .commit_page = pg_commit_page,
690 .extend = pg_extend,
691 .rerank = NULL,
692 };
693
694 void
695 728 vs_pg_storage_init_inspect(VsPgStorage *s, Relation index)
696 {
697 728 vs_pg_storage_init(s, index, NULL, DISTANCE_L2);
698 728 s->base.ops = &pg_storage_inspect_ops;
699 728 }
700
701 /* ----------------------------------------------------------------
702 * Initialization
703 * ---------------------------------------------------------------- */
704
705 void
706 21785 vs_pg_storage_init(
707 VsPgStorage *s, Relation index, Relation rel, DistanceMetric metric)
708 {
709
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 21785 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
21785 if (rel != NULL && RelationGetForm(rel)->relam == HEAP_TABLE_AM_OID)
710 ✗ s->base.ops = &pg_storage_readstream_ops;
711 else
712 21785 s->base.ops = &pg_storage_ops;
713 21785 s->index = index;
714 21785 s->rel = rel;
715 21785 s->snapshot = NULL;
716 21785 s->build_mode = false;
717 21785 s->cur_buf = InvalidBuffer;
718 21785 s->metric = metric;
719 21785 s->read_count = 0;
720 /*
721 * Left NULL: every caller inits with rel = NULL, and rerank -- the only
722 * consumer -- returns early without a heap relation. It is resolved in
723 * vs_pg_storage_set_rel, which is where the heap arrives and therefore
724 * where rerank becomes possible.
725 */
726 21785 s->type_info = NULL;
727 21785 }
728
729 void
730 349 vs_pg_storage_set_rel(VsPgStorage *s, Relation rel, Snapshot snapshot)
731 {
732 349 s->rel = rel;
733 349 s->snapshot = snapshot;
734
2/4
✓ Branch 0 taken 349 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 349 times.
✗ Branch 3 not taken.
349 if (rel != NULL && RelationGetForm(rel)->relam == HEAP_TABLE_AM_OID)
735 349 s->base.ops = &pg_storage_readstream_ops;
736
737 /*
738 * Rerank reads the heap attribute, which has the indexed column's type.
739 * From the per-backend cache: this runs on the scan path, so the index
740 * has a metadata page and the cache is populated.
741 */
742 349 if (rel != NULL)
743 349 s->type_info = prism_cache_type_info(s->index);
744 349 }
745