| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /* | ||
| 2 | * Copyright (c) 2026 Tiger Data, Inc. | ||
| 3 | * Licensed under the PostgreSQL License. See LICENSE for details. | ||
| 4 | * | ||
| 5 | * bufstorage.c - PG buffer cache VsStorage implementation | ||
| 6 | * | ||
| 7 | * Read path: ReadBuffer + LockBuffer(SHARE) -> return page | ||
| 8 | * UnlockReleaseBuffer on release | ||
| 9 | * | ||
| 10 | * Write path: ReadBufferExtended(P_NEW) + exclusive lock | ||
| 11 | * GenericXLogStart/RegisterBuffer/Finish for WAL | ||
| 12 | * Commit releases buffer | ||
| 13 | */ | ||
| 14 | |||
| 15 | #include <postgres.h> | ||
| 16 | |||
| 17 | #include <access/generic_xlog.h> | ||
| 18 | #include <access/heapam.h> | ||
| 19 | #include <access/htup_details.h> | ||
| 20 | #include <access/tableam.h> | ||
| 21 | #include <catalog/pg_am_d.h> | ||
| 22 | #include <executor/tuptable.h> | ||
| 23 | #include <pgstat.h> | ||
| 24 | #include <storage/buf_internals.h> | ||
| 25 | #include <storage/bufmgr.h> | ||
| 26 | #include <storage/read_stream.h> | ||
| 27 | #include <utils/memutils.h> | ||
| 28 | #include <utils/snapmgr.h> | ||
| 29 | |||
| 30 | #include "algo/distance.h" | ||
| 31 | #include "algo/topk.h" | ||
| 32 | #include "algo/vecops.h" | ||
| 33 | #include "amcache.h" | ||
| 34 | #include "index/posting_page.h" | ||
| 35 | #include "pg/bufstorage.h" | ||
| 36 | #include "support_pg.h" | ||
| 37 | #include "typeinfo.h" | ||
| 38 | |||
| 39 | /* Downcast from base to concrete type */ | ||
| 40 | #define PG_STORAGE(self) ((VsPgStorage *)(self)) | ||
| 41 | |||
| 42 | /* ---------------------------------------------------------------- | ||
| 43 | * Backend-local buffer-id cache (prism.recent_buffers) | ||
| 44 | * | ||
| 45 | * ~22% of warm query CPU is BufTableLookup hash probes inside | ||
| 46 | * ReadBuffer, for index pages that essentially never leave | ||
| 47 | * shared_buffers. Remember the buffer id per block (per backend) and | ||
| 48 | * re-pin it via ReadRecentBuffer, which validates the tag and pins | ||
| 49 | * without touching the buffer mapping table. A stale id (page evicted | ||
| 50 | * or buffer reused) just fails validation and falls back to | ||
| 51 | * ReadBuffer, which refreshes the cached id — correctness never | ||
| 52 | * depends on the cache. | ||
| 53 | * | ||
| 54 | * The cache stores 4-byte buffer ids, not page data, so it does not | ||
| 55 | * duplicate shared_buffers (~11 MB per backend for a 21 GB index). | ||
| 56 | * One relation is cached at a time per backend; switching indexes | ||
| 57 | * swaps the cache. | ||
| 58 | * ---------------------------------------------------------------- */ | ||
| 59 | static bool g_recent_buffers = true; | ||
| 60 | static RelFileLocator g_bufcache_locator; /* zeroed = invalid */ | ||
| 61 | static Buffer *g_bufcache = NULL; | ||
| 62 | static BlockNumber g_bufcache_len = 0; | ||
| 63 | static MemoryContext g_bufcache_ctx = NULL; | ||
| 64 | |||
| 65 | void | ||
| 66 | 293 | vs_pg_storage_set_recent_buffers(bool enabled) | |
| 67 | { | ||
| 68 | 293 | g_recent_buffers = enabled; | |
| 69 | 293 | } | |
| 70 | |||
| 71 | static inline Buffer * | ||
| 72 | 15786956 | bufcache_slot(Relation index, BlockNumber blkno) | |
| 73 | { | ||
| 74 | 15786956 | const RelFileLocator *loc = &index->rd_locator; | |
| 75 | |||
| 76 |
6/8✓ Branch 0 taken 15786757 times.
✓ Branch 1 taken 199 times.
✓ Branch 2 taken 15786472 times.
✓ Branch 3 taken 285 times.
✓ Branch 4 taken 15786472 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 15786472 times.
|
15786956 | if (unlikely( |
| 77 | g_bufcache == NULL || | ||
| 78 | !RelFileLocatorEquals(g_bufcache_locator, *loc))) | ||
| 79 | { | ||
| 80 | 484 | BlockNumber nblocks = RelationGetNumberOfBlocks(index); | |
| 81 | |||
| 82 | /* Headroom so post-build inserts don't invalidate the cache. */ | ||
| 83 | 484 | nblocks += nblocks / 8 + 1024; | |
| 84 | |||
| 85 | /* Backend-lifetime cache data belongs under CacheMemoryContext | ||
| 86 | * (as a named child, so it is attributed in | ||
| 87 | * pg_backend_memory_contexts rather than hiding in the top | ||
| 88 | * context). */ | ||
| 89 |
2/2✓ Branch 0 taken 199 times.
✓ Branch 1 taken 285 times.
|
484 | if (g_bufcache_ctx == NULL) |
| 90 | 199 | g_bufcache_ctx = AllocSetContextCreate( | |
| 91 | CacheMemoryContext, | ||
| 92 | "prism recent-buffers cache", | ||
| 93 | ALLOCSET_START_SMALL_SIZES); | ||
| 94 | |||
| 95 |
2/2✓ Branch 0 taken 285 times.
✓ Branch 1 taken 199 times.
|
484 | if (g_bufcache != NULL) |
| 96 | 285 | pfree(g_bufcache); | |
| 97 | 968 | g_bufcache = MemoryContextAlloc( | |
| 98 | 484 | g_bufcache_ctx, (Size)nblocks * sizeof(Buffer)); | |
| 99 |
2/2✓ Branch 0 taken 519825 times.
✓ Branch 1 taken 484 times.
|
520309 | for (BlockNumber i = 0; i < nblocks; i++) |
| 100 | 519825 | g_bufcache[i] = InvalidBuffer; | |
| 101 | 484 | g_bufcache_len = nblocks; | |
| 102 | 484 | g_bufcache_locator = *loc; | |
| 103 | |||
| 104 | /* | ||
| 105 | * Eagerly seed the cache from the buffer descriptors instead of | ||
| 106 | * populating one miss at a time: measured 38% of reads in a | ||
| 107 | * fresh backend (14% even warmed) fall through to the shared | ||
| 108 | * buffer-mapping hash purely because a slot was never | ||
| 109 | * populated. The tags are read WITHOUT the header lock -- a | ||
| 110 | * torn or stale id is harmless because ReadRecentBuffer | ||
| 111 | * re-validates the tag under its own pin, exactly as it does | ||
| 112 | * for ids that went stale after a normal miss fill. | ||
| 113 | */ | ||
| 114 |
2/2✓ Branch 0 taken 6856960 times.
✓ Branch 1 taken 484 times.
|
6857444 | for (int b = 0; b < NBuffers; b++) |
| 115 | { | ||
| 116 |
2/2✓ Branch 0 taken 902179 times.
✓ Branch 1 taken 5954781 times.
|
6856960 | BufferDesc *hdr = GetBufferDescriptor(b); |
| 117 | |||
| 118 |
2/2✓ Branch 0 taken 902179 times.
✓ Branch 1 taken 5954781 times.
|
7759139 | if (BufTagMatchesRelFileLocator(&hdr->tag, loc) && |
| 119 |
1/2✓ Branch 0 taken 16245 times.
✗ Branch 1 not taken.
|
16245 | BufTagGetForkNum(&hdr->tag) == MAIN_FORKNUM && |
| 120 |
1/2✓ Branch 0 taken 16245 times.
✗ Branch 1 not taken.
|
16245 | hdr->tag.blockNum < g_bufcache_len) |
| 121 | 16245 | g_bufcache[hdr->tag.blockNum] = BufferDescriptorGetBuffer(hdr); | |
| 122 | } | ||
| 123 | } | ||
| 124 | |||
| 125 |
1/2✓ Branch 0 taken 15786956 times.
✗ Branch 1 not taken.
|
15786956 | if (unlikely(blkno >= g_bufcache_len)) |
| 126 | return NULL; | ||
| 127 | 15786956 | return &g_bufcache[blkno]; | |
| 128 | } | ||
| 129 | |||
| 130 | /* ---------------------------------------------------------------- | ||
| 131 | * Read path | ||
| 132 | * ---------------------------------------------------------------- */ | ||
| 133 | |||
| 134 | /* Buffer-id cache effectiveness counters (diagnostic; exposed via | ||
| 135 | * vs_routing_stats). */ | ||
| 136 | uint64_t vs_bufcache_hits; | ||
| 137 | uint64_t vs_bufcache_cold; | ||
| 138 | uint64_t vs_bufcache_stale; | ||
| 139 | |||
| 140 | static Page | ||
| 141 | 15786956 | pg_read_page(VsStorage *self, BlockNumber blkno) | |
| 142 | { | ||
| 143 | 15786956 | VsPgStorage *s = PG_STORAGE(self); | |
| 144 | 15786956 | Buffer buf; | |
| 145 | |||
| 146 | /* One pinned buffer at a time: a read before the previous release | ||
| 147 | * would overwrite cur_buf and leak its pin. */ | ||
| 148 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 15786956 times.
|
15786956 | Assert(s->cur_buf == InvalidBuffer); |
| 149 | |||
| 150 |
1/2✓ Branch 0 taken 15786956 times.
✗ Branch 1 not taken.
|
15786956 | if (g_recent_buffers) |
| 151 | { | ||
| 152 | 15786956 | Buffer *slot = bufcache_slot(s->index, blkno); | |
| 153 | |||
| 154 |
5/6✓ Branch 0 taken 15786956 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 15781164 times.
✓ Branch 3 taken 5792 times.
✓ Branch 4 taken 15538829 times.
✓ Branch 5 taken 242335 times.
|
31568120 | if (slot != NULL && *slot != InvalidBuffer && |
| 155 | 15781164 | ReadRecentBuffer(s->index->rd_locator, MAIN_FORKNUM, blkno, *slot)) | |
| 156 | { | ||
| 157 | 15538829 | buf = *slot; | |
| 158 | 15538829 | vs_bufcache_hits++; | |
| 159 | /* | ||
| 160 | * ReadRecentBuffer counts the hit in the backend-wide | ||
| 161 | * pgBufferUsage (EXPLAIN BUFFERS) but, taking a locator rather | ||
| 162 | * than a Relation, cannot attribute it to the index the way | ||
| 163 | * ReadBuffer does. Do that here, or pg_statio_*_indexes shows a | ||
| 164 | * warm prism index with almost no block hits. | ||
| 165 | */ | ||
| 166 |
4/4✓ Branch 0 taken 3369610 times.
✓ Branch 1 taken 12169219 times.
✓ Branch 2 taken 72 times.
✓ Branch 3 taken 3369538 times.
|
15538829 | pgstat_count_buffer_hit(s->index); |
| 167 | } | ||
| 168 | else | ||
| 169 | { | ||
| 170 | /* Diagnostic: distinguish never-populated slots (first | ||
| 171 | * touch) from stale ids (eviction churn). */ | ||
| 172 |
2/2✓ Branch 0 taken 5792 times.
✓ Branch 1 taken 242335 times.
|
248127 | if (slot == NULL || *slot == InvalidBuffer) |
| 173 | 5792 | vs_bufcache_cold++; | |
| 174 | else | ||
| 175 | 242335 | vs_bufcache_stale++; | |
| 176 | 248127 | buf = ReadBuffer(s->index, blkno); | |
| 177 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 248127 times.
|
248127 | if (slot != NULL) |
| 178 | 248127 | *slot = buf; | |
| 179 | } | ||
| 180 | } | ||
| 181 | else | ||
| 182 | ✗ | buf = ReadBuffer(s->index, blkno); | |
| 183 | |||
| 184 | 15786956 | LockBuffer(buf, BUFFER_LOCK_SHARE); | |
| 185 | 15786956 | s->cur_buf = buf; | |
| 186 | 15786956 | s->read_count++; | |
| 187 | |||
| 188 | 15786956 | return BufferGetPage(buf); | |
| 189 | } | ||
| 190 | |||
| 191 | static void | ||
| 192 | 15814659 | pg_release_page(VsStorage *self, BlockNumber blkno) | |
| 193 | { | ||
| 194 | 15814659 | VsPgStorage *s = PG_STORAGE(self); | |
| 195 | |||
| 196 | 15814659 | (void)blkno; | |
| 197 | 15814659 | UnlockReleaseBuffer(s->cur_buf); | |
| 198 | 15814659 | s->cur_buf = InvalidBuffer; | |
| 199 | 15814659 | } | |
| 200 | |||
| 201 | /* | ||
| 202 | * Issue an async prefetch (posix_fadvise WILLNEED) for one index page so a | ||
| 203 | * batch of upcoming random reads can overlap instead of stalling serially on | ||
| 204 | * a cold cache. No-op for pages already in shared buffers. Gated on | ||
| 205 | * effective_io_concurrency so an operator can disable prefetching (0). | ||
| 206 | */ | ||
| 207 | static void | ||
| 208 | 11248 | pg_prefetch_page(VsStorage *self, BlockNumber blkno) | |
| 209 | { | ||
| 210 | 11248 | VsPgStorage *s = PG_STORAGE(self); | |
| 211 | |||
| 212 |
1/2✓ Branch 0 taken 11248 times.
✗ Branch 1 not taken.
|
11248 | if (effective_io_concurrency == 0) |
| 213 | return; | ||
| 214 | |||
| 215 | 11248 | PrefetchBuffer(s->index, MAIN_FORKNUM, blkno); | |
| 216 | } | ||
| 217 | |||
| 218 | /* ---------------------------------------------------------------- | ||
| 219 | * Write path | ||
| 220 | * ---------------------------------------------------------------- */ | ||
| 221 | |||
| 222 | static Page | ||
| 223 | 78271 | pg_write_page(VsStorage *self, BlockNumber blkno) | |
| 224 | { | ||
| 225 | 78271 | VsPgStorage *s = PG_STORAGE(self); | |
| 226 | |||
| 227 | 78271 | Buffer buf = ReadBuffer(s->index, blkno); | |
| 228 | 78271 | LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); | |
| 229 | 78271 | s->cur_buf = buf; | |
| 230 | |||
| 231 | 78271 | return BufferGetPage(buf); | |
| 232 | } | ||
| 233 | |||
| 234 | static Page | ||
| 235 | 2324 | pg_new_page(VsStorage *self, BlockNumber *blkno_out) | |
| 236 | { | ||
| 237 | 2324 | VsPgStorage *s = PG_STORAGE(self); | |
| 238 | |||
| 239 | 2324 | Buffer buf = ReadBufferExtended( | |
| 240 | s->index, MAIN_FORKNUM, P_NEW, RBM_NORMAL, NULL); | ||
| 241 | 2324 | LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); | |
| 242 | |||
| 243 | 2324 | *blkno_out = BufferGetBlockNumber(buf); | |
| 244 | 2324 | s->cur_buf = buf; | |
| 245 | |||
| 246 | 2324 | return BufferGetPage(buf); | |
| 247 | } | ||
| 248 | |||
| 249 | static void | ||
| 250 | 80595 | pg_commit_page(VsStorage *self, BlockNumber blkno) | |
| 251 | { | ||
| 252 | 80595 | VsPgStorage *s = PG_STORAGE(self); | |
| 253 | |||
| 254 | 80595 | (void)blkno; | |
| 255 | |||
| 256 |
2/2✓ Branch 0 taken 38153 times.
✓ Branch 1 taken 42442 times.
|
80595 | if (s->build_mode) |
| 257 | { | ||
| 258 | /* During index build, skip per-page WAL logging. | ||
| 259 | * log_newpage_range() at the end covers all pages. */ | ||
| 260 | 38153 | MarkBufferDirty(s->cur_buf); | |
| 261 | } | ||
| 262 | else | ||
| 263 | { | ||
| 264 | /* | ||
| 265 | * prism pages store their data in the content area between pd_lower | ||
| 266 | * and pd_upper — the region PostgreSQL treats as the free "hole" and | ||
| 267 | * omits from standard-layout full-page images (GenericXLog assumes the | ||
| 268 | * standard layout). Cover the hole (pd_lower = pd_upper) before | ||
| 269 | * logging so the entire page is preserved; prism never reads | ||
| 270 | * pd_lower back (scans locate data via PageGetContents / | ||
| 271 | * pd_special, and centroid pages derive their metadata cursor from | ||
| 272 | * entry_count -- see prism_centroid_meta_end, which exists because of | ||
| 273 | * this overwrite). | ||
| 274 | */ | ||
| 275 | 42442 | PageHeader ph = (PageHeader)BufferGetPage(s->cur_buf); | |
| 276 | 42442 | ph->pd_lower = ph->pd_upper; | |
| 277 | |||
| 278 | 42442 | GenericXLogState *state = GenericXLogStart(s->index); | |
| 279 | 42442 | GenericXLogRegisterBuffer(state, s->cur_buf, GENERIC_XLOG_FULL_IMAGE); | |
| 280 | 42442 | GenericXLogFinish(state); | |
| 281 | } | ||
| 282 | |||
| 283 | 80595 | UnlockReleaseBuffer(s->cur_buf); | |
| 284 | 80595 | s->cur_buf = InvalidBuffer; | |
| 285 | 80595 | } | |
| 286 | |||
| 287 | /* ---------------------------------------------------------------- | ||
| 288 | * Bulk extend: pre-allocate contiguous pages | ||
| 289 | * ---------------------------------------------------------------- */ | ||
| 290 | |||
| 291 | static BlockNumber | ||
| 292 | 249 | pg_extend(VsStorage *self, uint32_t npages) | |
| 293 | { | ||
| 294 | 249 | VsPgStorage *s = PG_STORAGE(self); | |
| 295 | |||
| 296 | 249 | BlockNumber start = InvalidBlockNumber; | |
| 297 | 249 | uint32_t remaining = npages; | |
| 298 | |||
| 299 |
2/2✓ Branch 0 taken 1127 times.
✓ Branch 1 taken 249 times.
|
1376 | while (remaining > 0) |
| 300 | { | ||
| 301 | 1127 | uint32_t batch = Min(remaining, 512); | |
| 302 | 1127 | Buffer *buffers = palloc(batch * sizeof(Buffer)); | |
| 303 | 1127 | uint32_t got = 0; | |
| 304 | |||
| 305 | 2254 | BlockNumber batch_start = ExtendBufferedRelBy( | |
| 306 | 1127 | BMR_REL(s->index), | |
| 307 | MAIN_FORKNUM, | ||
| 308 | NULL, | ||
| 309 | 0, | ||
| 310 | batch, | ||
| 311 | buffers, | ||
| 312 | &got); | ||
| 313 | |||
| 314 |
2/2✓ Branch 0 taken 249 times.
✓ Branch 1 taken 878 times.
|
1127 | if (start == InvalidBlockNumber) |
| 315 | 249 | start = batch_start; | |
| 316 | |||
| 317 |
2/2✓ Branch 0 taken 11903 times.
✓ Branch 1 taken 1127 times.
|
13030 | for (uint32_t i = 0; i < got; i++) |
| 318 | 11903 | ReleaseBuffer(buffers[i]); | |
| 319 | |||
| 320 | 1127 | pfree(buffers); | |
| 321 | 1127 | remaining -= got; | |
| 322 | } | ||
| 323 | |||
| 324 | 249 | return start; | |
| 325 | } | ||
| 326 | |||
| 327 | /* ---------------------------------------------------------------- | ||
| 328 | * Rerank: fetch vectors from heap, compute exact L2 | ||
| 329 | * ---------------------------------------------------------------- */ | ||
| 330 | |||
| 331 | /* Comparator for sorting candidate indices by TID block order */ | ||
| 332 | static int | ||
| 333 | 76119 | cmp_tid_order(const void *a, const void *b, void *arg) | |
| 334 | { | ||
| 335 | 76119 | const VsTopKEntry *cands = arg; | |
| 336 | |||
| 337 | /* Encoded ids are (block << 16) | offset — strictly monotonic in | ||
| 338 | * (block, offset), so raw id comparison IS TID order; no need to | ||
| 339 | * decode both TIDs on every comparison. */ | ||
| 340 | 76119 | uint64_t id_a = cands[*(const uint32_t *)a].id; | |
| 341 | 76119 | uint64_t id_b = cands[*(const uint32_t *)b].id; | |
| 342 | |||
| 343 |
2/2✓ Branch 0 taken 38582 times.
✓ Branch 1 taken 37537 times.
|
76119 | if (id_a < id_b) |
| 344 | return -1; | ||
| 345 |
1/2✓ Branch 0 taken 38582 times.
✗ Branch 1 not taken.
|
38582 | if (id_a > id_b) |
| 346 | 38582 | return 1; | |
| 347 | return 0; | ||
| 348 | } | ||
| 349 | |||
| 350 | /* | ||
| 351 | * pg_rerank - Rerank candidates with exact L2 distances. | ||
| 352 | * | ||
| 353 | * Fetches full-precision vectors from the heap table and computes | ||
| 354 | * exact L2 distance for candidates with nonzero error. Candidates | ||
| 355 | * are visited in TID block-number order to minimize buffer cache | ||
| 356 | * thrash — sequential block access avoids re-pinning the same | ||
| 357 | * buffer multiple times. | ||
| 358 | * | ||
| 359 | * Rows are fetched under the scan's snapshot, following HOT chains the way | ||
| 360 | * the executor's own heap fetch does. A candidate whose row is not visible is | ||
| 361 | * left out: the executor would discard it anyway, so keeping it would only | ||
| 362 | * take a top-k slot from a visible row. Its vector is never read either, | ||
| 363 | * which matters because an aborted insert or a vacuumed delete may already | ||
| 364 | * have made its TOAST chunks unreadable. | ||
| 365 | */ | ||
| 366 | static uint32_t | ||
| 367 | ✗ | pg_rerank( | |
| 368 | VsStorage *self, | ||
| 369 | const float *query, | ||
| 370 | Dimension dim, | ||
| 371 | const VsTopKEntry *candidates, | ||
| 372 | uint32_t count, | ||
| 373 | uint32_t keep, | ||
| 374 | uint32_t *out_indices, | ||
| 375 | Distance *out_distances) | ||
| 376 | { | ||
| 377 | ✗ | VsPgStorage *s = PG_STORAGE(self); | |
| 378 | |||
| 379 | ✗ | if (s->rel == NULL || count == 0) | |
| 380 | return 0; | ||
| 381 | |||
| 382 | ✗ | AttrNumber vec_attnum = s->index->rd_index->indkey.values[0]; | |
| 383 | |||
| 384 | /* | ||
| 385 | * Widening buffer for a vec16 heap column, allocated once per rerank | ||
| 386 | * call -- rerank runs once per query over the whole candidate set, so this | ||
| 387 | * is not a per-tuple allocation. | ||
| 388 | */ | ||
| 389 | ✗ | Vec32Access input = vec32_access(s->type_info, dim, CurrentMemoryContext); | |
| 390 | |||
| 391 | /* Sort by TID block order for sequential I/O */ | ||
| 392 | ✗ | uint32_t *order = palloc(count * sizeof(uint32_t)); | |
| 393 | ✗ | for (uint32_t i = 0; i < count; i++) | |
| 394 | ✗ | order[i] = i; | |
| 395 | ✗ | qsort_arg( | |
| 396 | order, count, sizeof(uint32_t), cmp_tid_order, (void *)candidates); | ||
| 397 | |||
| 398 | ✗ | VsTopK topk; | |
| 399 | ✗ | vs_topk_init(&topk, keep); | |
| 400 | |||
| 401 | ✗ | Assert(s->snapshot != NULL); | |
| 402 | ✗ | TupleTableSlot *slot = table_slot_create(s->rel, NULL); | |
| 403 | #if PG_VERSION_NUM >= 190000 | ||
| 404 | IndexFetchTableData *fetch = table_index_fetch_begin(s->rel, SO_NONE); | ||
| 405 | #else | ||
| 406 | ✗ | IndexFetchTableData *fetch = table_index_fetch_begin(s->rel); | |
| 407 | #endif | ||
| 408 | |||
| 409 | ✗ | for (uint32_t i = 0; i < count; i++) | |
| 410 | { | ||
| 411 | ✗ | uint32_t idx = order[i]; | |
| 412 | |||
| 413 | ✗ | Distance d; | |
| 414 | ✗ | if (candidates[idx].error == 0.0f) | |
| 415 | { | ||
| 416 | ✗ | d = candidates[idx].distance; | |
| 417 | } | ||
| 418 | else | ||
| 419 | { | ||
| 420 | ✗ | ItemPointerData tid = prism_posting_decode_tid(candidates[idx].id); | |
| 421 | ✗ | bool call_again = false; | |
| 422 | ✗ | bool all_dead = false; | |
| 423 | ✗ | if (!table_index_fetch_tuple( | |
| 424 | fetch, | ||
| 425 | &tid, | ||
| 426 | s->snapshot, | ||
| 427 | slot, | ||
| 428 | &call_again, | ||
| 429 | &all_dead)) | ||
| 430 | ✗ | continue; | |
| 431 | |||
| 432 | ✗ | bool isnull; | |
| 433 | ✗ | Datum val = slot_getattr(slot, vec_attnum, &isnull); | |
| 434 | ✗ | if (!isnull) | |
| 435 | { | ||
| 436 | ✗ | Vec32Ref qref = {.data = query, .dim = dim}; | |
| 437 | ✗ | Vec32Ref vref = vec32_read(&input, val); | |
| 438 | ✗ | d = vs_distance(qref, vref, s->metric); | |
| 439 | } | ||
| 440 | else | ||
| 441 | { | ||
| 442 | ✗ | d = candidates[idx].distance; | |
| 443 | } | ||
| 444 | ✗ | ExecClearTuple(slot); | |
| 445 | } | ||
| 446 | |||
| 447 | ✗ | vs_topk_insert_unique(&topk, d, 0.0f, (uint64_t)idx); | |
| 448 | } | ||
| 449 | |||
| 450 | ✗ | table_index_fetch_end(fetch); | |
| 451 | ✗ | ExecDropSingleTupleTableSlot(slot); | |
| 452 | |||
| 453 | ✗ | VsTopKEntry *entries = palloc(topk.cand_count * sizeof(VsTopKEntry)); | |
| 454 | ✗ | uint32_t nresults; | |
| 455 | ✗ | vs_topk_extract_sorted_unique(&topk, entries, &nresults); | |
| 456 | |||
| 457 | ✗ | for (uint32_t i = 0; i < nresults; i++) | |
| 458 | { | ||
| 459 | ✗ | out_indices[i] = (uint32_t)entries[i].id; | |
| 460 | ✗ | out_distances[i] = entries[i].distance; | |
| 461 | } | ||
| 462 | |||
| 463 | ✗ | pfree(entries); | |
| 464 | ✗ | vs_topk_cleanup(&topk); | |
| 465 | ✗ | pfree(order); | |
| 466 | |||
| 467 | ✗ | return nresults; | |
| 468 | } | ||
| 469 | |||
| 470 | /* ---------------------------------------------------------------- | ||
| 471 | * Heap AM rerank with read_stream | ||
| 472 | * | ||
| 473 | * Uses PostgreSQL's read_stream API for batched async I/O when | ||
| 474 | * fetching heap tuples for reranking. The stream callback yields | ||
| 475 | * block numbers in candidate order, and each buffer is processed | ||
| 476 | * directly as it arrives. | ||
| 477 | * ---------------------------------------------------------------- */ | ||
| 478 | |||
| 479 | typedef struct RerankStreamState | ||
| 480 | { | ||
| 481 | const VsTopKEntry *candidates; | ||
| 482 | const uint32_t *order; | ||
| 483 | VsTopK *topk; | ||
| 484 | VsPgStorage *storage; | ||
| 485 | uint32_t count; | ||
| 486 | uint32_t pos; | ||
| 487 | } RerankStreamState; | ||
| 488 | |||
| 489 | static BlockNumber | ||
| 490 | 15571 | rerank_stream_cb( | |
| 491 | ReadStream *stream, void *callback_private_data, void *per_buffer_data) | ||
| 492 | { | ||
| 493 | 15571 | RerankStreamState *st = callback_private_data; | |
| 494 | |||
| 495 |
2/2✓ Branch 0 taken 15425 times.
✓ Branch 1 taken 362 times.
|
15787 | while (st->pos < st->count) |
| 496 | { | ||
| 497 | 15425 | uint32_t idx = st->order[st->pos++]; | |
| 498 | |||
| 499 | /* Already exact — insert directly, no heap fetch */ | ||
| 500 |
2/2✓ Branch 0 taken 216 times.
✓ Branch 1 taken 15209 times.
|
15425 | if (st->candidates[idx].error == 0.0f) |
| 501 | { | ||
| 502 | 216 | vs_topk_insert_unique( | |
| 503 | st->topk, | ||
| 504 | 216 | st->candidates[idx].distance, | |
| 505 | 0.0f, | ||
| 506 | (uint64_t)idx); | ||
| 507 | 216 | continue; | |
| 508 | } | ||
| 509 | |||
| 510 | 15209 | *(uint32_t *)per_buffer_data = idx; | |
| 511 | 15209 | ItemPointerData tid = prism_posting_decode_tid(st->candidates[idx].id); | |
| 512 | 15209 | return ItemPointerGetBlockNumber(&tid); | |
| 513 | } | ||
| 514 | |||
| 515 | return InvalidBlockNumber; | ||
| 516 | } | ||
| 517 | |||
| 518 | static uint32_t | ||
| 519 | 362 | pg_rerank_readstream( | |
| 520 | VsStorage *self, | ||
| 521 | const float *query, | ||
| 522 | Dimension dim, | ||
| 523 | const VsTopKEntry *candidates, | ||
| 524 | uint32_t count, | ||
| 525 | uint32_t keep, | ||
| 526 | uint32_t *out_indices, | ||
| 527 | Distance *out_distances) | ||
| 528 | { | ||
| 529 | 362 | VsPgStorage *s = PG_STORAGE(self); | |
| 530 | |||
| 531 |
2/4✓ Branch 0 taken 362 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 362 times.
✗ Branch 3 not taken.
|
362 | if (s->rel == NULL || count == 0) |
| 532 | return 0; | ||
| 533 | |||
| 534 | 362 | AttrNumber vec_attnum = s->index->rd_index->indkey.values[0]; | |
| 535 | |||
| 536 | /* See pg_rerank: one widening buffer per call, not per candidate. */ | ||
| 537 | 362 | Vec32Access input = vec32_access(s->type_info, dim, CurrentMemoryContext); | |
| 538 | |||
| 539 | 362 | uint32_t *order = palloc(count * sizeof(uint32_t)); | |
| 540 |
2/2✓ Branch 1 taken 15425 times.
✓ Branch 2 taken 362 times.
|
15787 | for (uint32_t i = 0; i < count; i++) |
| 541 | 15425 | order[i] = i; | |
| 542 | 362 | qsort_arg( | |
| 543 | order, count, sizeof(uint32_t), cmp_tid_order, (void *)candidates); | ||
| 544 | |||
| 545 | 362 | VsTopK topk; | |
| 546 | 362 | vs_topk_init(&topk, keep); | |
| 547 | |||
| 548 | 362 | RerankStreamState state = { | |
| 549 | .candidates = candidates, | ||
| 550 | .order = order, | ||
| 551 | .topk = &topk, | ||
| 552 | .storage = s, | ||
| 553 | .count = count, | ||
| 554 | .pos = 0, | ||
| 555 | }; | ||
| 556 | |||
| 557 | 724 | TupleTableSlot *slot = MakeSingleTupleTableSlot( | |
| 558 | 362 | RelationGetDescr(s->rel), &TTSOpsBufferHeapTuple); | |
| 559 | |||
| 560 | 362 | ReadStream *stream = read_stream_begin_relation( | |
| 561 | READ_STREAM_DEFAULT, | ||
| 562 | NULL, | ||
| 563 | s->rel, | ||
| 564 | MAIN_FORKNUM, | ||
| 565 | rerank_stream_cb, | ||
| 566 | &state, | ||
| 567 | sizeof(uint32_t)); | ||
| 568 | |||
| 569 | 362 | void *per_buffer_data; | |
| 570 | 362 | Buffer buf; | |
| 571 | |||
| 572 |
2/2✓ Branch 1 taken 15209 times.
✓ Branch 2 taken 362 times.
|
15933 | while (BufferIsValid( |
| 573 | 15571 | buf = read_stream_next_buffer(stream, &per_buffer_data))) | |
| 574 | { | ||
| 575 | 15209 | uint32_t idx = *(uint32_t *)per_buffer_data; | |
| 576 | 15209 | ItemPointerData tid = prism_posting_decode_tid(candidates[idx].id); | |
| 577 | |||
| 578 | /* The executor's heap fetch, on the buffer the stream pinned: follow | ||
| 579 | * the HOT chain under the scan's snapshot, holding the page lock for | ||
| 580 | * the walk only. An invisible row is left out (see pg_rerank). */ | ||
| 581 | 15209 | HeapTupleData tuple; | |
| 582 | 15209 | bool all_dead; | |
| 583 | 15209 | LockBuffer(buf, BUFFER_LOCK_SHARE); | |
| 584 | 15209 | bool found = heap_hot_search_buffer( | |
| 585 | &tid, s->rel, buf, s->snapshot, &tuple, &all_dead, true); | ||
| 586 | 15209 | LockBuffer(buf, BUFFER_LOCK_UNLOCK); | |
| 587 | |||
| 588 |
2/2✓ Branch 0 taken 72 times.
✓ Branch 1 taken 15137 times.
|
15209 | if (!found) |
| 589 | { | ||
| 590 | 72 | ReleaseBuffer(buf); | |
| 591 | 72 | continue; | |
| 592 | } | ||
| 593 | |||
| 594 | 15137 | Distance d = candidates[idx].distance; | |
| 595 | 15137 | ExecStoreBufferHeapTuple(&tuple, slot, buf); | |
| 596 | |||
| 597 | 15137 | bool isnull; | |
| 598 | 15137 | Datum val = slot_getattr(slot, vec_attnum, &isnull); | |
| 599 |
1/2✓ Branch 0 taken 15137 times.
✗ Branch 1 not taken.
|
15137 | if (!isnull) |
| 600 | { | ||
| 601 | 15137 | Vec32Ref qref = {.data = query, .dim = dim}; | |
| 602 | 15137 | Vec32Ref vref = vec32_read(&input, val); | |
| 603 | 15137 | d = vs_distance(qref, vref, s->metric); | |
| 604 | } | ||
| 605 | 15137 | ExecClearTuple(slot); | |
| 606 | |||
| 607 | 15137 | vs_topk_insert_unique(&topk, d, 0.0f, (uint64_t)idx); | |
| 608 | 15137 | ReleaseBuffer(buf); | |
| 609 | } | ||
| 610 | |||
| 611 | 362 | read_stream_end(stream); | |
| 612 | 362 | ExecDropSingleTupleTableSlot(slot); | |
| 613 | |||
| 614 | 362 | VsTopKEntry *entries = palloc(topk.cand_count * sizeof(VsTopKEntry)); | |
| 615 | 362 | uint32_t nresults; | |
| 616 | 362 | vs_topk_extract_sorted_unique(&topk, entries, &nresults); | |
| 617 | |||
| 618 |
2/2✓ Branch 1 taken 11027 times.
✓ Branch 2 taken 362 times.
|
11389 | for (uint32_t i = 0; i < nresults; i++) |
| 619 | { | ||
| 620 | 11027 | out_indices[i] = (uint32_t)entries[i].id; | |
| 621 | 11027 | out_distances[i] = entries[i].distance; | |
| 622 | } | ||
| 623 | |||
| 624 | 362 | pfree(entries); | |
| 625 | 362 | vs_topk_cleanup(&topk); | |
| 626 | 362 | pfree(order); | |
| 627 | |||
| 628 | 362 | return nresults; | |
| 629 | } | ||
| 630 | |||
| 631 | /* ---------------------------------------------------------------- | ||
| 632 | * Static vtables | ||
| 633 | * ---------------------------------------------------------------- */ | ||
| 634 | |||
| 635 | static const VsStorageOps pg_storage_ops = { | ||
| 636 | .read_page = pg_read_page, | ||
| 637 | .release_page = pg_release_page, | ||
| 638 | .prefetch = pg_prefetch_page, | ||
| 639 | .write_page = pg_write_page, | ||
| 640 | .new_page = pg_new_page, | ||
| 641 | .commit_page = pg_commit_page, | ||
| 642 | .extend = pg_extend, | ||
| 643 | .rerank = pg_rerank, | ||
| 644 | }; | ||
| 645 | |||
| 646 | /* | ||
| 647 | * Read for a caller that sweeps the index rather than following a query's | ||
| 648 | * access pattern -- inspection. Identical to pg_read_page minus the | ||
| 649 | * recent-buffer cache, which for a sweep is worse than useless: its slots | ||
| 650 | * would fill with blocks no query asks for again, evicting the ones the | ||
| 651 | * query path reuses, and its hit/cold/stale counters would report the sweep | ||
| 652 | * instead of the workload they exist to measure. | ||
| 653 | * | ||
| 654 | * A separate entry in the ops table rather than a flag inside pg_read_page: | ||
| 655 | * the query path reads a page for every posting page it scans, and should | ||
| 656 | * not test a condition that only inspection can change. | ||
| 657 | */ | ||
| 658 | static Page | ||
| 659 | 27703 | pg_read_page_nocache(VsStorage *self, BlockNumber blkno) | |
| 660 | { | ||
| 661 | 27703 | VsPgStorage *s = PG_STORAGE(self); | |
| 662 | 27703 | Buffer buf = ReadBuffer(s->index, blkno); | |
| 663 | |||
| 664 | 27703 | LockBuffer(buf, BUFFER_LOCK_SHARE); | |
| 665 | 27703 | s->cur_buf = buf; | |
| 666 | 27703 | s->read_count++; | |
| 667 | |||
| 668 | 27703 | return BufferGetPage(buf); | |
| 669 | } | ||
| 670 | |||
| 671 | static const VsStorageOps pg_storage_readstream_ops = { | ||
| 672 | .read_page = pg_read_page, | ||
| 673 | .release_page = pg_release_page, | ||
| 674 | .prefetch = pg_prefetch_page, | ||
| 675 | .write_page = pg_write_page, | ||
| 676 | .new_page = pg_new_page, | ||
| 677 | .commit_page = pg_commit_page, | ||
| 678 | .extend = pg_extend, | ||
| 679 | .rerank = pg_rerank_readstream, | ||
| 680 | }; | ||
| 681 | |||
| 682 | /* Inspection: the standard ops with the cache-bypassing read. */ | ||
| 683 | static const VsStorageOps pg_storage_inspect_ops = { | ||
| 684 | .read_page = pg_read_page_nocache, | ||
| 685 | .release_page = pg_release_page, | ||
| 686 | .prefetch = pg_prefetch_page, | ||
| 687 | .write_page = pg_write_page, | ||
| 688 | .new_page = pg_new_page, | ||
| 689 | .commit_page = pg_commit_page, | ||
| 690 | .extend = pg_extend, | ||
| 691 | .rerank = NULL, | ||
| 692 | }; | ||
| 693 | |||
| 694 | void | ||
| 695 | 728 | vs_pg_storage_init_inspect(VsPgStorage *s, Relation index) | |
| 696 | { | ||
| 697 | 728 | vs_pg_storage_init(s, index, NULL, DISTANCE_L2); | |
| 698 | 728 | s->base.ops = &pg_storage_inspect_ops; | |
| 699 | 728 | } | |
| 700 | |||
| 701 | /* ---------------------------------------------------------------- | ||
| 702 | * Initialization | ||
| 703 | * ---------------------------------------------------------------- */ | ||
| 704 | |||
| 705 | void | ||
| 706 | 21785 | vs_pg_storage_init( | |
| 707 | VsPgStorage *s, Relation index, Relation rel, DistanceMetric metric) | ||
| 708 | { | ||
| 709 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 21785 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
21785 | if (rel != NULL && RelationGetForm(rel)->relam == HEAP_TABLE_AM_OID) |
| 710 | ✗ | s->base.ops = &pg_storage_readstream_ops; | |
| 711 | else | ||
| 712 | 21785 | s->base.ops = &pg_storage_ops; | |
| 713 | 21785 | s->index = index; | |
| 714 | 21785 | s->rel = rel; | |
| 715 | 21785 | s->snapshot = NULL; | |
| 716 | 21785 | s->build_mode = false; | |
| 717 | 21785 | s->cur_buf = InvalidBuffer; | |
| 718 | 21785 | s->metric = metric; | |
| 719 | 21785 | s->read_count = 0; | |
| 720 | /* | ||
| 721 | * Left NULL: every caller inits with rel = NULL, and rerank -- the only | ||
| 722 | * consumer -- returns early without a heap relation. It is resolved in | ||
| 723 | * vs_pg_storage_set_rel, which is where the heap arrives and therefore | ||
| 724 | * where rerank becomes possible. | ||
| 725 | */ | ||
| 726 | 21785 | s->type_info = NULL; | |
| 727 | 21785 | } | |
| 728 | |||
| 729 | void | ||
| 730 | 349 | vs_pg_storage_set_rel(VsPgStorage *s, Relation rel, Snapshot snapshot) | |
| 731 | { | ||
| 732 | 349 | s->rel = rel; | |
| 733 | 349 | s->snapshot = snapshot; | |
| 734 |
2/4✓ Branch 0 taken 349 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 349 times.
✗ Branch 3 not taken.
|
349 | if (rel != NULL && RelationGetForm(rel)->relam == HEAP_TABLE_AM_OID) |
| 735 | 349 | s->base.ops = &pg_storage_readstream_ops; | |
| 736 | |||
| 737 | /* | ||
| 738 | * Rerank reads the heap attribute, which has the indexed column's type. | ||
| 739 | * From the per-backend cache: this runs on the scan path, so the index | ||
| 740 | * has a metadata page and the cache is populated. | ||
| 741 | */ | ||
| 742 | 349 | if (rel != NULL) | |
| 743 | 349 | s->type_info = prism_cache_type_info(s->index); | |
| 744 | 349 | } | |
| 745 |