| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /* | ||
| 2 | * Copyright (c) 2026 Tiger Data, Inc. | ||
| 3 | * Licensed under the PostgreSQL License. See LICENSE for details. | ||
| 4 | * | ||
| 5 | * amcache.c - Per-index cached state for prism | ||
| 6 | * | ||
| 7 | * The rotation matrix P is O(dim³) to generate, so it lives in a | ||
| 8 | * process-local, reference-counted hash table (keyed by dim+seed) that | ||
| 9 | * survives relcache invalidation — see get_or_create_params / | ||
| 10 | * prism_release_params below. The table and the matrices it owns live in a | ||
| 11 | * dedicated child context of CacheMemoryContext, so the cache's footprint is | ||
| 12 | * visible as its own line in a memory-context dump. The global mean, | ||
| 13 | * P^T·global_mean, and an immutable PrismIndexBase template live in rd_amcache | ||
| 14 | * (allocated in rd_indexcxt) and are cheap to re-populate when invalidated. | ||
| 15 | * The metadata page is read at most once per backend, when rd_amcache is first | ||
| 16 | * populated. | ||
| 17 | */ | ||
| 18 | |||
| 19 | #include <postgres.h> | ||
| 20 | |||
| 21 | #include <storage/bufmgr.h> | ||
| 22 | #include <utils/hsearch.h> | ||
| 23 | #include <utils/memutils.h> | ||
| 24 | #include <utils/resowner.h> | ||
| 25 | |||
| 26 | #include "amcache.h" | ||
| 27 | #include "bufstorage.h" | ||
| 28 | #include "core/log.h" | ||
| 29 | #include "index/posting_page.h" | ||
| 30 | #include "meta.h" | ||
| 31 | #include "support_pg.h" | ||
| 32 | #include "typeinfo.h" | ||
| 33 | |||
| 34 | /* ---------------------------------------------------------------- | ||
| 35 | * Process-local RaBitQParams cache | ||
| 36 | * | ||
| 37 | * A backend can have several scans of differently-dimensioned (or | ||
| 38 | * differently-seeded) indexes open at once (e.g. a join across two | ||
| 39 | * vector-indexed tables). A fixed number of slots can't represent that | ||
| 40 | * without either dangling a live scan's pointer (freeing an in-use entry to | ||
| 41 | * make room for another) or hard-erroring once every slot is checked out. | ||
| 42 | * | ||
| 43 | * Instead this is a backend-private dynahash table (plain hash_create() in | ||
| 44 | * rabitq_cache_cxt, a child of CacheMemoryContext -- not shared memory, so | ||
| 45 | * no shared_preload_libraries entry is needed, and nothing here is shared | ||
| 46 | * across backends) keyed by dim+seed, sized to grow as needed: | ||
| 47 | * | ||
| 48 | * - get_or_create_params() looks up (dim, seed). On a hit, it bumps the | ||
| 49 | * entry's refcount and a decaying usage score and returns the cached | ||
| 50 | * matrix. On a miss, once the table is at or past a soft target size, | ||
| 51 | * it first tries to reclaim one idle (refcount == 0) entry -- the one | ||
| 52 | * with the lowest decayed usage -- before generating a new matrix. A | ||
| 53 | * held (refcount > 0) entry is *never* an eviction candidate, so a | ||
| 54 | * checked-out RaBitQParams* stays valid for as long as its owner holds | ||
| 55 | * it, no matter how many other dimensions get checked out around it. | ||
| 56 | * If nothing is currently idle, the table simply grows past the soft | ||
| 57 | * target rather than failing a live caller -- the target is not a | ||
| 58 | * hard cap. | ||
| 59 | * - prism_release_params() checks an entry back in (refcount--). | ||
| 60 | * Checkouts are additionally tracked by the checkout-time resource | ||
| 61 | * owner, so error paths that skip the release still return the | ||
| 62 | * refcount -- see rabitq_params_ref_desc below. | ||
| 63 | * ---------------------------------------------------------------- */ | ||
| 64 | |||
| 65 | /* Above this many live entries, try to reclaim an idle one before growing | ||
| 66 | * further. Not a hard cap -- see above. */ | ||
| 67 | #define PRISM_RABITQ_CACHE_TARGET_ENTRIES 8 | ||
| 68 | |||
| 69 | #define PRISM_RABITQ_USAGE_INCREMENT 1.0 | ||
| 70 | #define PRISM_RABITQ_USAGE_DECAY 0.99 | ||
| 71 | |||
| 72 | typedef struct RaBitQCacheKey | ||
| 73 | { | ||
| 74 | Dimension dim; | ||
| 75 | uint64_t seed; | ||
| 76 | } RaBitQCacheKey; | ||
| 77 | |||
| 78 | typedef struct RaBitQCacheEntry | ||
| 79 | { | ||
| 80 | RaBitQCacheKey key; /* hash key; must be first */ | ||
| 81 | RaBitQParams *params; /* NULL until fully initialized (see below) */ | ||
| 82 | int refcount; | ||
| 83 | double usage; | ||
| 84 | } RaBitQCacheEntry; | ||
| 85 | |||
| 86 | static HTAB *rabitq_cache = NULL; | ||
| 87 | |||
| 88 | /* Owns the hash table and every cached matrix; child of | ||
| 89 | * CacheMemoryContext, so it has backend lifetime but shows up as its own | ||
| 90 | * line in a memory-context dump. */ | ||
| 91 | static MemoryContext rabitq_cache_cxt = NULL; | ||
| 92 | |||
| 93 | /* | ||
| 94 | * Error safety for the refcounts: every checkout is also registered | ||
| 95 | * with the resource owner current at checkout time. If an error keeps | ||
| 96 | * the matching release from running -- an aborted scan's portal is | ||
| 97 | * dropped without amendscan, or an insert errors between checkout and | ||
| 98 | * release -- the owner's release sweep calls back here and returns the | ||
| 99 | * refcount, so no entry is left unevictable. A checkout leaked on the | ||
| 100 | * commit path additionally gets PostgreSQL's standard "resource was not | ||
| 101 | * closed" warning, turning a pairing bug into a visible failure. | ||
| 102 | */ | ||
| 103 | static void rabitq_params_ref_release(Datum res); | ||
| 104 | |||
| 105 | static const ResourceOwnerDesc rabitq_params_ref_desc = { | ||
| 106 | .name = "prism rabitq params ref", | ||
| 107 | .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS, | ||
| 108 | .release_priority = RELEASE_PRIO_FIRST, | ||
| 109 | .ReleaseResource = rabitq_params_ref_release, | ||
| 110 | .DebugPrint = NULL, | ||
| 111 | }; | ||
| 112 | |||
| 113 | static void | ||
| 114 | 6 | rabitq_params_ref_release(Datum res) | |
| 115 | { | ||
| 116 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 6 times.
|
6 | RaBitQCacheEntry *entry = (RaBitQCacheEntry *)DatumGetPointer(res); |
| 117 | |||
| 118 | /* Only reached for checkouts no normal release forgot: error paths. */ | ||
| 119 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 6 times.
|
6 | Assert(entry->refcount > 0); |
| 120 | 6 | if (entry->refcount > 0) | |
| 121 | 6 | entry->refcount--; | |
| 122 | 6 | } | |
| 123 | |||
| 124 | static void | ||
| 125 | 90 | rabitq_cache_init(void) | |
| 126 | { | ||
| 127 | 90 | HASHCTL ctl; | |
| 128 | |||
| 129 | /* The context survives prism_rabitq_cache_clear() resets. */ | ||
| 130 |
2/2✓ Branch 0 taken 89 times.
✓ Branch 1 taken 1 times.
|
90 | if (rabitq_cache_cxt == NULL) |
| 131 | 89 | rabitq_cache_cxt = AllocSetContextCreate( | |
| 132 | CacheMemoryContext, | ||
| 133 | "prism rabitq params cache", | ||
| 134 | ALLOCSET_DEFAULT_SIZES); | ||
| 135 | |||
| 136 | 90 | memset(&ctl, 0, sizeof(ctl)); | |
| 137 | 90 | ctl.keysize = sizeof(RaBitQCacheKey); | |
| 138 | 90 | ctl.entrysize = sizeof(RaBitQCacheEntry); | |
| 139 | 90 | ctl.hcxt = rabitq_cache_cxt; | |
| 140 | |||
| 141 | 90 | rabitq_cache = hash_create( | |
| 142 | "prism rabitq params cache", | ||
| 143 | PRISM_RABITQ_CACHE_TARGET_ENTRIES, | ||
| 144 | &ctl, | ||
| 145 | HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); | ||
| 146 | 90 | } | |
| 147 | |||
| 148 | /* | ||
| 149 | * Decay every entry's usage score and reclaim the least-used entry with no | ||
| 150 | * live checkouts, if one exists. A no-op (not an error) when every entry is | ||
| 151 | * currently held -- the caller grows the table instead. | ||
| 152 | */ | ||
| 153 | static void | ||
| 154 | 4 | rabitq_cache_evict_one(void) | |
| 155 | { | ||
| 156 | 4 | HASH_SEQ_STATUS seq; | |
| 157 | 4 | RaBitQCacheEntry *entry; | |
| 158 | 4 | RaBitQCacheEntry *victim = NULL; | |
| 159 | |||
| 160 | 4 | hash_seq_init(&seq, rabitq_cache); | |
| 161 |
2/2✓ Branch 2 taken 32 times.
✓ Branch 3 taken 4 times.
|
36 | while ((entry = hash_seq_search(&seq)) != NULL) |
| 162 | { | ||
| 163 | 32 | entry->usage *= PRISM_RABITQ_USAGE_DECAY; | |
| 164 |
4/4✓ Branch 0 taken 1 times.
✓ Branch 1 taken 31 times.
✓ Branch 2 taken 27 times.
✓ Branch 3 taken 4 times.
|
32 | if (entry->refcount == 0 && |
| 165 |
2/2✓ Branch 0 taken 25 times.
✓ Branch 1 taken 2 times.
|
27 | (victim == NULL || entry->usage < victim->usage)) |
| 166 | 32 | victim = entry; | |
| 167 | } | ||
| 168 | /* The scan above always runs to completion (hash_seq_search returned | ||
| 169 | * NULL) before anything below mutates the table -- deleting mid-scan | ||
| 170 | * is only safe for the entry hash_seq_search just returned, which is | ||
| 171 | * why the victim is removed after, not during, the loop. */ | ||
| 172 | |||
| 173 |
1/2✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
|
4 | if (victim != NULL) |
| 174 | { | ||
| 175 | 4 | pfree(victim->params); | |
| 176 | 4 | hash_search(rabitq_cache, &victim->key, HASH_REMOVE, NULL); | |
| 177 | } | ||
| 178 | 4 | } | |
| 179 | |||
| 180 | static RaBitQParams * | ||
| 181 | 20357 | get_or_create_params(Dimension dim, uint64_t seed) | |
| 182 | { | ||
| 183 | 20357 | RaBitQCacheKey key = {0}; /* zero incl. padding: HASH_BLOBS compares | |
| 184 | * the whole struct */ | ||
| 185 | 20357 | RaBitQCacheEntry *entry; | |
| 186 | 20357 | bool found; | |
| 187 | |||
| 188 |
2/2✓ Branch 0 taken 90 times.
✓ Branch 1 taken 20267 times.
|
20357 | if (rabitq_cache == NULL) |
| 189 | 90 | rabitq_cache_init(); | |
| 190 | |||
| 191 | 20357 | key.dim = dim; | |
| 192 | 20357 | key.seed = seed; | |
| 193 | |||
| 194 | 20357 | entry = hash_search(rabitq_cache, &key, HASH_FIND, &found); | |
| 195 |
3/4✓ Branch 0 taken 20236 times.
✓ Branch 1 taken 121 times.
✓ Branch 2 taken 20236 times.
✗ Branch 3 not taken.
|
20357 | if (found && entry->params != NULL) |
| 196 | { | ||
| 197 | /* Enlarge first: it can allocate, and nothing may fail between | ||
| 198 | * the refcount bump and the (no-fail) Remember. */ | ||
| 199 | 20236 | ResourceOwnerEnlarge(CurrentResourceOwner); | |
| 200 | 20236 | entry->usage += PRISM_RABITQ_USAGE_INCREMENT; | |
| 201 | 20236 | entry->refcount++; | |
| 202 | 20236 | ResourceOwnerRemember( | |
| 203 | CurrentResourceOwner, | ||
| 204 | PointerGetDatum(entry), | ||
| 205 | &rabitq_params_ref_desc); | ||
| 206 | 20236 | return entry->params; | |
| 207 | } | ||
| 208 | |||
| 209 | /* Miss (or a dead entry left by a failed init below -- params == NULL, | ||
| 210 | * refcount == 0, safe to recreate): try to make room first, but only | ||
| 211 | * when this is about to grow the table with a genuinely new key. */ | ||
| 212 |
3/4✓ Branch 0 taken 121 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 4 times.
✓ Branch 4 taken 117 times.
|
121 | if (!found && hash_get_num_entries(rabitq_cache) >= |
| 213 | PRISM_RABITQ_CACHE_TARGET_ENTRIES) | ||
| 214 | 4 | rabitq_cache_evict_one(); | |
| 215 | |||
| 216 | 121 | entry = hash_search(rabitq_cache, &key, HASH_ENTER, &found); | |
| 217 | |||
| 218 | /* Mark it dead/pending before any allocation that could throw: a | ||
| 219 | * failure below then leaves a well-defined, idle, re-creatable entry | ||
| 220 | * instead of one that looks live with garbage params. The matrix is | ||
| 221 | * generated into a local pointer and only published once fully | ||
| 222 | * initialized, for the same reason (an error out of vs_rabitq_init | ||
| 223 | * must not leave a live-looking entry holding a garbage matrix). */ | ||
| 224 | 121 | entry->params = NULL; | |
| 225 | 121 | entry->refcount = 0; | |
| 226 | 121 | entry->usage = 0; | |
| 227 | |||
| 228 | 121 | ResourceOwnerEnlarge(CurrentResourceOwner); | |
| 229 | |||
| 230 | 121 | RaBitQParams *params = | |
| 231 | 121 | MemoryContextAlloc(rabitq_cache_cxt, VS_RABITQ_PARAMS_SIZE(dim)); | |
| 232 | 121 | vs_rabitq_init(params, dim, seed); | |
| 233 | |||
| 234 | 121 | entry->params = params; | |
| 235 | 121 | entry->refcount = 1; | |
| 236 | 121 | entry->usage = PRISM_RABITQ_USAGE_INCREMENT; | |
| 237 | 121 | ResourceOwnerRemember( | |
| 238 | CurrentResourceOwner, | ||
| 239 | PointerGetDatum(entry), | ||
| 240 | &rabitq_params_ref_desc); | ||
| 241 | |||
| 242 | 121 | return entry->params; | |
| 243 | } | ||
| 244 | |||
| 245 | /* | ||
| 246 | * Check an entry back in (public API; see amcache.h). dim+seed must | ||
| 247 | * match a currently-held entry exactly as returned by a prior | ||
| 248 | * get_or_create_params call; a mismatch indicates a caller bug, not a | ||
| 249 | * runtime condition. | ||
| 250 | */ | ||
| 251 | void | ||
| 252 | 20351 | prism_release_params(Dimension dim, uint64_t seed, ResourceOwner owner) | |
| 253 | { | ||
| 254 | 20351 | RaBitQCacheKey key = {0}; | |
| 255 | 20351 | RaBitQCacheEntry *entry; | |
| 256 | 20351 | bool found; | |
| 257 | |||
| 258 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 20351 times.
|
20351 | if (rabitq_cache == NULL) |
| 259 | { | ||
| 260 | ✗ | Assert(false); | |
| 261 | 20351 | return; | |
| 262 | } | ||
| 263 | |||
| 264 | 20351 | key.dim = dim; | |
| 265 | 20351 | key.seed = seed; | |
| 266 | |||
| 267 | 20351 | entry = hash_search(rabitq_cache, &key, HASH_FIND, &found); | |
| 268 |
1/2✓ Branch 0 taken 20351 times.
✗ Branch 1 not taken.
|
20351 | if (found) |
| 269 | { | ||
| 270 | /* Forget first: it errors on a pairing bug (wrong owner), and | ||
| 271 | * the refcount must stay consistent with the registrations. */ | ||
| 272 | 20351 | ResourceOwnerForget( | |
| 273 | owner, PointerGetDatum(entry), &rabitq_params_ref_desc); | ||
| 274 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 20351 times.
|
20351 | Assert(entry->refcount > 0); |
| 275 | 20351 | if (entry->refcount > 0) | |
| 276 | 20351 | entry->refcount--; | |
| 277 | 20351 | return; | |
| 278 | } | ||
| 279 | |||
| 280 | /* Should be unreachable: every release_params call is paired with a | ||
| 281 | * prior successful get_or_create_params for the same dim/seed, and a | ||
| 282 | * held entry is never evicted. */ | ||
| 283 | ✗ | Assert(false); | |
| 284 | } | ||
| 285 | |||
| 286 | /* | ||
| 287 | * Test support: snapshot the cache into a caller-provided array and | ||
| 288 | * return the number of entries written. A plain exported symbol (no SQL | ||
| 289 | * surface): the test-only module test/pg/src/test_helpers.c | ||
| 290 | * wraps it in a set-returning function so the regression tests can | ||
| 291 | * observe refcounts, usage decay, and eviction. Inert otherwise. | ||
| 292 | */ | ||
| 293 | int | ||
| 294 | 18 | prism_rabitq_cache_stats(PrismRabitqCacheStat *stats, int max_stats) | |
| 295 | { | ||
| 296 | 18 | HASH_SEQ_STATUS seq; | |
| 297 | 18 | RaBitQCacheEntry *entry; | |
| 298 | 18 | int n = 0; | |
| 299 | |||
| 300 |
2/2✓ Branch 0 taken 17 times.
✓ Branch 1 taken 1 times.
|
18 | if (rabitq_cache == NULL) |
| 301 | return 0; | ||
| 302 | |||
| 303 | /* The scan must run to completion (see rabitq_cache_evict_one). */ | ||
| 304 | 17 | hash_seq_init(&seq, rabitq_cache); | |
| 305 |
2/2✓ Branch 2 taken 136 times.
✓ Branch 3 taken 17 times.
|
153 | while ((entry = hash_seq_search(&seq)) != NULL) |
| 306 | { | ||
| 307 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 136 times.
|
136 | if (n >= max_stats) |
| 308 | ✗ | continue; | |
| 309 | 136 | stats[n].dim = (int32_t)entry->key.dim; | |
| 310 | 136 | stats[n].refcount = entry->refcount; | |
| 311 | 136 | stats[n].usage = entry->usage; | |
| 312 | 136 | n++; | |
| 313 | } | ||
| 314 | return n; | ||
| 315 | } | ||
| 316 | |||
| 317 | /* | ||
| 318 | * Test support: reset the cache to its initial state, freeing the hash | ||
| 319 | * table and every cached matrix in one context reset (they all live in | ||
| 320 | * rabitq_cache_cxt). Returns the number of entries dropped. Refuses if | ||
| 321 | * any entry is currently checked out -- freeing a held matrix is | ||
| 322 | * exactly the use-after-free this cache exists to prevent. Exported | ||
| 323 | * for the test-only module test/pg/src/test_helpers.c; inert otherwise. | ||
| 324 | */ | ||
| 325 | int | ||
| 326 | 1 | prism_rabitq_cache_clear(void) | |
| 327 | { | ||
| 328 | 1 | HASH_SEQ_STATUS seq; | |
| 329 | 1 | RaBitQCacheEntry *entry; | |
| 330 | 1 | int n = 0; | |
| 331 | |||
| 332 |
1/2✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
|
1 | if (rabitq_cache == NULL) |
| 333 | return 0; | ||
| 334 | |||
| 335 | 1 | hash_seq_init(&seq, rabitq_cache); | |
| 336 |
2/2✓ Branch 2 taken 6 times.
✓ Branch 3 taken 1 times.
|
7 | while ((entry = hash_seq_search(&seq)) != NULL) |
| 337 | { | ||
| 338 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 6 times.
|
6 | if (entry->refcount > 0) |
| 339 | { | ||
| 340 | ✗ | hash_seq_term(&seq); | |
| 341 | ✗ | vs_error( | |
| 342 | "cannot clear the RaBitQ params cache: entry for " | ||
| 343 | "dimension %u has %d live checkout(s)", | ||
| 344 | (unsigned)entry->key.dim, | ||
| 345 | entry->refcount); | ||
| 346 | } | ||
| 347 | 6 | n++; | |
| 348 | } | ||
| 349 | |||
| 350 | 1 | rabitq_cache = NULL; | |
| 351 | 1 | MemoryContextReset(rabitq_cache_cxt); | |
| 352 | 1 | return n; | |
| 353 | } | ||
| 354 | |||
| 355 | /* ---------------------------------------------------------------- | ||
| 356 | * rd_amcache layout | ||
| 357 | * | ||
| 358 | * A fully-populated immutable PrismIndexBase template (params + storage left | ||
| 359 | * for the per-call rebind), the scan-planning scalars, and the inline | ||
| 360 | * global_mean + pt_global_mean vectors appended after the struct. | ||
| 361 | * | ||
| 362 | * The metapage-derived scalars (dim, metric, nlist, ...) are filled eagerly on | ||
| 363 | * first access. The rotated pt_global_mean — which needs the O(dim³) rotation | ||
| 364 | * matrix — is computed lazily (pt_ready) only when a caller actually needs it | ||
| 365 | * (the scan / insert path via prism_index_base_init), so metadata-only | ||
| 366 | * consumers like VACUUM's ambulkdelete get dim/metric from the cache without | ||
| 367 | * paying for rotation setup. | ||
| 368 | * | ||
| 369 | * Two separate constraints shape this, and they pull in different directions. | ||
| 370 | * | ||
| 371 | * ONE ALLOCATION, and that is not stylistic. PostgreSQL frees rd_amcache with | ||
| 372 | * a single targeted pfree in RelationInvalidateRelation, which does NOT delete | ||
| 373 | * rd_indexcxt — the relation survives with rd_isvalid = false. So a sub-object | ||
| 374 | * hung off this blob would leak into rd_indexcxt once per invalidation cycle. | ||
| 375 | * Relcache's own caches (RelationGetIndexList and friends) can be lazy | ||
| 376 | * per-allocation because relcache wrote per-field cleanup for each of them; an | ||
| 377 | * access method gets one line of cleanup written for it, so it gets one | ||
| 378 | * allocation. Nor can the blob be grown and swapped later: PrismIndexBase | ||
| 379 | * .pt_global_mean points into it. A scan copies the mean at beginscan, since | ||
| 380 | * it can process an invalidation between rescans; other callers use the | ||
| 381 | * pointer within a single call. | ||
| 382 | * | ||
| 383 | * LAZY CONTENT, which the single allocation does not prevent. The metapage | ||
| 384 | * read is the floor — dim comes from it and dim sizes the blob — but the work | ||
| 385 | * that follows it need not be eager. The rule for a new field: fill it here if | ||
| 386 | * computing it is cheap or it needs the pinned metapage buffer (the | ||
| 387 | * global_mean memcpy is both); otherwise give it a ready-flag and compute it | ||
| 388 | * on first use, as pt_global_mean does. Note that anything living in this blob | ||
| 389 | * inherits the metapage read as a precondition even when it would not | ||
| 390 | * otherwise need one -- type_info, which comes from the opclass, is the case | ||
| 391 | * in point. | ||
| 392 | * ---------------------------------------------------------------- */ | ||
| 393 | |||
| 394 | typedef struct AmCacheData | ||
| 395 | { | ||
| 396 | PrismIndexBase base; /* immutable template; params / fastscan / | ||
| 397 | * storage left zeroed (rebound per call) */ | ||
| 398 | const PrismIndexTypeInfo *type_info; /* indexed column's type, from the | ||
| 399 | * opclass */ | ||
| 400 | bool has_fastscan; /* index built with FASTSCAN posting pages */ | ||
| 401 | bool pt_ready; /* pt_global_mean computed (rotation done) */ | ||
| 402 | uint32_t nlist; | ||
| 403 | uint32_t global_mean_off; | ||
| 404 | uint32_t pt_global_mean_off; | ||
| 405 | } AmCacheData; | ||
| 406 | |||
| 407 | static inline float * | ||
| 408 | 606 | cache_global_mean(AmCacheData *c) | |
| 409 | { | ||
| 410 | 606 | return (float *)((char *)c + c->global_mean_off); | |
| 411 | } | ||
| 412 | |||
| 413 | static inline float * | ||
| 414 | 556 | cache_pt_global_mean(AmCacheData *c) | |
| 415 | { | ||
| 416 | 556 | return (float *)((char *)c + c->pt_global_mean_off); | |
| 417 | } | ||
| 418 | |||
| 419 | /* | ||
| 420 | * Return the per-backend cache, populating rd_amcache (one metapage read) on | ||
| 421 | * first use. pt_global_mean is stored in the cache; params are NOT frozen | ||
| 422 | * here (the process-local params cache is reference-counted per checkout, | ||
| 423 | * not per-index) — callers rebind via get_or_create_params / | ||
| 424 | * prism_release_params. | ||
| 425 | */ | ||
| 426 | static AmCacheData * | ||
| 427 | 41945 | get_cache_data(Relation index) | |
| 428 | { | ||
| 429 | 41945 | prism_check_index(index); | |
| 430 | |||
| 431 |
2/2✓ Branch 0 taken 328 times.
✓ Branch 1 taken 41617 times.
|
41945 | if (index->rd_amcache != NULL) |
| 432 | return (AmCacheData *)index->rd_amcache; | ||
| 433 | |||
| 434 | /* Read the metadata page once. */ | ||
| 435 | 328 | Buffer meta_buf = ReadBuffer(index, 0); | |
| 436 | 328 | LockBuffer(meta_buf, BUFFER_LOCK_SHARE); | |
| 437 | 328 | Page meta_page = BufferGetPage(meta_buf); | |
| 438 | |||
| 439 | 328 | const PrismMetaPage *meta = (const PrismMetaPage *)PageGetSpecialPointer( | |
| 440 | meta_page); | ||
| 441 | /* Reject an index whose metapage was written by an incompatible format | ||
| 442 | * (loud in release too, not just a debug Assert) — its layout would | ||
| 443 | * otherwise be misread. */ | ||
| 444 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 328 times.
|
328 | if (meta->magic != PRISM_META_MAGIC) |
| 445 | { | ||
| 446 | ✗ | uint32_t got = meta->magic; | |
| 447 | ✗ | UnlockReleaseBuffer(meta_buf); | |
| 448 | ✗ | ereport(ERROR, | |
| 449 | (errcode(ERRCODE_INDEX_CORRUPTED), | ||
| 450 | errmsg("index \"%s\" has an incompatible on-disk format " | ||
| 451 | "(metapage magic 0x%08X, expected 0x%08X)", | ||
| 452 | RelationGetRelationName(index), | ||
| 453 | got, | ||
| 454 | (uint32_t)PRISM_META_MAGIC), | ||
| 455 | errhint("REINDEX the index to rebuild it in the current " | ||
| 456 | "format."))); | ||
| 457 | } | ||
| 458 | |||
| 459 | 328 | Dimension dim = meta->dim; | |
| 460 | 328 | uint64_t seed = meta->rabitq_seed; | |
| 461 | |||
| 462 | 328 | uint32_t gm_off = MAXALIGN(sizeof(AmCacheData)); | |
| 463 | 328 | uint32_t pt_gm_off = gm_off + dim * sizeof(float); | |
| 464 | 328 | Size total = pt_gm_off + dim * sizeof(float); | |
| 465 | |||
| 466 | 328 | MemoryContext old = MemoryContextSwitchTo(index->rd_indexcxt); | |
| 467 | 328 | AmCacheData *c = palloc0(total); | |
| 468 |
2/2✓ Branch 0 taken 275 times.
✓ Branch 1 taken 53 times.
|
328 | MemoryContextSwitchTo(old); |
| 469 | |||
| 470 | 328 | c->global_mean_off = gm_off; | |
| 471 | 328 | c->pt_global_mean_off = pt_gm_off; | |
| 472 | 328 | c->has_fastscan = (meta->flags & PRISM_META_FLAG_FASTSCAN) != 0; | |
| 473 | 328 | c->nlist = meta->nlist; | |
| 474 | |||
| 475 | /* Immutable base template (storage / params / fastscan rebound per call). | ||
| 476 | */ | ||
| 477 | 328 | c->base.dim = dim; | |
| 478 | 328 | c->base.nlevels = meta->nlevels; | |
| 479 | 328 | c->base.fan_out = meta->fan_out; | |
| 480 | 328 | c->base.nlist = meta->nlist; | |
| 481 | 328 | c->base.first_centroid = meta->first_centroid; | |
| 482 | 328 | c->base.first_posting = meta->first_posting; | |
| 483 | 328 | c->base.ncentroid_pages = meta->ncentroid_pages; | |
| 484 | 328 | c->base.metric = (DistanceMetric)meta->metric; | |
| 485 | 328 | c->base.centroid_format = (PrismCentroidFormat)meta->centroid_format; | |
| 486 | 328 | c->base.rabitq_seed = seed; | |
| 487 | /* base.pt_global_mean stays NULL until prism_index_base_init computes it. | ||
| 488 | */ | ||
| 489 | |||
| 490 | /* Copy global mean; P^T * global_mean is computed lazily (see | ||
| 491 | * prism_index_base_init) so metadata-only callers skip the rotation. A | ||
| 492 | * mean too wide for the metapage is gathered from its reference pages | ||
| 493 | * after the metapage is released (the page layer pins one buffer). */ | ||
| 494 |
2/2✓ Branch 0 taken 275 times.
✓ Branch 1 taken 53 times.
|
328 | if (prism_meta_mean_inline(dim)) |
| 495 | 275 | memcpy(cache_global_mean(c), | |
| 496 | 275 | prism_meta_global_mean_const(meta), | |
| 497 | dim * sizeof(float)); | ||
| 498 | |||
| 499 | 328 | UnlockReleaseBuffer(meta_buf); | |
| 500 | |||
| 501 |
2/2✓ Branch 0 taken 53 times.
✓ Branch 1 taken 275 times.
|
328 | if (!prism_meta_mean_inline(dim)) |
| 502 | { | ||
| 503 | 53 | VsPgStorage st; | |
| 504 | |||
| 505 | 53 | vs_pg_storage_init_inspect(&st, index); | |
| 506 | 53 | prism_posting_reference_read( | |
| 507 | &st.base, PRISM_GLOBAL_MEAN_BLKNO, dim, cache_global_mean(c)); | ||
| 508 | } | ||
| 509 | |||
| 510 | /* Opclass-derived, so no metapage needed -- but resolved here so every | ||
| 511 | * post-build path reads it as a pointer instead of an fmgr call. */ | ||
| 512 | 328 | c->type_info = prism_index_type_info(index); | |
| 513 | |||
| 514 | 328 | index->rd_amcache = c; | |
| 515 | 328 | return c; | |
| 516 | } | ||
| 517 | |||
| 518 | /* ---------------------------------------------------------------- | ||
| 519 | * Public API | ||
| 520 | * ---------------------------------------------------------------- */ | ||
| 521 | |||
| 522 | void | ||
| 523 | 20357 | prism_index_base_init(Relation index, PrismIndexBase *base) | |
| 524 | { | ||
| 525 | 20357 | AmCacheData *c = get_cache_data(index); | |
| 526 | 20357 | RaBitQParams *params = | |
| 527 | 20357 | get_or_create_params(c->base.dim, c->base.rabitq_seed); | |
| 528 | |||
| 529 | /* Compute the rotated global mean on first use (needs the rotation | ||
| 530 | * matrix); cached thereafter for the backend. */ | ||
| 531 |
2/2✓ Branch 0 taken 278 times.
✓ Branch 1 taken 20079 times.
|
20357 | if (!c->pt_ready) |
| 532 | { | ||
| 533 | 278 | vs_rabitq_rotate( | |
| 534 | 278 | params, cache_global_mean(c), cache_pt_global_mean(c)); | |
| 535 | 278 | c->base.pt_global_mean = cache_pt_global_mean(c); | |
| 536 | 278 | c->pt_ready = true; | |
| 537 | } | ||
| 538 | |||
| 539 | /* Copy the immutable template, then rebind the fields that cannot be | ||
| 540 | * frozen for the backend's lifetime: params (checked out from the | ||
| 541 | * refcounted process-local cache; the caller must pair this with a | ||
| 542 | * matching prism_release_params(base->dim, base->rabitq_seed) once it's | ||
| 543 | * done with base->params) and fastscan (resolved from the session GUC). | ||
| 544 | * Storage pointers stay zeroed for the caller. */ | ||
| 545 | 20357 | *base = c->base; | |
| 546 | 20357 | base->params = params; | |
| 547 |
2/2✓ Branch 0 taken 14986 times.
✓ Branch 1 taken 5371 times.
|
20357 | base->fastscan = c->has_fastscan ? prism_fastscan_bits : 0; |
| 548 | 20357 | base->centroid_error_scale = (float)prism_centroid_error_scale; | |
| 549 | 20357 | base->centroid_beam_scale = (float)prism_centroid_beam_scale; | |
| 550 | 20357 | } | |
| 551 | |||
| 552 | void | ||
| 553 | 38 | prism_cache_meta( | |
| 554 | Relation index, | ||
| 555 | Dimension *dim, | ||
| 556 | DistanceMetric *metric, | ||
| 557 | BlockNumber *first_posting) | ||
| 558 | { | ||
| 559 | 38 | AmCacheData *c = get_cache_data(index); | |
| 560 | 38 | *dim = c->base.dim; | |
| 561 | 38 | *metric = c->base.metric; | |
| 562 | 38 | *first_posting = c->base.first_posting; | |
| 563 | 38 | } | |
| 564 | |||
| 565 | const PrismIndexTypeInfo * | ||
| 566 | 20704 | prism_cache_type_info(Relation index) | |
| 567 | { | ||
| 568 | 20704 | return get_cache_data(index)->type_info; | |
| 569 | } | ||
| 570 | |||
| 571 | PrismScanInfo | ||
| 572 | 846 | prism_cache_scan_info(Relation index) | |
| 573 | { | ||
| 574 | 846 | AmCacheData *c = get_cache_data(index); | |
| 575 | 846 | return (PrismScanInfo){ | |
| 576 | 846 | .nlist = c->nlist, | |
| 577 | 846 | .dim = c->base.dim, | |
| 578 | 846 | .nlevels = c->base.nlevels, | |
| 579 | 846 | .fan_out = c->base.fan_out, | |
| 580 | 846 | .first_centroid = c->base.first_centroid, | |
| 581 | 846 | .first_posting = c->base.first_posting, | |
| 582 | 846 | .ncentroid_pages = c->base.ncentroid_pages, | |
| 583 | 846 | .centroid_format = c->base.centroid_format, | |
| 584 | 846 | .has_fastscan = c->has_fastscan, | |
| 585 | }; | ||
| 586 | } | ||
| 587 |