| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /* | ||
| 2 | * Copyright (c) 2026 Tiger Data, Inc. | ||
| 3 | * Licensed under the PostgreSQL License. See LICENSE for details. | ||
| 4 | * | ||
| 5 | * amcache.c - Per-index cached state for prism | ||
| 6 | * | ||
| 7 | * The rotation matrix P is O(dim³) to generate, so it lives in a | ||
| 8 | * process-local, reference-counted hash table (keyed by dim+seed) that | ||
| 9 | * survives relcache invalidation — see get_or_create_params / | ||
| 10 | * prism_release_params below. The table and the matrices it owns live in a | ||
| 11 | * dedicated child context of CacheMemoryContext, so the cache's footprint is | ||
| 12 | * visible as its own line in a memory-context dump. The global mean, | ||
| 13 | * P^T·global_mean, and an immutable PrismIndexBase template live in rd_amcache | ||
| 14 | * (allocated in rd_indexcxt) and are cheap to re-populate when invalidated. | ||
| 15 | * The metadata page is read at most once per backend, when rd_amcache is first | ||
| 16 | * populated. | ||
| 17 | */ | ||
| 18 | |||
| 19 | #include <postgres.h> | ||
| 20 | |||
| 21 | #include <storage/bufmgr.h> | ||
| 22 | #include <utils/hsearch.h> | ||
| 23 | #include <utils/memutils.h> | ||
| 24 | #include <utils/resowner.h> | ||
| 25 | |||
| 26 | #include "amcache.h" | ||
| 27 | #include "core/log.h" | ||
| 28 | #include "meta.h" | ||
| 29 | #include "support_pg.h" | ||
| 30 | #include "typeinfo.h" | ||
| 31 | |||
| 32 | /* ---------------------------------------------------------------- | ||
| 33 | * Process-local RaBitQParams cache | ||
| 34 | * | ||
| 35 | * A backend can have several scans of differently-dimensioned (or | ||
| 36 | * differently-seeded) indexes open at once (e.g. a join across two | ||
| 37 | * vector-indexed tables). A fixed number of slots can't represent that | ||
| 38 | * without either dangling a live scan's pointer (freeing an in-use entry to | ||
| 39 | * make room for another) or hard-erroring once every slot is checked out. | ||
| 40 | * | ||
| 41 | * Instead this is a backend-private dynahash table (plain hash_create() in | ||
| 42 | * rabitq_cache_cxt, a child of CacheMemoryContext -- not shared memory, so | ||
| 43 | * no shared_preload_libraries entry is needed, and nothing here is shared | ||
| 44 | * across backends) keyed by dim+seed, sized to grow as needed: | ||
| 45 | * | ||
| 46 | * - get_or_create_params() looks up (dim, seed). On a hit, it bumps the | ||
| 47 | * entry's refcount and a decaying usage score and returns the cached | ||
| 48 | * matrix. On a miss, once the table is at or past a soft target size, | ||
| 49 | * it first tries to reclaim one idle (refcount == 0) entry -- the one | ||
| 50 | * with the lowest decayed usage -- before generating a new matrix. A | ||
| 51 | * held (refcount > 0) entry is *never* an eviction candidate, so a | ||
| 52 | * checked-out RaBitQParams* stays valid for as long as its owner holds | ||
| 53 | * it, no matter how many other dimensions get checked out around it. | ||
| 54 | * If nothing is currently idle, the table simply grows past the soft | ||
| 55 | * target rather than failing a live caller -- the target is not a | ||
| 56 | * hard cap. | ||
| 57 | * - prism_release_params() checks an entry back in (refcount--). | ||
| 58 | * Checkouts are additionally tracked by the checkout-time resource | ||
| 59 | * owner, so error paths that skip the release still return the | ||
| 60 | * refcount -- see rabitq_params_ref_desc below. | ||
| 61 | * ---------------------------------------------------------------- */ | ||
| 62 | |||
| 63 | /* Above this many live entries, try to reclaim an idle one before growing | ||
| 64 | * further. Not a hard cap -- see above. */ | ||
| 65 | #define PRISM_RABITQ_CACHE_TARGET_ENTRIES 8 | ||
| 66 | |||
| 67 | #define PRISM_RABITQ_USAGE_INCREMENT 1.0 | ||
| 68 | #define PRISM_RABITQ_USAGE_DECAY 0.99 | ||
| 69 | |||
| 70 | typedef struct RaBitQCacheKey | ||
| 71 | { | ||
| 72 | Dimension dim; | ||
| 73 | uint64_t seed; | ||
| 74 | } RaBitQCacheKey; | ||
| 75 | |||
| 76 | typedef struct RaBitQCacheEntry | ||
| 77 | { | ||
| 78 | RaBitQCacheKey key; /* hash key; must be first */ | ||
| 79 | RaBitQParams *params; /* NULL until fully initialized (see below) */ | ||
| 80 | int refcount; | ||
| 81 | double usage; | ||
| 82 | } RaBitQCacheEntry; | ||
| 83 | |||
| 84 | static HTAB *rabitq_cache = NULL; | ||
| 85 | |||
| 86 | /* Owns the hash table and every cached matrix; child of | ||
| 87 | * CacheMemoryContext, so it has backend lifetime but shows up as its own | ||
| 88 | * line in a memory-context dump. */ | ||
| 89 | static MemoryContext rabitq_cache_cxt = NULL; | ||
| 90 | |||
| 91 | /* | ||
| 92 | * Error safety for the refcounts: every checkout is also registered | ||
| 93 | * with the resource owner current at checkout time. If an error keeps | ||
| 94 | * the matching release from running -- an aborted scan's portal is | ||
| 95 | * dropped without amendscan, or an insert errors between checkout and | ||
| 96 | * release -- the owner's release sweep calls back here and returns the | ||
| 97 | * refcount, so no entry is left unevictable. A checkout leaked on the | ||
| 98 | * commit path additionally gets PostgreSQL's standard "resource was not | ||
| 99 | * closed" warning, turning a pairing bug into a visible failure. | ||
| 100 | */ | ||
| 101 | static void rabitq_params_ref_release(Datum res); | ||
| 102 | |||
| 103 | static const ResourceOwnerDesc rabitq_params_ref_desc = { | ||
| 104 | .name = "prism rabitq params ref", | ||
| 105 | .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS, | ||
| 106 | .release_priority = RELEASE_PRIO_FIRST, | ||
| 107 | .ReleaseResource = rabitq_params_ref_release, | ||
| 108 | .DebugPrint = NULL, | ||
| 109 | }; | ||
| 110 | |||
| 111 | static void | ||
| 112 | 5 | rabitq_params_ref_release(Datum res) | |
| 113 | { | ||
| 114 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 5 times.
|
5 | RaBitQCacheEntry *entry = (RaBitQCacheEntry *)DatumGetPointer(res); |
| 115 | |||
| 116 | /* Only reached for checkouts no normal release forgot: error paths. */ | ||
| 117 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 5 times.
|
5 | Assert(entry->refcount > 0); |
| 118 | 5 | if (entry->refcount > 0) | |
| 119 | 5 | entry->refcount--; | |
| 120 | 5 | } | |
| 121 | |||
| 122 | static void | ||
| 123 | 74 | rabitq_cache_init(void) | |
| 124 | { | ||
| 125 | 74 | HASHCTL ctl; | |
| 126 | |||
| 127 | /* The context survives prism_rabitq_cache_clear() resets. */ | ||
| 128 |
2/2✓ Branch 0 taken 73 times.
✓ Branch 1 taken 1 times.
|
74 | if (rabitq_cache_cxt == NULL) |
| 129 | 73 | rabitq_cache_cxt = AllocSetContextCreate( | |
| 130 | CacheMemoryContext, | ||
| 131 | "prism rabitq params cache", | ||
| 132 | ALLOCSET_DEFAULT_SIZES); | ||
| 133 | |||
| 134 | 74 | memset(&ctl, 0, sizeof(ctl)); | |
| 135 | 74 | ctl.keysize = sizeof(RaBitQCacheKey); | |
| 136 | 74 | ctl.entrysize = sizeof(RaBitQCacheEntry); | |
| 137 | 74 | ctl.hcxt = rabitq_cache_cxt; | |
| 138 | |||
| 139 | 74 | rabitq_cache = hash_create( | |
| 140 | "prism rabitq params cache", | ||
| 141 | PRISM_RABITQ_CACHE_TARGET_ENTRIES, | ||
| 142 | &ctl, | ||
| 143 | HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); | ||
| 144 | 74 | } | |
| 145 | |||
| 146 | /* | ||
| 147 | * Decay every entry's usage score and reclaim the least-used entry with no | ||
| 148 | * live checkouts, if one exists. A no-op (not an error) when every entry is | ||
| 149 | * currently held -- the caller grows the table instead. | ||
| 150 | */ | ||
| 151 | static void | ||
| 152 | 2 | rabitq_cache_evict_one(void) | |
| 153 | { | ||
| 154 | 2 | HASH_SEQ_STATUS seq; | |
| 155 | 2 | RaBitQCacheEntry *entry; | |
| 156 | 2 | RaBitQCacheEntry *victim = NULL; | |
| 157 | |||
| 158 | 2 | hash_seq_init(&seq, rabitq_cache); | |
| 159 |
2/2✓ Branch 2 taken 16 times.
✓ Branch 3 taken 2 times.
|
18 | while ((entry = hash_seq_search(&seq)) != NULL) |
| 160 | { | ||
| 161 | 16 | entry->usage *= PRISM_RABITQ_USAGE_DECAY; | |
| 162 |
4/4✓ Branch 0 taken 1 times.
✓ Branch 1 taken 15 times.
✓ Branch 2 taken 13 times.
✓ Branch 3 taken 2 times.
|
16 | if (entry->refcount == 0 && |
| 163 |
2/2✓ Branch 0 taken 11 times.
✓ Branch 1 taken 2 times.
|
13 | (victim == NULL || entry->usage < victim->usage)) |
| 164 | 16 | victim = entry; | |
| 165 | } | ||
| 166 | /* The scan above always runs to completion (hash_seq_search returned | ||
| 167 | * NULL) before anything below mutates the table -- deleting mid-scan | ||
| 168 | * is only safe for the entry hash_seq_search just returned, which is | ||
| 169 | * why the victim is removed after, not during, the loop. */ | ||
| 170 | |||
| 171 |
1/2✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
|
2 | if (victim != NULL) |
| 172 | { | ||
| 173 | 2 | pfree(victim->params); | |
| 174 | 2 | hash_search(rabitq_cache, &victim->key, HASH_REMOVE, NULL); | |
| 175 | } | ||
| 176 | 2 | } | |
| 177 | |||
| 178 | static RaBitQParams * | ||
| 179 | 17772 | get_or_create_params(Dimension dim, uint64_t seed) | |
| 180 | { | ||
| 181 | 17772 | RaBitQCacheKey key = {0}; /* zero incl. padding: HASH_BLOBS compares | |
| 182 | * the whole struct */ | ||
| 183 | 17772 | RaBitQCacheEntry *entry; | |
| 184 | 17772 | bool found; | |
| 185 | |||
| 186 |
2/2✓ Branch 0 taken 74 times.
✓ Branch 1 taken 17698 times.
|
17772 | if (rabitq_cache == NULL) |
| 187 | 74 | rabitq_cache_init(); | |
| 188 | |||
| 189 | 17772 | key.dim = dim; | |
| 190 | 17772 | key.seed = seed; | |
| 191 | |||
| 192 | 17772 | entry = hash_search(rabitq_cache, &key, HASH_FIND, &found); | |
| 193 |
3/4✓ Branch 0 taken 17677 times.
✓ Branch 1 taken 95 times.
✓ Branch 2 taken 17677 times.
✗ Branch 3 not taken.
|
17772 | if (found && entry->params != NULL) |
| 194 | { | ||
| 195 | /* Enlarge first: it can allocate, and nothing may fail between | ||
| 196 | * the refcount bump and the (no-fail) Remember. */ | ||
| 197 | 17677 | ResourceOwnerEnlarge(CurrentResourceOwner); | |
| 198 | 17677 | entry->usage += PRISM_RABITQ_USAGE_INCREMENT; | |
| 199 | 17677 | entry->refcount++; | |
| 200 | 17677 | ResourceOwnerRemember( | |
| 201 | CurrentResourceOwner, | ||
| 202 | PointerGetDatum(entry), | ||
| 203 | &rabitq_params_ref_desc); | ||
| 204 | 17677 | return entry->params; | |
| 205 | } | ||
| 206 | |||
| 207 | /* Miss (or a dead entry left by a failed init below -- params == NULL, | ||
| 208 | * refcount == 0, safe to recreate): try to make room first, but only | ||
| 209 | * when this is about to grow the table with a genuinely new key. */ | ||
| 210 |
3/4✓ Branch 0 taken 95 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 2 times.
✓ Branch 4 taken 93 times.
|
95 | if (!found && hash_get_num_entries(rabitq_cache) >= |
| 211 | PRISM_RABITQ_CACHE_TARGET_ENTRIES) | ||
| 212 | 2 | rabitq_cache_evict_one(); | |
| 213 | |||
| 214 | 95 | entry = hash_search(rabitq_cache, &key, HASH_ENTER, &found); | |
| 215 | |||
| 216 | /* Mark it dead/pending before any allocation that could throw: a | ||
| 217 | * failure below then leaves a well-defined, idle, re-creatable entry | ||
| 218 | * instead of one that looks live with garbage params. The matrix is | ||
| 219 | * generated into a local pointer and only published once fully | ||
| 220 | * initialized, for the same reason (an error out of vs_rabitq_init | ||
| 221 | * must not leave a live-looking entry holding a garbage matrix). */ | ||
| 222 | 95 | entry->params = NULL; | |
| 223 | 95 | entry->refcount = 0; | |
| 224 | 95 | entry->usage = 0; | |
| 225 | |||
| 226 | 95 | ResourceOwnerEnlarge(CurrentResourceOwner); | |
| 227 | |||
| 228 | 95 | RaBitQParams *params = | |
| 229 | 95 | MemoryContextAlloc(rabitq_cache_cxt, VS_RABITQ_PARAMS_SIZE(dim)); | |
| 230 | 95 | vs_rabitq_init(params, dim, seed); | |
| 231 | |||
| 232 | 95 | entry->params = params; | |
| 233 | 95 | entry->refcount = 1; | |
| 234 | 95 | entry->usage = PRISM_RABITQ_USAGE_INCREMENT; | |
| 235 | 95 | ResourceOwnerRemember( | |
| 236 | CurrentResourceOwner, | ||
| 237 | PointerGetDatum(entry), | ||
| 238 | &rabitq_params_ref_desc); | ||
| 239 | |||
| 240 | 95 | return entry->params; | |
| 241 | } | ||
| 242 | |||
| 243 | /* | ||
| 244 | * Check an entry back in (public API; see amcache.h). dim+seed must | ||
| 245 | * match a currently-held entry exactly as returned by a prior | ||
| 246 | * get_or_create_params call; a mismatch indicates a caller bug, not a | ||
| 247 | * runtime condition. | ||
| 248 | */ | ||
| 249 | void | ||
| 250 | 17767 | prism_release_params(Dimension dim, uint64_t seed, ResourceOwner owner) | |
| 251 | { | ||
| 252 | 17767 | RaBitQCacheKey key = {0}; | |
| 253 | 17767 | RaBitQCacheEntry *entry; | |
| 254 | 17767 | bool found; | |
| 255 | |||
| 256 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 17767 times.
|
17767 | if (rabitq_cache == NULL) |
| 257 | { | ||
| 258 | ✗ | Assert(false); | |
| 259 | 17767 | return; | |
| 260 | } | ||
| 261 | |||
| 262 | 17767 | key.dim = dim; | |
| 263 | 17767 | key.seed = seed; | |
| 264 | |||
| 265 | 17767 | entry = hash_search(rabitq_cache, &key, HASH_FIND, &found); | |
| 266 |
1/2✓ Branch 0 taken 17767 times.
✗ Branch 1 not taken.
|
17767 | if (found) |
| 267 | { | ||
| 268 | /* Forget first: it errors on a pairing bug (wrong owner), and | ||
| 269 | * the refcount must stay consistent with the registrations. */ | ||
| 270 | 17767 | ResourceOwnerForget( | |
| 271 | owner, PointerGetDatum(entry), &rabitq_params_ref_desc); | ||
| 272 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 17767 times.
|
17767 | Assert(entry->refcount > 0); |
| 273 | 17767 | if (entry->refcount > 0) | |
| 274 | 17767 | entry->refcount--; | |
| 275 | 17767 | return; | |
| 276 | } | ||
| 277 | |||
| 278 | /* Should be unreachable: every release_params call is paired with a | ||
| 279 | * prior successful get_or_create_params for the same dim/seed, and a | ||
| 280 | * held entry is never evicted. */ | ||
| 281 | ✗ | Assert(false); | |
| 282 | } | ||
| 283 | |||
| 284 | /* | ||
| 285 | * Test support: snapshot the cache into a caller-provided array and | ||
| 286 | * return the number of entries written. A plain exported symbol (no SQL | ||
| 287 | * surface): the test-only module test/pg/src/test_helpers.c | ||
| 288 | * wraps it in a set-returning function so the regression tests can | ||
| 289 | * observe refcounts, usage decay, and eviction. Inert otherwise. | ||
| 290 | */ | ||
| 291 | int | ||
| 292 | 18 | prism_rabitq_cache_stats(PrismRabitqCacheStat *stats, int max_stats) | |
| 293 | { | ||
| 294 | 18 | HASH_SEQ_STATUS seq; | |
| 295 | 18 | RaBitQCacheEntry *entry; | |
| 296 | 18 | int n = 0; | |
| 297 | |||
| 298 |
2/2✓ Branch 0 taken 17 times.
✓ Branch 1 taken 1 times.
|
18 | if (rabitq_cache == NULL) |
| 299 | return 0; | ||
| 300 | |||
| 301 | /* The scan must run to completion (see rabitq_cache_evict_one). */ | ||
| 302 | 17 | hash_seq_init(&seq, rabitq_cache); | |
| 303 |
2/2✓ Branch 2 taken 136 times.
✓ Branch 3 taken 17 times.
|
153 | while ((entry = hash_seq_search(&seq)) != NULL) |
| 304 | { | ||
| 305 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 136 times.
|
136 | if (n >= max_stats) |
| 306 | ✗ | continue; | |
| 307 | 136 | stats[n].dim = (int32_t)entry->key.dim; | |
| 308 | 136 | stats[n].refcount = entry->refcount; | |
| 309 | 136 | stats[n].usage = entry->usage; | |
| 310 | 136 | n++; | |
| 311 | } | ||
| 312 | return n; | ||
| 313 | } | ||
| 314 | |||
| 315 | /* | ||
| 316 | * Test support: reset the cache to its initial state, freeing the hash | ||
| 317 | * table and every cached matrix in one context reset (they all live in | ||
| 318 | * rabitq_cache_cxt). Returns the number of entries dropped. Refuses if | ||
| 319 | * any entry is currently checked out -- freeing a held matrix is | ||
| 320 | * exactly the use-after-free this cache exists to prevent. Exported | ||
| 321 | * for the test-only module test/pg/src/test_helpers.c; inert otherwise. | ||
| 322 | */ | ||
| 323 | int | ||
| 324 | 1 | prism_rabitq_cache_clear(void) | |
| 325 | { | ||
| 326 | 1 | HASH_SEQ_STATUS seq; | |
| 327 | 1 | RaBitQCacheEntry *entry; | |
| 328 | 1 | int n = 0; | |
| 329 | |||
| 330 |
1/2✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
|
1 | if (rabitq_cache == NULL) |
| 331 | return 0; | ||
| 332 | |||
| 333 | 1 | hash_seq_init(&seq, rabitq_cache); | |
| 334 |
2/2✓ Branch 2 taken 4 times.
✓ Branch 3 taken 1 times.
|
5 | while ((entry = hash_seq_search(&seq)) != NULL) |
| 335 | { | ||
| 336 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
|
4 | if (entry->refcount > 0) |
| 337 | { | ||
| 338 | ✗ | hash_seq_term(&seq); | |
| 339 | ✗ | vs_error( | |
| 340 | "cannot clear the RaBitQ params cache: entry for " | ||
| 341 | "dimension %u has %d live checkout(s)", | ||
| 342 | (unsigned)entry->key.dim, | ||
| 343 | entry->refcount); | ||
| 344 | } | ||
| 345 | 4 | n++; | |
| 346 | } | ||
| 347 | |||
| 348 | 1 | rabitq_cache = NULL; | |
| 349 | 1 | MemoryContextReset(rabitq_cache_cxt); | |
| 350 | 1 | return n; | |
| 351 | } | ||
| 352 | |||
| 353 | /* ---------------------------------------------------------------- | ||
| 354 | * rd_amcache layout | ||
| 355 | * | ||
| 356 | * A fully-populated immutable PrismIndexBase template (params + storage left | ||
| 357 | * for the per-call rebind), the scan-planning scalars, and the inline | ||
| 358 | * global_mean + pt_global_mean vectors appended after the struct. | ||
| 359 | * | ||
| 360 | * The metapage-derived scalars (dim, metric, nlist, ...) are filled eagerly on | ||
| 361 | * first access. The rotated pt_global_mean — which needs the O(dim³) rotation | ||
| 362 | * matrix — is computed lazily (pt_ready) only when a caller actually needs it | ||
| 363 | * (the scan / insert path via prism_index_base_init), so metadata-only | ||
| 364 | * consumers like VACUUM's ambulkdelete get dim/metric from the cache without | ||
| 365 | * paying for rotation setup. | ||
| 366 | * | ||
| 367 | * Two separate constraints shape this, and they pull in different directions. | ||
| 368 | * | ||
| 369 | * ONE ALLOCATION, and that is not stylistic. PostgreSQL frees rd_amcache with | ||
| 370 | * a single targeted pfree in RelationInvalidateRelation, which does NOT delete | ||
| 371 | * rd_indexcxt — the relation survives with rd_isvalid = false. So a sub-object | ||
| 372 | * hung off this blob would leak into rd_indexcxt once per invalidation cycle. | ||
| 373 | * Relcache's own caches (RelationGetIndexList and friends) can be lazy | ||
| 374 | * per-allocation because relcache wrote per-field cleanup for each of them; an | ||
| 375 | * access method gets one line of cleanup written for it, so it gets one | ||
| 376 | * allocation. Nor can the blob be grown and swapped later: PrismIndexBase | ||
| 377 | * .pt_global_mean points into it and a live scan holds that pointer for the | ||
| 378 | * duration of the scan. | ||
| 379 | * | ||
| 380 | * LAZY CONTENT, which the single allocation does not prevent. The metapage | ||
| 381 | * read is the floor — dim comes from it and dim sizes the blob — but the work | ||
| 382 | * that follows it need not be eager. The rule for a new field: fill it here if | ||
| 383 | * computing it is cheap or it needs the pinned metapage buffer (the | ||
| 384 | * global_mean memcpy is both); otherwise give it a ready-flag and compute it | ||
| 385 | * on first use, as pt_global_mean does. Note that anything living in this blob | ||
| 386 | * inherits the metapage read as a precondition even when it would not | ||
| 387 | * otherwise need one -- type_info, which comes from the opclass, is the case | ||
| 388 | * in point. | ||
| 389 | * ---------------------------------------------------------------- */ | ||
| 390 | |||
| 391 | typedef struct AmCacheData | ||
| 392 | { | ||
| 393 | PrismIndexBase base; /* immutable template; params / fastscan / | ||
| 394 | * storage left zeroed (rebound per call) */ | ||
| 395 | const PrismIndexTypeInfo *type_info; /* indexed column's type, from the | ||
| 396 | * opclass */ | ||
| 397 | bool has_fastscan; /* index built with FASTSCAN posting pages */ | ||
| 398 | bool pt_ready; /* pt_global_mean computed (rotation done) */ | ||
| 399 | uint32_t nlist; | ||
| 400 | uint32_t global_mean_off; | ||
| 401 | uint32_t pt_global_mean_off; | ||
| 402 | } AmCacheData; | ||
| 403 | |||
| 404 | static inline float * | ||
| 405 | 450 | cache_global_mean(AmCacheData *c) | |
| 406 | { | ||
| 407 | 450 | return (float *)((char *)c + c->global_mean_off); | |
| 408 | } | ||
| 409 | |||
| 410 | static inline float * | ||
| 411 | 414 | cache_pt_global_mean(AmCacheData *c) | |
| 412 | { | ||
| 413 | 414 | return (float *)((char *)c + c->pt_global_mean_off); | |
| 414 | } | ||
| 415 | |||
| 416 | /* | ||
| 417 | * Return the per-backend cache, populating rd_amcache (one metapage read) on | ||
| 418 | * first use. pt_global_mean is stored in the cache; params are NOT frozen | ||
| 419 | * here (the process-local params cache is reference-counted per checkout, | ||
| 420 | * not per-index) — callers rebind via get_or_create_params / | ||
| 421 | * prism_release_params. | ||
| 422 | */ | ||
| 423 | static AmCacheData * | ||
| 424 | 36478 | get_cache_data(Relation index) | |
| 425 | { | ||
| 426 |
2/2✓ Branch 0 taken 243 times.
✓ Branch 1 taken 36235 times.
|
36478 | if (index->rd_amcache != NULL) |
| 427 | return (AmCacheData *)index->rd_amcache; | ||
| 428 | |||
| 429 | /* Read the metadata page once. */ | ||
| 430 | 243 | Buffer meta_buf = ReadBuffer(index, 0); | |
| 431 | 243 | LockBuffer(meta_buf, BUFFER_LOCK_SHARE); | |
| 432 | 243 | Page meta_page = BufferGetPage(meta_buf); | |
| 433 | |||
| 434 | 243 | const PrismMetaPage *meta = (const PrismMetaPage *)PageGetSpecialPointer( | |
| 435 | meta_page); | ||
| 436 | /* Reject an index whose metapage was written by an incompatible format | ||
| 437 | * (loud in release too, not just a debug Assert) — its layout would | ||
| 438 | * otherwise be misread. */ | ||
| 439 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 243 times.
|
243 | if (meta->magic != PRISM_META_MAGIC) |
| 440 | { | ||
| 441 | ✗ | uint32_t got = meta->magic; | |
| 442 | ✗ | UnlockReleaseBuffer(meta_buf); | |
| 443 | ✗ | ereport(ERROR, | |
| 444 | (errcode(ERRCODE_INDEX_CORRUPTED), | ||
| 445 | errmsg("index \"%s\" has an incompatible on-disk format " | ||
| 446 | "(metapage magic 0x%08X, expected 0x%08X)", | ||
| 447 | RelationGetRelationName(index), | ||
| 448 | got, | ||
| 449 | (uint32_t)PRISM_META_MAGIC), | ||
| 450 | errhint("REINDEX the index to rebuild it in the current " | ||
| 451 | "format."))); | ||
| 452 | } | ||
| 453 | |||
| 454 | 243 | Dimension dim = meta->dim; | |
| 455 | 243 | uint64_t seed = meta->rabitq_seed; | |
| 456 | |||
| 457 | 243 | uint32_t gm_off = MAXALIGN(sizeof(AmCacheData)); | |
| 458 | 243 | uint32_t pt_gm_off = gm_off + dim * sizeof(float); | |
| 459 | 243 | Size total = pt_gm_off + dim * sizeof(float); | |
| 460 | |||
| 461 | 243 | MemoryContext old = MemoryContextSwitchTo(index->rd_indexcxt); | |
| 462 | 243 | AmCacheData *c = palloc0(total); | |
| 463 | 243 | MemoryContextSwitchTo(old); | |
| 464 | |||
| 465 | 243 | c->global_mean_off = gm_off; | |
| 466 | 243 | c->pt_global_mean_off = pt_gm_off; | |
| 467 | 243 | c->has_fastscan = (meta->flags & PRISM_META_FLAG_FASTSCAN) != 0; | |
| 468 | 243 | c->nlist = meta->nlist; | |
| 469 | |||
| 470 | /* Immutable base template (storage / params / fastscan rebound per call). | ||
| 471 | */ | ||
| 472 | 243 | c->base.dim = dim; | |
| 473 | 243 | c->base.nlevels = meta->nlevels; | |
| 474 | 243 | c->base.fan_out = meta->fan_out; | |
| 475 | 243 | c->base.nlist = meta->nlist; | |
| 476 | 243 | c->base.first_centroid = meta->first_centroid; | |
| 477 | 243 | c->base.first_posting = meta->first_posting; | |
| 478 | 243 | c->base.ncentroid_pages = meta->ncentroid_pages; | |
| 479 | 243 | c->base.metric = (DistanceMetric)meta->metric; | |
| 480 | 243 | c->base.centroid_format = (PrismCentroidFormat)meta->centroid_format; | |
| 481 | 243 | c->base.rabitq_seed = seed; | |
| 482 | /* base.pt_global_mean stays NULL until prism_index_base_init computes it. | ||
| 483 | */ | ||
| 484 | |||
| 485 | /* Copy global mean; P^T * global_mean is computed lazily (see | ||
| 486 | * prism_index_base_init) so metadata-only callers skip the rotation. */ | ||
| 487 | 243 | memcpy(cache_global_mean(c), | |
| 488 | 243 | prism_meta_global_mean_const(meta), | |
| 489 | dim * sizeof(float)); | ||
| 490 | |||
| 491 | 243 | UnlockReleaseBuffer(meta_buf); | |
| 492 | |||
| 493 | /* Opclass-derived, so no metapage needed -- but resolved here so every | ||
| 494 | * post-build path reads it as a pointer instead of an fmgr call. */ | ||
| 495 | 243 | c->type_info = prism_index_type_info(index); | |
| 496 | |||
| 497 | 243 | index->rd_amcache = c; | |
| 498 | 243 | return c; | |
| 499 | } | ||
| 500 | |||
| 501 | /* ---------------------------------------------------------------- | ||
| 502 | * Public API | ||
| 503 | * ---------------------------------------------------------------- */ | ||
| 504 | |||
| 505 | void | ||
| 506 | 17772 | prism_index_base_init(Relation index, PrismIndexBase *base) | |
| 507 | { | ||
| 508 | 17772 | AmCacheData *c = get_cache_data(index); | |
| 509 | 17772 | RaBitQParams *params = | |
| 510 | 17772 | get_or_create_params(c->base.dim, c->base.rabitq_seed); | |
| 511 | |||
| 512 | /* Compute the rotated global mean on first use (needs the rotation | ||
| 513 | * matrix); cached thereafter for the backend. */ | ||
| 514 |
2/2✓ Branch 0 taken 207 times.
✓ Branch 1 taken 17565 times.
|
17772 | if (!c->pt_ready) |
| 515 | { | ||
| 516 | 207 | vs_rabitq_rotate( | |
| 517 | 207 | params, cache_global_mean(c), cache_pt_global_mean(c)); | |
| 518 | 207 | c->base.pt_global_mean = cache_pt_global_mean(c); | |
| 519 | 207 | c->pt_ready = true; | |
| 520 | } | ||
| 521 | |||
| 522 | /* Copy the immutable template, then rebind the fields that cannot be | ||
| 523 | * frozen for the backend's lifetime: params (checked out from the | ||
| 524 | * refcounted process-local cache; the caller must pair this with a | ||
| 525 | * matching prism_release_params(base->dim, base->rabitq_seed) once it's | ||
| 526 | * done with base->params) and fastscan (resolved from the session GUC). | ||
| 527 | * Storage pointers stay zeroed for the caller. */ | ||
| 528 | 17772 | *base = c->base; | |
| 529 | 17772 | base->params = params; | |
| 530 |
2/2✓ Branch 0 taken 14879 times.
✓ Branch 1 taken 2893 times.
|
17772 | base->fastscan = c->has_fastscan ? prism_fastscan_bits : 0; |
| 531 | 17772 | base->centroid_error_scale = (float)prism_centroid_error_scale; | |
| 532 | 17772 | base->centroid_beam_scale = (float)prism_centroid_beam_scale; | |
| 533 | 17772 | } | |
| 534 | |||
| 535 | void | ||
| 536 | 20 | prism_cache_meta( | |
| 537 | Relation index, | ||
| 538 | Dimension *dim, | ||
| 539 | DistanceMetric *metric, | ||
| 540 | BlockNumber *first_posting) | ||
| 541 | { | ||
| 542 | 20 | AmCacheData *c = get_cache_data(index); | |
| 543 | 20 | *dim = c->base.dim; | |
| 544 | 20 | *metric = c->base.metric; | |
| 545 | 20 | *first_posting = c->base.first_posting; | |
| 546 | 20 | } | |
| 547 | |||
| 548 | const PrismIndexTypeInfo * | ||
| 549 | 18037 | prism_cache_type_info(Relation index) | |
| 550 | { | ||
| 551 | 18037 | return get_cache_data(index)->type_info; | |
| 552 | } | ||
| 553 | |||
| 554 | PrismScanInfo | ||
| 555 | 649 | prism_cache_scan_info(Relation index) | |
| 556 | { | ||
| 557 | 649 | AmCacheData *c = get_cache_data(index); | |
| 558 | 649 | return (PrismScanInfo){ | |
| 559 | 649 | .nlist = c->nlist, | |
| 560 | 649 | .dim = c->base.dim, | |
| 561 | 649 | .nlevels = c->base.nlevels, | |
| 562 | 649 | .fan_out = c->base.fan_out, | |
| 563 | 649 | .first_centroid = c->base.first_centroid, | |
| 564 | 649 | .first_posting = c->base.first_posting, | |
| 565 | 649 | .ncentroid_pages = c->base.ncentroid_pages, | |
| 566 | 649 | .centroid_format = c->base.centroid_format, | |
| 567 | 649 | .has_fastscan = c->has_fastscan, | |
| 568 | }; | ||
| 569 | } | ||
| 570 |