| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /* | ||
| 2 | * Copyright (c) 2026 Tiger Data, Inc. | ||
| 3 | * Licensed under the PostgreSQL License. See LICENSE for details. | ||
| 4 | * | ||
| 5 | * build_progress.c - PostgreSQL body of the build-progress reporting seam. | ||
| 6 | * | ||
| 7 | * Live phase + % go to pg_stat_progress_create_index (always). Per-phase | ||
| 8 | * resource stats + summary go to the server log only when | ||
| 9 | * prism.log_build_stats is on, modeled on core btree's log_btree_build_stats | ||
| 10 | * (ResetUsage/ShowUsage). The planned-allocation line is always emitted so an | ||
| 11 | * OOM is pre-explained. The standalone no-op bodies live in | ||
| 12 | * src/index/build_progress.c. | ||
| 13 | */ | ||
| 14 | |||
| 15 | #include <postgres.h> | ||
| 16 | |||
| 17 | #include "vs_config.h" | ||
| 18 | |||
| 19 | #include <access/parallel.h> | ||
| 20 | #include <commands/progress.h> | ||
| 21 | #include <miscadmin.h> | ||
| 22 | #include <pgstat.h> | ||
| 23 | #include <portability/instr_time.h> | ||
| 24 | #include <tcop/tcopprot.h> /* ResetUsage / ShowUsage */ | ||
| 25 | #include <utils/backend_progress.h> | ||
| 26 | #include <utils/injection_point.h> | ||
| 27 | #include <utils/memutils.h> | ||
| 28 | |||
| 29 | #include "index/build_progress.h" | ||
| 30 | #include "index/index_build.h" /* PrismBuildStats */ | ||
| 31 | |||
| 32 | static int64 | ||
| 33 | 4290 | now_ns(void) | |
| 34 | { | ||
| 35 | 4290 | instr_time t; | |
| 36 | |||
| 37 | 4290 | INSTR_TIME_SET_CURRENT(t); | |
| 38 | 4290 | return INSTR_TIME_GET_NANOSEC(t); | |
| 39 | } | ||
| 40 | |||
| 41 | /* | ||
| 42 | * Test hook name fired at each phase boundary, so an isolation test can pause | ||
| 43 | * a build at any phase and read pg_stat_progress_create_index. SCAN and | ||
| 44 | * SCAN_PARALLEL share the long-standing "prism-build-load" name (the | ||
| 45 | * build_progress isolation test depends on it); the other phases get their | ||
| 46 | * own. Returns NULL for phases without a hook. No-op unless PG was built with | ||
| 47 | * injection points and a test attached an action. | ||
| 48 | */ | ||
| 49 | static const char * | ||
| 50 | 1771 | injection_name_for_phase(int phase) | |
| 51 | { | ||
| 52 |
7/7✓ Branch 0 taken 250 times.
✓ Branch 1 taken 20 times.
✓ Branch 2 taken 5 times.
✓ Branch 3 taken 249 times.
✓ Branch 4 taken 249 times.
✓ Branch 5 taken 748 times.
✓ Branch 6 taken 250 times.
|
1771 | switch (phase) |
| 53 | { | ||
| 54 | case PRISM_BUILD_PHASE_SAMPLE: | ||
| 55 | return "prism-build-sample"; | ||
| 56 | 250 | case PRISM_BUILD_PHASE_KMEANS: | |
| 57 | 250 | return "prism-build-kmeans"; | |
| 58 | 20 | case PRISM_BUILD_PHASE_SUBTREES: | |
| 59 | 20 | return "prism-build-subtrees"; | |
| 60 | 5 | case PRISM_BUILD_PHASE_REFINE: | |
| 61 | 5 | return "prism-build-refine"; | |
| 62 | 249 | case PRISM_BUILD_PHASE_SCAN: | |
| 63 | case PRISM_BUILD_PHASE_SCAN_PARALLEL: | ||
| 64 | 249 | return "prism-build-load"; | |
| 65 | 249 | case PRISM_BUILD_PHASE_POSTING: | |
| 66 | 249 | return "prism-build-posting"; | |
| 67 | 748 | default: | |
| 68 | 748 | return NULL; | |
| 69 | } | ||
| 70 | } | ||
| 71 | |||
| 72 | /* Add a finished phase's elapsed time to the matching PrismBuildStats field. | ||
| 73 | */ | ||
| 74 | static void | ||
| 75 | 2020 | accumulate_stats(PrismBuildStats *s, int phase, double ms) | |
| 76 | { | ||
| 77 |
1/2✓ Branch 0 taken 2020 times.
✗ Branch 1 not taken.
|
2020 | if (s == NULL) |
| 78 | return; | ||
| 79 | |||
| 80 |
7/7✓ Branch 0 taken 250 times.
✓ Branch 1 taken 269 times.
✓ Branch 2 taken 5 times.
✓ Branch 3 taken 250 times.
✓ Branch 4 taken 498 times.
✓ Branch 5 taken 249 times.
✓ Branch 6 taken 499 times.
|
2020 | switch (phase) |
| 81 | { | ||
| 82 | 250 | case PRISM_BUILD_PHASE_SAMPLE: | |
| 83 | 250 | s->ms_sample += ms; | |
| 84 | 250 | break; | |
| 85 | 269 | case PRISM_BUILD_PHASE_KMEANS: | |
| 86 | case PRISM_BUILD_PHASE_SUBTREES: | ||
| 87 | 269 | s->ms_kmeans += ms; | |
| 88 | 269 | break; | |
| 89 | 5 | case PRISM_BUILD_PHASE_REFINE: | |
| 90 | 5 | s->ms_refine += ms; | |
| 91 | 5 | break; | |
| 92 | 250 | case PRISM_BUILD_PHASE_SETUP: | |
| 93 | 250 | s->ms_setup += ms; | |
| 94 | 250 | break; | |
| 95 | 498 | case PRISM_BUILD_PHASE_SCAN: | |
| 96 | case PRISM_BUILD_PHASE_SCAN_PARALLEL: | ||
| 97 | case PRISM_BUILD_PHASE_POSTING: | ||
| 98 | 498 | s->ms_posting += ms; | |
| 99 | 498 | break; | |
| 100 | 249 | case PRISM_BUILD_PHASE_CENTROID: | |
| 101 | 249 | s->ms_centroid += ms; | |
| 102 | 249 | break; | |
| 103 | default: | ||
| 104 | break; | ||
| 105 | } | ||
| 106 | } | ||
| 107 | |||
| 108 | /* Emit the just-finished phase's resource line(s) and reset the usage window. | ||
| 109 | */ | ||
| 110 | static void | ||
| 111 | 2020 | flush_phase(PrismBuildProgress *p) | |
| 112 | { | ||
| 113 | 2020 | double ms = (double)(now_ns() - p->phase_start_ns) / 1e6; | |
| 114 | |||
| 115 | 2020 | accumulate_stats(p->stats, p->cur_phase, ms); | |
| 116 | |||
| 117 |
4/4✓ Branch 0 taken 9 times.
✓ Branch 1 taken 2011 times.
✓ Branch 2 taken 8 times.
✓ Branch 3 taken 1 times.
|
2020 | if (!p->log_stats || p->cur_phase == 0) |
| 118 | return; | ||
| 119 | |||
| 120 | 8 | double heap_mb = p->heap_ctx ? (double)MemoryContextMemAllocated( | |
| 121 | (MemoryContext)p->heap_ctx, true) / | ||
| 122 | (1024.0 * 1024.0) | ||
| 123 |
1/2✓ Branch 0 taken 8 times.
✗ Branch 1 not taken.
|
8 | : 0.0; |
| 124 | |||
| 125 |
1/2✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
|
8 | ereport(LOG, |
| 126 | (errmsg(VS_AM_NAME " build: phase \"%s\" done in %.0f ms " | ||
| 127 | "(build heap %.1f MB, DSM %.1f MB)", | ||
| 128 | prism_build_phase_name(p->cur_phase), | ||
| 129 | ms, | ||
| 130 | heap_mb, | ||
| 131 | (double)p->dsm_bytes / (1024.0 * 1024.0)), | ||
| 132 | errhidestmt(true))); | ||
| 133 | |||
| 134 | /* btree-style CPU + maxrss for the phase, then reset for the next one. */ | ||
| 135 | 8 | ShowUsage(VS_AM_NAME " build phase resource usage"); | |
| 136 | 8 | ResetUsage(); | |
| 137 | } | ||
| 138 | |||
| 139 | void | ||
| 140 | 250 | prism_build_progress_begin( | |
| 141 | PrismBuildProgress *p, | ||
| 142 | bool is_parallel, | ||
| 143 | bool log_stats, | ||
| 144 | void *heap_ctx, | ||
| 145 | struct PrismBuildStats *stats, | ||
| 146 | double tuples_total) | ||
| 147 | { | ||
| 148 | 250 | int64 t = now_ns(); | |
| 149 | |||
| 150 | 250 | p->cur_phase = 0; | |
| 151 | 250 | p->phase_start_ns = t; | |
| 152 | 250 | p->build_start_ns = t; | |
| 153 | 250 | p->log_stats = log_stats; | |
| 154 | 250 | p->is_parallel = is_parallel; | |
| 155 | 250 | p->total_set = false; | |
| 156 | 250 | p->heap_ctx = heap_ctx; | |
| 157 | 250 | p->dsm_bytes = 0; | |
| 158 | 250 | p->tuples_total = tuples_total; | |
| 159 | 250 | p->stats = stats; | |
| 160 | |||
| 161 | /* Publish the row total up front so percent_complete is meaningful for the | ||
| 162 | * whole build (the parallel path has no leader-side loop to publish it | ||
| 163 | * mid-scan). */ | ||
| 164 |
2/2✓ Branch 0 taken 242 times.
✓ Branch 1 taken 8 times.
|
250 | if (tuples_total > 0) |
| 165 | { | ||
| 166 | 242 | pgstat_progress_update_param( | |
| 167 | 242 | PROGRESS_CREATEIDX_TUPLES_TOTAL, (int64)tuples_total); | |
| 168 | 242 | p->total_set = true; | |
| 169 | } | ||
| 170 | |||
| 171 |
2/2✓ Branch 0 taken 1 times.
✓ Branch 1 taken 249 times.
|
250 | if (log_stats) |
| 172 | 1 | ResetUsage(); | |
| 173 | 250 | } | |
| 174 | |||
| 175 | void | ||
| 176 | 1771 | prism_build_report_phase(PrismBuildProgress *p, int phase) | |
| 177 | { | ||
| 178 | 1771 | flush_phase(p); | |
| 179 | 1771 | p->cur_phase = phase; | |
| 180 | 1771 | p->phase_start_ns = now_ns(); | |
| 181 | 1771 | pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE, phase); | |
| 182 | |||
| 183 | /* Each scan-shaped phase walks the heap from the start, so its | ||
| 184 | * tuples-done count restarts; the phases in between leave the previous | ||
| 185 | * scan's final count standing. */ | ||
| 186 |
2/2✓ Branch 0 taken 499 times.
✓ Branch 1 taken 1272 times.
|
1771 | if (phase == PRISM_BUILD_PHASE_SAMPLE || phase == PRISM_BUILD_PHASE_SCAN || |
| 187 | phase == PRISM_BUILD_PHASE_SCAN_PARALLEL) | ||
| 188 | 499 | pgstat_progress_update_param(PROGRESS_CREATEIDX_TUPLES_DONE, 0); | |
| 189 | |||
| 190 | 1771 | const char *ip = injection_name_for_phase(phase); | |
| 191 |
2/2✓ Branch 0 taken 1023 times.
✓ Branch 1 taken 748 times.
|
1771 | if (ip != NULL) |
| 192 | 1023 | INJECTION_POINT(ip, NULL); | |
| 193 | 1770 | } | |
| 194 | |||
| 195 | void | ||
| 196 | 381 | prism_build_progress_incr_tuples(int64_t n) | |
| 197 | { | ||
| 198 | /* A scan whose row count divides evenly into the batch size has nothing | ||
| 199 | * left to flush, and one that stole no rows never had anything. Reporting | ||
| 200 | * it anyway costs a worker a parallel message for the leader to add zero | ||
| 201 | * to. */ | ||
| 202 |
2/2✓ Branch 0 taken 305 times.
✓ Branch 1 taken 76 times.
|
381 | if (n == 0) |
| 203 | return; | ||
| 204 | |||
| 205 | /* From a worker this piggybacks over the parallel message queue and the | ||
| 206 | * leader applies it (also while blocked at a barrier -- interrupt | ||
| 207 | * processing runs inside its condition-variable sleeps); in the leader | ||
| 208 | * it applies directly. */ | ||
| 209 | 305 | pgstat_progress_parallel_incr_param(PROGRESS_CREATEIDX_TUPLES_DONE, n); | |
| 210 | } | ||
| 211 | |||
| 212 | void | ||
| 213 | 60 | prism_build_report_progress(PrismBuildProgress *p, double done) | |
| 214 | { | ||
| 215 |
3/4✓ Branch 0 taken 1 times.
✓ Branch 1 taken 59 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
|
60 | if (!p->total_set && p->tuples_total > 0) |
| 216 | { | ||
| 217 | ✗ | pgstat_progress_update_param( | |
| 218 | ✗ | PROGRESS_CREATEIDX_TUPLES_TOTAL, (int64)p->tuples_total); | |
| 219 | ✗ | p->total_set = true; | |
| 220 | } | ||
| 221 | /* | ||
| 222 | * Test hook: the parallel leader reaches this after its workers have | ||
| 223 | * finished, and tuples_done still holds only what live reporting | ||
| 224 | * produced -- this call is what replaces it with the exact count. So a | ||
| 225 | * watcher that stops the build here sees a nonzero counter if and only | ||
| 226 | * if progress was reported during the scan. | ||
| 227 | * | ||
| 228 | * Here rather than at a flush, because the leader arrives whatever share | ||
| 229 | * of the scan work-stealing gave it, including none: the counter it | ||
| 230 | * pauses with is the workers', relayed over the parallel message queue | ||
| 231 | * and necessarily already applied, since the leader cannot see a worker | ||
| 232 | * finish without first draining that worker's earlier messages. | ||
| 233 | * | ||
| 234 | * The leader is also the backend running the CREATE INDEX, so it is the | ||
| 235 | * one a test driving the build through a session can see stop. | ||
| 236 | * | ||
| 237 | * Tested against the phase rather than p->is_parallel. That flag holds | ||
| 238 | * the worker count the planner asked for, so it stays set when no worker | ||
| 239 | * launches and the build falls back to the serial scan -- which reports | ||
| 240 | * every 10000 rows and would reach a 'wait' here repeatedly, against the | ||
| 241 | * one wake-up that releases it. The parallel leader's publish is the | ||
| 242 | * only caller during SCAN_PARALLEL, and it happens once. | ||
| 243 | * | ||
| 244 | * Only ever in the leader: a worker has no PrismBuildProgress to report | ||
| 245 | * through, which the assertion records. | ||
| 246 | */ | ||
| 247 | #ifdef USE_INJECTION_POINTS | ||
| 248 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 60 times.
|
60 | Assert(!IsParallelWorker()); |
| 249 |
2/2✓ Branch 0 taken 45 times.
✓ Branch 1 taken 15 times.
|
60 | if (p->cur_phase == PRISM_BUILD_PHASE_SCAN_PARALLEL) |
| 250 | 45 | INJECTION_POINT("prism-scan-progress", NULL); | |
| 251 | #endif | ||
| 252 | 60 | pgstat_progress_update_param(PROGRESS_CREATEIDX_TUPLES_DONE, (int64)done); | |
| 253 | 60 | } | |
| 254 | |||
| 255 | void | ||
| 256 | 47 | prism_build_report_dsm_bytes(PrismBuildProgress *p, uint64_t dsm_bytes) | |
| 257 | { | ||
| 258 | 47 | p->dsm_bytes = dsm_bytes; | |
| 259 | 47 | } | |
| 260 | |||
| 261 | void | ||
| 262 | 47 | prism_build_report_planned_alloc( | |
| 263 | PrismBuildProgress *p, | ||
| 264 | uint64_t sample_bytes, | ||
| 265 | uint64_t centroid_tree_bytes, | ||
| 266 | uint64_t dsm_total_bytes) | ||
| 267 | { | ||
| 268 | 47 | (void)p; | |
| 269 | 47 | const double mb = 1024.0 * 1024.0; | |
| 270 | |||
| 271 | /* Always emitted (not gated by the GUC) so an OOM in any of these is | ||
| 272 | * pre-explained in the log even on a default-configured server. */ | ||
| 273 |
1/2✓ Branch 1 taken 47 times.
✗ Branch 2 not taken.
|
47 | ereport(LOG, |
| 274 | (errmsg("prism build: planned allocations — samples %.0f MB, " | ||
| 275 | "centroid tree ~%.0f MB, DSM %.0f MB " | ||
| 276 | "(maintenance_work_mem %d kB)", | ||
| 277 | (double)sample_bytes / mb, | ||
| 278 | (double)centroid_tree_bytes / mb, | ||
| 279 | (double)dsm_total_bytes / mb, | ||
| 280 | maintenance_work_mem), | ||
| 281 | errhidestmt(true))); | ||
| 282 | 47 | } | |
| 283 | |||
| 284 | void | ||
| 285 | 249 | prism_build_progress_end(PrismBuildProgress *p) | |
| 286 | { | ||
| 287 | 249 | flush_phase(p); | |
| 288 | 249 | p->cur_phase = 0; | |
| 289 | |||
| 290 | /* Final per-phase summary, always logged (like the planned-allocation | ||
| 291 | * line), via the shared prism_build_stats_print so the PostgreSQL and | ||
| 292 | * standalone builds report identically. */ | ||
| 293 |
1/2✓ Branch 0 taken 249 times.
✗ Branch 1 not taken.
|
249 | if (p->stats != NULL) |
| 294 | { | ||
| 295 | 249 | p->stats->ms_total = (double)(now_ns() - p->build_start_ns) / 1e6; | |
| 296 | 249 | prism_build_stats_print(p->stats); | |
| 297 | } | ||
| 298 | 249 | } | |
| 299 |