social_media_100m: reset dir "../data/social_media"
social_media_100m: start dir="../data/social_media" docs=10000000 writers=32 read_threads=32 batch=256 report_secs=5 nodes=4 partitions_per_node=8 shards=32 target_ops=40000 target_wps=10000 target_mix_rps=30000 read_mix=hot_id92/first_id6/text2
social_media_100m: phase_start phase=mixed elapsed_secs=5 target_wps=10000 target_rps=30000 evaluate_writes=true evaluate_reads=true
social_media_100m: progress docs=51200 txns=203 phase=mixed writes/s=10240 avg_writes/s=10212 roll_wps=10240/10240/10240 reads/s=26092 avg_reads/s=26022 roll_rps=26092/26092/26092 total_ops/s=36332 avg_total_ops/s=36234 roll_ops=36332/36332/36332 read_band=yes read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:24005/0-0ms, first:1568/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:519/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=171ms p99=325ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=37MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=36MB/0s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p0 lag=238 q=0 imm=0 dirty=0MB trend=0 emerg=0 budget=1MB hot_table=0 fg_tokens=256 commit_ms=[wal:118620 mem:62 wait:0 manifest:0] wal_ms=[queue:1 group:50939 append:4223 fsync:4974 batches:1450 txns:7168 batch_mb:39 avg_txn_ms=q0 g7 a0 f0]
social_media_100m: progress docs=106496 txns=416 phase=mixed writes/s=11059 avg_writes/s=10613 roll_wps=10240/11059/11059 reads/s=24830 avg_reads/s=25373 roll_rps=24830/26092/26092 total_ops/s=35889 avg_total_ops/s=35986 roll_ops=35889/36332/36332 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22805/0-0ms, first:1509/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:515/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=125ms p99=198ms band=no slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=70MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=68MB/0s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=419 q=0 imm=0 dirty=0MB trend=237 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:79001 mem:12 wait:0 manifest:0] wal_ms=[queue:2 group:40562 append:2707 fsync:2400 batches:1115 txns:6142 batch_mb:35 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=2MB dirty=0MB imm=0 lag=419 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=237 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=2MB dirty=0MB imm=0 lag=416 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=185 emerg=4 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=2MB dirty=0MB imm=0 lag=422 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=202 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=155648 txns=608 phase=mixed writes/s=9830 avg_writes/s=10341 roll_wps=9830/10240/11059 reads/s=24812 avg_reads/s=25158 roll_rps=24812/24830/26092 total_ops/s=34642 avg_total_ops/s=35499 roll_ops=34642/35889/36332 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22805/0-0ms, first:1507/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:500/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=106ms p99=177ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=102MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=100MB/0s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=611 q=0 imm=0 dirty=0MB trend=498 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:86735 mem:22 wait:0 manifest:0] wal_ms=[queue:4 group:40927 append:2934 fsync:3127 batches:1083 txns:6144 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=3MB dirty=0MB imm=0 lag=611 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=498 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=3MB dirty=0MB imm=0 lag=614 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=430 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=3MB dirty=0MB imm=0 lag=612 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=412 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=204800 txns=800 phase=mixed writes/s=9830 avg_writes/s=10207 roll_wps=9830/10240/11059 reads/s=24788 avg_reads/s=25050 roll_rps=24788/24830/26092 total_ops/s=34618 avg_total_ops/s=35257 roll_ops=34618/35889/36332 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22806/0-0ms, first:1470/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:511/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=95ms p99=152ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=134MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=131MB/0s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=803 q=0 imm=0 dirty=0MB trend=769 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:89008 mem:9 wait:0 manifest:0] wal_ms=[queue:0 group:41365 append:2854 fsync:3169 batches:1075 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=4MB dirty=0MB imm=0 lag=803 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=769 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=4MB dirty=0MB imm=0 lag=806 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=659 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=4MB dirty=0MB imm=0 lag=804 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=627 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=253952 txns=992 phase=mixed writes/s=9830 avg_writes/s=10125 roll_wps=9830/9830/11059 reads/s=24810 avg_reads/s=24985 roll_rps=24788/24812/26092 total_ops/s=34640 avg_total_ops/s=35110 roll_ops=34618/34642/36332 read_band=no read_target_rps=27500 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22836/0-0ms, first:1475/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:497/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=84ms p99=131ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=166MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=163MB/0s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=995 q=0 imm=0 dirty=0MB trend=1005 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:84065 mem:48 wait:0 manifest:0] wal_ms=[queue:2 group:40571 append:3150 fsync:2812 batches:1091 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=5MB dirty=0MB imm=0 lag=995 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1005 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=5MB dirty=0MB imm=0 lag=998 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=887 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=5MB dirty=0MB imm=0 lag=995 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=857 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=303104 txns=1184 phase=mixed writes/s=9830 avg_writes/s=10070 roll_wps=9830/9830/11059 reads/s=27146 avg_reads/s=25330 roll_rps=24788/24830/27146 total_ops/s=36976 avg_total_ops/s=35400 roll_ops=34618/35889/36976 read_band=yes read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:24998/0-0ms, first:1620/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:527/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=84ms p99=140ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=198MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=195MB/0s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=1187 q=0 imm=0 dirty=0MB trend=1265 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:92061 mem:28 wait:0 manifest:0] wal_ms=[queue:0 group:41241 append:2785 fsync:3760 batches:1097 txns:6144 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=6MB dirty=0MB imm=0 lag=1187 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1265 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=6MB dirty=0MB imm=0 lag=1190 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1130 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=6MB dirty=0MB imm=0 lag=1187 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1079 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=352256 txns=1376 phase=mixed writes/s=9830 avg_writes/s=10031 roll_wps=9830/9830/11059 reads/s=24801 avg_reads/s=25241 roll_rps=24788/24812/27146 total_ops/s=34631 avg_total_ops/s=35272 roll_ops=34618/34642/36976 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22813/0-0ms, first:1505/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:483/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=79ms p99=117ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=230MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=226MB/0s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=1379 q=0 imm=0 dirty=0MB trend=1500 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:106966 mem:36 wait:0 manifest:0] wal_ms=[queue:0 group:40107 append:3080 fsync:4797 batches:979 txns:6144 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=7MB dirty=0MB imm=0 lag=1379 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1500 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=7MB dirty=0MB imm=0 lag=1382 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1371 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=7MB dirty=0MB imm=0 lag=1379 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1291 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=3 low_cached_read_windows=2 p99=120 writes/s=9881 reads/s=24808 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=401664 txns=1572 phase=mixed writes/s=9881 avg_writes/s=10008 roll_wps=9830/9830/11059 reads/s=24808 avg_reads/s=25176 roll_rps=24788/24812/27146 total_ops/s=34689 avg_total_ops/s=35184 roll_ops=34618/34689/36976 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22846/0-0ms, first:1473/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:488/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=77ms p99=120ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=265MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=260MB/0s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=1587 q=0 imm=0 dirty=0MB trend=1755 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:108959 mem:63 wait:0 manifest:0] wal_ms=[queue:5 group:44025 append:2983 fsync:4845 batches:1091 txns:6655 batch_mb:37 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=8MB dirty=0MB imm=0 lag=1587 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1755 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=8MB dirty=0MB imm=0 lag=1590 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1609 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=8MB dirty=0MB imm=0 lag=1588 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1503 emerg=4 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=4 low_cached_read_windows=3 p99=127 writes/s=10598 reads/s=24800 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=454656 txns=1776 phase=mixed writes/s=10598 avg_writes/s=10070 roll_wps=9830/9830/11059 reads/s=24800 avg_reads/s=25126 roll_rps=24788/24810/27146 total_ops/s=35398 avg_total_ops/s=35196 roll_ops=34618/34689/36976 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22842/0-0ms, first:1468/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:490/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=80ms p99=127ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=297MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=292MB/0s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=1790 q=0 imm=0 dirty=0MB trend=2005 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:117942 mem:75 wait:0 manifest:0] wal_ms=[queue:0 group:43971 append:3445 fsync:5454 batches:1019 txns:6585 batch_mb:36 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=9MB dirty=0MB imm=0 lag=1790 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2005 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=9MB dirty=0MB imm=0 lag=1793 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1851 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=9MB dirty=0MB imm=0 lag=1791 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1728 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=5 low_cached_read_windows=4 p99=126 writes/s=10649 reads/s=24882 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=507904 txns=1984 phase=mixed writes/s=10649 avg_writes/s=10118 roll_wps=9830/9881/11059 reads/s=24882 avg_reads/s=25079 roll_rps=24788/24812/27146 total_ops/s=35531 avg_total_ops/s=35197 roll_ops=34618/35398/36976 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22887/0-0ms, first:1493/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:502/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=82ms p99=126ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=332MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=326MB/0s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=1987 q=0 imm=0 dirty=0MB trend=2242 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:105809 mem:58 wait:0 manifest:0] wal_ms=[queue:0 group:40672 append:3662 fsync:4173 batches:1080 txns:6215 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=10MB dirty=0MB imm=0 lag=1987 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2242 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=10MB dirty=0MB imm=0 lag=1990 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2095 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=10MB dirty=0MB imm=0 lag=1988 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1956 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=6 low_cached_read_windows=5 p99=130 writes/s=9830 reads/s=24826 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=557056 txns=2176 phase=mixed writes/s=9830 avg_writes/s=10087 roll_wps=9830/9830/11059 reads/s=24826 avg_reads/s=25042 roll_rps=24788/24812/27146 total_ops/s=34656 avg_total_ops/s=35129 roll_ops=34618/34689/36976 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22819/0-0ms, first:1524/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:482/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=79ms p99=130ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=364MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=358MB/0s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=2179 q=0 imm=0 dirty=0MB trend=2478 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:95042 mem:26 wait:0 manifest:0] wal_ms=[queue:2 group:41564 append:3675 fsync:3225 batches:1041 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=11MB dirty=0MB imm=0 lag=2179 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2478 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=11MB dirty=0MB imm=0 lag=2182 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2339 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=11MB dirty=0MB imm=0 lag=2180 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2178 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=7 low_cached_read_windows=6 p99=132 writes/s=9830 reads/s=24778 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=606208 txns=2368 phase=mixed writes/s=9830 avg_writes/s=10062 roll_wps=9830/9830/11059 reads/s=24778 avg_reads/s=25012 roll_rps=24778/24812/27146 total_ops/s=34608 avg_total_ops/s=35074 roll_ops=34608/34689/36976 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22818/0-0ms, first:1466/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:494/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=76ms p99=132ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=396MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=389MB/0s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=2371 q=0 imm=0 dirty=0MB trend=2737 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:86336 mem:50 wait:0 manifest:0] wal_ms=[queue:1 group:39405 append:3224 fsync:3024 batches:1070 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=12MB dirty=0MB imm=0 lag=2371 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2737 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=12MB dirty=0MB imm=0 lag=2374 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2579 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=12MB dirty=0MB imm=0 lag=2372 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2389 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=8 low_cached_read_windows=7 p99=136 writes/s=9830 reads/s=24778 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=655360 txns=2560 phase=mixed writes/s=9830 avg_writes/s=10041 roll_wps=9830/9830/11059 reads/s=24778 avg_reads/s=24987 roll_rps=24778/24810/27146 total_ops/s=34608 avg_total_ops/s=35028 roll_ops=34608/34656/36976 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22820/0-0ms, first:1455/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:502/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=85ms p99=136ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=428MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=421MB/0s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=2563 q=0 imm=0 dirty=0MB trend=2986 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:100061 mem:87 wait:0 manifest:0] wal_ms=[queue:0 group:40625 append:3625 fsync:4020 batches:1027 txns:6144 batch_mb:35 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=13MB dirty=0MB imm=0 lag=2563 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2986 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=13MB dirty=0MB imm=0 lag=2566 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2813 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=13MB dirty=0MB imm=0 lag=2564 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2607 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=9 low_cached_read_windows=8 p99=124 writes/s=9830 reads/s=24784 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=704512 txns=2752 phase=mixed writes/s=9830 avg_writes/s=10024 roll_wps=9830/9830/10649 reads/s=24784 avg_reads/s=24966 roll_rps=24778/24808/27146 total_ops/s=34614 avg_total_ops/s=34990 roll_ops=34608/34642/36976 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22832/0-0ms, first:1459/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:494/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=83ms p99=124ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=461MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=453MB/0s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=2755 q=0 imm=0 dirty=0MB trend=3225 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:92482 mem:60 wait:0 manifest:0] wal_ms=[queue:0 group:39953 append:3371 fsync:3337 batches:1041 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=14MB dirty=0MB imm=0 lag=2755 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3225 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=14MB dirty=0MB imm=0 lag=2758 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3054 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=14MB dirty=0MB imm=0 lag=2756 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2828 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=10 low_cached_read_windows=9 p99=142 writes/s=9932 reads/s=24790 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=754176 txns=2948 phase=mixed writes/s=9932 avg_writes/s=10015 roll_wps=9830/9830/10649 reads/s=24790 avg_reads/s=24948 roll_rps=24778/24801/27146 total_ops/s=34722 avg_total_ops/s=34963 roll_ops=34608/34656/36976 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22783/0-0ms, first:1496/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:509/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=81ms p99=142ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=496MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=486MB/0s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=2963 q=0 imm=0 dirty=0MB trend=3479 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:106856 mem:76 wait:0 manifest:0] wal_ms=[queue:3 group:44509 append:3986 fsync:3809 batches:1120 txns:6655 batch_mb:37 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=15MB dirty=0MB imm=0 lag=2963 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3479 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=15MB dirty=0MB imm=0 lag=2966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3291 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=15MB dirty=0MB imm=0 lag=2964 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3064 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=11 low_cached_read_windows=10 p99=129 writes/s=10547 reads/s=24780 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=806912 txns=3152 phase=mixed writes/s=10547 avg_writes/s=10046 roll_wps=9830/9830/10649 reads/s=24780 avg_reads/s=24931 roll_rps=24778/24801/27146 total_ops/s=35327 avg_total_ops/s=34977 roll_ops=34608/34689/36976 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22779/0-0ms, first:1510/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:491/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=85ms p99=129ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=527MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=518MB/1s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=3155 q=0 imm=0 dirty=0MB trend=3716 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:112591 mem:87 wait:0 manifest:0] wal_ms=[queue:0 group:41691 append:3369 fsync:4785 batches:1045 txns:6143 batch_mb:33 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=16MB dirty=0MB imm=0 lag=3155 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3716 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=16MB dirty=0MB imm=0 lag=3158 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3536 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=16MB dirty=0MB imm=0 lag=3156 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3269 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=12 low_cached_read_windows=11 p99=146 writes/s=9830 reads/s=24771 wal_group_avg_ms=7 bottleneck=checkpoint_advance
social_media_100m: progress docs=856064 txns=3344 phase=mixed writes/s=9830 avg_writes/s=10031 roll_wps=9830/9830/10649 reads/s=24771 avg_reads/s=24916 roll_rps=24771/24800/27146 total_ops/s=34601 avg_total_ops/s=34947 roll_ops=34601/34689/36976 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22798/0-0ms, first:1491/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:481/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=89ms p99=146ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=560MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=549MB/1s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=3347 q=0 imm=0 dirty=0MB trend=3963 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:113743 mem:103 wait:0 manifest:0] wal_ms=[queue:0 group:44576 append:3780 fsync:5095 batches:1099 txns:6144 batch_mb:35 avg_txn_ms=q0 g7 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=17MB dirty=0MB imm=0 lag=3347 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3963 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=17MB dirty=0MB imm=0 lag=3350 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3770 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=17MB dirty=0MB imm=0 lag=3347 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3486 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=13 low_cached_read_windows=12 p99=136 writes/s=10393 reads/s=24841 wal_group_avg_ms=7 bottleneck=checkpoint_advance
social_media_100m: progress docs=908032 txns=3549 phase=mixed writes/s=10393 avg_writes/s=10047 roll_wps=9830/9881/10649 reads/s=24841 avg_reads/s=24903 roll_rps=24771/24800/24882 total_ops/s=35234 avg_total_ops/s=34950 roll_ops=34601/34689/35531 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22841/0-0ms, first:1499/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:501/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=89ms p99=136ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=595MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=583MB/1s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=3555 q=0 imm=0 dirty=0MB trend=4203 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:118589 mem:86 wait:0 manifest:0] wal_ms=[queue:4 group:46591 append:3559 fsync:5078 batches:1119 txns:6654 batch_mb:36 avg_txn_ms=q0 g7 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=18MB dirty=0MB imm=0 lag=3555 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4203 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=18MB dirty=0MB imm=0 lag=3557 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4016 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=18MB dirty=0MB imm=0 lag=3553 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3723 emerg=4 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=14 low_cached_read_windows=13 p99=148 writes/s=10086 reads/s=24831 wal_group_avg_ms=7 bottleneck=checkpoint_advance
social_media_100m: progress docs=958464 txns=3744 phase=mixed writes/s=10086 avg_writes/s=10046 roll_wps=9830/9932/10649 reads/s=24831 avg_reads/s=24892 roll_rps=24771/24800/24882 total_ops/s=34917 avg_total_ops/s=34938 roll_ops=34601/34722/35531 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22866/0-0ms, first:1468/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:496/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=93ms p99=148ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=627MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=615MB/1s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=3747 q=0 imm=0 dirty=0MB trend=4432 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:109907 mem:101 wait:0 manifest:0] wal_ms=[queue:2 group:43046 append:3162 fsync:4849 batches:1043 txns:6142 batch_mb:35 avg_txn_ms=q0 g7 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=19MB dirty=0MB imm=0 lag=3747 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4432 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=19MB dirty=0MB imm=0 lag=3749 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4244 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=19MB dirty=0MB imm=0 lag=3745 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3934 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=15 low_cached_read_windows=14 p99=130 writes/s=9830 reads/s=24805 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=1007616 txns=3936 phase=mixed writes/s=9830 avg_writes/s=10034 roll_wps=9830/9932/10649 reads/s=24805 avg_reads/s=24882 roll_rps=24771/24800/24882 total_ops/s=34635 avg_total_ops/s=34916 roll_ops=34601/34722/35531 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22838/0-0ms, first:1483/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:483/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=85ms p99=130ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=659MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=646MB/1s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=3939 q=0 imm=0 dirty=0MB trend=4668 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:83019 mem:96 wait:0 manifest:0] wal_ms=[queue:1 group:37974 append:3572 fsync:2273 batches:1026 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=20MB dirty=0MB imm=0 lag=3939 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4668 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=20MB dirty=0MB imm=0 lag=3941 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4488 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=20MB dirty=0MB imm=0 lag=3937 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4159 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=16 low_cached_read_windows=15 p99=151 writes/s=9830 reads/s=24771 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=1056768 txns=4128 phase=mixed writes/s=9830 avg_writes/s=10022 roll_wps=9830/9830/10649 reads/s=24771 avg_reads/s=24872 roll_rps=24771/24790/24882 total_ops/s=34601 avg_total_ops/s=34894 roll_ops=34601/34656/35531 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22777/0-0ms, first:1494/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:500/0-1ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=95ms p99=151ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=692MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=678MB/1s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=4131 q=0 imm=0 dirty=0MB trend=4903 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:80711 mem:39 wait:0 manifest:0] wal_ms=[queue:9 group:40040 append:3593 fsync:2176 batches:1103 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=21MB dirty=0MB imm=0 lag=4131 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4903 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=21MB dirty=0MB imm=0 lag=4133 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4717 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=21MB dirty=0MB imm=0 lag=4129 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4374 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=17 low_cached_read_windows=16 p99=154 writes/s=9932 reads/s=24800 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=1106432 txns=4324 phase=mixed writes/s=9932 avg_writes/s=10015 roll_wps=9830/9830/10547 reads/s=24800 avg_reads/s=24863 roll_rps=24771/24790/24841 total_ops/s=34732 avg_total_ops/s=34878 roll_ops=34601/34656/35327 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22822/0-0ms, first:1504/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:474/0-1ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=95ms p99=154ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=726MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=712MB/1s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=4339 q=0 imm=0 dirty=0MB trend=5172 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:85638 mem:57 wait:0 manifest:0] wal_ms=[queue:0 group:41897 append:3544 fsync:2405 batches:1186 txns:6654 batch_mb:37 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=22MB dirty=0MB imm=0 lag=4339 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5172 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=22MB dirty=0MB imm=0 lag=4341 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4955 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=22MB dirty=0MB imm=0 lag=4337 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4587 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=18 low_cached_read_windows=17 p99=157 writes/s=10547 reads/s=24838 wal_group_avg_ms=7 bottleneck=checkpoint_advance
social_media_100m: progress docs=1159168 txns=4528 phase=mixed writes/s=10547 avg_writes/s=10036 roll_wps=9830/9932/10547 reads/s=24838 avg_reads/s=24856 roll_rps=24771/24790/24841 total_ops/s=35385 avg_total_ops/s=34892 roll_ops=34601/34722/35385 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22830/0-0ms, first:1520/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:487/0-1ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=98ms p99=157ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=759MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=744MB/1s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=4531 q=0 imm=0 dirty=0MB trend=5397 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:92197 mem:33 wait:0 manifest:0] wal_ms=[queue:0 group:45089 append:3968 fsync:2479 batches:1064 txns:6144 batch_mb:34 avg_txn_ms=q0 g7 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=23MB dirty=0MB imm=0 lag=4531 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5397 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=23MB dirty=0MB imm=0 lag=4533 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5194 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=23MB dirty=0MB imm=0 lag=4529 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4811 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=19 low_cached_read_windows=18 p99=163 writes/s=9830 reads/s=24833 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=1208320 txns=4720 phase=mixed writes/s=9830 avg_writes/s=10025 roll_wps=9830/9932/10547 reads/s=24833 avg_reads/s=24850 roll_rps=24771/24800/24841 total_ops/s=34663 avg_total_ops/s=34875 roll_ops=34601/34722/35385 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22839/0-0ms, first:1505/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:488/0-2ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=103ms p99=163ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=791MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=776MB/1s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=4723 q=0 imm=0 dirty=0MB trend=5651 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:80859 mem:62 wait:0 manifest:0] wal_ms=[queue:2 group:37855 append:3730 fsync:2352 batches:1105 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=24MB dirty=0MB imm=0 lag=4723 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5651 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=24MB dirty=0MB imm=0 lag=4725 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5431 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=24MB dirty=0MB imm=0 lag=4721 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5041 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=20 low_cached_read_windows=19 p99=178 writes/s=9830 reads/s=24772 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=1257472 txns=4912 phase=mixed writes/s=9830 avg_writes/s=10015 roll_wps=9830/9932/10547 reads/s=24772 avg_reads/s=24841 roll_rps=24771/24800/24841 total_ops/s=34602 avg_total_ops/s=34856 roll_ops=34601/34722/35385 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22818/0-0ms, first:1481/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:472/0-2ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=116ms p99=178ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=823MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=807MB/1s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=4919 q=0 imm=0 dirty=0MB trend=5887 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:88773 mem:74 wait:0 manifest:0] wal_ms=[queue:2 group:39882 append:3789 fsync:2986 batches:1097 txns:6208 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=25MB dirty=0MB imm=0 lag=4919 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5887 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=25MB dirty=0MB imm=0 lag=4921 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5671 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=25MB dirty=0MB imm=0 lag=4916 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5267 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=21 low_cached_read_windows=20 p99=187 writes/s=10240 reads/s=24858 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=1308672 txns=5114 phase=mixed writes/s=10240 avg_writes/s=10020 roll_wps=9830/9932/10547 reads/s=24858 avg_reads/s=24833 roll_rps=24771/24805/24858 total_ops/s=35098 avg_total_ops/s=34853 roll_ops=34601/34732/35385 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22893/0-0ms, first:1476/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:488/0-2ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=118ms p99=187ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=858MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=841MB/1s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=5123 q=0 imm=0 dirty=0MB trend=6118 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:102608 mem:95 wait:0 manifest:0] wal_ms=[queue:3 group:42389 append:5120 fsync:3628 batches:1191 txns:6588 batch_mb:37 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=26MB dirty=0MB imm=0 lag=5123 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6118 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=26MB dirty=0MB imm=0 lag=5124 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5897 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=26MB dirty=0MB imm=0 lag=5121 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5479 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=22 low_cached_read_windows=21 p99=192 writes/s=10240 reads/s=24772 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=1359872 txns=5312 phase=mixed writes/s=10240 avg_writes/s=10027 roll_wps=9830/10086/10547 reads/s=24772 avg_reads/s=24826 roll_rps=24771/24805/24858 total_ops/s=35012 avg_total_ops/s=34853 roll_ops=34601/34917/35385 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22792/0-0ms, first:1485/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:494/0-2ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=120ms p99=192ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=890MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=873MB/1s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=5315 q=0 imm=0 dirty=0MB trend=6357 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:82759 mem:86 wait:0 manifest:0] wal_ms=[queue:0 group:40213 append:3537 fsync:2655 batches:1146 txns:6142 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=27MB dirty=0MB imm=0 lag=5315 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6357 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=27MB dirty=0MB imm=0 lag=5316 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6119 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=27MB dirty=0MB imm=0 lag=5313 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5687 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=23 low_cached_read_windows=22 p99=205 writes/s=9830 reads/s=24803 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=1409024 txns=5504 phase=mixed writes/s=9830 avg_writes/s=10017 roll_wps=9830/9932/10547 reads/s=24803 avg_reads/s=24819 roll_rps=24771/24805/24858 total_ops/s=34633 avg_total_ops/s=34836 roll_ops=34601/34732/35385 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22822/0-0ms, first:1492/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:488/0-3ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=134ms p99=205ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=922MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=904MB/1s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=5507 q=0 imm=0 dirty=0MB trend=6583 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:89026 mem:51 wait:0 manifest:0] wal_ms=[queue:0 group:41678 append:3822 fsync:2848 batches:1120 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=28MB dirty=0MB imm=0 lag=5507 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6583 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=28MB dirty=0MB imm=0 lag=5508 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6363 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=28MB dirty=0MB imm=0 lag=5505 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5914 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=24 low_cached_read_windows=23 p99=216 writes/s=10035 reads/s=24735 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=1459200 txns=5701 phase=mixed writes/s=10035 avg_writes/s=10016 roll_wps=9830/10035/10547 reads/s=24735 avg_reads/s=24812 roll_rps=24735/24805/24858 total_ops/s=34770 avg_total_ops/s=34828 roll_ops=34601/34770/35385 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22746/0-0ms, first:1493/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:496/0-3ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=136ms p99=216ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=957MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=938MB/1s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=5715 q=0 imm=0 dirty=0MB trend=6827 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:106756 mem:90 wait:0 manifest:0] wal_ms=[queue:5 group:45521 append:4187 fsync:4017 batches:1148 txns:6655 batch_mb:37 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=29MB dirty=0MB imm=0 lag=5715 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6827 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=29MB dirty=0MB imm=0 lag=5716 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6612 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=29MB dirty=0MB imm=0 lag=5713 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6139 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=25 low_cached_read_windows=24 p99=219 writes/s=10393 reads/s=24760 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=1511168 txns=5904 phase=mixed writes/s=10393 avg_writes/s=10027 roll_wps=9830/10035/10547 reads/s=24760 avg_reads/s=24806 roll_rps=24735/24803/24858 total_ops/s=35153 avg_total_ops/s=34833 roll_ops=34601/34770/35385 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22759/0-0ms, first:1502/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:499/0-3ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=138ms p99=219ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=988MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=969MB/1s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=5907 q=0 imm=0 dirty=0MB trend=7049 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:96608 mem:87 wait:0 manifest:0] wal_ms=[queue:1 group:39464 append:3390 fsync:4036 batches:1056 txns:6144 batch_mb:33 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=30MB dirty=0MB imm=0 lag=5907 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7049 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=30MB dirty=0MB imm=0 lag=5908 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6855 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=30MB dirty=0MB imm=0 lag=5905 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6369 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=26 low_cached_read_windows=25 p99=220 writes/s=9881 reads/s=24805 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=1560576 txns=6096 phase=mixed writes/s=9881 avg_writes/s=10021 roll_wps=9830/9932/10547 reads/s=24805 avg_reads/s=24802 roll_rps=24735/24803/24858 total_ops/s=34686 avg_total_ops/s=34823 roll_ops=34601/34732/35385 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22831/0-0ms, first:1476/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:497/0-3ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=133ms p99=220ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1021MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1001MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=6099 q=0 imm=0 dirty=0MB trend=7282 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:99088 mem:71 wait:0 manifest:0] wal_ms=[queue:3 group:40701 append:3377 fsync:4180 batches:1049 txns:6144 batch_mb:35 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=31MB dirty=0MB imm=0 lag=6099 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7282 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=31MB dirty=0MB imm=0 lag=6100 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7087 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=32MB dirty=0MB imm=0 lag=6097 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6589 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=27 low_cached_read_windows=26 p99=222 writes/s=9830 reads/s=24794 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=1609728 txns=6288 phase=mixed writes/s=9830 avg_writes/s=10013 roll_wps=9830/9932/10547 reads/s=24794 avg_reads/s=24797 roll_rps=24735/24800/24858 total_ops/s=34624 avg_total_ops/s=34810 roll_ops=34601/34732/35385 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22846/0-0ms, first:1459/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:489/0-3ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=137ms p99=222ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1053MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1031MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=6291 q=0 imm=0 dirty=0MB trend=7517 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:114795 mem:108 wait:0 manifest:0] wal_ms=[queue:4 group:40286 append:3812 fsync:4746 batches:978 txns:6142 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=32MB dirty=0MB imm=0 lag=6291 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7517 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=32MB dirty=0MB imm=0 lag=6292 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7323 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=33MB dirty=0MB imm=0 lag=6289 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6809 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=28 low_cached_read_windows=27 p99=224 writes/s=9830 reads/s=24829 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=1658880 txns=6481 phase=mixed writes/s=9830 avg_writes/s=10005 roll_wps=9830/9932/10547 reads/s=24829 avg_reads/s=24793 roll_rps=24735/24803/24858 total_ops/s=34659 avg_total_ops/s=34798 roll_ops=34602/34732/35385 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22855/0-0ms, first:1478/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:495/0-3ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=136ms p99=224ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1088MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1064MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=6499 q=0 imm=0 dirty=0MB trend=7761 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:111912 mem:85 wait:0 manifest:0] wal_ms=[queue:2 group:44681 append:4776 fsync:4130 batches:1144 txns:6652 batch_mb:37 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=33MB dirty=0MB imm=0 lag=6499 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7761 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=34MB dirty=0MB imm=0 lag=6500 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7568 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=34MB dirty=0MB imm=0 lag=6497 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7043 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=29 low_cached_read_windows=28 p99=219 writes/s=10342 reads/s=24788 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=1710592 txns=6683 phase=mixed writes/s=10342 avg_writes/s=10014 roll_wps=9830/10035/10547 reads/s=24788 avg_reads/s=24789 roll_rps=24735/24803/24858 total_ops/s=35130 avg_total_ops/s=34803 roll_ops=34602/34770/35385 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22808/0-0ms, first:1474/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:506/0-3ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=132ms p99=219ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1120MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1096MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=6691 q=0 imm=0 dirty=0MB trend=8000 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:96930 mem:104 wait:0 manifest:0] wal_ms=[queue:2 group:39836 append:3403 fsync:3604 batches:1032 txns:6143 batch_mb:33 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=34MB dirty=0MB imm=0 lag=6691 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8000 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=35MB dirty=0MB imm=0 lag=6692 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7793 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=35MB dirty=0MB imm=0 lag=6689 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7251 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=30 low_cached_read_windows=29 p99=222 writes/s=10137 reads/s=24807 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=1761280 txns=6880 phase=mixed writes/s=10137 avg_writes/s=10015 roll_wps=9830/10035/10393 reads/s=24807 avg_reads/s=24786 roll_rps=24735/24803/24858 total_ops/s=34944 avg_total_ops/s=34801 roll_ops=34602/34770/35153 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22823/0-0ms, first:1496/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:487/0-3ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=139ms p99=222ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1153MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1128MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=6894 q=0 imm=0 dirty=0MB trend=8238 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:102745 mem:80 wait:0 manifest:0] wal_ms=[queue:0 group:40779 append:3292 fsync:4497 batches:1056 txns:6463 batch_mb:37 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=35MB dirty=0MB imm=0 lag=6894 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8238 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=36MB dirty=0MB imm=0 lag=6895 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8030 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=36MB dirty=0MB imm=0 lag=6890 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7474 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: phase_start phase=reads_only elapsed_secs=180 target_wps=0 target_rps=30000 evaluate_writes=false evaluate_reads=true
social_media_100m: progress docs=1812224 txns=7080 phase=reads_only writes/s=10188 avg_writes/s=10017 roll_wps=9830/10137/10393 reads/s=24886 avg_reads/s=24780 roll_rps=24735/24803/24886 total_ops/s=35074 avg_total_ops/s=34797 roll_ops=34602/34944/35153 read_band=no read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22905/0-0ms, first:1495/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:485/0-4ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=150ms p99=234ms band=yes slo=fail mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1187MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1161MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=read_generator_limit cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=7091 q=0 imm=0 dirty=0MB trend=8468 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:114433 mem:352 wait:0 manifest:0] wal_ms=[queue:0 group:40523 append:4166 fsync:4651 batches:1116 txns:6335 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7091 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8468 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7092 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8265 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7089 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7689 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=1689 avg_writes/s=9791 roll_wps=1689/10137/10393 reads/s=29451 avg_reads/s=24901 roll_rps=24735/24805/29451 total_ops/s=31140 avg_total_ops/s=34692 roll_ops=31140/34944/35153 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27085/0-0ms, first:1764/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:600/0-3ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=158ms p99=239ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=7115 q=0 imm=0 dirty=0MB trend=8650 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:13069 mem:6 wait:0 manifest:0] wal_ms=[queue:0 group:4892 append:544 fsync:619 batches:140 txns:768 batch_mb:5 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8650 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8501 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7908 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=9533 roll_wps=0/10035/10393 reads/s=29456 avg_reads/s=25016 roll_rps=24735/24805/29456 total_ops/s=29456 avg_total_ops/s=34549 roll_ops=29456/34770/35153 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27090/0-0ms, first:1769/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:596/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=7115 q=0 imm=0 dirty=0MB trend=8835 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8835 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8736 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8119 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=9289 roll_wps=0/9881/10393 reads/s=29488 avg_reads/s=25127 roll_rps=24735/24807/29488 total_ops/s=29488 avg_total_ops/s=34416 roll_ops=29456/34686/35153 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27154/0-0ms, first:1746/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:587/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=7115 q=0 imm=0 dirty=0MB trend=9021 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9021 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8978 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8342 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=9056 roll_wps=0/9881/10393 reads/s=29492 avg_reads/s=25232 roll_rps=24735/24829/29492 total_ops/s=29492 avg_total_ops/s=34288 roll_ops=29456/34686/35153 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27141/0-0ms, first:1766/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:583/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=7115 q=0 imm=0 dirty=0MB trend=9203 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9203 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9202 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8549 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=8835 roll_wps=0/9830/10393 reads/s=29415 avg_reads/s=25330 roll_rps=24760/24886/29492 total_ops/s=29415 avg_total_ops/s=34165 roll_ops=29415/34659/35153 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27031/0-0ms, first:1794/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:590/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=9438 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9438 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9385 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8770 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=8625 roll_wps=0/9830/10342 reads/s=29446 avg_reads/s=25424 roll_rps=24788/29415/29492 total_ops/s=29446 avg_total_ops/s=34049 roll_ops=29415/34624/35130 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27076/0-0ms, first:1784/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:586/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=9680 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9680 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9570 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8997 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=8424 roll_wps=0/1689/10342 reads/s=29404 avg_reads/s=25512 roll_rps=24788/29415/29492 total_ops/s=29404 avg_total_ops/s=33936 roll_ops=29404/31140/35130 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27016/0-0ms, first:1788/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:599/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=9917 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9917 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9753 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9220 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=8233 roll_wps=0/0/10342 reads/s=29455 avg_reads/s=25598 roll_rps=24788/29446/29492 total_ops/s=29455 avg_total_ops/s=33831 roll_ops=29404/29492/35130 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27125/0-0ms, first:1737/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:592/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=10159 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10159 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9933 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9452 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=8049 roll_wps=0/0/10342 reads/s=29423 avg_reads/s=25679 roll_rps=24788/29446/29492 total_ops/s=29423 avg_total_ops/s=33728 roll_ops=29404/29488/35130 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27086/0-0ms, first:1745/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:591/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=10390 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10390 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10118 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9665 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=7874 roll_wps=0/0/10188 reads/s=29423 avg_reads/s=25757 roll_rps=24807/29446/29492 total_ops/s=29423 avg_total_ops/s=33631 roll_ops=29404/29456/35074 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27082/0-0ms, first:1756/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:585/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=10628 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10628 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10301 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9890 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=7707 roll_wps=0/0/10188 reads/s=29442 avg_reads/s=25832 roll_rps=24886/29446/29492 total_ops/s=29442 avg_total_ops/s=33539 roll_ops=29404/29455/35074 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27097/0-0ms, first:1756/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:588/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=10874 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10874 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10476 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10123 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=7546 roll_wps=0/0/1689 reads/s=29471 avg_reads/s=25904 roll_rps=29404/29451/29492 total_ops/s=29471 avg_total_ops/s=33450 roll_ops=29404/29455/31140 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27090/0-0ms, first:1793/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:587/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=11114 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=11114 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10656 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10346 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=7392 roll_wps=0/0/0 reads/s=29317 avg_reads/s=25971 roll_rps=29317/29446/29492 total_ops/s=29317 avg_total_ops/s=33363 roll_ops=29317/29446/29492 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26994/0-0ms, first:1744/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:578/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=11339 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=11339 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10844 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10559 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=7244 roll_wps=0/0/0 reads/s=29420 avg_reads/s=26036 roll_rps=29317/29442/29492 total_ops/s=29420 avg_total_ops/s=33280 roll_ops=29317/29442/29492 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27073/0-0ms, first:1752/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:594/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=11567 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=11567 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=11032 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10773 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=7102 roll_wps=0/0/0 reads/s=29394 avg_reads/s=26099 roll_rps=29317/29423/29492 total_ops/s=29394 avg_total_ops/s=33201 roll_ops=29317/29423/29492 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27065/0-0ms, first:1747/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:582/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=11802 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=11802 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=11216 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10990 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=6965 roll_wps=0/0/0 reads/s=29440 avg_reads/s=26160 roll_rps=29317/29423/29471 total_ops/s=29440 avg_total_ops/s=33125 roll_ops=29317/29423/29471 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27083/0-0ms, first:1778/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:579/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=12041 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=12041 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=11393 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=11212 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=6834 roll_wps=0/0/0 reads/s=29495 avg_reads/s=26219 roll_rps=29317/29440/29495 total_ops/s=29495 avg_total_ops/s=33053 roll_ops=29317/29440/29495 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27160/0-0ms, first:1752/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:583/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=12269 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=12269 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=11578 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=11427 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=6707 roll_wps=0/0/0 reads/s=29493 avg_reads/s=26276 roll_rps=29317/29440/29495 total_ops/s=29493 avg_total_ops/s=32983 roll_ops=29317/29440/29495 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27118/0-0ms, first:1769/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:605/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=12505 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=12505 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=11762 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=11643 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=6585 roll_wps=0/0/0 reads/s=29455 avg_reads/s=26331 roll_rps=29317/29442/29495 total_ops/s=29455 avg_total_ops/s=32916 roll_ops=29317/29442/29495 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27134/0-0ms, first:1743/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:577/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=12739 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=12739 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=11947 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=11858 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=6468 roll_wps=0/0/0 reads/s=29458 avg_reads/s=26384 roll_rps=29317/29442/29495 total_ops/s=29458 avg_total_ops/s=32852 roll_ops=29317/29442/29495 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27081/0-0ms, first:1780/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:596/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=12975 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=12975 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=12136 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=12075 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=6354 roll_wps=0/0/0 reads/s=29443 avg_reads/s=26435 roll_rps=29317/29443/29495 total_ops/s=29443 avg_total_ops/s=32789 roll_ops=29317/29443/29495 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27086/0-0ms, first:1768/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:589/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=13208 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=13208 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=12316 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=12296 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=6244 roll_wps=0/0/0 reads/s=29474 avg_reads/s=26484 roll_rps=29317/29455/29495 total_ops/s=29474 avg_total_ops/s=32728 roll_ops=29317/29455/29495 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27135/0-0ms, first:1766/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:573/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=13446 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=13446 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=12520 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=12502 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1820672 txns=7112 phase=reads_only writes/s=0 avg_writes/s=6138 roll_wps=0/0/0 reads/s=29443 avg_reads/s=26531 roll_rps=29317/29455/29495 total_ops/s=29443 avg_total_ops/s=32669 roll_ops=29317/29455/29495 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27084/0-0ms, first:1767/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:591/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=steady assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=13686 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=13686 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=12747 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=12687 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: phase_start phase=writes_only elapsed_secs=301 target_wps=10000 target_rps=0 evaluate_writes=true evaluate_reads=false
social_media_100m: progress docs=1820672 txns=7112 phase=writes_only writes/s=0 avg_writes/s=6036 roll_wps=0/0/0 reads/s=29438 avg_reads/s=26576 roll_rps=29317/29443/29495 total_ops/s=29438 avg_total_ops/s=32612 roll_ops=29317/29443/29495 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27073/0-0ms, first:1777/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:588/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=no slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1191MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1165MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=healthy cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7116 q=0 imm=0 dirty=0MB trend=13920 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7116 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=13920 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=37MB dirty=0MB imm=0 lag=7113 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=12966 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=36MB dirty=0MB imm=0 lag=7115 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=12877 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1872128 txns=7314 phase=writes_only writes/s=10291 avg_writes/s=6104 roll_wps=0/0/10291 reads/s=32 avg_reads/s=26139 roll_rps=32/29443/29495 total_ops/s=10323 avg_total_ops/s=32243 roll_ops=10323/29443/29495 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:29/0-0ms, first:2/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=199ms p99=290ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1229MB wal_growth=8MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1202MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7340 q=0 imm=0 dirty=0MB trend=14151 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:153396 mem:472 wait:0 manifest:0] wal_ms=[queue:2 group:47359 append:4087 fsync:6189 batches:1264 txns:7168 batch_mb:40 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=38MB dirty=0MB imm=0 lag=7340 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=14151 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=38MB dirty=0MB imm=0 lag=7337 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=13175 emerg=4 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=38MB dirty=0MB imm=0 lag=7339 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=13182 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1925376 txns=7523 phase=writes_only writes/s=10649 avg_writes/s=6177 roll_wps=0/0/10649 reads/s=0 avg_reads/s=25718 roll_rps=0/29443/29495 total_ops/s=10649 avg_total_ops/s=31895 roll_ops=10323/29443/29495 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=174ms p99=293ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1261MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1233MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=wal_fsync cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7532 q=0 imm=0 dirty=0MB trend=14379 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:100730 mem:116 wait:0 manifest:0] wal_ms=[queue:6 group:36419 append:2855 fsync:4147 batches:1130 txns:6141 batch_mb:34 avg_txn_ms=q0 g5 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=39MB dirty=0MB imm=0 lag=7532 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=14379 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=39MB dirty=0MB imm=0 lag=7531 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=13460 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=39MB dirty=0MB imm=0 lag=7529 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=13378 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=1976320 txns=7720 phase=writes_only writes/s=10188 avg_writes/s=6240 roll_wps=0/0/10649 reads/s=0 avg_reads/s=25310 roll_rps=0/29443/29495 total_ops/s=10188 avg_total_ops/s=31550 roll_ops=10188/29443/29495 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=165ms p99=294ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1293MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1265MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7724 q=0 imm=0 dirty=0MB trend=14599 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:88995 mem:73 wait:0 manifest:0] wal_ms=[queue:0 group:38687 append:3688 fsync:3508 batches:1156 txns:6144 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=40MB dirty=0MB imm=0 lag=7724 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=14599 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=40MB dirty=0MB imm=0 lag=7723 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=13734 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=40MB dirty=0MB imm=0 lag=7721 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=13581 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: progress docs=2025472 txns=7912 phase=writes_only writes/s=9830 avg_writes/s=6295 roll_wps=0/0/10649 reads/s=0 avg_reads/s=24914 roll_rps=0/29443/29495 total_ops/s=9830 avg_total_ops/s=31209 roll_ops=9830/29443/29495 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=148ms p99=296ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1325MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1295MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=7916 q=0 imm=0 dirty=0MB trend=14837 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:112917 mem:158 wait:0 manifest:0] wal_ms=[queue:4 group:41522 append:4015 fsync:4645 batches:1103 txns:6141 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=41MB dirty=0MB imm=0 lag=7916 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=14837 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=41MB dirty=0MB imm=0 lag=7915 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=14017 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=41MB dirty=0MB imm=0 lag=7912 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=13802 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: milestone_docs=2000000 writes/s=9830 reads/s=0 txn_p99_ms=296 read_p99_ms=0 total_mb=2668 wal_mb=1325 data_mb=0 meta_mb=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=3 low_cached_read_windows=0 p99=253 writes/s=9728 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=2074112 txns=8102 phase=writes_only writes/s=9728 avg_writes/s=6347 roll_wps=0/0/10649 reads/s=0 avg_reads/s=24530 roll_rps=0/29443/29493 total_ops/s=9728 avg_total_ops/s=30877 roll_ops=9728/29443/29493 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=141ms p99=253ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1358MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1326MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=8108 q=0 imm=0 dirty=0MB trend=15071 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:92193 mem:28 wait:0 manifest:0] wal_ms=[queue:1 group:40665 append:3654 fsync:3383 batches:1166 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=42MB dirty=0MB imm=0 lag=8108 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=15071 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=42MB dirty=0MB imm=0 lag=8107 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=14297 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=42MB dirty=0MB imm=0 lag=8103 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=14010 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=4 low_cached_read_windows=0 p99=283 writes/s=9932 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=2123776 txns=8296 phase=writes_only writes/s=9932 avg_writes/s=6400 roll_wps=0/9728/10649 reads/s=0 avg_reads/s=24159 roll_rps=0/29438/29474 total_ops/s=9932 avg_total_ops/s=30559 roll_ops=9728/29438/29474 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=156ms p99=283ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1390MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1357MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=8300 q=0 imm=0 dirty=0MB trend=15294 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:110419 mem:56 wait:0 manifest:0] wal_ms=[queue:2 group:39452 append:3649 fsync:4754 batches:1147 txns:6141 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=43MB dirty=0MB imm=0 lag=8300 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=15294 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=43MB dirty=0MB imm=0 lag=8299 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=14589 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=43MB dirty=0MB imm=0 lag=8295 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=14211 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=5 low_cached_read_windows=0 p99=202 writes/s=9881 reads/s=0 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=2173184 txns=8490 phase=writes_only writes/s=9881 avg_writes/s=6451 roll_wps=0/9830/10649 reads/s=0 avg_reads/s=23798 roll_rps=0/32/29474 total_ops/s=9881 avg_total_ops/s=30249 roll_ops=9728/10649/29474 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=117ms p99=202ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1424MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1389MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=8508 q=0 imm=0 dirty=0MB trend=15538 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:124734 mem:54 wait:0 manifest:0] wal_ms=[queue:1 group:42655 append:4962 fsync:5186 batches:1182 txns:6655 batch_mb:37 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=44MB dirty=0MB imm=0 lag=8508 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=15538 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=44MB dirty=0MB imm=0 lag=8507 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=14890 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=44MB dirty=0MB imm=0 lag=8503 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=14433 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=6 low_cached_read_windows=0 p99=182 writes/s=10547 reads/s=0 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=2225920 txns=8696 phase=writes_only writes/s=10547 avg_writes/s=6511 roll_wps=0/9881/10649 reads/s=0 avg_reads/s=23448 roll_rps=0/0/29474 total_ops/s=10547 avg_total_ops/s=29959 roll_ops=9728/10547/29474 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=110ms p99=182ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1458MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1422MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=8709 q=0 imm=0 dirty=0MB trend=15764 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:136053 mem:198 wait:0 manifest:0] wal_ms=[queue:0 group:44270 append:4732 fsync:7209 batches:1231 txns:6516 batch_mb:36 avg_txn_ms=q0 g6 a0 f1]
social_media_100m: hot node3:p4 backlog=0MB wal=45MB dirty=0MB imm=0 lag=8709 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=15764 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=45MB dirty=0MB imm=0 lag=8711 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=15175 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=45MB dirty=0MB imm=0 lag=8706 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=14639 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=7 low_cached_read_windows=0 p99=189 writes/s=10598 reads/s=0 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=2278912 txns=8903 phase=writes_only writes/s=10598 avg_writes/s=6569 roll_wps=0/9932/10649 reads/s=0 avg_reads/s=23106 roll_rps=0/0/29474 total_ops/s=10598 avg_total_ops/s=29675 roll_ops=9728/10547/29474 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=109ms p99=189ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1492MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1454MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=8908 q=0 imm=0 dirty=0MB trend=15986 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:112778 mem:144 wait:0 manifest:0] wal_ms=[queue:0 group:38452 append:4148 fsync:5290 batches:1101 txns:6280 batch_mb:35 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=46MB dirty=0MB imm=0 lag=8908 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=15986 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=46MB dirty=0MB imm=0 lag=8907 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=15454 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=46MB dirty=0MB imm=0 lag=8903 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=14829 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=8 low_cached_read_windows=0 p99=191 writes/s=9932 reads/s=0 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=2328576 txns=9096 phase=writes_only writes/s=9932 avg_writes/s=6616 roll_wps=0/9932/10649 reads/s=0 avg_reads/s=22776 roll_rps=0/0/29443 total_ops/s=9932 avg_total_ops/s=29392 roll_ops=9728/10323/29443 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=112ms p99=191ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1524MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1484MB/2s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=9100 q=0 imm=0 dirty=0MB trend=16223 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:151620 mem:120 wait:0 manifest:0] wal_ms=[queue:0 group:40735 append:3692 fsync:8474 batches:1093 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a0 f1]
social_media_100m: hot node3:p4 backlog=0MB wal=47MB dirty=0MB imm=0 lag=9100 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=16223 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=47MB dirty=0MB imm=0 lag=9099 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=15739 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=47MB dirty=0MB imm=0 lag=9095 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=15051 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=9 low_cached_read_windows=0 p99=196 writes/s=9830 reads/s=0 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=2377728 txns=9288 phase=writes_only writes/s=9830 avg_writes/s=6660 roll_wps=0/9932/10649 reads/s=0 avg_reads/s=22455 roll_rps=0/0/29438 total_ops/s=9830 avg_total_ops/s=29115 roll_ops=9728/10188/29438 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=108ms p99=196ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=15MB wal=1556MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1514MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:6 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=9292 q=1 imm=1 dirty=0MB trend=16469 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:120949 mem:1547 wait:0 manifest:0] wal_ms=[queue:0 group:39269 append:4425 fsync:5673 batches:1128 txns:6142 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p4 backlog=2MB wal=48MB dirty=0MB imm=1 lag=9292 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=3 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=16469 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=48MB dirty=0MB imm=0 lag=9291 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=16001 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=48MB dirty=0MB imm=0 lag=9286 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=15050 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=10 low_cached_read_windows=0 p99=182 writes/s=9830 reads/s=0 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=2426880 txns=9480 phase=writes_only writes/s=9830 avg_writes/s=6703 roll_wps=9728/9932/10649 reads/s=0 avg_reads/s=22143 roll_rps=0/0/32 total_ops/s=9830 avg_total_ops/s=28846 roll_ops=9728/9932/10649 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=107ms p99=182ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=24MB wal=1588MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1545MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:6 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=9483 q=1 imm=1 dirty=0MB trend=16251 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:96338 mem:1421 wait:0 manifest:0] wal_ms=[queue:2 group:38040 append:4831 fsync:3377 batches:1129 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p6 backlog=4MB wal=49MB dirty=0MB imm=1 lag=9483 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=2 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=16251 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=49MB dirty=0MB imm=0 lag=9478 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=15261 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=49MB dirty=0MB imm=0 lag=9486 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=0 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=14616 emerg=1 budget=1MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=11 low_cached_read_windows=0 p99=199 writes/s=9830 reads/s=0 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=2476032 txns=9672 phase=writes_only writes/s=9830 avg_writes/s=6745 roll_wps=9728/9932/10649 reads/s=0 avg_reads/s=21839 roll_rps=0/0/0 total_ops/s=9830 avg_total_ops/s=28584 roll_ops=9728/9932/10649 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=110ms p99=199ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1620MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1577MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=9677 q=0 imm=0 dirty=0MB trend=2347 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:120637 mem:214 wait:0 manifest:0] wal_ms=[queue:4 group:37353 append:4432 fsync:5576 batches:1128 txns:6195 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=50MB dirty=0MB imm=0 lag=9677 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2347 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=50MB dirty=0MB imm=0 lag=9672 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=513 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=50MB dirty=0MB imm=0 lag=9678 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=456 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=12 low_cached_read_windows=0 p99=224 writes/s=10393 reads/s=0 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=2528000 txns=9877 phase=writes_only writes/s=10393 avg_writes/s=6794 roll_wps=9728/9932/10598 reads/s=0 avg_reads/s=21543 roll_rps=0/0/0 total_ops/s=10393 avg_total_ops/s=28337 roll_ops=9728/9932/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=115ms p99=224ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1655MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1610MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=9883 q=0 imm=0 dirty=0MB trend=2489 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:143663 mem:107 wait:0 manifest:0] wal_ms=[queue:3 group:40143 append:4530 fsync:7436 batches:1115 txns:6601 batch_mb:37 avg_txn_ms=q0 g6 a0 f1]
social_media_100m: hot node3:p7 backlog=0MB wal=51MB dirty=0MB imm=0 lag=9883 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2489 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=51MB dirty=0MB imm=0 lag=9878 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=729 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=51MB dirty=0MB imm=0 lag=9884 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=687 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=13 low_cached_read_windows=0 p99=174 writes/s=10086 reads/s=0 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=2578432 txns=10072 phase=writes_only writes/s=10086 avg_writes/s=6837 roll_wps=9728/9932/10598 reads/s=0 avg_reads/s=21256 roll_rps=0/0/0 total_ops/s=10086 avg_total_ops/s=28093 roll_ops=9728/9932/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=108ms p99=174ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1688MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1643MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=10091 q=0 imm=0 dirty=0MB trend=2607 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:127864 mem:103 wait:0 manifest:0] wal_ms=[queue:0 group:42684 append:4765 fsync:6548 batches:1209 txns:6637 batch_mb:36 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=53MB dirty=0MB imm=0 lag=10091 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2607 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=53MB dirty=0MB imm=0 lag=10086 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=942 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=52MB dirty=0MB imm=0 lag=10092 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=920 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=14 low_cached_read_windows=0 p99=162 writes/s=10547 reads/s=0 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=2631168 txns=10280 phase=writes_only writes/s=10547 avg_writes/s=6884 roll_wps=9728/9932/10598 reads/s=0 avg_reads/s=20975 roll_rps=0/0/0 total_ops/s=10547 avg_total_ops/s=27859 roll_ops=9728/9932/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=99ms p99=162ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1722MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1676MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=10283 q=0 imm=0 dirty=0MB trend=2720 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:112580 mem:104 wait:0 manifest:0] wal_ms=[queue:1 group:37340 append:4215 fsync:5359 batches:1067 txns:6161 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=53MB dirty=0MB imm=0 lag=10283 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2720 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=54MB dirty=0MB imm=0 lag=10278 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1159 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=53MB dirty=0MB imm=0 lag=10284 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1153 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=15 low_cached_read_windows=0 p99=241 writes/s=9932 reads/s=0 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=2680832 txns=10472 phase=writes_only writes/s=9932 avg_writes/s=6923 roll_wps=9830/9932/10598 reads/s=0 avg_reads/s=20702 roll_rps=0/0/0 total_ops/s=9932 avg_total_ops/s=27625 roll_ops=9830/9932/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=111ms p99=241ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1754MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1708MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=10475 q=0 imm=0 dirty=0MB trend=2839 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:123103 mem:156 wait:0 manifest:0] wal_ms=[queue:5 group:38798 append:4840 fsync:5263 batches:1031 txns:6142 batch_mb:35 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=55MB dirty=0MB imm=0 lag=10475 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2839 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=54MB dirty=0MB imm=0 lag=10476 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1384 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=55MB dirty=0MB imm=0 lag=10470 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1369 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=16 low_cached_read_windows=0 p99=191 writes/s=9830 reads/s=0 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=2729984 txns=10664 phase=writes_only writes/s=9830 avg_writes/s=6960 roll_wps=9830/9932/10598 reads/s=0 avg_reads/s=20437 roll_rps=0/0/0 total_ops/s=9830 avg_total_ops/s=27397 roll_ops=9830/9932/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=102ms p99=191ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1786MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1739MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=10667 q=0 imm=0 dirty=0MB trend=2976 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:111575 mem:132 wait:0 manifest:0] wal_ms=[queue:3 group:38810 append:4076 fsync:5104 batches:1034 txns:6143 batch_mb:33 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=56MB dirty=0MB imm=0 lag=10667 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2976 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=55MB dirty=0MB imm=0 lag=10668 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1626 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=55MB dirty=0MB imm=0 lag=10667 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1619 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=17 low_cached_read_windows=0 p99=202 writes/s=9830 reads/s=0 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=2779136 txns=10856 phase=writes_only writes/s=9830 avg_writes/s=6995 roll_wps=9830/9932/10598 reads/s=0 avg_reads/s=20178 roll_rps=0/0/0 total_ops/s=9830 avg_total_ops/s=27173 roll_ops=9830/9932/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=101ms p99=202ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1819MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1771MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=10859 q=0 imm=0 dirty=0MB trend=3110 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:108268 mem:106 wait:0 manifest:0] wal_ms=[queue:2 group:38647 append:4880 fsync:4097 batches:1069 txns:6143 batch_mb:35 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=56MB dirty=0MB imm=0 lag=10859 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3110 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=56MB dirty=0MB imm=0 lag=10859 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1890 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=56MB dirty=0MB imm=0 lag=10860 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1853 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=18 low_cached_read_windows=0 p99=251 writes/s=10035 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=2829312 txns=11054 phase=writes_only writes/s=10035 avg_writes/s=7033 roll_wps=9830/9932/10598 reads/s=0 avg_reads/s=19926 roll_rps=0/0/0 total_ops/s=10035 avg_total_ops/s=26959 roll_ops=9830/9932/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=105ms p99=251ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1854MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1804MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=11067 q=0 imm=0 dirty=0MB trend=4247 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:128382 mem:148 wait:0 manifest:0] wal_ms=[queue:2 group:40578 append:5107 fsync:5679 batches:1218 txns:6655 batch_mb:37 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=58MB dirty=0MB imm=0 lag=11067 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4247 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=57MB dirty=0MB imm=0 lag=11067 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2188 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=57MB dirty=0MB imm=0 lag=11068 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2108 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=19 low_cached_read_windows=0 p99=194 writes/s=10444 reads/s=0 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=2881536 txns=11256 phase=writes_only writes/s=10444 avg_writes/s=7074 roll_wps=9830/9932/10547 reads/s=0 avg_reads/s=19679 roll_rps=0/0/0 total_ops/s=10444 avg_total_ops/s=26753 roll_ops=9830/9932/10547 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=119ms p99=194ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1886MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1837MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=11259 q=0 imm=0 dirty=0MB trend=4654 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:108947 mem:89 wait:0 manifest:0] wal_ms=[queue:1 group:38565 append:5003 fsync:4788 batches:1198 txns:6144 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=59MB dirty=0MB imm=0 lag=11259 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4654 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=58MB dirty=0MB imm=0 lag=11259 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2479 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=58MB dirty=0MB imm=0 lag=11260 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2339 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=2930688 txns=11448 phase=writes_only writes/s=9830 avg_writes/s=7107 roll_wps=9830/9932/10547 reads/s=0 avg_reads/s=19439 roll_rps=0/0/0 total_ops/s=9830 avg_total_ops/s=26546 roll_ops=9830/9932/10547 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=101ms p99=165ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1918MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1869MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=11451 q=0 imm=0 dirty=0MB trend=4768 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:104009 mem:61 wait:0 manifest:0] wal_ms=[queue:4 group:35409 append:4946 fsync:3997 batches:1070 txns:6142 batch_mb:34 avg_txn_ms=q0 g5 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=60MB dirty=0MB imm=0 lag=11451 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4768 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=59MB dirty=0MB imm=0 lag=11451 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2778 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=59MB dirty=0MB imm=0 lag=11451 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2585 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=2981376 txns=11647 phase=writes_only writes/s=10137 avg_writes/s=7142 roll_wps=9830/10035/10547 reads/s=0 avg_reads/s=19205 roll_rps=0/0/0 total_ops/s=10137 avg_total_ops/s=26347 roll_ops=9830/10035/10547 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=101ms p99=189ms band=yes slo=pass mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1953MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=100MB/s cleanup_debt=1902MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=11659 q=0 imm=0 dirty=0MB trend=4879 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:118403 mem:205 wait:0 manifest:0] wal_ms=[queue:5 group:43741 append:6658 fsync:3750 batches:1178 txns:6655 batch_mb:37 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=61MB dirty=0MB imm=0 lag=11659 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4879 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=60MB dirty=0MB imm=0 lag=11659 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3058 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=61MB dirty=0MB imm=0 lag=11659 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2853 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: phase_start phase=mixed elapsed_secs=422 target_wps=10000 target_rps=30000 evaluate_writes=true evaluate_reads=true
social_media_100m: progress docs=3033088 txns=11848 phase=mixed writes/s=10342 avg_writes/s=7180 roll_wps=9830/10086/10547 reads/s=0 avg_reads/s=18976 roll_rps=0/0/0 total_ops/s=10342 avg_total_ops/s=26156 roll_ops=9830/10086/10547 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=97ms p99=153ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=1985MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=17MB/s cleanup_debt=1934MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=11851 q=0 imm=0 dirty=0MB trend=5003 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:89136 mem:93 wait:0 manifest:0] wal_ms=[queue:1 group:36809 append:5251 fsync:2921 batches:1166 txns:6143 batch_mb:34 avg_txn_ms=q0 g5 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=62MB dirty=0MB imm=0 lag=11851 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5003 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=61MB dirty=0MB imm=0 lag=11851 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3325 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=62MB dirty=0MB imm=0 lag=11851 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3072 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=3082240 txns=12040 phase=mixed writes/s=9830 avg_writes/s=7210 roll_wps=9830/10086/10547 reads/s=24688 avg_reads/s=19041 roll_rps=0/0/24688 total_ops/s=34518 avg_total_ops/s=26251 roll_ops=9830/10137/34518 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22698/0-0ms, first:1492/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:497/0-1ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=109ms p99=171ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2017MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=1966MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=12043 q=0 imm=0 dirty=0MB trend=5911 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:83714 mem:101 wait:0 manifest:0] wal_ms=[queue:2 group:39795 append:4961 fsync:2583 batches:1323 txns:6142 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=63MB dirty=0MB imm=0 lag=12043 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5911 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=62MB dirty=0MB imm=0 lag=12043 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3586 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=63MB dirty=0MB imm=0 lag=12043 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3296 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=2 low_cached_read_windows=3 p99=223 writes/s=9830 reads/s=24723 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3131392 txns=12232 phase=mixed writes/s=9830 avg_writes/s=7240 roll_wps=9830/10035/10547 reads/s=24723 avg_reads/s=19105 roll_rps=0/0/24723 total_ops/s=34553 avg_total_ops/s=26345 roll_ops=9830/10137/34553 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22747/0-0ms, first:1489/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:486/0-3ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=128ms p99=223ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2051MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=17MB/s cleanup_debt=1999MB/3s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=12248 q=0 imm=0 dirty=0MB trend=6995 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:91996 mem:93 wait:0 manifest:0] wal_ms=[queue:0 group:43705 append:5407 fsync:3627 batches:1454 txns:6567 batch_mb:36 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=64MB dirty=0MB imm=0 lag=12248 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6995 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=63MB dirty=0MB imm=0 lag=12249 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3858 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=64MB dirty=0MB imm=0 lag=12247 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3524 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=3 low_cached_read_windows=4 p99=184 writes/s=10598 reads/s=24808 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3184384 txns=12440 phase=mixed writes/s=10598 avg_writes/s=7277 roll_wps=9830/10035/10598 reads/s=24808 avg_reads/s=19168 roll_rps=0/0/24808 total_ops/s=35406 avg_total_ops/s=26445 roll_ops=9830/10342/35406 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22838/0-0ms, first:1483/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:486/0-2ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=111ms p99=184ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2084MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2031MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=12443 q=0 imm=0 dirty=0MB trend=7254 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:101315 mem:191 wait:0 manifest:0] wal_ms=[queue:1 group:41878 append:5583 fsync:3812 batches:1271 txns:6232 batch_mb:35 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=65MB dirty=0MB imm=0 lag=12443 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7254 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=64MB dirty=0MB imm=0 lag=12443 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4131 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=65MB dirty=0MB imm=0 lag=12443 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3738 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=4 low_cached_read_windows=5 p99=228 writes/s=9881 reads/s=24638 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3233792 txns=12632 phase=mixed writes/s=9881 avg_writes/s=7306 roll_wps=9830/9932/10598 reads/s=24638 avg_reads/s=19229 roll_rps=0/0/24808 total_ops/s=34519 avg_total_ops/s=26535 roll_ops=9830/10342/35406 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22676/0-0ms, first:1472/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:489/0-3ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=126ms p99=228ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=63MB wal=2116MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=35MB/s cleanup_debt=2061MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:14 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=12635 q=1 imm=1 dirty=0MB trend=7438 emerg=2 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:104382 mem:3288 wait:0 manifest:0] wal_ms=[queue:10 group:39734 append:5905 fsync:4142 batches:1303 txns:6141 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p7 backlog=6MB wal=66MB dirty=0MB imm=1 lag=12635 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7438 emerg=2 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=65MB dirty=0MB imm=0 lag=12635 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4373 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=66MB dirty=0MB imm=0 lag=12635 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3988 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=5 low_cached_read_windows=6 p99=227 writes/s=9830 reads/s=24648 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3282944 txns=12824 phase=mixed writes/s=9830 avg_writes/s=7334 roll_wps=9830/9881/10598 reads/s=24648 avg_reads/s=19287 roll_rps=0/0/24808 total_ops/s=34478 avg_total_ops/s=26621 roll_ops=9830/10444/35406 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22662/0-0ms, first:1492/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:493/0-4ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=132ms p99=227ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2148MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2095MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=12827 q=0 imm=0 dirty=0MB trend=4596 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:106462 mem:1236 wait:0 manifest:0] wal_ms=[queue:2 group:38435 append:7688 fsync:3314 batches:1307 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=66MB dirty=0MB imm=0 lag=12827 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=5 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4596 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p4 backlog=0MB wal=66MB dirty=0MB imm=0 lag=12830 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=4 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1982 emerg=1 budget=1MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node0:p1 backlog=0MB wal=67MB dirty=0MB imm=0 lag=12824 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=10 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=335 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=6 low_cached_read_windows=7 p99=193 writes/s=10035 reads/s=24819 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3333120 txns=13022 phase=mixed writes/s=10035 avg_writes/s=7363 roll_wps=9830/10035/10598 reads/s=24819 avg_reads/s=19346 roll_rps=0/24638/24819 total_ops/s=34854 avg_total_ops/s=26709 roll_ops=9830/34478/35406 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22829/0-0ms, first:1498/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:491/0-2ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=110ms p99=193ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2183MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2129MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p5 lag=13036 q=0 imm=0 dirty=0MB trend=537 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256 commit_ms=[wal:114074 mem:501 wait:0 manifest:0] wal_ms=[queue:3 group:42124 append:6796 fsync:4039 batches:1306 txns:6654 batch_mb:36 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node2:p5 backlog=0MB wal=68MB dirty=0MB imm=0 lag=13036 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=11 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=537 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node0:p7 backlog=0MB wal=68MB dirty=0MB imm=0 lag=13036 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=10 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=459 emerg=4 budget=2MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node0:p1 backlog=0MB wal=68MB dirty=0MB imm=0 lag=13032 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=10 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=480 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=7 low_cached_read_windows=8 p99=196 writes/s=10444 reads/s=24787 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3385344 txns=13224 phase=mixed writes/s=10444 avg_writes/s=7396 roll_wps=9830/10035/10598 reads/s=24787 avg_reads/s=19404 roll_rps=0/24648/24819 total_ops/s=35231 avg_total_ops/s=26800 roll_ops=9830/34518/35406 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22790/0-0ms, first:1510/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:486/0-2ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=114ms p99=196ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2215MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=26MB/s cleanup_debt=2161MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p5 lag=13228 q=0 imm=0 dirty=0MB trend=764 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256 commit_ms=[wal:92012 mem:120 wait:0 manifest:0] wal_ms=[queue:8 group:41466 append:5920 fsync:2610 batches:1296 txns:6142 batch_mb:35 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node2:p5 backlog=0MB wal=69MB dirty=0MB imm=0 lag=13228 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=11 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=764 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node0:p7 backlog=0MB wal=69MB dirty=0MB imm=0 lag=13228 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=10 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=632 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node0:p1 backlog=0MB wal=69MB dirty=0MB imm=0 lag=13224 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=10 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=625 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=8 low_cached_read_windows=9 p99=198 writes/s=9830 reads/s=24814 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3434496 txns=13416 phase=mixed writes/s=9830 avg_writes/s=7421 roll_wps=9830/10035/10598 reads/s=24814 avg_reads/s=19461 roll_rps=0/24688/24819 total_ops/s=34644 avg_total_ops/s=26882 roll_ops=9830/34519/35406 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22855/0-0ms, first:1466/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:493/0-2ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=110ms p99=198ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2249MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2193MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p5 lag=13433 q=0 imm=0 dirty=0MB trend=980 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256 commit_ms=[wal:99847 mem:99 wait:0 manifest:0] wal_ms=[queue:0 group:43398 append:6888 fsync:2855 batches:1404 txns:6573 batch_mb:36 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node2:p5 backlog=0MB wal=70MB dirty=0MB imm=0 lag=13433 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=11 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=980 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node0:p7 backlog=0MB wal=70MB dirty=0MB imm=0 lag=13433 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=10 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=812 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=70MB dirty=0MB imm=0 lag=13425 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=9 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=785 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=9 low_cached_read_windows=10 p99=171 writes/s=10547 reads/s=24914 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3487232 txns=13624 phase=mixed writes/s=10547 avg_writes/s=7454 roll_wps=9830/10035/10598 reads/s=24914 avg_reads/s=19517 roll_rps=0/24723/24914 total_ops/s=35461 avg_total_ops/s=26971 roll_ops=9830/34553/35461 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22915/0-0ms, first:1494/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:504/0-2ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=109ms p99=171ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2282MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2226MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p5 lag=13628 q=0 imm=0 dirty=0MB trend=1203 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256 commit_ms=[wal:94616 mem:98 wait:0 manifest:0] wal_ms=[queue:0 group:37946 append:6475 fsync:2684 batches:1270 txns:6225 batch_mb:35 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node2:p5 backlog=0MB wal=71MB dirty=0MB imm=0 lag=13628 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=11 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1203 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=71MB dirty=0MB imm=0 lag=13620 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=9 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1007 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=71MB dirty=0MB imm=0 lag=13626 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=9 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1004 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=10 low_cached_read_windows=11 p99=184 writes/s=9932 reads/s=24785 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3536896 txns=13816 phase=mixed writes/s=9932 avg_writes/s=7479 roll_wps=9830/10035/10598 reads/s=24785 avg_reads/s=19571 roll_rps=0/24785/24914 total_ops/s=34717 avg_total_ops/s=27050 roll_ops=10137/34644/35461 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22790/0-0ms, first:1487/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:507/0-2ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=112ms p99=184ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2314MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2259MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=13819 q=0 imm=0 dirty=0MB trend=1746 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256 commit_ms=[wal:93756 mem:64 wait:0 manifest:0] wal_ms=[queue:2 group:39891 append:5534 fsync:3514 batches:1283 txns:6141 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=72MB dirty=0MB imm=0 lag=13819 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=9 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1746 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=72MB dirty=0MB imm=0 lag=13820 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=11 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1428 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=72MB dirty=0MB imm=0 lag=13812 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=9 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1233 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=11 low_cached_read_windows=12 p99=200 writes/s=9830 reads/s=24721 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3586048 txns=14008 phase=mixed writes/s=9830 avg_writes/s=7503 roll_wps=9830/9932/10598 reads/s=24721 avg_reads/s=19623 roll_rps=0/24785/24914 total_ops/s=34551 avg_total_ops/s=27126 roll_ops=10342/34644/35461 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22762/0-0ms, first:1483/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:475/0-2ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=120ms p99=200ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2347MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2290MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=14011 q=0 imm=0 dirty=0MB trend=3153 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256 commit_ms=[wal:89227 mem:103 wait:0 manifest:0] wal_ms=[queue:0 group:38860 append:5671 fsync:2806 batches:1324 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=73MB dirty=0MB imm=0 lag=14011 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=9 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3153 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=73MB dirty=0MB imm=0 lag=14012 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=11 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1649 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=72MB dirty=0MB imm=0 lag=14011 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=10 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1490 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=12 low_cached_read_windows=13 p99=194 writes/s=9830 reads/s=24790 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3635200 txns=14200 phase=mixed writes/s=9830 avg_writes/s=7527 roll_wps=9830/9881/10598 reads/s=24790 avg_reads/s=19675 roll_rps=24638/24787/24914 total_ops/s=34620 avg_total_ops/s=27202 roll_ops=34478/34644/35461 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22811/0-0ms, first:1477/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:502/0-2ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=109ms p99=194ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2379MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2322MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=14203 q=0 imm=0 dirty=0MB trend=3350 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256 commit_ms=[wal:91724 mem:88 wait:0 manifest:0] wal_ms=[queue:0 group:38237 append:6068 fsync:2617 batches:1266 txns:6142 batch_mb:34 avg_txn_ms=q0 g6 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=74MB dirty=0MB imm=0 lag=14203 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=9 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3350 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=74MB dirty=0MB imm=0 lag=14204 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=11 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1862 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=73MB dirty=0MB imm=0 lag=14203 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=10 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1750 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=13 low_cached_read_windows=14 p99=202 writes/s=10188 reads/s=24792 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3686144 txns=14401 phase=mixed writes/s=10188 avg_writes/s=7553 roll_wps=9830/9932/10598 reads/s=24792 avg_reads/s=19725 roll_rps=24638/24790/24914 total_ops/s=34980 avg_total_ops/s=27278 roll_ops=34478/34717/35461 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22787/0-0ms, first:1499/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:506/0-3ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=121ms p99=202ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=30MB wal=2414MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=29MB/s cleanup_debt=2354MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:7 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=14411 q=0 imm=0 dirty=0MB trend=1986 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256 commit_ms=[wal:97766 mem:2297 wait:0 manifest:0] wal_ms=[queue:5 group:42474 append:7186 fsync:2638 batches:1476 txns:6655 batch_mb:37 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=75MB dirty=0MB imm=0 lag=14411 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=10 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1986 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=75MB dirty=0MB imm=0 lag=14409 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=10 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1906 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node2:p2 backlog=7MB wal=75MB dirty=0MB imm=1 lag=14404 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=10 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1861 emerg=2 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=14 low_cached_read_windows=15 p99=232 writes/s=10240 reads/s=24548 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3737344 txns=14601 phase=mixed writes/s=10240 avg_writes/s=7580 roll_wps=9830/10035/10598 reads/s=24548 avg_reads/s=19773 roll_rps=24548/24790/24914 total_ops/s=34788 avg_total_ops/s=27353 roll_ops=34478/34788/35461 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22577/0-0ms, first:1472/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:498/0-5ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=142ms p99=232ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=55MB wal=2447MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=75MB/s cleanup_debt=2383MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:12 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p7 lag=14616 q=0 imm=0 dirty=0MB trend=2132 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256 commit_ms=[wal:111711 mem:2305 wait:0 manifest:0] wal_ms=[queue:3 group:41027 append:9638 fsync:2645 batches:1413 txns:6577 batch_mb:36 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node1:p7 backlog=0MB wal=76MB dirty=0MB imm=0 lag=14616 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=10 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2132 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node0:p6 backlog=2MB wal=76MB dirty=0MB imm=1 lag=14616 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=12 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2083 emerg=2 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=76MB dirty=0MB imm=0 lag=14613 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=10 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1966 emerg=1 budget=1MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=15 low_cached_read_windows=16 p99=242 writes/s=10393 reads/s=24743 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3789312 txns=14804 phase=mixed writes/s=10393 avg_writes/s=7607 roll_wps=9830/10035/10547 reads/s=24743 avg_reads/s=19820 roll_rps=24548/24787/24914 total_ops/s=35136 avg_total_ops/s=27427 roll_ops=34478/34788/35461 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22768/0-0ms, first:1482/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:493/0-6ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=134ms p99=242ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2481MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=17MB/s cleanup_debt=2422MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=14811 q=0 imm=0 dirty=0MB trend=842 emerg=1 budget=1MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:122226 mem:324 wait:0 manifest:0] wal_ms=[queue:3 group:41547 append:8922 fsync:4165 batches:1356 txns:6221 batch_mb:35 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=77MB dirty=0MB imm=0 lag=14811 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=14 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=842 emerg=1 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=77MB dirty=0MB imm=0 lag=14812 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=16 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=562 emerg=1 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node0:p7 backlog=0MB wal=77MB dirty=0MB imm=0 lag=14812 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=15 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=453 emerg=1 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=16 low_cached_read_windows=17 p99=211 writes/s=10137 reads/s=24766 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3840000 txns=15000 phase=mixed writes/s=10137 avg_writes/s=7632 roll_wps=9830/10137/10547 reads/s=24766 avg_reads/s=19868 roll_rps=24548/24787/24914 total_ops/s=34903 avg_total_ops/s=27500 roll_ops=34478/34854/35461 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22784/0-0ms, first:1489/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:492/0-2ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=119ms p99=211ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2513MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2453MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=15003 q=0 imm=0 dirty=0MB trend=992 emerg=1 budget=1MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:106658 mem:109 wait:0 manifest:0] wal_ms=[queue:0 group:38748 append:8318 fsync:2804 batches:1250 txns:6142 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=78MB dirty=0MB imm=0 lag=15003 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=14 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=992 emerg=1 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=78MB dirty=0MB imm=0 lag=15004 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=16 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=786 emerg=1 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node0:p7 backlog=0MB wal=78MB dirty=0MB imm=0 lag=15004 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=15 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=633 emerg=1 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=17 low_cached_read_windows=18 p99=226 writes/s=9830 reads/s=24749 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3889152 txns=15192 phase=mixed writes/s=9830 avg_writes/s=7653 roll_wps=9830/10137/10547 reads/s=24749 avg_reads/s=19915 roll_rps=24548/24787/24914 total_ops/s=34579 avg_total_ops/s=27568 roll_ops=34551/34854/35461 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22745/0-0ms, first:1512/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:492/0-3ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=117ms p99=226ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2545MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2484MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=15195 q=0 imm=0 dirty=0MB trend=1124 emerg=1 budget=1MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:108542 mem:123 wait:0 manifest:0] wal_ms=[queue:3 group:40027 append:8844 fsync:2635 batches:1297 txns:6141 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=79MB dirty=0MB imm=0 lag=15195 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=14 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1124 emerg=1 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=79MB dirty=0MB imm=0 lag=15196 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=16 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1008 emerg=1 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=79MB dirty=0MB imm=0 lag=15193 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=14 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=848 emerg=1 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=18 low_cached_read_windows=19 p99=218 writes/s=9830 reads/s=24762 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3938304 txns=15384 phase=mixed writes/s=9830 avg_writes/s=7674 roll_wps=9830/10137/10547 reads/s=24762 avg_reads/s=19960 roll_rps=24548/24785/24914 total_ops/s=34592 avg_total_ops/s=27634 roll_ops=34551/34788/35461 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22772/0-0ms, first:1471/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:519/0-6ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=122ms p99=218ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2578MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2516MB/4s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=15387 q=0 imm=0 dirty=0MB trend=1248 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:101982 mem:65 wait:0 manifest:0] wal_ms=[queue:0 group:39734 append:7624 fsync:2705 batches:1276 txns:6141 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=80MB dirty=0MB imm=0 lag=15387 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=14 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1248 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=80MB dirty=0MB imm=0 lag=15388 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=16 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1216 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=80MB dirty=0MB imm=0 lag=15385 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=14 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1073 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=19 low_cached_read_windows=20 p99=236 writes/s=9830 reads/s=24788 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=3987456 txns=15576 phase=mixed writes/s=9830 avg_writes/s=7694 roll_wps=9830/9932/10547 reads/s=24788 avg_reads/s=20005 roll_rps=24548/24785/24914 total_ops/s=34618 avg_total_ops/s=27699 roll_ops=34551/34717/35461 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22798/0-0ms, first:1498/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:491/0-6ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=137ms p99=236ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=2610MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2547MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p5 lag=15587 q=0 imm=0 dirty=0MB trend=1428 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:99662 mem:66 wait:0 manifest:0] wal_ms=[queue:2 group:40159 append:7198 fsync:3150 batches:1331 txns:6393 batch_mb:35 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node2:p5 backlog=0MB wal=81MB dirty=0MB imm=0 lag=15587 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=16 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1428 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node3:p7 backlog=0MB wal=81MB dirty=0MB imm=0 lag=15588 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=14 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1379 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=81MB dirty=0MB imm=0 lag=15586 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=14 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1285 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=20 low_cached_read_windows=21 p99=259 writes/s=10240 reads/s=19582 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=4038656 txns=15778 phase=mixed writes/s=10240 avg_writes/s=7717 roll_wps=9830/10137/10547 reads/s=19582 avg_reads/s=19999 roll_rps=19582/24766/24914 total_ops/s=29822 avg_total_ops/s=27716 roll_ops=29822/34717/35461 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:18020/0-0ms, first:1185/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:376/0-7ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=162ms p99=259ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=6MB wal=2647MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2581MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:1 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p5 lag=15804 q=0 imm=0 dirty=0MB trend=1639 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:121543 mem:732 wait:0 manifest:0] wal_ms=[queue:1 group:44757 append:9624 fsync:2887 batches:1459 txns:6912 batch_mb:39 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node2:p5 backlog=0MB wal=82MB dirty=0MB imm=0 lag=15804 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=16 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1639 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=82MB dirty=0MB imm=0 lag=15801 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=14 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1512 emerg=5 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=82MB dirty=0MB imm=0 lag=15802 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=16 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1503 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=21 low_cached_read_windows=22 p99=272 writes/s=10496 reads/s=14728 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=4091136 txns=15982 phase=mixed writes/s=10496 avg_writes/s=7743 roll_wps=9830/10137/10496 reads/s=14728 avg_reads/s=19947 roll_rps=14728/24762/24792 total_ops/s=25224 avg_total_ops/s=27690 roll_ops=25224/34620/35136 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:13552/0-0ms, first:865/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:311/0-8ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=177ms p99=272ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=35MB wal=2679MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2612MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:7 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p5 lag=15995 q=0 imm=0 dirty=0MB trend=1841 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:107526 mem:908 wait:0 manifest:0] wal_ms=[queue:7 group:42135 append:8352 fsync:2317 batches:1309 txns:6147 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node2:p5 backlog=0MB wal=83MB dirty=0MB imm=0 lag=15995 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=16 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1841 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=83MB dirty=0MB imm=0 lag=15994 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=16 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1708 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=83MB dirty=0MB imm=0 lag=15994 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=14 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1453 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=22 low_cached_read_windows=23 p99=294 writes/s=10342 reads/s=5709 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=4142848 txns=16184 phase=mixed writes/s=10342 avg_writes/s=7767 roll_wps=9830/10188/10496 reads/s=5709 avg_reads/s=19812 roll_rps=5709/24749/24792 total_ops/s=16051 avg_total_ops/s=27579 roll_ops=16051/34618/35136 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:5260/0-1ms, first:336/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:113/0-9ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=202ms p99=294ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=11MB wal=2711MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2645MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:2 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p1 lag=16182 q=1 imm=1 dirty=0MB trend=1413 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:111287 mem:702 wait:0 manifest:0] wal_ms=[queue:0 group:40857 append:9786 fsync:2402 batches:1395 txns:6140 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node0:p1 backlog=5MB wal=84MB dirty=0MB imm=1 lag=16182 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=17 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1413 emerg=2 budget=1MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node3:p7 backlog=0MB wal=85MB dirty=0MB imm=0 lag=16187 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=18 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=827 emerg=2 budget=1MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=84MB dirty=0MB imm=0 lag=16182 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=20 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=511 emerg=2 budget=1MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=23 low_cached_read_windows=24 p99=305 writes/s=9881 reads/s=21980 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=4192256 txns=16376 phase=mixed writes/s=9881 avg_writes/s=7786 roll_wps=9830/10188/10496 reads/s=21980 avg_reads/s=19831 roll_rps=5709/24749/24792 total_ops/s=31861 avg_total_ops/s=27617 roll_ops=16051/34618/35136 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:20240/0-0ms, first:1318/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:421/0-10ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=203ms p99=305ms band=yes slo=fail mode=steady assist=no target_wps=10000 active_hot=0/32 backlog=22MB wal=2744MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=16MB/s cleanup_debt=2677MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:6 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=16379 q=0 imm=0 dirty=0MB trend=992 emerg=2 budget=1MB hot_table=17095084847128333938 fg_tokens=256 commit_ms=[wal:111091 mem:765 wait:0 manifest:0] wal_ms=[queue:0 group:41832 append:10642 fsync:2532 batches:1521 txns:6144 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=86MB dirty=0MB imm=0 lag=16379 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=18 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=992 emerg=2 budget=1MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=85MB dirty=0MB imm=0 lag=16374 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=20 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=730 emerg=2 budget=1MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=85MB dirty=0MB imm=0 lag=16372 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=19 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=701 emerg=2 budget=1MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=24 low_cached_read_windows=25 p99=407 writes/s=9830 reads/s=1110 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=4241408 txns=16568 phase=mixed writes/s=9830 avg_writes/s=7804 roll_wps=9830/10188/10496 reads/s=1110 avg_reads/s=19657 roll_rps=1110/24743/24792 total_ops/s=10940 avg_total_ops/s=27461 roll_ops=10940/34592/35136 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:1024/0-2ms, first:60/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:25/0-11ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=306ms p99=407ms band=yes slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=30/32 backlog=611MB wal=2776MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=111MB/s cleanup_debt=2677MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:153 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p2 lag=16564 q=5 imm=5 dirty=0MB trend=908 emerg=2 budget=1MB hot_table=17095084847128333938 fg_tokens=256 commit_ms=[wal:134092 mem:19802 wait:0 manifest:0] wal_ms=[queue:6 group:46348 append:14528 fsync:2728 batches:1591 txns:6144 batch_mb:33 avg_txn_ms=q0 g7 a2 f0]
social_media_100m: hot node2:p2 backlog=21MB wal=86MB dirty=0MB imm=5 lag=16564 q=5 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=21 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=908 emerg=2 budget=1MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node0:p0 backlog=16MB wal=86MB dirty=1MB imm=4 lag=16570 q=4 durable=0 truncate=0 primary_q=4 secondary_q=0 primary_done=25 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=698 emerg=2 budget=1MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node1:p7 backlog=21MB wal=86MB dirty=1MB imm=4 lag=16569 q=4 durable=0 truncate=0 primary_q=4 secondary_q=0 primary_done=23 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=659 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=25 low_cached_read_windows=26 p99=450 writes/s=9830 reads/s=705 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=4290560 txns=16760 phase=mixed writes/s=9830 avg_writes/s=7822 roll_wps=9830/10137/10496 reads/s=705 avg_reads/s=19483 roll_rps=705/24548/24788 total_ops/s=10535 avg_total_ops/s=27305 roll_ops=10535/34579/35136 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=2ms ops=[id:654/0-4ms, first:38/0-2ms, f1:0/0-0ms, fm:0/0-0ms, txt:13/0-12ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=283ms p99=450ms band=yes slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=16/32 backlog=498MB wal=2808MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=225MB/s cleanup_debt=2677MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:145 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p6 lag=16762 q=5 imm=5 dirty=1MB trend=772 emerg=2 budget=1MB hot_table=17095084847128333938 fg_tokens=256 commit_ms=[wal:133056 mem:55 wait:0 manifest:0] wal_ms=[queue:2 group:47470 append:14102 fsync:1779 batches:1472 txns:6143 batch_mb:35 avg_txn_ms=q0 g7 a2 f0]
social_media_100m: hot node1:p6 backlog=17MB wal=87MB dirty=1MB imm=5 lag=16762 q=5 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=25 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=772 emerg=2 budget=1MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node3:p2 backlog=17MB wal=87MB dirty=0MB imm=5 lag=16763 q=5 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=25 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=457 emerg=2 budget=1MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node2:p4 backlog=18MB wal=87MB dirty=1MB imm=5 lag=16764 q=5 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=24 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=421 emerg=2 budget=1MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: RED partition_stuck node0:p1 windows=3 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=5 q=5 lag=16972 durable_delta=0 truncate_delta=0 primary_flush_delta=2 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=26 low_cached_read_windows=27 p99=481 writes/s=9830 reads/s=5479 wal_group_avg_ms=9 bottleneck=wal_group_wait
social_media_100m: progress docs=4339712 txns=16953 phase=mixed writes/s=9830 avg_writes/s=7839 roll_wps=9830/9881/10496 reads/s=5479 avg_reads/s=19355 roll_rps=705/21980/24788 total_ops/s=15309 avg_total_ops/s=27194 roll_ops=10535/31861/35136 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=3ms ops=[id:5035/0-1ms, first:325/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:118/0-13ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=305ms p99=481ms band=yes slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=2/32 backlog=382MB wal=2842MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2677MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:130 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=16976 q=4 imm=4 dirty=2MB trend=406 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256 commit_ms=[wal:171838 mem:291 wait:0 manifest:0] wal_ms=[queue:0 group:61393 append:15940 fsync:1746 batches:1416 txns:6802 batch_mb:38 avg_txn_ms=q0 g9 a2 f0]
social_media_100m: hot node3:p6 backlog=20MB wal=88MB dirty=2MB imm=4 lag=16976 q=4 durable=0 truncate=0 primary_q=4 secondary_q=0 primary_done=27 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=406 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256
social_media_100m: hot node0:p7 backlog=19MB wal=89MB dirty=2MB imm=4 lag=16976 q=4 durable=0 truncate=0 primary_q=4 secondary_q=0 primary_done=29 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=88 emerg=2 budget=2MB hot_table=14124715428051363376 fg_tokens=256
social_media_100m: hot node1:p7 backlog=12MB wal=88MB dirty=2MB imm=3 lag=16976 q=3 durable=0 truncate=0 primary_q=3 secondary_q=0 primary_done=28 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1058 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: RED partition_stuck node2:p6 windows=4 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=5 q=5 lag=17178 durable_delta=0 truncate_delta=0 primary_flush_delta=2 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=27 low_cached_read_windows=28 p99=494 writes/s=10291 reads/s=2352 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=4391168 txns=17155 phase=mixed writes/s=10291 avg_writes/s=7860 roll_wps=9830/9881/10496 reads/s=2352 avg_reads/s=19199 roll_rps=705/19582/24788 total_ops/s=12643 avg_total_ops/s=27059 roll_ops=10535/29822/34903 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=4ms ops=[id:2153/0-5ms, first:149/0-3ms, f1:0/0-0ms, fm:0/0-0ms, txt:50/0-14ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=298ms p99=494ms band=yes slo=fail mode=cleanup assist=yes target_wps=10000 active_hot=2/32 backlog=313MB wal=2878MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2677MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:119 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p0 lag=17174 q=5 imm=5 dirty=2MB trend=186 emerg=2 budget=2MB hot_table=14124715428051363376 fg_tokens=256 commit_ms=[wal:152224 mem:1288 wait:0 manifest:0] wal_ms=[queue:3 group:51601 append:13712 fsync:1499 batches:1366 txns:6507 batch_mb:35 avg_txn_ms=q0 g7 a2 f0]
social_media_100m: hot node2:p0 backlog=16MB wal=90MB dirty=2MB imm=5 lag=17174 q=5 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=29 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=186 emerg=2 budget=2MB hot_table=14124715428051363376 fg_tokens=256
social_media_100m: hot node0:p7 backlog=17MB wal=90MB dirty=0MB imm=5 lag=17179 q=5 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=32 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=64 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node3:p4 backlog=8MB wal=89MB dirty=2MB imm=4 lag=17177 q=4 durable=0 truncate=0 primary_q=4 secondary_q=0 primary_done=30 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=864 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED partition_stuck node3:p2 windows=5 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=5 q=5 lag=17370 durable_delta=0 truncate_delta=0 primary_flush_delta=2 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=1 low_wps_windows=0 wal_group_wait_windows=28 low_cached_read_windows=29 p99=508 writes/s=10240 reads/s=1719 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=4442368 txns=17354 phase=mixed writes/s=10240 avg_writes/s=7880 roll_wps=9830/9881/10496 reads/s=1719 avg_reads/s=19043 roll_rps=705/14728/24788 total_ops/s=11959 avg_total_ops/s=26923 roll_ops=10535/25224/34618 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=5ms ops=[id:1582/0-14ms, first:102/0-6ms, f1:0/0-0ms, fm:0/0-0ms, txt:34/0-15ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=315ms p99=508ms band=yes slo=fail mode=cleanup assist=yes target_wps=10000 active_hot=0/32 backlog=215MB wal=2910MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2677MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:97 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=17369 q=4 imm=4 dirty=3MB trend=1092 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:153215 mem:3098 wait:0 manifest:0] wal_ms=[queue:0 group:51406 append:14107 fsync:1289 batches:1177 txns:6142 batch_mb:34 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node3:p4 backlog=5MB wal=90MB dirty=3MB imm=4 lag=17369 q=4 durable=0 truncate=0 primary_q=4 secondary_q=0 primary_done=33 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1092 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p2 backlog=10MB wal=90MB dirty=2MB imm=5 lag=17370 q=5 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=33 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=920 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node1:p1 backlog=2MB wal=90MB dirty=3MB imm=2 lag=17368 q=2 durable=0 truncate=0 primary_q=2 secondary_q=0 primary_done=35 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=706 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: RED partition_stuck node1:p2 windows=3 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=5 q=5 lag=17563 durable_delta=0 truncate_delta=0 primary_flush_delta=3 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=2 low_wps_windows=0 wal_group_wait_windows=29 low_cached_read_windows=30 p99=522 writes/s=10240 reads/s=5803 wal_group_avg_ms=9 bottleneck=wal_group_wait
social_media_100m: progress docs=4493568 txns=17554 phase=mixed writes/s=10240 avg_writes/s=7900 roll_wps=9830/10240/10496 reads/s=5803 avg_reads/s=18925 roll_rps=705/5803/24788 total_ops/s=16043 avg_total_ops/s=26825 roll_ops=10535/16051/34618 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=3ms ops=[id:5340/0-0ms, first:349/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:113/0-17ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=322ms p99=522ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=191MB wal=2942MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2677MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:94 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p3 lag=17563 q=3 imm=3 dirty=3MB trend=589 emerg=2 budget=2MB hot_table=14124715428051363376 fg_tokens=256 commit_ms=[wal:168625 mem:4715 wait:0 manifest:0] wal_ms=[queue:0 group:57886 append:14972 fsync:1652 batches:1202 txns:6144 batch_mb:35 avg_txn_ms=q0 g9 a2 f0]
social_media_100m: hot node1:p3 backlog=3MB wal=91MB dirty=3MB imm=3 lag=17563 q=3 durable=0 truncate=0 primary_q=3 secondary_q=0 primary_done=38 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=589 emerg=2 budget=2MB hot_table=14124715428051363376 fg_tokens=256
social_media_100m: hot node2:p1 backlog=2MB wal=91MB dirty=4MB imm=2 lag=17561 q=2 durable=0 truncate=0 primary_q=2 secondary_q=0 primary_done=36 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=481 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node2:p0 backlog=10MB wal=92MB dirty=3MB imm=2 lag=17558 q=2 durable=0 truncate=0 primary_q=2 secondary_q=0 primary_done=35 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=458 emerg=2 budget=2MB hot_table=14124715428051363376 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=3 low_wps_windows=0 wal_group_wait_windows=30 low_cached_read_windows=31 p99=530 writes/s=10188 reads/s=4855 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=4544512 txns=17752 phase=mixed writes/s=10188 avg_writes/s=7920 roll_wps=9830/10240/10496 reads/s=4855 avg_reads/s=18800 roll_rps=705/5709/24788 total_ops/s=15043 avg_total_ops/s=26720 roll_ops=10535/16043/34618 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=3ms ops=[id:4463/0-1ms, first:295/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:96/0-18ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=330ms p99=530ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=89MB wal=2974MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2677MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:53 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p6 lag=17753 q=1 imm=1 dirty=4MB trend=670 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=256 commit_ms=[wal:133871 mem:4235 wait:0 manifest:0] wal_ms=[queue:0 group:43780 append:13226 fsync:2393 batches:1390 txns:6143 batch_mb:34 avg_txn_ms=q0 g7 a2 f0]
social_media_100m: hot node0:p6 backlog=2MB wal=92MB dirty=4MB imm=1 lag=17753 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=40 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=670 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=92MB dirty=4MB imm=1 lag=17752 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=40 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=590 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p6 backlog=7MB wal=92MB dirty=0MB imm=2 lag=17754 q=2 durable=0 truncate=0 primary_q=2 secondary_q=0 primary_done=43 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=584 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=4 low_wps_windows=0 wal_group_wait_windows=31 low_cached_read_windows=32 p99=535 writes/s=9830 reads/s=6103 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=4593664 txns=17944 phase=mixed writes/s=9830 avg_writes/s=7936 roll_wps=9830/10240/10496 reads/s=6103 avg_reads/s=18689 roll_rps=705/5709/21980 total_ops/s=15933 avg_total_ops/s=26625 roll_ops=10535/15933/31861 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=2ms ops=[id:5618/0-0ms, first:360/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:124/0-19ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=315ms p99=535ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=10MB wal=3007MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2685MB/5s lsn_delta=[durable:16384 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:4 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p0 lag=17951 q=1 imm=1 dirty=0MB trend=266 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=256 commit_ms=[wal:142101 mem:2117 wait:0 manifest:0] wal_ms=[queue:2 group:48657 append:14486 fsync:2629 batches:1458 txns:6141 batch_mb:34 avg_txn_ms=q0 g7 a2 f0]
social_media_100m: hot node1:p0 backlog=3MB wal=94MB dirty=0MB imm=1 lag=17951 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=42 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=266 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=256
social_media_100m: hot node3:p7 backlog=0MB wal=94MB dirty=0MB imm=0 lag=17947 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=39 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=260 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node0:p6 backlog=2MB wal=93MB dirty=0MB imm=1 lag=17945 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=43 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=206 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=32 low_cached_read_windows=33 p99=481 writes/s=9984 reads/s=19769 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=4643584 txns=18141 phase=mixed writes/s=9984 avg_writes/s=7952 roll_wps=9830/10188/10496 reads/s=19769 avg_reads/s=18697 roll_rps=705/5709/21980 total_ops/s=29753 avg_total_ops/s=26649 roll_ops=10535/15933/31861 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:18187/0-0ms, first:1175/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:406/0-19ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=260ms p99=481ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3042MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2685MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=18165 q=0 imm=0 dirty=0MB trend=1305 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:135557 mem:1900 wait:0 manifest:0] wal_ms=[queue:0 group:50309 append:11526 fsync:3310 batches:1534 txns:7000 batch_mb:39 avg_txn_ms=q0 g7 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=95MB dirty=0MB imm=0 lag=18165 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=39 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1305 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node0:p0 backlog=0MB wal=95MB dirty=0MB imm=0 lag=18160 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=39 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=316 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=95MB dirty=0MB imm=0 lag=18162 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=46 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=313 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=33 low_cached_read_windows=34 p99=443 writes/s=10547 reads/s=24526 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=4696320 txns=18347 phase=mixed writes/s=10547 avg_writes/s=7973 roll_wps=9830/10188/10547 reads/s=24526 avg_reads/s=18744 roll_rps=705/5709/24526 total_ops/s=35073 avg_total_ops/s=26717 roll_ops=10535/15933/35073 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22566/0-0ms, first:1470/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:489/0-7ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=216ms p99=443ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3077MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2685MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=18363 q=0 imm=0 dirty=0MB trend=2966 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:103016 mem:1673 wait:0 manifest:0] wal_ms=[queue:0 group:41736 append:8541 fsync:3407 batches:1600 txns:6310 batch_mb:35 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=96MB dirty=0MB imm=0 lag=18363 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=39 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2966 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=96MB dirty=0MB imm=0 lag=18362 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=46 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=544 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=95MB dirty=0MB imm=0 lag=18361 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=41 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=542 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=34 low_cached_read_windows=35 p99=317 writes/s=10496 reads/s=24416 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=4748800 txns=18552 phase=mixed writes/s=10496 avg_writes/s=7994 roll_wps=9830/10188/10547 reads/s=24416 avg_reads/s=18790 roll_rps=705/5803/24526 total_ops/s=34912 avg_total_ops/s=26784 roll_ops=10535/15933/35073 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22456/0-0ms, first:1483/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:476/0-11ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=195ms p99=317ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3108MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2685MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=18555 q=0 imm=0 dirty=0MB trend=4756 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:106210 mem:1715 wait:0 manifest:0] wal_ms=[queue:1 group:42488 append:10320 fsync:2795 batches:1584 txns:6141 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=97MB dirty=0MB imm=0 lag=18555 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=39 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4756 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=96MB dirty=0MB imm=0 lag=18553 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=41 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=767 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=97MB dirty=0MB imm=0 lag=18554 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=46 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=761 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=35 low_cached_read_windows=36 p99=353 writes/s=9932 reads/s=24434 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=4798464 txns=18744 phase=mixed writes/s=9932 avg_writes/s=8010 roll_wps=9830/10188/10547 reads/s=24434 avg_reads/s=18836 roll_rps=705/5803/24526 total_ops/s=34366 avg_total_ops/s=26846 roll_ops=10535/15933/35073 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22482/0-0ms, first:1467/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:484/0-12ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=194ms p99=353ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=1MB wal=3141MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2685MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=18747 q=0 imm=0 dirty=0MB trend=5132 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:105833 mem:2064 wait:0 manifest:0] wal_ms=[queue:7 group:43645 append:9986 fsync:2989 batches:1595 txns:6140 batch_mb:34 avg_txn_ms=q0 g7 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18747 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=39 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5132 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=97MB dirty=0MB imm=0 lag=18745 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=41 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=984 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18745 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=933 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: phase_start phase=reads_only elapsed_secs=604 target_wps=0 target_rps=30000 evaluate_writes=false evaluate_reads=true
social_media_100m: progress docs=4847616 txns=18936 phase=reads_only writes/s=9830 avg_writes/s=8024 roll_wps=9830/10188/10547 reads/s=24420 avg_reads/s=18881 roll_rps=705/6103/24526 total_ops/s=34250 avg_total_ops/s=26905 roll_ops=10535/16043/35073 read_band=no read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22483/0-0ms, first:1443/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:493/0-14ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=188ms p99=298ms band=yes slo=fail mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3173MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:1194 truncate:0] bottleneck=read_generator_limit cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=18939 q=0 imm=0 dirty=0MB trend=5818 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:123358 mem:1142 wait:0 manifest:0] wal_ms=[queue:0 group:46112 append:11931 fsync:2747 batches:1512 txns:6143 batch_mb:34 avg_txn_ms=q0 g7 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18939 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=39 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5818 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18937 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1157 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18936 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=965 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=1638 avg_writes/s=7971 roll_wps=1638/10188/10547 reads/s=29197 avg_reads/s=18963 roll_rps=1719/19769/29197 total_ops/s=30835 avg_total_ops/s=26934 roll_ops=11959/29753/35073 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26830/0-0ms, first:1771/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:595/0-2ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=171ms p99=292ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=1MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=18971 q=0 imm=0 dirty=0MB trend=1258 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:19054 mem:142 wait:0 manifest:0] wal_ms=[queue:0 group:6848 append:1862 fsync:482 batches:251 txns:1024 batch_mb:6 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18971 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=42 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1258 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18968 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1142 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=992 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7905 roll_wps=0/10188/10547 reads/s=29203 avg_reads/s=19045 roll_rps=1719/24416/29203 total_ops/s=29203 avg_total_ops/s=26950 roll_ops=11959/29753/35073 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26890/0-0ms, first:1739/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:573/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=18971 q=0 imm=0 dirty=0MB trend=1463 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18971 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=42 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1463 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18968 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1305 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1216 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7841 roll_wps=0/9984/10547 reads/s=29259 avg_reads/s=19126 roll_rps=1719/24420/29259 total_ops/s=29259 avg_total_ops/s=26967 roll_ops=11959/29753/35073 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26914/0-0ms, first:1762/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:582/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=18971 q=0 imm=0 dirty=0MB trend=1657 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18971 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=42 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1657 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18968 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1473 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1438 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7777 roll_wps=0/9932/10547 reads/s=29228 avg_reads/s=19206 roll_rps=4855/24434/29259 total_ops/s=29228 avg_total_ops/s=26983 roll_ops=15043/29753/35073 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26897/0-0ms, first:1761/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:569/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=18971 q=0 imm=0 dirty=0MB trend=1848 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18971 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=42 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1848 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1662 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18968 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1639 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7715 roll_wps=0/9830/10547 reads/s=29163 avg_reads/s=19283 roll_rps=4855/24526/29259 total_ops/s=29163 avg_total_ops/s=26998 roll_ops=15043/29753/35073 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26786/0-0ms, first:1771/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:605/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=18971 q=0 imm=0 dirty=0MB trend=2054 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18971 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=42 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2054 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1872 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18968 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1805 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7654 roll_wps=0/9830/10547 reads/s=29238 avg_reads/s=19360 roll_rps=6103/29163/29259 total_ops/s=29238 avg_total_ops/s=27014 roll_ops=15933/29753/35073 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26904/0-0ms, first:1746/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:587/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=18971 q=0 imm=0 dirty=0MB trend=2211 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18971 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=42 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2211 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2095 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1985 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7593 roll_wps=0/1638/10547 reads/s=29324 avg_reads/s=19437 roll_rps=19769/29197/29324 total_ops/s=29324 avg_total_ops/s=27030 roll_ops=29163/29753/35073 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26986/0-0ms, first:1745/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:593/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=18971 q=0 imm=0 dirty=0MB trend=2345 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18971 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=42 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2345 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2322 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2213 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7534 roll_wps=0/0/10547 reads/s=29320 avg_reads/s=19512 roll_rps=24416/29203/29324 total_ops/s=29320 avg_total_ops/s=27046 roll_ops=29163/29324/35073 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26950/0-0ms, first:1785/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:584/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=2541 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2541 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p7 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18971 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=42 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2466 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2435 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7475 roll_wps=0/0/10496 reads/s=29287 avg_reads/s=19586 roll_rps=24416/29228/29324 total_ops/s=29287 avg_total_ops/s=27061 roll_ops=29163/29320/34912 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26947/0-0ms, first:1762/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:577/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=2761 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2761 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2656 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p7 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18971 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=42 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2596 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7418 roll_wps=0/0/9932 reads/s=29297 avg_reads/s=19659 roll_rps=24420/29238/29324 total_ops/s=29297 avg_total_ops/s=27077 roll_ops=29163/29297/34366 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26990/0-0ms, first:1723/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:584/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=2996 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2996 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2890 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p7 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18971 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=42 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2718 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7361 roll_wps=0/0/9830 reads/s=29275 avg_reads/s=19730 roll_rps=24420/29259/29324 total_ops/s=29275 avg_total_ops/s=27091 roll_ops=29163/29287/34250 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26983/0-0ms, first:1729/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:562/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=3206 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3206 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3099 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p7 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18971 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=42 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2861 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7305 roll_wps=0/0/1638 reads/s=29337 avg_reads/s=19801 roll_rps=29163/29275/29337 total_ops/s=29337 avg_total_ops/s=27106 roll_ops=29163/29287/30835 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26988/0-0ms, first:1754/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:594/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=3435 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3435 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3329 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p7 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18971 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=42 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3001 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7250 roll_wps=0/0/0 reads/s=29254 avg_reads/s=19871 roll_rps=29163/29275/29337 total_ops/s=29254 avg_total_ops/s=27121 roll_ops=29163/29275/29337 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26965/0-0ms, first:1726/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:562/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=3661 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3661 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3554 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p7 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18971 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=42 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3135 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7196 roll_wps=0/0/0 reads/s=29273 avg_reads/s=19939 roll_rps=29163/29275/29337 total_ops/s=29273 avg_total_ops/s=27135 roll_ops=29163/29275/29337 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26945/0-0ms, first:1734/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:594/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=3882 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3882 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3775 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18968 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3296 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7143 roll_wps=0/0/0 reads/s=29284 avg_reads/s=20007 roll_rps=29163/29284/29337 total_ops/s=29284 avg_total_ops/s=27150 roll_ops=29163/29284/29337 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26945/0-0ms, first:1770/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:569/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=4102 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4102 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3997 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18968 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3461 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7090 roll_wps=0/0/0 reads/s=29304 avg_reads/s=20073 roll_rps=29163/29287/29337 total_ops/s=29304 avg_total_ops/s=27163 roll_ops=29163/29287/29337 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26978/0-0ms, first:1755/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:569/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=4336 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4336 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4232 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18962 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3636 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=7038 roll_wps=0/0/0 reads/s=29305 avg_reads/s=20139 roll_rps=29238/29297/29337 total_ops/s=29305 avg_total_ops/s=27177 roll_ops=29238/29297/29337 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26984/0-0ms, first:1726/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:593/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=4556 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4556 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4452 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18962 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3845 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=6987 roll_wps=0/0/0 reads/s=29295 avg_reads/s=20203 roll_rps=29254/29297/29337 total_ops/s=29295 avg_total_ops/s=27190 roll_ops=29254/29297/29337 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26962/0-0ms, first:1758/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:574/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=4764 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4764 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4658 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18962 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4072 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=6937 roll_wps=0/0/0 reads/s=29314 avg_reads/s=20267 roll_rps=29254/29297/29337 total_ops/s=29314 avg_total_ops/s=27204 roll_ops=29254/29297/29337 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26979/0-0ms, first:1737/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:597/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=4979 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4979 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4873 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18962 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4300 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=6887 roll_wps=0/0/0 reads/s=29327 avg_reads/s=20330 roll_rps=29254/29297/29337 total_ops/s=29327 avg_total_ops/s=27217 roll_ops=29254/29297/29337 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27021/0-0ms, first:1735/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:570/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=5189 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5189 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5084 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18962 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4502 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=6838 roll_wps=0/0/0 reads/s=29337 avg_reads/s=20392 roll_rps=29254/29304/29337 total_ops/s=29337 avg_total_ops/s=27230 roll_ops=29254/29304/29337 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26996/0-0ms, first:1747/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:594/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=5424 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5424 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5318 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18962 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4722 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: progress docs=4855808 txns=18968 phase=reads_only writes/s=0 avg_writes/s=6790 roll_wps=0/0/0 reads/s=29269 avg_reads/s=20453 roll_rps=29254/29304/29337 total_ops/s=29269 avg_total_ops/s=27243 roll_ops=29254/29304/29337 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26903/0-0ms, first:1759/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:607/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=5646 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5646 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5543 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4948 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: phase_start phase=writes_only elapsed_secs=720 target_wps=10000 target_rps=0 evaluate_writes=true evaluate_reads=false
social_media_100m: progress docs=4855808 txns=18968 phase=writes_only writes/s=0 avg_writes/s=6742 roll_wps=0/0/0 reads/s=29332 avg_reads/s=20513 roll_rps=29254/29305/29337 total_ops/s=29332 avg_total_ops/s=27255 roll_ops=29254/29305/29337 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26999/0-0ms, first:1754/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:578/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3178MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=18969 q=0 imm=0 dirty=0MB trend=5851 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5851 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=98MB dirty=0MB imm=0 lag=18966 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5747 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=99MB dirty=0MB imm=0 lag=18969 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5177 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: progress docs=4906496 txns=19167 phase=writes_only writes/s=10137 avg_writes/s=6765 roll_wps=0/0/10137 reads/s=59 avg_reads/s=20370 roll_rps=59/29304/29337 total_ops/s=10196 avg_total_ops/s=27135 roll_ops=10196/29304/29337 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:53/0-0ms, first:4/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:1/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=278ms p99=488ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=3MB wal=3216MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:1 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p5 lag=19190 q=1 imm=1 dirty=0MB trend=5968 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:139359 mem:159 wait:0 manifest:0] wal_ms=[queue:2 group:47023 append:10197 fsync:3144 batches:1292 txns:7163 batch_mb:39 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node1:p5 backlog=3MB wal=100MB dirty=0MB imm=1 lag=19190 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=45 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5968 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=100MB dirty=0MB imm=0 lag=19193 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5412 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=100MB dirty=0MB imm=0 lag=19186 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5395 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: progress docs=4959488 txns=19375 phase=writes_only writes/s=10598 avg_writes/s=6791 roll_wps=0/0/10598 reads/s=0 avg_reads/s=20230 roll_rps=0/29304/29337 total_ops/s=10598 avg_total_ops/s=27021 roll_ops=10196/29304/29337 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=243ms p99=498ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3248MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=19385 q=0 imm=0 dirty=0MB trend=5622 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:114774 mem:60 wait:0 manifest:0] wal_ms=[queue:2 group:37666 append:8049 fsync:3425 batches:1138 txns:6144 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=101MB dirty=0MB imm=0 lag=19385 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5622 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=101MB dirty=0MB imm=0 lag=19378 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5589 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=101MB dirty=0MB imm=0 lag=19383 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5511 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=3 low_cached_read_windows=0 p99=340 writes/s=10393 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5011456 txns=19576 phase=writes_only writes/s=10393 avg_writes/s=6815 roll_wps=0/0/10598 reads/s=0 avg_reads/s=20092 roll_rps=0/29304/29337 total_ops/s=10393 avg_total_ops/s=26907 roll_ops=10196/29304/29337 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=206ms p99=340ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3280MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=19577 q=0 imm=0 dirty=0MB trend=5856 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:118626 mem:78 wait:0 manifest:0] wal_ms=[queue:4 group:37963 append:7318 fsync:3093 batches:1007 txns:6144 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=102MB dirty=0MB imm=0 lag=19577 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5856 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=102MB dirty=0MB imm=0 lag=19570 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5805 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=102MB dirty=0MB imm=0 lag=19575 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5708 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=4 low_cached_read_windows=0 p99=339 writes/s=9830 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5060608 txns=19768 phase=writes_only writes/s=9830 avg_writes/s=6835 roll_wps=0/0/10598 reads/s=0 avg_reads/s=19955 roll_rps=0/29304/29337 total_ops/s=9830 avg_total_ops/s=26790 roll_ops=9830/29304/29337 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=197ms p99=339ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3312MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=19769 q=0 imm=0 dirty=0MB trend=6079 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:117824 mem:39 wait:0 manifest:0] wal_ms=[queue:0 group:38932 append:8423 fsync:3132 batches:1145 txns:6142 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=103MB dirty=0MB imm=0 lag=19769 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6079 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=103MB dirty=0MB imm=0 lag=19762 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6018 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=103MB dirty=0MB imm=0 lag=19767 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5916 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=5 low_cached_read_windows=0 p99=336 writes/s=9830 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5109760 txns=19960 phase=writes_only writes/s=9830 avg_writes/s=6855 roll_wps=0/0/10598 reads/s=0 avg_reads/s=19820 roll_rps=0/29295/29337 total_ops/s=9830 avg_total_ops/s=26675 roll_ops=9830/29295/29337 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=189ms p99=336ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3344MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=19961 q=0 imm=0 dirty=0MB trend=6309 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:115829 mem:49 wait:0 manifest:0] wal_ms=[queue:1 group:39722 append:8590 fsync:2154 batches:1072 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=104MB dirty=0MB imm=0 lag=19961 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6309 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=104MB dirty=0MB imm=0 lag=19954 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6238 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=104MB dirty=0MB imm=0 lag=19959 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6131 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=6 low_cached_read_windows=0 p99=354 writes/s=9830 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5158912 txns=20152 phase=writes_only writes/s=9830 avg_writes/s=6875 roll_wps=0/9830/10598 reads/s=0 avg_reads/s=19687 roll_rps=0/29269/29337 total_ops/s=9830 avg_total_ops/s=26562 roll_ops=9830/29269/29337 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=198ms p99=354ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3376MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=20153 q=0 imm=0 dirty=0MB trend=6542 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:114104 mem:45 wait:0 manifest:0] wal_ms=[queue:2 group:38277 append:8981 fsync:1739 batches:1091 txns:6142 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=105MB dirty=0MB imm=0 lag=20153 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6542 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=105MB dirty=0MB imm=0 lag=20146 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6458 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=105MB dirty=0MB imm=0 lag=20151 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6341 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=7 low_cached_read_windows=0 p99=292 writes/s=10086 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5209344 txns=20351 phase=writes_only writes/s=10086 avg_writes/s=6895 roll_wps=0/9830/10598 reads/s=0 avg_reads/s=19556 roll_rps=0/59/29337 total_ops/s=10086 avg_total_ops/s=26451 roll_ops=9830/10598/29337 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=180ms p99=292ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3411MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=20361 q=0 imm=0 dirty=0MB trend=6760 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:119324 mem:51 wait:0 manifest:0] wal_ms=[queue:5 group:42458 append:9157 fsync:2004 batches:1208 txns:6650 batch_mb:37 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=106MB dirty=0MB imm=0 lag=20361 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6760 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=106MB dirty=0MB imm=0 lag=20359 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6554 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=106MB dirty=0MB imm=0 lag=20359 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6474 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=8 low_cached_read_windows=0 p99=284 writes/s=10444 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5261568 txns=20554 phase=writes_only writes/s=10444 avg_writes/s=6918 roll_wps=0/9830/10598 reads/s=0 avg_reads/s=19426 roll_rps=0/0/29337 total_ops/s=10444 avg_total_ops/s=26344 roll_ops=9830/10444/29337 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=172ms p99=284ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3446MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=20569 q=0 imm=0 dirty=0MB trend=6990 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:125293 mem:96 wait:0 manifest:0] wal_ms=[queue:6 group:42256 append:9503 fsync:2498 batches:1160 txns:6653 batch_mb:37 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=107MB dirty=0MB imm=0 lag=20569 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6990 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=107MB dirty=0MB imm=0 lag=20567 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6762 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=107MB dirty=0MB imm=0 lag=20567 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6692 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=9 low_cached_read_windows=0 p99=293 writes/s=10393 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5313536 txns=20757 phase=writes_only writes/s=10393 avg_writes/s=6941 roll_wps=0/10086/10598 reads/s=0 avg_reads/s=19298 roll_rps=0/0/29337 total_ops/s=10393 avg_total_ops/s=26239 roll_ops=9830/10393/29337 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=178ms p99=293ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3478MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=20761 q=0 imm=0 dirty=0MB trend=7221 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:122053 mem:84 wait:0 manifest:0] wal_ms=[queue:3 group:40900 append:9275 fsync:1660 batches:1020 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=108MB dirty=0MB imm=0 lag=20761 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7221 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=108MB dirty=0MB imm=0 lag=20759 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6962 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=108MB dirty=0MB imm=0 lag=20758 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=44 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6898 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=10 low_cached_read_windows=0 p99=289 writes/s=10035 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5363712 txns=20952 phase=writes_only writes/s=10035 avg_writes/s=6960 roll_wps=0/10086/10598 reads/s=0 avg_reads/s=19172 roll_rps=0/0/29332 total_ops/s=10035 avg_total_ops/s=26132 roll_ops=9830/10393/29332 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=165ms p99=289ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3511MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p6 lag=20951 q=0 imm=0 dirty=0MB trend=7165 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:121062 mem:151 wait:0 manifest:0] wal_ms=[queue:0 group:41572 append:9798 fsync:2112 batches:1144 txns:6144 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node0:p6 backlog=0MB wal=109MB dirty=0MB imm=0 lag=20951 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=48 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7165 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node1:p1 backlog=0MB wal=109MB dirty=0MB imm=0 lag=20951 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6593 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node0:p7 backlog=0MB wal=109MB dirty=0MB imm=0 lag=20955 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=49 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6126 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=11 low_cached_read_windows=0 p99=311 writes/s=9830 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5412864 txns=21144 phase=writes_only writes/s=9830 avg_writes/s=6979 roll_wps=0/10086/10598 reads/s=0 avg_reads/s=19047 roll_rps=0/0/29332 total_ops/s=9830 avg_total_ops/s=26026 roll_ops=9830/10196/29332 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=170ms p99=311ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=51MB wal=3542MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2690MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:15 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=21145 q=0 imm=0 dirty=0MB trend=5921 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256 commit_ms=[wal:129601 mem:1204 wait:0 manifest:0] wal_ms=[queue:2 group:41938 append:9781 fsync:2214 batches:1079 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=111MB dirty=0MB imm=0 lag=21145 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=42 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5921 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node1:p4 backlog=0MB wal=111MB dirty=0MB imm=0 lag=21141 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=51 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4475 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node3:p3 backlog=0MB wal=110MB dirty=0MB imm=0 lag=21141 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=46 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3904 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=12 low_cached_read_windows=0 p99=364 writes/s=9830 reads/s=0 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=5462016 txns=21336 phase=writes_only writes/s=9830 avg_writes/s=6996 roll_wps=9830/10086/10598 reads/s=0 avg_reads/s=18924 roll_rps=0/0/59 total_ops/s=9830 avg_total_ops/s=25920 roll_ops=9830/10086/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=236ms p99=364ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=24MB wal=3575MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2703MB/5s lsn_delta=[durable:3526 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:10 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p5 lag=21338 q=1 imm=1 dirty=2MB trend=354 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:127183 mem:1071 wait:0 manifest:0] wal_ms=[queue:2 group:44744 append:12207 fsync:1913 batches:1359 txns:6143 batch_mb:34 avg_txn_ms=q0 g7 a1 f0]
social_media_100m: hot node2:p5 backlog=2MB wal=111MB dirty=2MB imm=1 lag=21338 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=51 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=354 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p7 backlog=2MB wal=112MB dirty=0MB imm=1 lag=21337 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=43 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=302 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=111MB dirty=0MB imm=0 lag=21334 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=245 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=13 low_cached_read_windows=0 p99=291 writes/s=10086 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5512448 txns=21534 phase=writes_only writes/s=10086 avg_writes/s=7016 roll_wps=9830/10086/10598 reads/s=0 avg_reads/s=18803 roll_rps=0/0/0 total_ops/s=10086 avg_total_ops/s=25819 roll_ops=9830/10086/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=155ms p99=291ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=2MB wal=3609MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2703MB/6s lsn_delta=[durable:99 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:1 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=21543 q=0 imm=0 dirty=0MB trend=473 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256 commit_ms=[wal:129358 mem:1733 wait:0 manifest:0] wal_ms=[queue:5 group:44673 append:10272 fsync:2674 batches:1282 txns:6654 batch_mb:37 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=112MB dirty=0MB imm=0 lag=21543 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=50 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=473 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=112MB dirty=0MB imm=0 lag=21542 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=47 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=462 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=112MB dirty=0MB imm=0 lag=21541 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=54 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=204 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=14 low_cached_read_windows=0 p99=374 writes/s=10035 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5562624 txns=21731 phase=writes_only writes/s=10035 avg_writes/s=7034 roll_wps=9830/10035/10444 reads/s=0 avg_reads/s=18683 roll_rps=0/0/0 total_ops/s=10035 avg_total_ops/s=25717 roll_ops=9830/10035/10444 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=148ms p99=374ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=9MB wal=3642MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2704MB/6s lsn_delta=[durable:195 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:3 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p5 lag=21737 q=0 imm=0 dirty=0MB trend=422 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256 commit_ms=[wal:122428 mem:1333 wait:0 manifest:0] wal_ms=[queue:2 group:41144 append:9319 fsync:1992 batches:1105 txns:6210 batch_mb:35 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node1:p5 backlog=0MB wal=113MB dirty=0MB imm=0 lag=21737 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=54 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=422 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: hot node3:p5 backlog=2MB wal=113MB dirty=0MB imm=1 lag=21743 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=58 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=291 emerg=2 budget=2MB hot_table=16823683620448006766 fg_tokens=256
social_media_100m: hot node0:p3 backlog=0MB wal=114MB dirty=0MB imm=0 lag=21733 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=59 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=288 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=15 low_cached_read_windows=0 p99=307 writes/s=10598 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5615616 txns=21937 phase=writes_only writes/s=10598 avg_writes/s=7056 roll_wps=9830/10035/10598 reads/s=0 avg_reads/s=18564 roll_rps=0/0/0 total_ops/s=10598 avg_total_ops/s=25620 roll_ops=9830/10035/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=150ms p99=307ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3677MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2704MB/6s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=21945 q=0 imm=0 dirty=0MB trend=1156 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:122372 mem:1360 wait:0 manifest:0] wal_ms=[queue:1 group:40465 append:9743 fsync:1848 batches:1161 txns:6587 batch_mb:36 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=115MB dirty=0MB imm=0 lag=21945 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=57 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1156 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node0:p3 backlog=0MB wal=115MB dirty=0MB imm=0 lag=21939 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=59 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=431 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: hot node0:p1 backlog=0MB wal=115MB dirty=0MB imm=0 lag=21941 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=58 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=323 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=16 low_cached_read_windows=0 p99=301 writes/s=10188 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5666560 txns=22136 phase=writes_only writes/s=10188 avg_writes/s=7076 roll_wps=9830/10086/10598 reads/s=0 avg_reads/s=18447 roll_rps=0/0/0 total_ops/s=10188 avg_total_ops/s=25523 roll_ops=9830/10086/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=151ms p99=301ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3709MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2704MB/6s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=22137 q=0 imm=0 dirty=0MB trend=2867 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:122724 mem:380 wait:0 manifest:0] wal_ms=[queue:2 group:41829 append:10110 fsync:2094 batches:1163 txns:6144 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=116MB dirty=0MB imm=0 lag=22137 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=57 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2867 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node0:p3 backlog=0MB wal=116MB dirty=0MB imm=0 lag=22131 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=59 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=583 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=115MB dirty=0MB imm=0 lag=22134 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=57 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=495 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=17 low_cached_read_windows=0 p99=315 writes/s=9881 reads/s=0 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=5715968 txns=22328 phase=writes_only writes/s=9881 avg_writes/s=7093 roll_wps=9830/10086/10598 reads/s=0 avg_reads/s=18332 roll_rps=0/0/0 total_ops/s=9881 avg_total_ops/s=25425 roll_ops=9830/10086/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=173ms p99=315ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3741MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2708MB/5s lsn_delta=[durable:130 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=22329 q=0 imm=0 dirty=0MB trend=3044 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:128770 mem:1052 wait:0 manifest:0] wal_ms=[queue:2 group:43941 append:10531 fsync:1801 batches:1128 txns:6142 batch_mb:35 avg_txn_ms=q0 g7 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=117MB dirty=0MB imm=0 lag=22329 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=57 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3044 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node0:p3 backlog=0MB wal=117MB dirty=0MB imm=0 lag=22323 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=59 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=736 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=116MB dirty=0MB imm=0 lag=22329 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=59 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=667 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=18 low_cached_read_windows=0 p99=283 writes/s=9830 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5765120 txns=22520 phase=writes_only writes/s=9830 avg_writes/s=7109 roll_wps=9830/10086/10598 reads/s=0 avg_reads/s=18218 roll_rps=0/0/0 total_ops/s=9830 avg_total_ops/s=25327 roll_ops=9830/10086/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=162ms p99=283ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3774MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2708MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=22521 q=0 imm=0 dirty=0MB trend=3161 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:115138 mem:168 wait:0 manifest:0] wal_ms=[queue:3 group:39166 append:9330 fsync:2219 batches:1161 txns:6142 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=118MB dirty=0MB imm=0 lag=22521 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=57 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3161 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=117MB dirty=0MB imm=0 lag=22519 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=828 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=118MB dirty=0MB imm=0 lag=22514 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=762 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=19 low_cached_read_windows=0 p99=273 writes/s=9932 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5814784 txns=22715 phase=writes_only writes/s=9932 avg_writes/s=7126 roll_wps=9830/10035/10598 reads/s=0 avg_reads/s=18106 roll_rps=0/0/0 total_ops/s=9932 avg_total_ops/s=25232 roll_ops=9830/10035/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=159ms p99=273ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3808MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2708MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=22729 q=0 imm=0 dirty=0MB trend=3295 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:124564 mem:151 wait:0 manifest:0] wal_ms=[queue:4 group:43781 append:10095 fsync:2393 batches:1282 txns:6654 batch_mb:36 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=119MB dirty=0MB imm=0 lag=22729 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=57 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3295 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=118MB dirty=0MB imm=0 lag=22727 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1049 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=119MB dirty=0MB imm=0 lag=22722 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=975 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=20 low_cached_read_windows=0 p99=286 writes/s=10342 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5866496 txns=22917 phase=writes_only writes/s=10342 avg_writes/s=7145 roll_wps=9830/10035/10598 reads/s=0 avg_reads/s=17995 roll_rps=0/0/0 total_ops/s=10342 avg_total_ops/s=25140 roll_ops=9830/10035/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=165ms p99=286ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3840MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2708MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=22921 q=0 imm=0 dirty=0MB trend=3615 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:139843 mem:87 wait:0 manifest:0] wal_ms=[queue:0 group:42692 append:10985 fsync:2493 batches:1076 txns:6144 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=120MB dirty=0MB imm=0 lag=22921 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=57 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3615 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=119MB dirty=0MB imm=0 lag=22919 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1289 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=120MB dirty=0MB imm=0 lag=22914 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1200 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=21 low_cached_read_windows=0 p99=278 writes/s=10035 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5916672 txns=23112 phase=writes_only writes/s=10035 avg_writes/s=7163 roll_wps=9830/10035/10598 reads/s=0 avg_reads/s=17885 roll_rps=0/0/0 total_ops/s=10035 avg_total_ops/s=25048 roll_ops=9830/10035/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=154ms p99=278ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3872MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2708MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=23123 q=0 imm=0 dirty=0MB trend=3861 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:133498 mem:80 wait:0 manifest:0] wal_ms=[queue:0 group:43433 append:10573 fsync:3316 batches:1223 txns:6383 batch_mb:36 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=121MB dirty=0MB imm=0 lag=23123 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=57 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3861 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=120MB dirty=0MB imm=0 lag=23119 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1522 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=121MB dirty=0MB imm=0 lag=23110 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1414 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=22 low_cached_read_windows=0 p99=321 writes/s=10291 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=5968128 txns=23315 phase=writes_only writes/s=10291 avg_writes/s=7181 roll_wps=9830/10035/10598 reads/s=0 avg_reads/s=17776 roll_rps=0/0/0 total_ops/s=10291 avg_total_ops/s=24957 roll_ops=9830/10035/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=139ms p99=321ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3907MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2708MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=23321 q=0 imm=0 dirty=0MB trend=4006 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:143278 mem:158 wait:0 manifest:0] wal_ms=[queue:0 group:43386 append:10870 fsync:1934 batches:1117 txns:6417 batch_mb:35 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=122MB dirty=0MB imm=0 lag=23321 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=57 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4006 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=121MB dirty=0MB imm=0 lag=23319 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1771 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=122MB dirty=0MB imm=0 lag=23314 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1652 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=23 low_cached_read_windows=0 p99=243 writes/s=10188 reads/s=0 wal_group_avg_ms=6 bottleneck=checkpoint_advance
social_media_100m: progress docs=6019072 txns=23512 phase=writes_only writes/s=10188 avg_writes/s=7198 roll_wps=9830/10086/10598 reads/s=0 avg_reads/s=17669 roll_rps=0/0/0 total_ops/s=10188 avg_total_ops/s=24867 roll_ops=9830/10086/10598 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=140ms p99=243ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3939MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2708MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=23513 q=0 imm=0 dirty=0MB trend=4297 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:124020 mem:110 wait:0 manifest:0] wal_ms=[queue:1 group:42274 append:10278 fsync:1872 batches:1146 txns:6144 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=123MB dirty=0MB imm=0 lag=23513 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=57 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4297 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=122MB dirty=0MB imm=0 lag=23511 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2004 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=123MB dirty=0MB imm=0 lag=23506 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1877 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: phase_start phase=mixed elapsed_secs=841 target_wps=10000 target_rps=30000 evaluate_writes=true evaluate_reads=true
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=24 low_cached_read_windows=1 p99=282 writes/s=9830 reads/s=0 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=6068224 txns=23704 phase=mixed writes/s=9830 avg_writes/s=7214 roll_wps=9830/10086/10598 reads/s=0 avg_reads/s=17563 roll_rps=0/0/0 total_ops/s=9830 avg_total_ops/s=24777 roll_ops=9830/10086/10598 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=151ms p99=282ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=3971MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2708MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=23705 q=0 imm=0 dirty=0MB trend=4953 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:130019 mem:155 wait:0 manifest:0] wal_ms=[queue:2 group:42776 append:10790 fsync:2643 batches:1207 txns:6144 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=124MB dirty=0MB imm=0 lag=23705 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=57 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4953 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=123MB dirty=0MB imm=0 lag=23703 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2225 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=124MB dirty=0MB imm=0 lag=23698 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2082 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=25 low_cached_read_windows=2 p99=284 writes/s=9881 reads/s=22027 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=6117632 txns=23898 phase=mixed writes/s=9881 avg_writes/s=7229 roll_wps=9830/10035/10598 reads/s=22027 avg_reads/s=17589 roll_rps=0/0/22027 total_ops/s=31908 avg_total_ops/s=24818 roll_ops=9830/10188/31908 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:20267/0-0ms, first:1307/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:452/0-2ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=168ms p99=284ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=4MB wal=4006MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2708MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:1 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p5 lag=23913 q=0 imm=0 dirty=0MB trend=2074 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:110235 mem:283 wait:0 manifest:0] wal_ms=[queue:6 group:45753 append:9220 fsync:3328 batches:1625 txns:6654 batch_mb:37 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node2:p5 backlog=0MB wal=125MB dirty=0MB imm=0 lag=23913 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2074 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=125MB dirty=0MB imm=0 lag=23909 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2061 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=125MB dirty=0MB imm=0 lag=23912 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1732 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=26 low_cached_read_windows=3 p99=292 writes/s=10342 reads/s=18883 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=6169344 txns=24100 phase=mixed writes/s=10342 avg_writes/s=7247 roll_wps=9830/10188/10598 reads/s=18883 avg_reads/s=17595 roll_rps=0/0/22027 total_ops/s=29225 avg_total_ops/s=24842 roll_ops=9830/10188/31908 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:17400/0-0ms, first:1095/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:387/0-3ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=171ms p99=292ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4039MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2717MB/5s lsn_delta=[durable:760 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p1 lag=24100 q=0 imm=0 dirty=0MB trend=1371 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:109437 mem:416 wait:0 manifest:0] wal_ms=[queue:3 group:43627 append:9398 fsync:3027 batches:1442 txns:6143 batch_mb:34 avg_txn_ms=q0 g7 a1 f0]
social_media_100m: hot node0:p1 backlog=0MB wal=126MB dirty=0MB imm=0 lag=24100 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=61 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1371 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node3:p2 backlog=0MB wal=126MB dirty=0MB imm=0 lag=24104 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=65 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1338 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=126MB dirty=0MB imm=0 lag=24098 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=65 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=421 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=27 low_cached_read_windows=4 p99=278 writes/s=10086 reads/s=24214 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=6219776 txns=24296 phase=mixed writes/s=10086 avg_writes/s=7263 roll_wps=9830/10086/10342 reads/s=24214 avg_reads/s=17633 roll_rps=0/0/24214 total_ops/s=34300 avg_total_ops/s=24896 roll_ops=9830/10188/34300 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22258/0-0ms, first:1466/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:490/0-4ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=169ms p99=278ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4071MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2717MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=24297 q=0 imm=0 dirty=0MB trend=1821 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:103409 mem:235 wait:0 manifest:0] wal_ms=[queue:4 group:42863 append:8937 fsync:2735 batches:1465 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=127MB dirty=0MB imm=0 lag=24297 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=60 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1821 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p2 backlog=0MB wal=127MB dirty=0MB imm=0 lag=24296 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=65 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1500 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=126MB dirty=0MB imm=0 lag=24293 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=63 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=581 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=28 low_cached_read_windows=5 p99=279 writes/s=9830 reads/s=24303 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=6268928 txns=24489 phase=mixed writes/s=9830 avg_writes/s=7278 roll_wps=9830/10035/10342 reads/s=24303 avg_reads/s=17671 roll_rps=0/0/24303 total_ops/s=34133 avg_total_ops/s=24949 roll_ops=9830/10291/34300 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22368/0-0ms, first:1459/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:475/0-4ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=165ms p99=279ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4105MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2717MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=24505 q=0 imm=0 dirty=0MB trend=3410 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:119575 mem:167 wait:0 manifest:0] wal_ms=[queue:1 group:44729 append:11071 fsync:3491 batches:1599 txns:6646 batch_mb:37 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=128MB dirty=0MB imm=0 lag=24505 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=60 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3410 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p1 backlog=0MB wal=128MB dirty=0MB imm=0 lag=24502 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=65 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=698 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=128MB dirty=0MB imm=0 lag=24501 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=63 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=664 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=29 low_cached_read_windows=6 p99=266 writes/s=10291 reads/s=24666 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=6320384 txns=24690 phase=mixed writes/s=10291 avg_writes/s=7295 roll_wps=9830/10086/10342 reads/s=24666 avg_reads/s=17710 roll_rps=0/0/24666 total_ops/s=34957 avg_total_ops/s=25005 roll_ops=9830/10342/34957 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22669/0-0ms, first:1514/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:482/0-4ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=158ms p99=266ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4138MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2717MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=24697 q=0 imm=0 dirty=0MB trend=5048 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:119544 mem:165 wait:0 manifest:0] wal_ms=[queue:9 group:41360 append:9359 fsync:4464 batches:1341 txns:6151 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=129MB dirty=0MB imm=0 lag=24697 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=60 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5048 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p1 backlog=0MB wal=129MB dirty=0MB imm=0 lag=24694 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=65 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=921 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=129MB dirty=0MB imm=0 lag=24693 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=63 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=888 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=30 low_cached_read_windows=7 p99=272 writes/s=10188 reads/s=24607 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=6371328 txns=24888 phase=mixed writes/s=10188 avg_writes/s=7311 roll_wps=9830/10188/10342 reads/s=24607 avg_reads/s=17748 roll_rps=0/18883/24666 total_ops/s=34795 avg_total_ops/s=25059 roll_ops=9830/29225/34957 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22599/0-0ms, first:1492/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:514/0-4ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=162ms p99=272ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4170MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2717MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=24889 q=0 imm=0 dirty=0MB trend=6227 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:108218 mem:114 wait:0 manifest:0] wal_ms=[queue:2 group:40700 append:10161 fsync:2125 batches:1420 txns:6143 batch_mb:34 avg_txn_ms=q0 g6 a1 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=130MB dirty=0MB imm=0 lag=24889 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=60 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6227 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p1 backlog=0MB wal=130MB dirty=0MB imm=0 lag=24886 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=65 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1131 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=130MB dirty=0MB imm=0 lag=24885 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=63 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1116 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=31 low_cached_read_windows=8 p99=278 writes/s=10035 reads/s=24570 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=6421504 txns=25085 phase=mixed writes/s=10035 avg_writes/s=7326 roll_wps=9830/10188/10342 reads/s=24570 avg_reads/s=17786 roll_rps=0/22027/24666 total_ops/s=34605 avg_total_ops/s=25112 roll_ops=9830/31908/34957 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22614/0-0ms, first:1469/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:486/0-4ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=163ms p99=278ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=7MB wal=4205MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2723MB/5s lsn_delta=[durable:1702 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:2 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=25097 q=0 imm=0 dirty=0MB trend=6625 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:148666 mem:550 wait:0 manifest:0] wal_ms=[queue:2 group:51776 append:13843 fsync:3890 batches:1568 txns:6653 batch_mb:37 avg_txn_ms=q0 g7 a2 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=131MB dirty=0MB imm=0 lag=25097 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=60 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6625 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=131MB dirty=0MB imm=0 lag=25093 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=63 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1345 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p1 backlog=0MB wal=131MB dirty=0MB imm=0 lag=25094 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=65 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1336 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=32 low_cached_read_windows=9 p99=301 writes/s=10188 reads/s=24246 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=6472448 txns=25285 phase=mixed writes/s=10188 avg_writes/s=7342 roll_wps=9830/10188/10342 reads/s=24246 avg_reads/s=17822 roll_rps=0/24214/24666 total_ops/s=34434 avg_total_ops/s=25164 roll_ops=9830/34133/34957 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:22343/0-0ms, first:1426/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:477/0-5ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=190ms p99=301ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=23MB wal=4237MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2723MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:9 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=25289 q=1 imm=1 dirty=0MB trend=6777 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:153930 mem:1035 wait:0 manifest:0] wal_ms=[queue:2 group:50956 append:15035 fsync:2214 batches:1351 txns:6143 batch_mb:34 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node3:p7 backlog=1MB wal=132MB dirty=0MB imm=1 lag=25289 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=62 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6777 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=4MB wal=132MB dirty=0MB imm=1 lag=25288 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=65 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1336 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=3MB wal=132MB dirty=0MB imm=1 lag=25282 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=66 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1308 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=33 low_cached_read_windows=10 p99=317 writes/s=10086 reads/s=24549 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=6522880 txns=25480 phase=mixed writes/s=10086 avg_writes/s=7357 roll_wps=9830/10188/10342 reads/s=24549 avg_reads/s=17859 roll_rps=0/24246/24666 total_ops/s=34635 avg_total_ops/s=25216 roll_ops=9830/34300/34957 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22564/0-0ms, first:1489/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:494/0-6ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=181ms p99=317ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4269MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2723MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p0 lag=25485 q=0 imm=0 dirty=0MB trend=401 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:134951 mem:671 wait:0 manifest:0] wal_ms=[queue:11 group:48106 append:12977 fsync:1580 batches:1358 txns:6144 batch_mb:34 avg_txn_ms=q0 g7 a2 f0]
social_media_100m: hot node1:p0 backlog=0MB wal=133MB dirty=0MB imm=0 lag=25485 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=68 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=401 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p6 backlog=0MB wal=133MB dirty=0MB imm=0 lag=25482 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=65 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=379 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node0:p1 backlog=0MB wal=133MB dirty=0MB imm=0 lag=25476 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=67 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=373 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=34 low_cached_read_windows=11 p99=290 writes/s=9830 reads/s=24653 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=6572032 txns=25672 phase=mixed writes/s=9830 avg_writes/s=7370 roll_wps=9830/10086/10342 reads/s=24653 avg_reads/s=17896 roll_rps=0/24303/24666 total_ops/s=34483 avg_total_ops/s=25266 roll_ops=9830/34434/34957 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22677/0-0ms, first:1484/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:492/0-3ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=161ms p99=290ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4302MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2723MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p0 lag=25677 q=0 imm=0 dirty=0MB trend=578 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:139817 mem:343 wait:0 manifest:0] wal_ms=[queue:0 group:45776 append:13185 fsync:2895 batches:1274 txns:6143 batch_mb:34 avg_txn_ms=q0 g7 a2 f0]
social_media_100m: hot node1:p0 backlog=0MB wal=134MB dirty=0MB imm=0 lag=25677 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=68 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=578 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node0:p1 backlog=0MB wal=134MB dirty=0MB imm=0 lag=25668 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=67 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=528 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node0:p7 backlog=0MB wal=134MB dirty=0MB imm=0 lag=25673 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=72 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=525 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=35 low_cached_read_windows=12 p99=313 writes/s=10137 reads/s=18990 wal_group_avg_ms=6 bottleneck=wal_group_wait
social_media_100m: progress docs=6622720 txns=25872 phase=mixed writes/s=10137 avg_writes/s=7385 roll_wps=9830/10086/10342 reads/s=18990 avg_reads/s=17900 roll_rps=0/24303/24666 total_ops/s=29127 avg_total_ops/s=25285 roll_ops=9830/34434/34957 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:17459/0-0ms, first:1140/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:390/0-6ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=168ms p99=313ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=3MB wal=4337MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2723MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:1 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p1 lag=25878 q=0 imm=0 dirty=0MB trend=660 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:137833 mem:713 wait:0 manifest:0] wal_ms=[queue:4 group:46327 append:13674 fsync:2445 batches:1462 txns:6653 batch_mb:37 avg_txn_ms=q0 g6 a2 f0]
social_media_100m: hot node1:p1 backlog=0MB wal=135MB dirty=0MB imm=0 lag=25878 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=68 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=660 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=135MB dirty=0MB imm=0 lag=25874 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=68 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=618 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node1:p3 backlog=0MB wal=135MB dirty=0MB imm=0 lag=25881 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=68 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=612 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=36 low_cached_read_windows=13 p99=330 writes/s=10188 reads/s=18127 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=6673664 txns=26070 phase=mixed writes/s=10188 avg_writes/s=7400 roll_wps=9830/10137/10342 reads/s=18127 avg_reads/s=17901 roll_rps=18127/24303/24666 total_ops/s=28315 avg_total_ops/s=25301 roll_ops=28315/34434/34957 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:16676/0-0ms, first:1095/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:354/0-7ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=199ms p99=330ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4370MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2727MB/8s lsn_delta=[durable:1933 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=26088 q=0 imm=0 dirty=0MB trend=376 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256 commit_ms=[wal:150378 mem:797 wait:0 manifest:0] wal_ms=[queue:7 group:52379 append:14491 fsync:2218 batches:1399 txns:6621 batch_mb:37 avg_txn_ms=q0 g7 a2 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=136MB dirty=0MB imm=0 lag=26088 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=73 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=376 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=136MB dirty=0MB imm=0 lag=26084 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=69 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=363 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=136MB dirty=0MB imm=0 lag=26088 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=72 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=331 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=37 low_cached_read_windows=14 p99=333 writes/s=10240 reads/s=24438 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=6724864 txns=26271 phase=mixed writes/s=10240 avg_writes/s=7415 roll_wps=9830/10188/10342 reads/s=24438 avg_reads/s=17935 roll_rps=18127/24438/24666 total_ops/s=34678 avg_total_ops/s=25350 roll_ops=28315/34483/34957 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22467/0-0ms, first:1474/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:496/0-9ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=198ms p99=333ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4403MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2727MB/8s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:1 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=26280 q=0 imm=0 dirty=0MB trend=606 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256 commit_ms=[wal:146739 mem:653 wait:0 manifest:0] wal_ms=[queue:0 group:50409 append:13947 fsync:1895 batches:1323 txns:6177 batch_mb:34 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=137MB dirty=0MB imm=0 lag=26280 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=73 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=606 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=137MB dirty=0MB imm=0 lag=26277 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=69 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=589 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=137MB dirty=0MB imm=0 lag=26275 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=72 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=534 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=38 low_cached_read_windows=15 p99=347 writes/s=10188 reads/s=24295 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=6775808 txns=26470 phase=mixed writes/s=10188 avg_writes/s=7430 roll_wps=9830/10188/10291 reads/s=24295 avg_reads/s=17969 roll_rps=18127/24438/24666 total_ops/s=34483 avg_total_ops/s=25399 roll_ops=28315/34483/34957 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:22382/0-0ms, first:1440/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:471/0-12ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=214ms p99=347ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4436MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2727MB/8s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=26471 q=0 imm=0 dirty=0MB trend=833 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256 commit_ms=[wal:133639 mem:558 wait:0 manifest:0] wal_ms=[queue:3 group:44928 append:13135 fsync:1735 batches:1293 txns:6143 batch_mb:34 avg_txn_ms=q0 g7 a2 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=138MB dirty=0MB imm=0 lag=26471 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=73 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=833 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=138MB dirty=0MB imm=0 lag=26469 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=69 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=829 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=138MB dirty=0MB imm=0 lag=26467 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=72 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=755 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=39 low_cached_read_windows=16 p99=403 writes/s=10035 reads/s=23801 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=6825984 txns=26664 phase=mixed writes/s=10035 avg_writes/s=7444 roll_wps=9830/10188/10291 reads/s=23801 avg_reads/s=18000 roll_rps=18127/24438/24666 total_ops/s=33836 avg_total_ops/s=25444 roll_ops=28315/34483/34957 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:21870/0-0ms, first:1447/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:483/0-15ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=280ms p99=403ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4468MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2731MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=26663 q=0 imm=0 dirty=0MB trend=1061 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256 commit_ms=[wal:149075 mem:1835 wait:0 manifest:0] wal_ms=[queue:0 group:50336 append:14172 fsync:4064 batches:1493 txns:6142 batch_mb:34 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=139MB dirty=0MB imm=0 lag=26663 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=73 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1061 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=139MB dirty=0MB imm=0 lag=26665 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=71 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=832 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node1:p1 backlog=0MB wal=139MB dirty=0MB imm=0 lag=26662 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=71 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=756 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=40 low_cached_read_windows=17 p99=485 writes/s=9830 reads/s=23498 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=6875136 txns=26856 phase=mixed writes/s=9830 avg_writes/s=7457 roll_wps=9830/10188/10291 reads/s=23498 avg_reads/s=18029 roll_rps=18127/24438/24666 total_ops/s=33328 avg_total_ops/s=25486 roll_ops=28315/34483/34957 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:21618/0-0ms, first:1402/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:478/0-17ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=307ms p99=485ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4500MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2731MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p3 lag=26856 q=0 imm=0 dirty=0MB trend=369 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256 commit_ms=[wal:153012 mem:1390 wait:0 manifest:0] wal_ms=[queue:2 group:53841 append:16785 fsync:2792 batches:1551 txns:6144 batch_mb:34 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node1:p3 backlog=0MB wal=140MB dirty=0MB imm=0 lag=26856 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=75 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=369 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=140MB dirty=0MB imm=0 lag=26853 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=73 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=332 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256
social_media_100m: hot node0:p5 backlog=0MB wal=140MB dirty=0MB imm=0 lag=26855 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=77 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=295 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=1 low_wps_windows=0 wal_group_wait_windows=41 low_cached_read_windows=18 p99=512 writes/s=10035 reads/s=23404 wal_group_avg_ms=9 bottleneck=wal_group_wait
social_media_100m: progress docs=6925312 txns=27053 phase=mixed writes/s=10035 avg_writes/s=7470 roll_wps=9830/10137/10240 reads/s=23404 avg_reads/s=18056 roll_rps=18127/24295/24653 total_ops/s=33439 avg_total_ops/s=25526 roll_ops=28315/34483/34795 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:21531/0-0ms, first:1402/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:471/0-19ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=350ms p99=512ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4534MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2731MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p5 lag=27061 q=0 imm=0 dirty=0MB trend=556 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256 commit_ms=[wal:208472 mem:498 wait:0 manifest:0] wal_ms=[queue:2 group:63121 append:19433 fsync:8926 batches:1734 txns:6655 batch_mb:37 avg_txn_ms=q0 g9 a2 f1]
social_media_100m: hot node1:p5 backlog=0MB wal=141MB dirty=0MB imm=0 lag=27061 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=73 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=556 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=141MB dirty=0MB imm=0 lag=27062 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=76 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=470 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=141MB dirty=0MB imm=0 lag=27059 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=76 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=429 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=2 low_wps_windows=0 wal_group_wait_windows=42 low_cached_read_windows=19 p99=522 writes/s=10035 reads/s=19530 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=6975488 txns=27249 phase=mixed writes/s=10035 avg_writes/s=7483 roll_wps=9830/10086/10240 reads/s=19530 avg_reads/s=18063 roll_rps=18127/24246/24653 total_ops/s=29565 avg_total_ops/s=25546 roll_ops=28315/34434/34678 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:17970/0-0ms, first:1167/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:393/0-20ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=298ms p99=522ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=4MB wal=4570MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2734MB/5s lsn_delta=[durable:700 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:1 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p5 lag=27269 q=0 imm=0 dirty=0MB trend=776 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256 commit_ms=[wal:170835 mem:516 wait:0 manifest:0] wal_ms=[queue:3 group:57993 append:17983 fsync:3351 batches:1598 txns:6655 batch_mb:37 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node1:p5 backlog=0MB wal=142MB dirty=0MB imm=0 lag=27269 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=73 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=776 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256
social_media_100m: hot node3:p7 backlog=0MB wal=143MB dirty=0MB imm=0 lag=27272 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=73 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=737 emerg=2 budget=2MB hot_table=14124715428051363376 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=142MB dirty=0MB imm=0 lag=27267 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=76 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=659 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=3 low_wps_windows=0 wal_group_wait_windows=43 low_cached_read_windows=20 p99=540 writes/s=10035 reads/s=19167 wal_group_avg_ms=9 bottleneck=wal_group_wait
social_media_100m: progress docs=7025664 txns=27445 phase=mixed writes/s=10035 avg_writes/s=7496 roll_wps=9830/10086/10240 reads/s=19167 avg_reads/s=18068 roll_rps=18127/23801/24653 total_ops/s=29202 avg_total_ops/s=25564 roll_ops=28315/33836/34678 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:17623/0-0ms, first:1158/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:386/0-21ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=376ms p99=540ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=83MB wal=4602MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2734MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:26 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p6 lag=27459 q=1 imm=1 dirty=0MB trend=876 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256 commit_ms=[wal:180111 mem:1670 wait:0 manifest:0] wal_ms=[queue:4 group:59767 append:19334 fsync:3458 batches:1523 txns:6144 batch_mb:34 avg_txn_ms=q0 g9 a3 f0]
social_media_100m: hot node0:p6 backlog=3MB wal=143MB dirty=0MB imm=1 lag=27459 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=77 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=876 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256
social_media_100m: hot node3:p4 backlog=1MB wal=143MB dirty=0MB imm=1 lag=27464 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=81 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=742 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256
social_media_100m: hot node3:p2 backlog=2MB wal=143MB dirty=0MB imm=1 lag=27463 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=83 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=635 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=4 low_wps_windows=0 wal_group_wait_windows=44 low_cached_read_windows=21 p99=562 writes/s=10240 reads/s=18691 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=7076864 txns=27645 phase=mixed writes/s=10240 avg_writes/s=7510 roll_wps=9830/10086/10240 reads/s=18691 avg_reads/s=18070 roll_rps=18127/23498/24653 total_ops/s=28931 avg_total_ops/s=25580 roll_ops=28315/33439/34678 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=2ms ops=[id:17189/0-0ms, first:1116/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:385/1-23ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=384ms p99=562ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=76MB wal=4634MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2735MB/5s lsn_delta=[durable:553 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:30 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p6 lag=27651 q=1 imm=1 dirty=0MB trend=1074 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256 commit_ms=[wal:162143 mem:569 wait:0 manifest:0] wal_ms=[queue:3 group:55202 append:20044 fsync:1655 batches:1655 txns:6140 batch_mb:34 avg_txn_ms=q0 g8 a3 f0]
social_media_100m: hot node0:p6 backlog=0MB wal=144MB dirty=0MB imm=1 lag=27651 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=78 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1074 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=256
social_media_100m: hot node3:p6 backlog=4MB wal=144MB dirty=0MB imm=1 lag=27654 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=79 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=481 emerg=2 budget=2MB hot_table=14124715428051363376 fg_tokens=256
social_media_100m: hot node1:p1 backlog=0MB wal=144MB dirty=0MB imm=0 lag=27654 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=85 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=306 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=5 low_wps_windows=0 wal_group_wait_windows=45 low_cached_read_windows=22 p99=570 writes/s=10444 reads/s=19424 wal_group_avg_ms=9 bottleneck=wal_group_wait
social_media_100m: progress docs=7129088 txns=27848 phase=mixed writes/s=10444 avg_writes/s=7526 roll_wps=9830/10137/10444 reads/s=19424 avg_reads/s=18077 roll_rps=18127/23404/24653 total_ops/s=29868 avg_total_ops/s=25603 roll_ops=28315/33328/34678 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:17870/0-0ms, first:1156/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:398/0-24ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=357ms p99=570ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4666MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2735MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p1 lag=27845 q=0 imm=0 dirty=0MB trend=507 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256 commit_ms=[wal:167646 mem:398 wait:0 manifest:0] wal_ms=[queue:0 group:57599 append:18016 fsync:2234 batches:1484 txns:6142 batch_mb:34 avg_txn_ms=q0 g9 a2 f0]
social_media_100m: hot node1:p1 backlog=0MB wal=145MB dirty=0MB imm=0 lag=27845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=85 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=507 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node3:p7 backlog=0MB wal=146MB dirty=0MB imm=0 lag=27848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=79 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=348 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=145MB dirty=0MB imm=0 lag=27845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=81 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=321 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=6 low_wps_windows=0 wal_group_wait_windows=46 low_cached_read_windows=23 p99=571 writes/s=9830 reads/s=19442 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=7178240 txns=28040 phase=mixed writes/s=9830 avg_writes/s=7537 roll_wps=9830/10137/10444 reads/s=19442 avg_reads/s=18083 roll_rps=18127/19530/24438 total_ops/s=29272 avg_total_ops/s=25620 roll_ops=28315/29868/34678 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:17897/0-0ms, first:1167/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:378/0-26ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=327ms p99=571ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4698MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2735MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=28040 q=0 imm=0 dirty=0MB trend=1297 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256 commit_ms=[wal:157892 mem:185 wait:0 manifest:0] wal_ms=[queue:0 group:54688 append:16135 fsync:1761 batches:1447 txns:6142 batch_mb:34 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=147MB dirty=0MB imm=0 lag=28040 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=79 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1297 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: hot node1:p1 backlog=0MB wal=146MB dirty=0MB imm=0 lag=28037 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=85 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=708 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=146MB dirty=0MB imm=0 lag=28037 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=81 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=547 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=7 low_wps_windows=0 wal_group_wait_windows=47 low_cached_read_windows=24 p99=572 writes/s=9830 reads/s=19543 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=7227392 txns=28234 phase=mixed writes/s=9830 avg_writes/s=7549 roll_wps=9830/10035/10444 reads/s=19543 avg_reads/s=18090 roll_rps=18127/19543/24438 total_ops/s=29373 avg_total_ops/s=25639 roll_ops=28315/29868/34678 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:17973/0-0ms, first:1188/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:381/0-23ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=336ms p99=572ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4732MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2735MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=28247 q=0 imm=0 dirty=0MB trend=2253 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256 commit_ms=[wal:161811 mem:211 wait:0 manifest:0] wal_ms=[queue:0 group:58138 append:17400 fsync:2003 batches:1647 txns:6694 batch_mb:37 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=148MB dirty=0MB imm=0 lag=28247 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=79 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2253 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: hot node1:p1 backlog=0MB wal=147MB dirty=0MB imm=0 lag=28243 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=85 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=906 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=147MB dirty=0MB imm=0 lag=28243 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=81 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=772 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=8 low_wps_windows=0 wal_group_wait_windows=48 low_cached_read_windows=25 p99=575 writes/s=10240 reads/s=19747 wal_group_avg_ms=9 bottleneck=wal_group_wait
social_media_100m: progress docs=7278592 txns=28433 phase=mixed writes/s=10240 avg_writes/s=7562 roll_wps=9830/10035/10444 reads/s=19747 avg_reads/s=18097 roll_rps=18691/19747/24438 total_ops/s=29987 avg_total_ops/s=25659 roll_ops=28931/29987/34678 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:18139/0-0ms, first:1199/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:408/0-27ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=334ms p99=575ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4768MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2735MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=28456 q=0 imm=0 dirty=0MB trend=3229 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256 commit_ms=[wal:185200 mem:164 wait:0 manifest:0] wal_ms=[queue:5 group:59797 append:17613 fsync:2156 batches:1410 txns:6616 batch_mb:37 avg_txn_ms=q0 g9 a2 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=149MB dirty=0MB imm=0 lag=28456 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=79 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3229 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: hot node1:p1 backlog=0MB wal=148MB dirty=0MB imm=0 lag=28453 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=85 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1114 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=148MB dirty=0MB imm=0 lag=28453 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=81 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1000 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=9 low_wps_windows=0 wal_group_wait_windows=49 low_cached_read_windows=26 p99=580 writes/s=10188 reads/s=19436 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=7329536 txns=28632 phase=mixed writes/s=10188 avg_writes/s=7575 roll_wps=9830/10035/10444 reads/s=19436 avg_reads/s=18103 roll_rps=18691/19543/24295 total_ops/s=29624 avg_total_ops/s=25678 roll_ops=28931/29868/34483 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:17914/0-0ms, first:1132/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:389/0-24ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=356ms p99=580ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=4800MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2735MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p7 lag=28645 q=0 imm=0 dirty=0MB trend=1219 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256 commit_ms=[wal:152718 mem:516 wait:0 manifest:0] wal_ms=[queue:0 group:51489 append:16210 fsync:1628 batches:1418 txns:6144 batch_mb:34 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node1:p7 backlog=0MB wal=149MB dirty=0MB imm=0 lag=28645 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=81 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1219 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=149MB dirty=0MB imm=0 lag=28649 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=80 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1111 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=150MB dirty=0MB imm=0 lag=28646 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=85 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1054 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=10 low_wps_windows=0 wal_group_wait_windows=50 low_cached_read_windows=27 p99=590 writes/s=10035 reads/s=19438 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=7379712 txns=28828 phase=mixed writes/s=10035 avg_writes/s=7588 roll_wps=9830/10035/10444 reads/s=19438 avg_reads/s=18109 roll_rps=18691/19530/23801 total_ops/s=29473 avg_total_ops/s=25697 roll_ops=28931/29624/33836 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=2ms ops=[id:17875/0-0ms, first:1170/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:392/0-24ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=379ms p99=590ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=15MB wal=4833MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2742MB/5s lsn_delta=[durable:167 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:5 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=28838 q=0 imm=0 dirty=0MB trend=1052 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256 commit_ms=[wal:152118 mem:855 wait:0 manifest:0] wal_ms=[queue:2 group:52222 append:15401 fsync:1201 batches:1352 txns:6144 batch_mb:34 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node3:p6 backlog=0MB wal=150MB dirty=0MB imm=0 lag=28838 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=81 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1052 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node2:p7 backlog=0MB wal=151MB dirty=0MB imm=0 lag=28840 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=86 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=948 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node0:p0 backlog=0MB wal=150MB dirty=0MB imm=0 lag=28840 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=73 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=911 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=11 low_wps_windows=0 wal_group_wait_windows=51 low_cached_read_windows=28 p99=605 writes/s=10188 reads/s=19414 wal_group_avg_ms=9 bottleneck=wal_group_wait
social_media_100m: progress docs=7430656 txns=29027 phase=mixed writes/s=10188 avg_writes/s=7601 roll_wps=9830/10035/10444 reads/s=19414 avg_reads/s=18114 roll_rps=18691/19442/23498 total_ops/s=29602 avg_total_ops/s=25715 roll_ops=28931/29602/33439 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=2ms ops=[id:17871/0-0ms, first:1150/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:393/0-17ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=438ms p99=605ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=17MB wal=4865MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2742MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:7 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p7 lag=29032 q=0 imm=0 dirty=0MB trend=1084 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256 commit_ms=[wal:178799 mem:726 wait:0 manifest:0] wal_ms=[queue:0 group:58300 append:15855 fsync:2718 batches:1379 txns:6143 batch_mb:34 avg_txn_ms=q0 g9 a2 f0]
social_media_100m: hot node2:p7 backlog=0MB wal=152MB dirty=0MB imm=0 lag=29032 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=86 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1084 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node0:p7 backlog=0MB wal=152MB dirty=0MB imm=0 lag=29031 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=89 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=448 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: hot node0:p3 backlog=0MB wal=152MB dirty=0MB imm=0 lag=29026 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=83 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=406 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=12 low_wps_windows=0 wal_group_wait_windows=52 low_cached_read_windows=29 p99=635 writes/s=9984 reads/s=19078 wal_group_avg_ms=10 bottleneck=wal_group_wait
social_media_100m: progress docs=7480576 txns=29224 phase=mixed writes/s=9984 avg_writes/s=7612 roll_wps=9830/10035/10444 reads/s=19078 avg_reads/s=18118 roll_rps=18691/19438/23404 total_ops/s=29062 avg_total_ops/s=25730 roll_ops=28931/29565/33439 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=3ms ops=[id:17512/0-0ms, first:1177/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:389/1-26ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=452ms p99=635ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=36MB wal=4897MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2742MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:15 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p7 lag=29223 q=0 imm=0 dirty=0MB trend=605 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256 commit_ms=[wal:188157 mem:1044 wait:0 manifest:0] wal_ms=[queue:0 group:62822 append:19648 fsync:1534 batches:1446 txns:6142 batch_mb:34 avg_txn_ms=q0 g10 a3 f0]
social_media_100m: hot node0:p7 backlog=0MB wal=153MB dirty=0MB imm=0 lag=29223 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=89 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=605 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: hot node3:p7 backlog=3MB wal=153MB dirty=0MB imm=1 lag=29224 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=83 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=577 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: hot node0:p3 backlog=0MB wal=153MB dirty=0MB imm=0 lag=29218 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=83 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=549 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=13 low_wps_windows=0 wal_group_wait_windows=53 low_cached_read_windows=30 p99=647 writes/s=10240 reads/s=19511 wal_group_avg_ms=9 bottleneck=wal_group_wait
social_media_100m: progress docs=7531776 txns=29422 phase=mixed writes/s=10240 avg_writes/s=7624 roll_wps=9830/10188/10444 reads/s=19511 avg_reads/s=18122 roll_rps=18691/19438/19747 total_ops/s=29751 avg_total_ops/s=25746 roll_ops=28931/29565/29987 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=2ms ops=[id:17966/0-0ms, first:1159/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:384/0-28ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=408ms p99=647ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=7MB wal=4935MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2745MB/5s lsn_delta=[durable:1365 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:3 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=29448 q=0 imm=0 dirty=0MB trend=713 emerg=6 budget=4MB hot_table=10150445572359035967 fg_tokens=256 commit_ms=[wal:195252 mem:725 wait:0 manifest:0] wal_ms=[queue:0 group:68958 append:21499 fsync:2407 batches:1816 txns:7168 batch_mb:40 avg_txn_ms=q0 g9 a2 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=153MB dirty=0MB imm=0 lag=29448 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=87 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=713 emerg=6 budget=4MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node1:p1 backlog=0MB wal=154MB dirty=0MB imm=0 lag=29445 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=88 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=505 emerg=5 budget=4MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=154MB dirty=0MB imm=0 lag=29441 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=88 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=490 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=14 low_wps_windows=0 wal_group_wait_windows=54 low_cached_read_windows=31 p99=648 writes/s=10342 reads/s=19583 wal_group_avg_ms=7 bottleneck=wal_group_wait
social_media_100m: progress docs=7583488 txns=29624 phase=mixed writes/s=10342 avg_writes/s=7637 roll_wps=9830/10188/10444 reads/s=19583 avg_reads/s=18128 roll_rps=18691/19438/19747 total_ops/s=29925 avg_total_ops/s=25765 roll_ops=28931/29602/29987 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:18030/0-0ms, first:1165/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:388/0-21ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=365ms p99=648ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=2MB wal=4967MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2745MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:1 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=29640 q=0 imm=0 dirty=0MB trend=1168 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256 commit_ms=[wal:139465 mem:409 wait:0 manifest:0] wal_ms=[queue:5 group:49079 append:16151 fsync:2949 batches:1696 txns:6142 batch_mb:34 avg_txn_ms=q0 g7 a2 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=155MB dirty=0MB imm=0 lag=29640 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=85 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1168 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=154MB dirty=0MB imm=0 lag=29640 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=87 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=950 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=155MB dirty=0MB imm=0 lag=29633 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=88 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=723 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=15 low_wps_windows=0 wal_group_wait_windows=55 low_cached_read_windows=32 p99=659 writes/s=10035 reads/s=19082 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=7633664 txns=29821 phase=mixed writes/s=10035 avg_writes/s=7649 roll_wps=9830/10188/10444 reads/s=19082 avg_reads/s=18132 roll_rps=18691/19438/19747 total_ops/s=29117 avg_total_ops/s=25781 roll_ops=28931/29602/29987 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=2ms ops=[id:17591/0-0ms, first:1114/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:376/0-24ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=428ms p99=659ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=13MB wal=4999MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2747MB/5s lsn_delta=[durable:505 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:6 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=29832 q=0 imm=0 dirty=0MB trend=1507 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256 commit_ms=[wal:196710 mem:1578 wait:0 manifest:0] wal_ms=[queue:8 group:53152 append:17449 fsync:10636 batches:1682 txns:6143 batch_mb:34 avg_txn_ms=q0 g8 a2 f1]
social_media_100m: hot node3:p7 backlog=0MB wal=156MB dirty=0MB imm=0 lag=29832 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=85 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1507 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node2:p2 backlog=3MB wal=156MB dirty=0MB imm=1 lag=29825 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=89 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=940 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=155MB dirty=0MB imm=0 lag=29828 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=81 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=662 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=16 low_wps_windows=0 wal_group_wait_windows=56 low_cached_read_windows=33 p99=662 writes/s=10496 reads/s=19550 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=7686144 txns=30024 phase=mixed writes/s=10496 avg_writes/s=7663 roll_wps=9830/10188/10496 reads/s=19550 avg_reads/s=18138 roll_rps=19078/19442/19747 total_ops/s=30046 avg_total_ops/s=25801 roll_ops=29062/29624/30046 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:17991/0-0ms, first:1178/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:381/0-26ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=413ms p99=662ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=5031MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2747MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=30022 q=0 imm=0 dirty=0MB trend=843 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256 commit_ms=[wal:165991 mem:506 wait:0 manifest:0] wal_ms=[queue:12 group:55065 append:18655 fsync:3816 batches:1573 txns:6143 batch_mb:34 avg_txn_ms=q0 g8 a3 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=157MB dirty=0MB imm=0 lag=30022 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=91 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=843 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=157MB dirty=0MB imm=0 lag=30021 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=87 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=776 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=157MB dirty=0MB imm=0 lag=30025 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=89 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=663 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=17 low_wps_windows=0 wal_group_wait_windows=57 low_cached_read_windows=34 p99=642 writes/s=9830 reads/s=19606 wal_group_avg_ms=9 bottleneck=wal_group_wait
social_media_100m: progress docs=7735296 txns=30216 phase=mixed writes/s=9830 avg_writes/s=7673 roll_wps=9830/10188/10496 reads/s=19606 avg_reads/s=18144 roll_rps=19078/19511/19747 total_ops/s=29436 avg_total_ops/s=25817 roll_ops=29062/29602/30046 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:18019/0-0ms, first:1201/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:385/0-22ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=326ms p99=642ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=5063MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2747MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:1 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30231 q=0 imm=0 dirty=0MB trend=1043 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:169227 mem:491 wait:0 manifest:0] wal_ms=[queue:3 group:61035 append:17615 fsync:1672 batches:1553 txns:6524 batch_mb:36 avg_txn_ms=q0 g9 a2 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=158MB dirty=0MB imm=0 lag=30231 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=89 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1043 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=158MB dirty=0MB imm=0 lag=30231 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=89 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=881 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node0:p3 backlog=0MB wal=158MB dirty=0MB imm=0 lag=30223 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=90 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=579 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=18 low_wps_windows=0 wal_group_wait_windows=58 low_cached_read_windows=35 p99=514 writes/s=9984 reads/s=12901 wal_group_avg_ms=9 bottleneck=wal_group_wait
social_media_100m: progress docs=7785216 txns=30412 phase=mixed writes/s=9984 avg_writes/s=7684 roll_wps=9830/10188/10496 reads/s=12901 avg_reads/s=18117 roll_rps=12901/19511/19747 total_ops/s=22885 avg_total_ops/s=25801 roll_ops=22885/29602/30046 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:11870/0-0ms, first:769/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:260/0-24ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=334ms p99=514ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=12MB wal=5100MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2747MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:3 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30439 q=0 imm=0 dirty=0MB trend=1808 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:190692 mem:613 wait:0 manifest:0] wal_ms=[queue:0 group:61663 append:18395 fsync:2946 batches:1582 txns:6752 batch_mb:37 avg_txn_ms=q0 g9 a2 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=159MB dirty=0MB imm=0 lag=30439 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=89 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1808 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=159MB dirty=0MB imm=0 lag=30439 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=89 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1094 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node0:p3 backlog=0MB wal=159MB dirty=0MB imm=0 lag=30433 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=90 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=730 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=19 low_wps_windows=0 wal_group_wait_windows=59 low_cached_read_windows=36 p99=540 writes/s=10291 reads/s=4339 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=7836672 txns=30612 phase=mixed writes/s=10291 avg_writes/s=7696 roll_wps=9830/10188/10496 reads/s=4339 avg_reads/s=18048 roll_rps=4339/19438/19747 total_ops/s=14630 avg_total_ops/s=25744 roll_ops=14630/29602/30046 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=2ms ops=[id:3975/0-1ms, first:275/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:88/0-28ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=335ms p99=540ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=43MB wal=5133MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2747MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:14 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30632 q=1 imm=1 dirty=0MB trend=2229 emerg=5 budget=4MB hot_table=11207436792058616795 fg_tokens=256 commit_ms=[wal:154052 mem:587 wait:0 manifest:0] wal_ms=[queue:2 group:54744 append:17541 fsync:2026 batches:1657 txns:6172 batch_mb:35 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node3:p7 backlog=2MB wal=160MB dirty=0MB imm=1 lag=30632 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=90 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2229 emerg=5 budget=4MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node2:p5 backlog=0MB wal=160MB dirty=0MB imm=0 lag=30632 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=89 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1296 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: hot node3:p0 backlog=3MB wal=160MB dirty=0MB imm=1 lag=30637 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=94 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=751 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=256
social_media_100m: phase_start phase=reads_only elapsed_secs=1023 target_wps=0 target_rps=30000 evaluate_writes=false evaluate_reads=true
social_media_100m: progress docs=7888896 txns=30817 phase=reads_only writes/s=10444 avg_writes/s=7709 roll_wps=9830/10188/10496 reads/s=7231 avg_reads/s=17994 roll_rps=4339/19436/19606 total_ops/s=17675 avg_total_ops/s=25703 roll_ops=14630/29473/30046 read_band=no read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:6644/0-0ms, first:451/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:135/0-27ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=360ms p99=589ms band=yes slo=fail mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5165MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:419 truncate:0] bottleneck=read_generator_limit cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=30821 q=0 imm=0 dirty=0MB trend=614 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:148099 mem:109 wait:0 manifest:0] wal_ms=[queue:2 group:53861 append:17815 fsync:2097 batches:1768 txns:6144 batch_mb:34 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30821 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=614 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30821 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=467 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30817 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=94 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=461 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=1638 avg_writes/s=7679 roll_wps=1638/10188/10496 reads/s=28916 avg_reads/s=18046 roll_rps=4339/19438/28916 total_ops/s=30554 avg_total_ops/s=25725 roll_ops=14630/29473/30554 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26570/0-0ms, first:1757/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:588/0-5ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=279ms p99=457ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=3469 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:18506 mem:75 wait:0 manifest:0] wal_ms=[queue:0 group:6427 append:1811 fsync:137 batches:163 txns:768 batch_mb:4 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3469 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=814 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=683 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7642 roll_wps=0/10188/10496 reads/s=29138 avg_reads/s=18099 roll_rps=4339/19511/29138 total_ops/s=29138 avg_total_ops/s=25741 roll_ops=14630/29436/30554 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26800/0-0ms, first:1754/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:583/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=6064 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6064 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1033 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30841 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=94 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=891 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7604 roll_wps=0/10035/10496 reads/s=29286 avg_reads/s=18152 roll_rps=4339/19550/29286 total_ops/s=29286 avg_total_ops/s=25756 roll_ops=14630/29286/30554 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26927/0-0ms, first:1764/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:594/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=6277 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6277 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1253 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1105 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7568 roll_wps=0/10035/10496 reads/s=29245 avg_reads/s=18204 roll_rps=4339/19583/29286 total_ops/s=29245 avg_total_ops/s=25772 roll_ops=14630/29286/30554 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26877/0-0ms, first:1775/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:592/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=6510 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6510 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1475 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1326 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7531 roll_wps=0/9984/10496 reads/s=29326 avg_reads/s=18256 roll_rps=4339/19606/29326 total_ops/s=29326 avg_total_ops/s=25787 roll_ops=14630/29286/30554 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26959/0-0ms, first:1768/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:599/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=6682 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6682 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1687 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30841 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=94 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1554 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7495 roll_wps=0/9830/10496 reads/s=29321 avg_reads/s=18308 roll_rps=4339/28916/29326 total_ops/s=29321 avg_total_ops/s=25803 roll_ops=14630/29286/30554 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26962/0-0ms, first:1773/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:586/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=6885 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6885 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1906 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p7 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1763 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7459 roll_wps=0/1638/10496 reads/s=29273 avg_reads/s=18359 roll_rps=4339/29138/29326 total_ops/s=29273 avg_total_ops/s=25818 roll_ops=14630/29286/30554 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26871/0-0ms, first:1811/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:590/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=7077 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7077 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2124 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30841 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=94 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1991 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7424 roll_wps=0/0/10444 reads/s=29286 avg_reads/s=18409 roll_rps=4339/29245/29326 total_ops/s=29286 avg_total_ops/s=25833 roll_ops=14630/29286/30554 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26959/0-0ms, first:1745/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:582/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=7252 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7252 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2352 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30841 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=94 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2215 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7389 roll_wps=0/0/10444 reads/s=29277 avg_reads/s=18459 roll_rps=4339/29273/29326 total_ops/s=29277 avg_total_ops/s=25848 roll_ops=14630/29277/30554 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26939/0-0ms, first:1770/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:568/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=7466 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7466 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2578 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30841 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=94 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2437 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7354 roll_wps=0/0/10444 reads/s=29299 avg_reads/s=18509 roll_rps=4339/29277/29326 total_ops/s=29299 avg_total_ops/s=25863 roll_ops=14630/29286/30554 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26963/0-0ms, first:1778/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:557/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=7657 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7657 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2806 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30841 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=94 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2654 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7320 roll_wps=0/0/10444 reads/s=29316 avg_reads/s=18558 roll_rps=7231/29286/29326 total_ops/s=29316 avg_total_ops/s=25878 roll_ops=17675/29286/30554 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27001/0-0ms, first:1736/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:578/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=7852 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=7852 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3034 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30841 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=94 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2887 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7285 roll_wps=0/0/1638 reads/s=29294 avg_reads/s=18607 roll_rps=28916/29286/29326 total_ops/s=29294 avg_total_ops/s=25892 roll_ops=29138/29294/30554 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26953/0-0ms, first:1752/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:588/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=8048 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8048 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3245 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30841 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=94 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3094 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7252 roll_wps=0/0/0 reads/s=29279 avg_reads/s=18655 roll_rps=29138/29286/29326 total_ops/s=29279 avg_total_ops/s=25907 roll_ops=29138/29286/29326 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26914/0-0ms, first:1759/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:605/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=8224 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8224 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3468 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30841 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=94 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3317 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7218 roll_wps=0/0/0 reads/s=29280 avg_reads/s=18703 roll_rps=29245/29286/29326 total_ops/s=29280 avg_total_ops/s=25921 roll_ops=29245/29286/29326 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26930/0-0ms, first:1753/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:596/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=8423 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8423 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3677 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30841 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=94 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3525 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7185 roll_wps=0/0/0 reads/s=29272 avg_reads/s=18750 roll_rps=29245/29286/29326 total_ops/s=29272 avg_total_ops/s=25935 roll_ops=29245/29286/29326 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26929/0-0ms, first:1753/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:590/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=8611 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8611 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3890 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30847 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3745 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7152 roll_wps=0/0/0 reads/s=29312 avg_reads/s=18796 roll_rps=29272/29294/29326 total_ops/s=29312 avg_total_ops/s=25948 roll_ops=29272/29294/29326 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26936/0-0ms, first:1776/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:599/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=8806 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=8806 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4091 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30847 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3977 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7119 roll_wps=0/0/0 reads/s=29242 avg_reads/s=18843 roll_rps=29242/29286/29321 total_ops/s=29242 avg_total_ops/s=25962 roll_ops=29242/29286/29321 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26884/0-0ms, first:1787/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:570/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=9018 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9018 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4296 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30847 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4200 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7087 roll_wps=0/0/0 reads/s=29285 avg_reads/s=18889 roll_rps=29242/29285/29316 total_ops/s=29285 avg_total_ops/s=25976 roll_ops=29242/29285/29316 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26907/0-0ms, first:1775/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:601/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=9236 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9236 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4511 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30847 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4433 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7055 roll_wps=0/0/0 reads/s=29291 avg_reads/s=18934 roll_rps=29242/29286/29316 total_ops/s=29291 avg_total_ops/s=25989 roll_ops=29242/29286/29316 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26974/0-0ms, first:1742/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:574/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=9385 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9385 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4734 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30847 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4661 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=7023 roll_wps=0/0/0 reads/s=29402 avg_reads/s=18980 roll_rps=29242/29291/29402 total_ops/s=29402 avg_total_ops/s=26003 roll_ops=29242/29291/29402 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27042/0-0ms, first:1773/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:586/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=9508 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9508 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4951 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30847 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4904 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=6992 roll_wps=0/0/0 reads/s=29305 avg_reads/s=19025 roll_rps=29242/29294/29402 total_ops/s=29305 avg_total_ops/s=26017 roll_ops=29242/29294/29402 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26960/0-0ms, first:1765/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:579/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=9645 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9645 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5161 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30847 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5126 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=6961 roll_wps=0/0/0 reads/s=29369 avg_reads/s=19069 roll_rps=29242/29294/29402 total_ops/s=29369 avg_total_ops/s=26030 roll_ops=29242/29294/29402 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:27012/0-0ms, first:1776/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:580/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=9771 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9771 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5383 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30847 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5366 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: progress docs=7897088 txns=30848 phase=reads_only writes/s=0 avg_writes/s=6930 roll_wps=0/0/0 reads/s=29294 avg_reads/s=19113 roll_rps=29242/29294/29402 total_ops/s=29294 avg_total_ops/s=26043 roll_ops=29242/29294/29402 read_band=yes read_target_rps=30000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26950/0-0ms, first:1754/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:589/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=yes slo=pass mode=cleanup assist=no target_wps=0 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=9896 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:0 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=9896 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5597 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30847 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5586 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: phase_start phase=writes_only elapsed_secs=1144 target_wps=10000 target_rps=0 evaluate_writes=true evaluate_reads=false
social_media_100m: progress docs=7897088 txns=30848 phase=writes_only writes/s=0 avg_writes/s=6899 roll_wps=0/0/0 reads/s=29302 avg_reads/s=19157 roll_rps=29242/29294/29402 total_ops/s=29302 avg_total_ops/s=26056 roll_ops=29242/29294/29402 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:26966/0-0ms, first:1759/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:576/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=0ms p99=0ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=5169MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=30848 q=0 imm=0 dirty=0MB trend=10028 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:0 mem:0 wait:0 manifest:0] wal_ms=[queue:0 group:22 append:0 fsync:0 batches:0 txns:0 batch_mb:0 avg_txn_ms=q0 g0 a0 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=162MB dirty=0MB imm=0 lag=30848 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10028 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30847 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5817 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=161MB dirty=0MB imm=0 lag=30845 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5812 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=1 low_wps_windows=0 wal_group_wait_windows=1 low_cached_read_windows=0 p99=601 writes/s=9932 reads/s=54 wal_group_avg_ms=10 bottleneck=wal_group_wait
social_media_100m: progress docs=7946752 txns=31043 phase=writes_only writes/s=9932 avg_writes/s=6912 roll_wps=0/0/9932 reads/s=54 avg_reads/s=19073 roll_rps=54/29294/29402 total_ops/s=9986 avg_total_ops/s=25985 roll_ops=9986/29294/29402 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:48/0-0ms, first:4/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:1/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=355ms p99=601ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=1MB wal=5207MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2753MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:1 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=31072 q=0 imm=0 dirty=0MB trend=10156 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:220706 mem:1099 wait:0 manifest:0] wal_ms=[queue:0 group:73256 append:18601 fsync:1723 batches:1355 txns:7165 batch_mb:40 avg_txn_ms=q0 g10 a2 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=163MB dirty=0MB imm=0 lag=31072 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=92 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=10156 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p6 backlog=0MB wal=163MB dirty=0MB imm=0 lag=31069 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6024 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=163MB dirty=0MB imm=0 lag=31065 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=94 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5993 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=2 low_wps_windows=0 wal_group_wait_windows=2 low_cached_read_windows=0 p99=611 writes/s=10393 reads/s=0 wal_group_avg_ms=9 bottleneck=wal_group_wait
social_media_100m: progress docs=7998720 txns=31246 phase=writes_only writes/s=10393 avg_writes/s=6927 roll_wps=0/0/10393 reads/s=0 avg_reads/s=18989 roll_rps=0/29294/29402 total_ops/s=10393 avg_total_ops/s=25916 roll_ops=9986/29294/29402 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=352ms p99=611ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=4MB wal=5238MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2755MB/10s lsn_delta=[durable:1006 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:2 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=31261 q=0 imm=0 dirty=0MB trend=6230 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:175499 mem:1830 wait:0 manifest:0] wal_ms=[queue:2 group:59912 append:17360 fsync:1888 batches:1359 txns:6143 batch_mb:34 avg_txn_ms=q0 g9 a2 f0]
social_media_100m: hot node2:p6 backlog=0MB wal=164MB dirty=0MB imm=0 lag=31261 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6230 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p0 backlog=0MB wal=163MB dirty=0MB imm=0 lag=31269 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=96 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3714 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p4 backlog=0MB wal=163MB dirty=0MB imm=0 lag=31263 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=99 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=268 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=3 low_wps_windows=0 wal_group_wait_windows=3 low_cached_read_windows=0 p99=604 writes/s=9984 reads/s=0 wal_group_avg_ms=9 bottleneck=wal_group_wait
social_media_100m: progress docs=8048640 txns=31441 phase=writes_only writes/s=9984 avg_writes/s=6940 roll_wps=0/0/10393 reads/s=0 avg_reads/s=18907 roll_rps=0/29294/29402 total_ops/s=9984 avg_total_ops/s=25847 roll_ops=9984/29294/29402 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=361ms p99=604ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=21MB wal=5271MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2756MB/5s lsn_delta=[durable:477 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:8 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p4 lag=31455 q=0 imm=0 dirty=0MB trend=479 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:172095 mem:1169 wait:0 manifest:0] wal_ms=[queue:2 group:56555 append:18040 fsync:1792 batches:1430 txns:6144 batch_mb:34 avg_txn_ms=q0 g9 a2 f0]
social_media_100m: hot node3:p4 backlog=0MB wal=164MB dirty=0MB imm=0 lag=31455 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=99 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=479 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node1:p5 backlog=0MB wal=164MB dirty=0MB imm=0 lag=31452 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=94 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=351 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=164MB dirty=0MB imm=0 lag=31454 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=289 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=4 low_wps_windows=0 wal_group_wait_windows=4 low_cached_read_windows=0 p99=613 writes/s=10496 reads/s=0 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=8101120 txns=31646 phase=writes_only writes/s=10496 avg_writes/s=6955 roll_wps=0/0/10496 reads/s=0 avg_reads/s=18825 roll_rps=0/29291/29402 total_ops/s=10496 avg_total_ops/s=25780 roll_ops=9984/29291/29402 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=362ms p99=613ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=8MB wal=5303MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2756MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:3 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p5 lag=31644 q=0 imm=0 dirty=0MB trend=546 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256 commit_ms=[wal:164078 mem:494 wait:0 manifest:0] wal_ms=[queue:5 group:52693 append:17905 fsync:1724 batches:1475 txns:6144 batch_mb:34 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node1:p5 backlog=0MB wal=165MB dirty=0MB imm=0 lag=31644 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=94 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=546 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: hot node3:p6 backlog=0MB wal=165MB dirty=0MB imm=0 lag=31646 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=462 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p4 backlog=0MB wal=165MB dirty=0MB imm=0 lag=31649 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=374 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=5 low_wps_windows=0 wal_group_wait_windows=5 low_cached_read_windows=0 p99=562 writes/s=9984 reads/s=0 wal_group_avg_ms=9 bottleneck=wal_group_wait
social_media_100m: progress docs=8151040 txns=31840 phase=writes_only writes/s=9984 avg_writes/s=6968 roll_wps=0/0/10496 reads/s=0 avg_reads/s=18744 roll_rps=0/29291/29402 total_ops/s=9984 avg_total_ops/s=25712 roll_ops=9984/29291/29402 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=334ms p99=562ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=4MB wal=5335MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2756MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:4 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p0 lag=31843 q=0 imm=0 dirty=0MB trend=493 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256 commit_ms=[wal:167063 mem:300 wait:0 manifest:0] wal_ms=[queue:7 group:56079 append:17204 fsync:1795 batches:1378 txns:6142 batch_mb:34 avg_txn_ms=q0 g9 a2 f0]
social_media_100m: hot node1:p0 backlog=0MB wal=166MB dirty=0MB imm=0 lag=31843 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=104 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=493 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node0:p6 backlog=0MB wal=166MB dirty=0MB imm=0 lag=31833 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=101 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=447 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node2:p3 backlog=1MB wal=166MB dirty=0MB imm=1 lag=31838 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=101 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=414 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=6 low_wps_windows=0 wal_group_wait_windows=6 low_cached_read_windows=0 p99=610 writes/s=9830 reads/s=0 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=8200192 txns=32033 phase=writes_only writes/s=9830 avg_writes/s=6979 roll_wps=0/9830/10496 reads/s=0 avg_reads/s=18663 roll_rps=0/29291/29402 total_ops/s=9830 avg_total_ops/s=25642 roll_ops=9830/29291/29402 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=352ms p99=610ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=37MB wal=5368MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:220 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:12 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=32045 q=0 imm=0 dirty=0MB trend=930 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=256 commit_ms=[wal:149593 mem:321 wait:0 manifest:0] wal_ms=[queue:4 group:52650 append:17688 fsync:1890 batches:1694 txns:6490 batch_mb:36 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=168MB dirty=0MB imm=0 lag=32045 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=99 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=930 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=256
social_media_100m: hot node1:p0 backlog=0MB wal=168MB dirty=0MB imm=0 lag=32048 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=104 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=645 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: hot node0:p7 backlog=3MB wal=167MB dirty=0MB imm=1 lag=32041 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=103 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=560 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=7 low_wps_windows=0 wal_group_wait_windows=7 low_cached_read_windows=0 p99=624 writes/s=10188 reads/s=0 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=8251136 txns=32232 phase=writes_only writes/s=10188 avg_writes/s=6993 roll_wps=0/9932/10496 reads/s=0 avg_reads/s=18583 roll_rps=0/54/29402 total_ops/s=10188 avg_total_ops/s=25576 roll_ops=9830/10496/29402 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=388ms p99=624ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=4MB wal=5405MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:1 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=32256 q=0 imm=0 dirty=0MB trend=1267 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=256 commit_ms=[wal:180879 mem:440 wait:0 manifest:0] wal_ms=[queue:2 group:58994 append:20944 fsync:2303 batches:1728 txns:6819 batch_mb:38 avg_txn_ms=q0 g8 a3 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=169MB dirty=0MB imm=0 lag=32256 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=99 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1267 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=169MB dirty=0MB imm=0 lag=32249 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=101 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=622 emerg=5 budget=4MB hot_table=17248333830116347718 fg_tokens=256
social_media_100m: hot node3:p2 backlog=4MB wal=168MB dirty=0MB imm=1 lag=32254 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=107 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=543 emerg=2 budget=2MB hot_table=13077696131643367010 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=8 low_wps_windows=0 wal_group_wait_windows=8 low_cached_read_windows=0 p99=625 writes/s=10291 reads/s=0 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=8302592 txns=32433 phase=writes_only writes/s=10291 avg_writes/s=7006 roll_wps=0/9984/10496 reads/s=0 avg_reads/s=18504 roll_rps=0/0/29369 total_ops/s=10291 avg_total_ops/s=25510 roll_ops=9830/10393/29369 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=286ms p99=625ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=5437MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=32448 q=0 imm=0 dirty=0MB trend=1770 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=256 commit_ms=[wal:167413 mem:112 wait:0 manifest:0] wal_ms=[queue:3 group:53618 append:15589 fsync:1529 batches:1219 txns:6141 batch_mb:34 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=170MB dirty=0MB imm=0 lag=32448 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=99 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1770 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=170MB dirty=0MB imm=0 lag=32441 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=101 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=833 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=256
social_media_100m: hot node1:p6 backlog=0MB wal=170MB dirty=0MB imm=0 lag=32447 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=100 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=654 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=9 low_cached_read_windows=0 p99=459 writes/s=10342 reads/s=0 wal_group_avg_ms=8 bottleneck=wal_group_wait
social_media_100m: progress docs=8354304 txns=32635 phase=writes_only writes/s=10342 avg_writes/s=7020 roll_wps=0/9984/10496 reads/s=0 avg_reads/s=18425 roll_rps=0/0/29369 total_ops/s=10342 avg_total_ops/s=25445 roll_ops=9830/10342/29369 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=280ms p99=459ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=5469MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=32640 q=0 imm=0 dirty=0MB trend=2071 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=256 commit_ms=[wal:152264 mem:420 wait:0 manifest:0] wal_ms=[queue:0 group:50742 append:15965 fsync:1822 batches:1410 txns:6142 batch_mb:34 avg_txn_ms=q0 g8 a2 f0]
social_media_100m: hot node3:p7 backlog=0MB wal=171MB dirty=0MB imm=0 lag=32640 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=99 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2071 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=256
social_media_100m: hot node2:p2 backlog=0MB wal=171MB dirty=0MB imm=0 lag=32633 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=101 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1036 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=256
social_media_100m: hot node1:p6 backlog=0MB wal=171MB dirty=0MB imm=0 lag=32639 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=100 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=792 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=256
social_media_100m: RED acceptance_gate_failed high_p99_windows=1 low_wps_windows=1 wal_group_wait_windows=10 low_cached_read_windows=0 p99=664 writes/s=8960 reads/s=0 wal_group_avg_ms=9 bottleneck=checkpoint_advance
social_media_100m: progress docs=8399104 txns=32811 phase=writes_only writes/s=8960 avg_writes/s=7028 roll_wps=0/9984/10496 reads/s=0 avg_reads/s=18347 roll_rps=0/0/29302 total_ops/s=8960 avg_total_ops/s=25375 roll_ops=8960/10291/29302 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=315ms p99=664ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=934MB wal=5501MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=32827 q=12 imm=12 dirty=0MB trend=2750 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0 commit_ms=[wal:167936 mem:98629 wait:89861 manifest:0] wal_ms=[queue:4 group:57259 append:20059 fsync:3146 batches:1690 txns:5976 batch_mb:34 avg_txn_ms=q0 g9 a3 f0]
social_media_100m: hot node3:p7 backlog=26MB wal=172MB dirty=0MB imm=12 lag=32827 q=12 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=102 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2750 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node2:p2 backlog=30MB wal=172MB dirty=0MB imm=12 lag=32820 q=12 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=101 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1255 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node2:p5 backlog=28MB wal=171MB dirty=0MB imm=12 lag=32826 q=12 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=100 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=975 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=2 low_wps_windows=2 wal_group_wait_windows=11 low_cached_read_windows=0 p99=696 writes/s=8243 reads/s=0 wal_group_avg_ms=11 bottleneck=checkpoint_advance
social_media_100m: progress docs=8440320 txns=32971 phase=writes_only writes/s=8243 avg_writes/s=7032 roll_wps=0/9984/10496 reads/s=0 avg_reads/s=18270 roll_rps=0/0/29302 total_ops/s=8243 avg_total_ops/s=25302 roll_ops=8243/10188/29302 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=563ms p99=696ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=911MB wal=5529MB wal_growth=5MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p2 lag=32989 q=12 imm=12 dirty=0MB trend=1633 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0 commit_ms=[wal:120841 mem:283812 wait:256032 manifest:0] wal_ms=[queue:0 group:63100 append:23050 fsync:6480 batches:3329 txns:5425 batch_mb:29 avg_txn_ms=q0 g11 a4 f1]
social_media_100m: hot node2:p2 backlog=30MB wal=173MB dirty=0MB imm=12 lag=32989 q=12 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=101 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1633 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node2:p5 backlog=29MB wal=172MB dirty=0MB imm=13 lag=32995 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=100 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1327 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node1:p6 backlog=29MB wal=173MB dirty=0MB imm=13 lag=32995 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=101 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1295 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: RED partition_stuck node0:p0 windows=3 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=13 q=13 lag=33154 durable_delta=0 truncate_delta=0 primary_flush_delta=1 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=3 low_wps_windows=3 wal_group_wait_windows=12 low_cached_read_windows=0 p99=760 writes/s=8345 reads/s=0 wal_group_avg_ms=11 bottleneck=checkpoint_advance
social_media_100m: progress docs=8482048 txns=33134 phase=writes_only writes/s=8345 avg_writes/s=7037 roll_wps=8243/9984/10496 reads/s=0 avg_reads/s=18192 roll_rps=0/0/54 total_ops/s=8345 avg_total_ops/s=25229 roll_ops=8243/9986/10496 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=665ms p99=760ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=889MB wal=5556MB wal_growth=5MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p5 lag=33154 q=13 imm=13 dirty=0MB trend=1685 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0 commit_ms=[wal:109776 mem:285892 wait:258318 manifest:0] wal_ms=[queue:8 group:58464 append:22384 fsync:6256 batches:3290 txns:5075 batch_mb:28 avg_txn_ms=q0 g11 a4 f1]
social_media_100m: hot node2:p5 backlog=29MB wal=173MB dirty=0MB imm=13 lag=33154 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=100 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1685 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node1:p6 backlog=29MB wal=173MB dirty=0MB imm=13 lag=33154 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=101 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1614 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node3:p4 backlog=30MB wal=173MB dirty=0MB imm=13 lag=33153 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=106 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1574 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: monotonic_growth_fail node=0 partition=0 windows=3 immutables=13 backlog_mb=28 wal_mb=173 checkpoint_lag_lsn=33154 flush_queue_depth=13
social_media_100m: RED partition_stuck node0:p0 windows=4 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=13 q=13 lag=33314 durable_delta=0 truncate_delta=0 primary_flush_delta=1 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=4 low_wps_windows=4 wal_group_wait_windows=13 low_cached_read_windows=0 p99=828 writes/s=7884 reads/s=0 wal_group_avg_ms=11 bottleneck=checkpoint_advance
social_media_100m: progress docs=8521472 txns=33288 phase=writes_only writes/s=7884 avg_writes/s=7040 roll_wps=7884/9984/10496 reads/s=0 avg_reads/s=18116 roll_rps=0/0/0 total_ops/s=7884 avg_total_ops/s=25156 roll_ops=7884/9984/10496 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=545ms p99=828ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=853MB wal=5583MB wal_growth=5MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p5 lag=33314 q=13 imm=13 dirty=0MB trend=2039 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0 commit_ms=[wal:116352 mem:277541 wait:250415 manifest:0] wal_ms=[queue:6 group:59375 append:23013 fsync:6730 batches:3159 txns:5120 batch_mb:29 avg_txn_ms=q0 g11 a4 f1]
social_media_100m: hot node2:p5 backlog=27MB wal=174MB dirty=0MB imm=13 lag=33314 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=101 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2039 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node1:p6 backlog=27MB wal=174MB dirty=1MB imm=12 lag=33314 q=12 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=102 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1940 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node0:p0 backlog=27MB wal=174MB dirty=0MB imm=13 lag=33314 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=98 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1866 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: RED partition_stuck node0:p0 windows=5 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=13 q=13 lag=33466 durable_delta=0 truncate_delta=0 primary_flush_delta=1 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=5 low_wps_windows=5 wal_group_wait_windows=14 low_cached_read_windows=0 p99=995 writes/s=8192 reads/s=0 wal_group_avg_ms=11 bottleneck=checkpoint_advance
social_media_100m: progress docs=8562432 txns=33448 phase=writes_only writes/s=8192 avg_writes/s=7044 roll_wps=7884/9984/10496 reads/s=0 avg_reads/s=18039 roll_rps=0/0/0 total_ops/s=8192 avg_total_ops/s=25083 roll_ops=7884/9984/10496 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=630ms p99=995ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=780MB wal=5608MB wal_growth=5MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p5 lag=33466 q=13 imm=13 dirty=0MB trend=2399 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0 commit_ms=[wal:103439 mem:263327 wait:237791 manifest:0] wal_ms=[queue:5 group:54046 append:22792 fsync:6676 batches:3325 txns:4860 batch_mb:27 avg_txn_ms=q0 g11 a4 f1]
social_media_100m: hot node2:p5 backlog=25MB wal=175MB dirty=0MB imm=13 lag=33466 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=102 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2399 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node1:p6 backlog=23MB wal=175MB dirty=1MB imm=12 lag=33465 q=12 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=104 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2259 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node0:p0 backlog=25MB wal=175MB dirty=0MB imm=13 lag=33466 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=99 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2199 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: RED partition_stuck node0:p0 windows=6 reason="queue/immutable plateau while checkpoint lag grows and no flush/durable progress" imm=13 q=13 lag=33619 durable_delta=0 truncate_delta=0 primary_flush_delta=0 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=6 low_wps_windows=6 wal_group_wait_windows=15 low_cached_read_windows=0 p99=1155 writes/s=7526 reads/s=0 wal_group_avg_ms=12 bottleneck=checkpoint_advance
social_media_100m: progress docs=8600064 txns=33597 phase=writes_only writes/s=7526 avg_writes/s=7046 roll_wps=7526/9830/10496 reads/s=0 avg_reads/s=17964 roll_rps=0/0/0 total_ops/s=7526 avg_total_ops/s=25010 roll_ops=7526/9830/10496 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=605ms p99=1155ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=737MB wal=5634MB wal_growth=5MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p6 lag=33618 q=13 imm=13 dirty=1MB trend=2572 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0 commit_ms=[wal:109905 mem:265984 wait:240258 manifest:0] wal_ms=[queue:0 group:59795 append:24453 fsync:7612 batches:3518 txns:4911 batch_mb:27 avg_txn_ms=q0 g12 a4 f1]
social_media_100m: hot node1:p6 backlog=24MB wal=176MB dirty=1MB imm=13 lag=33618 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=104 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2572 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node0:p0 backlog=25MB wal=175MB dirty=0MB imm=13 lag=33619 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=99 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2537 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node1:p7 backlog=25MB wal=176MB dirty=1MB imm=12 lag=33616 q=12 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=103 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2425 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: RED partition_stuck node0:p0 windows=7 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=14 q=14 lag=33781 durable_delta=0 truncate_delta=0 primary_flush_delta=1 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=7 low_wps_windows=7 wal_group_wait_windows=16 low_cached_read_windows=0 p99=1231 writes/s=8089 reads/s=0 wal_group_avg_ms=11 bottleneck=checkpoint_advance
social_media_100m: progress docs=8640512 txns=33754 phase=writes_only writes/s=8089 avg_writes/s=7049 roll_wps=7526/8960/10342 reads/s=0 avg_reads/s=17888 roll_rps=0/0/0 total_ops/s=8089 avg_total_ops/s=24937 roll_ops=7526/8960/10342 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=468ms p99=1231ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=701MB wal=5661MB wal_growth=5MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p0 lag=33781 q=14 imm=14 dirty=1MB trend=2871 emerg=6 budget=4MB hot_table=17248333830116347718 fg_tokens=0 commit_ms=[wal:116785 mem:280596 wait:253117 manifest:0] wal_ms=[queue:1 group:57109 append:23902 fsync:8826 batches:3273 txns:5150 batch_mb:28 avg_txn_ms=q0 g11 a4 f1]
social_media_100m: hot node0:p0 backlog=25MB wal=176MB dirty=1MB imm=14 lag=33781 q=14 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=100 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2871 emerg=6 budget=4MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node1:p6 backlog=21MB wal=177MB dirty=1MB imm=13 lag=33780 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=106 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2880 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node1:p7 backlog=23MB wal=176MB dirty=1MB imm=12 lag=33778 q=12 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=104 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2768 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: RED partition_stuck node0:p0 windows=8 reason="queue/immutable plateau while checkpoint lag grows and no flush/durable progress" imm=15 q=15 lag=33926 durable_delta=0 truncate_delta=0 primary_flush_delta=0 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=8 low_wps_windows=8 wal_group_wait_windows=17 low_cached_read_windows=0 p99=1297 writes/s=7731 reads/s=0 wal_group_avg_ms=12 bottleneck=checkpoint_advance
social_media_100m: progress docs=8679168 txns=33905 phase=writes_only writes/s=7731 avg_writes/s=7051 roll_wps=7526/8345/10342 reads/s=0 avg_reads/s=17814 roll_rps=0/0/0 total_ops/s=7731 avg_total_ops/s=24865 roll_ops=7526/8345/10342 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=618ms p99=1297ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=712MB wal=5685MB wal_growth=5MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p6 lag=33925 q=14 imm=14 dirty=1MB trend=3188 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0 commit_ms=[wal:111029 mem:252034 wait:227376 manifest:0] wal_ms=[queue:6 group:59514 append:22765 fsync:6606 batches:3153 txns:4665 batch_mb:26 avg_txn_ms=q0 g12 a4 f1]
social_media_100m: hot node1:p6 backlog=22MB wal=178MB dirty=1MB imm=14 lag=33925 q=14 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=106 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3188 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node0:p0 backlog=26MB wal=177MB dirty=1MB imm=15 lag=33926 q=15 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=100 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3184 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node1:p7 backlog=24MB wal=177MB dirty=1MB imm=13 lag=33923 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=104 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3101 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: monotonic_growth_fail node=0 partition=0 windows=3 immutables=15 backlog_mb=26 wal_mb=177 checkpoint_lag_lsn=33926 flush_queue_depth=15
social_media_100m: RED partition_stuck node0:p0 windows=9 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=15 q=15 lag=34064 durable_delta=0 truncate_delta=0 primary_flush_delta=1 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=9 low_wps_windows=9 wal_group_wait_windows=18 low_cached_read_windows=0 p99=1379 writes/s=7168 reads/s=0 wal_group_avg_ms=12 bottleneck=checkpoint_advance
social_media_100m: progress docs=8715008 txns=34045 phase=writes_only writes/s=7168 avg_writes/s=7051 roll_wps=7168/8243/10342 reads/s=0 avg_reads/s=17740 roll_rps=0/0/0 total_ops/s=7168 avg_total_ops/s=24791 roll_ops=7168/8243/10342 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=631ms p99=1379ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=680MB wal=5708MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p6 lag=34063 q=14 imm=14 dirty=2MB trend=3490 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0 commit_ms=[wal:107116 mem:240369 wait:216241 manifest:0] wal_ms=[queue:12 group:54551 append:22580 fsync:6623 batches:2918 txns:4407 batch_mb:24 avg_txn_ms=q0 g12 a5 f1]
social_media_100m: hot node1:p6 backlog=22MB wal=178MB dirty=2MB imm=14 lag=34063 q=14 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=106 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3490 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node2:p6 backlog=19MB wal=178MB dirty=3MB imm=13 lag=34062 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=115 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3110 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node0:p1 backlog=21MB wal=178MB dirty=3MB imm=15 lag=34056 q=15 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=108 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2913 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: monotonic_growth_fail node=0 partition=2 windows=3 immutables=15 backlog_mb=23 wal_mb=178 checkpoint_lag_lsn=34063 flush_queue_depth=15
social_media_100m: RED partition_stuck node0:p0 windows=10 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=15 q=15 lag=34204 durable_delta=0 truncate_delta=0 primary_flush_delta=1 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=10 low_wps_windows=10 wal_group_wait_windows=19 low_cached_read_windows=0 p99=1501 writes/s=7065 reads/s=0 wal_group_avg_ms=12 bottleneck=checkpoint_advance
social_media_100m: progress docs=8750336 txns=34183 phase=writes_only writes/s=7065 avg_writes/s=7050 roll_wps=7065/8192/10342 reads/s=0 avg_reads/s=17667 roll_rps=0/0/0 total_ops/s=7065 avg_total_ops/s=24717 roll_ops=7065/8192/10342 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=449ms p99=1501ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=653MB wal=5731MB wal_growth=5MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p6 lag=34205 q=14 imm=14 dirty=2MB trend=3807 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0 commit_ms=[wal:110659 mem:243175 wait:218539 manifest:0] wal_ms=[queue:5 group:56399 append:25102 fsync:8415 batches:3251 txns:4576 batch_mb:25 avg_txn_ms=q0 g12 a5 f1]
social_media_100m: hot node1:p6 backlog=22MB wal=179MB dirty=2MB imm=14 lag=34205 q=14 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=106 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3807 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node2:p6 backlog=20MB wal=179MB dirty=3MB imm=14 lag=34204 q=14 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=115 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3428 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node0:p1 backlog=21MB wal=179MB dirty=4MB imm=15 lag=34196 q=15 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=108 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3232 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: RED partition_stuck node0:p0 windows=11 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=15 q=14 lag=34350 durable_delta=0 truncate_delta=0 primary_flush_delta=1 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=11 low_wps_windows=11 wal_group_wait_windows=20 low_cached_read_windows=0 p99=1574 writes/s=7321 reads/s=0 wal_group_avg_ms=12 bottleneck=wal_fsync
social_media_100m: progress docs=8786944 txns=34326 phase=writes_only writes/s=7321 avg_writes/s=7050 roll_wps=7065/8089/10342 reads/s=0 avg_reads/s=17593 roll_rps=0/0/0 total_ops/s=7321 avg_total_ops/s=24643 roll_ops=7065/8089/10342 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=678ms p99=1574ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=621MB wal=5756MB wal_growth=5MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_fsync cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p6 lag=34349 q=13 imm=14 dirty=4MB trend=4123 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0 commit_ms=[wal:131340 mem:250530 wait:225348 manifest:0] wal_ms=[queue:0 group:59171 append:26287 fsync:13040 batches:3268 txns:4576 batch_mb:26 avg_txn_ms=q0 g12 a5 f2]
social_media_100m: hot node1:p6 backlog=20MB wal=180MB dirty=4MB imm=14 lag=34349 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=107 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4123 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node2:p6 backlog=20MB wal=180MB dirty=3MB imm=14 lag=34347 q=14 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=115 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3739 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node0:p1 backlog=22MB wal=180MB dirty=4MB imm=16 lag=34342 q=16 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=108 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3542 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: RED partition_stuck node0:p0 windows=12 reason="queue/immutable plateau while checkpoint lag grows and no flush/durable progress" imm=15 q=14 lag=34497 durable_delta=0 truncate_delta=0 primary_flush_delta=0 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=12 low_wps_windows=12 wal_group_wait_windows=0 low_cached_read_windows=0 p99=1681 writes/s=7731 reads/s=0 wal_group_avg_ms=12 bottleneck=checkpoint_advance
social_media_100m: progress docs=8825600 txns=34476 phase=writes_only writes/s=7731 avg_writes/s=7052 roll_wps=7065/7884/8960 reads/s=0 avg_reads/s=17520 roll_rps=0/0/0 total_ops/s=7731 avg_total_ops/s=24572 roll_ops=7065/7884/8960 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=637ms p99=1681ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=607MB wal=5781MB wal_growth=5MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p6 lag=34496 q=14 imm=15 dirty=4MB trend=4416 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0 commit_ms=[wal:343535 mem:253336 wait:227387 manifest:0] wal_ms=[queue:5 group:61097 append:25752 fsync:43306 batches:3116 txns:4700 batch_mb:26 avg_txn_ms=q0 g12 a5 f9]
social_media_100m: hot node1:p6 backlog=19MB wal=180MB dirty=4MB imm=15 lag=34496 q=14 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=108 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4416 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node0:p1 backlog=19MB wal=181MB dirty=4MB imm=15 lag=34489 q=14 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=110 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3814 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node3:p7 backlog=14MB wal=181MB dirty=5MB imm=12 lag=34497 q=12 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=112 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3576 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: RED partition_stuck node0:p0 windows=13 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=16 q=15 lag=34629 durable_delta=0 truncate_delta=0 primary_flush_delta=1 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=13 low_wps_windows=13 wal_group_wait_windows=0 low_cached_read_windows=0 p99=1800 writes/s=6553 reads/s=0 wal_group_avg_ms=13 bottleneck=checkpoint_advance
social_media_100m: progress docs=8858368 txns=34604 phase=writes_only writes/s=6553 avg_writes/s=7049 roll_wps=6553/7731/8345 reads/s=0 avg_reads/s=17449 roll_rps=0/0/0 total_ops/s=6553 avg_total_ops/s=24498 roll_ops=6553/7731/8345 read_band=yes read_target_rps=0 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=804ms p99=1800ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=590MB wal=5803MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p1 lag=34621 q=14 imm=15 dirty=4MB trend=4044 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0 commit_ms=[wal:617733 mem:228137 wait:203080 manifest:0] wal_ms=[queue:17 group:58880 append:32860 fsync:88533 batches:3012 txns:4256 batch_mb:24 avg_txn_ms=q0 g13 a7 f20]
social_media_100m: hot node0:p1 backlog=19MB wal=181MB dirty=4MB imm=15 lag=34621 q=14 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=110 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4044 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node3:p7 backlog=14MB wal=181MB dirty=5MB imm=14 lag=34630 q=14 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=113 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3803 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node1:p3 backlog=20MB wal=180MB dirty=5MB imm=15 lag=34629 q=14 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=114 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3576 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: phase_start phase=mixed elapsed_secs=1261 target_wps=10000 target_rps=30000 evaluate_writes=true evaluate_reads=true
social_media_100m: RED partition_stuck node0:p0 windows=14 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=17 q=16 lag=34701 durable_delta=0 truncate_delta=0 primary_flush_delta=2 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=14 low_wps_windows=14 wal_group_wait_windows=0 low_cached_read_windows=1 p99=1877 writes/s=3430 reads/s=0 wal_group_avg_ms=21 bottleneck=wal_fsync
social_media_100m: progress docs=8875520 txns=34670 phase=mixed writes/s=3430 avg_writes/s=7034 roll_wps=3430/7731/8345 reads/s=0 avg_reads/s=17379 roll_rps=0/0/0 total_ops/s=3430 avg_total_ops/s=24413 roll_ops=3430/7731/8345 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=0ms ops=[id:0/0-0ms, first:0/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:0/0-0ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=1262ms p99=1877ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=528MB wal=5811MB wal_growth=2MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_fsync cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p1 lag=34693 q=14 imm=16 dirty=6MB trend=4162 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0 commit_ms=[wal:412455 mem:77374 wait:67145 manifest:0] wal_ms=[queue:0 group:47982 append:38235 fsync:45424 batches:1692 txns:2269 batch_mb:13 avg_txn_ms=q0 g21 a16 f20]
social_media_100m: hot node0:p1 backlog=17MB wal=181MB dirty=6MB imm=16 lag=34693 q=14 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=113 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4162 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node3:p7 backlog=11MB wal=182MB dirty=7MB imm=13 lag=34701 q=11 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=116 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3922 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node1:p3 backlog=17MB wal=181MB dirty=5MB imm=14 lag=34701 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=116 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3731 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: RED partition_stuck node0:p0 windows=15 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=17 q=16 lag=34719 durable_delta=0 truncate_delta=0 primary_flush_delta=3 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=15 low_wps_windows=15 wal_group_wait_windows=0 low_cached_read_windows=2 p99=1950 writes/s=921 reads/s=989 wal_group_avg_ms=13 bottleneck=wal_fsync
social_media_100m: progress docs=8880128 txns=34689 phase=mixed writes/s=921 avg_writes/s=7002 roll_wps=921/7526/8192 reads/s=989 avg_reads/s=17294 roll_rps=0/0/989 total_ops/s=1910 avg_total_ops/s=24296 roll_ops=1910/7526/8192 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:915/0-1ms, first:58/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:15/0-1ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=1915ms p99=1950ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=452MB wal=5818MB wal_growth=0MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_fsync cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p1 lag=34711 q=13 imm=16 dirty=7MB trend=4362 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0 commit_ms=[wal:7498159 mem:68879 wait:61389 manifest:0] wal_ms=[queue:0 group:7878 append:4335 fsync:276351 batches:494 txns:576 batch_mb:3 avg_txn_ms=q0 g13 a7 f479]
social_media_100m: hot node0:p1 backlog=15MB wal=182MB dirty=7MB imm=16 lag=34711 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=115 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4362 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node3:p7 backlog=9MB wal=182MB dirty=7MB imm=11 lag=34719 q=10 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=118 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4142 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node1:p3 backlog=14MB wal=181MB dirty=7MB imm=14 lag=34719 q=12 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=119 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3907 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: RED partition_stuck node0:p0 windows=16 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=17 q=15 lag=34836 durable_delta=0 truncate_delta=0 primary_flush_delta=2 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=16 low_wps_windows=16 wal_group_wait_windows=0 low_cached_read_windows=3 p99=2131 writes/s=6041 reads/s=2168 wal_group_avg_ms=14 bottleneck=wal_fsync
social_media_100m: progress docs=8910336 txns=34808 phase=mixed writes/s=6041 avg_writes/s=6998 roll_wps=921/7321/8192 reads/s=2168 avg_reads/s=17233 roll_rps=0/0/2168 total_ops/s=8209 avg_total_ops/s=24231 roll_ops=1910/7526/8209 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=2ms ops=[id:1995/0-2ms, first:125/0-2ms, f1:0/0-0ms, fm:0/0-0ms, txt:47/0-15ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=1145ms p99=2131ms band=no slo=fail mode=deep_cleanup assist=yes target_wps=10000 active_hot=32/32 backlog=401MB wal=5836MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_fsync cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=34836 q=9 imm=10 dirty=8MB trend=4293 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0 commit_ms=[wal:1096864 mem:189328 wait:166685 manifest:0] wal_ms=[queue:12 group:52726 append:36877 fsync:124606 batches:2782 txns:3742 batch_mb:21 avg_txn_ms=q0 g14 a9 f33]
social_media_100m: hot node3:p7 backlog=9MB wal=182MB dirty=8MB imm=10 lag=34836 q=9 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=119 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4293 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node1:p3 backlog=13MB wal=182MB dirty=7MB imm=14 lag=34836 q=11 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=120 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4131 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node0:p3 backlog=15MB wal=182MB dirty=5MB imm=16 lag=34829 q=14 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=115 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3896 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: RED partition_stuck node0:p0 windows=17 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=17 q=14 lag=34920 durable_delta=0 truncate_delta=0 primary_flush_delta=2 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=17 low_wps_windows=17 wal_group_wait_windows=0 low_cached_read_windows=4 p99=2191 writes/s=4249 reads/s=1034 wal_group_avg_ms=22 bottleneck=wal_fsync
social_media_100m: progress docs=8931584 txns=34889 phase=mixed writes/s=4249 avg_writes/s=6986 roll_wps=921/7168/8089 reads/s=1034 avg_reads/s=17168 roll_rps=0/0/2168 total_ops/s=5283 avg_total_ops/s=24154 roll_ops=1910/7321/8209 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=3ms ops=[id:952/0-3ms, first:61/0-3ms, f1:0/0-0ms, fm:0/0-0ms, txt:21/0-20ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=1012ms p99=2191ms band=no slo=fail mode=cleanup assist=yes target_wps=10000 active_hot=31/32 backlog=318MB wal=5848MB wal_growth=2MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_fsync cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=34920 q=7 imm=9 dirty=9MB trend=4483 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0 commit_ms=[wal:543375 mem:118361 wait:104823 manifest:0] wal_ms=[queue:0 group:59579 append:20699 fsync:74767 batches:1853 txns:2687 batch_mb:15 avg_txn_ms=q0 g22 a7 f27]
social_media_100m: hot node3:p7 backlog=6MB wal=183MB dirty=9MB imm=9 lag=34920 q=7 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=122 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4483 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node1:p3 backlog=10MB wal=182MB dirty=7MB imm=14 lag=34920 q=11 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=123 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4311 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node0:p3 backlog=13MB wal=183MB dirty=6MB imm=16 lag=34913 q=13 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=117 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4060 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: RED partition_stuck node0:p0 windows=18 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=17 q=14 lag=34960 durable_delta=0 truncate_delta=0 primary_flush_delta=2 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=18 low_wps_windows=18 wal_group_wait_windows=0 low_cached_read_windows=5 p99=2365 writes/s=2099 reads/s=757 wal_group_avg_ms=14 bottleneck=wal_fsync
social_media_100m: progress docs=8942080 txns=34931 phase=mixed writes/s=2099 avg_writes/s=6965 roll_wps=921/7065/8089 reads/s=757 avg_reads/s=17099 roll_rps=0/0/2168 total_ops/s=2856 avg_total_ops/s=24064 roll_ops=1910/7168/8209 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=4ms ops=[id:697/0-4ms, first:46/0-4ms, f1:0/0-0ms, fm:0/0-0ms, txt:13/0-26ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=2246ms p99=2365ms band=no slo=fail mode=cleanup assist=yes target_wps=10000 active_hot=29/32 backlog=263MB wal=5857MB wal_growth=1MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_fsync cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p3 lag=34960 q=10 imm=13 dirty=8MB trend=4496 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0 commit_ms=[wal:5070196 mem:92767 wait:82189 manifest:0] wal_ms=[queue:0 group:18675 append:10781 fsync:214066 batches:1044 txns:1296 batch_mb:7 avg_txn_ms=q0 g14 a8 f165]
social_media_100m: hot node1:p3 backlog=8MB wal=182MB dirty=8MB imm=13 lag=34960 q=10 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=125 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4496 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node0:p3 backlog=11MB wal=183MB dirty=7MB imm=15 lag=34953 q=12 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=119 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4256 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node0:p6 backlog=7MB wal=182MB dirty=9MB imm=11 lag=34954 q=10 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=124 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4242 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: RED partition_stuck node0:p0 windows=19 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=17 q=14 lag=35091 durable_delta=0 truncate_delta=0 primary_flush_delta=2 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=19 low_wps_windows=19 wal_group_wait_windows=0 low_cached_read_windows=6 p99=2423 writes/s=6912 reads/s=677 wal_group_avg_ms=13 bottleneck=checkpoint_advance
social_media_100m: progress docs=8976640 txns=35066 phase=mixed writes/s=6912 avg_writes/s=6964 roll_wps=921/6912/7731 reads/s=677 avg_reads/s=17033 roll_rps=0/0/2168 total_ops/s=7589 avg_total_ops/s=23997 roll_ops=1910/7168/8209 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=5ms ops=[id:621/0-5ms, first:43/0-5ms, f1:0/0-0ms, fm:0/0-0ms, txt:13/4-29ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=1002ms p99=2423ms band=no slo=fail mode=cleanup assist=yes target_wps=10000 active_hot=28/32 backlog=235MB wal=5881MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p6 lag=35085 q=8 imm=9 dirty=9MB trend=4514 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0 commit_ms=[wal:1205431 mem:238676 wait:212124 manifest:0] wal_ms=[queue:3 group:56971 append:34720 fsync:130717 batches:3317 txns:4200 batch_mb:23 avg_txn_ms=q0 g13 a8 f31]
social_media_100m: hot node0:p6 backlog=5MB wal=183MB dirty=9MB imm=9 lag=35085 q=8 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=126 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4514 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node2:p5 backlog=6MB wal=183MB dirty=10MB imm=10 lag=35090 q=8 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=121 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3885 emerg=2 budget=2MB hot_table=14927820125186777592 fg_tokens=0
social_media_100m: hot node1:p2 backlog=9MB wal=184MB dirty=10MB imm=13 lag=35090 q=11 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=123 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=3092 emerg=2 budget=2MB hot_table=16161175518796973571 fg_tokens=0
social_media_100m: RED partition_stuck node0:p1 windows=8 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=15 q=13 lag=35217 durable_delta=0 truncate_delta=0 primary_flush_delta=2 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=20 low_wps_windows=20 wal_group_wait_windows=0 low_cached_read_windows=0 p99=2477 writes/s=6707 reads/s=253 wal_group_avg_ms=13 bottleneck=wal_fsync
social_media_100m: progress docs=9010176 txns=35199 phase=mixed writes/s=6707 avg_writes/s=6963 roll_wps=921/6707/7731 reads/s=253 avg_reads/s=16967 roll_rps=0/253/2168 total_ops/s=6960 avg_total_ops/s=23930 roll_ops=1910/7065/8209 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=6ms ops=[id:235/0-6ms, first:12/0-6ms, f1:0/0-0ms, fm:0/0-0ms, txt:5/19-30ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=965ms p99=2477ms band=no slo=fail mode=cleanup assist=yes target_wps=10000 active_hot=28/32 backlog=204MB wal=5903MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_fsync cluster_flush_q=[p:160 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p6 lag=35219 q=8 imm=9 dirty=10MB trend=4812 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0 commit_ms=[wal:988734 mem:233377 wait:207600 manifest:0] wal_ms=[queue:8 group:56428 append:29806 fsync:103585 batches:3072 txns:4292 batch_mb:24 avg_txn_ms=q0 g13 a6 f24]
social_media_100m: hot node0:p6 backlog=5MB wal=184MB dirty=10MB imm=9 lag=35219 q=8 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=128 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=4812 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node1:p6 backlog=4MB wal=184MB dirty=9MB imm=10 lag=35224 q=9 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=125 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1813 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0
social_media_100m: hot node1:p5 backlog=6MB wal=184MB dirty=10MB imm=14 lag=35219 q=11 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=122 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1539 emerg=2 budget=2MB hot_table=16161175518796973571 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=21 low_wps_windows=21 wal_group_wait_windows=0 low_cached_read_windows=0 p99=2181 writes/s=2355 reads/s=727 wal_group_avg_ms=14 bottleneck=wal_fsync
social_media_100m: progress docs=9021952 txns=35242 phase=mixed writes/s=2355 avg_writes/s=6944 roll_wps=921/6553/7731 reads/s=727 avg_reads/s=16903 roll_rps=0/677/2168 total_ops/s=3082 avg_total_ops/s=23847 roll_ops=1910/6960/8209 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=7ms ops=[id:669/0-7ms, first:43/0-7ms, f1:0/0-0ms, fm:0/0-0ms, txt:14/0-31ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=1154ms p99=2181ms band=no slo=fail mode=cleanup assist=yes target_wps=10000 active_hot=20/32 backlog=129MB wal=5908MB wal_growth=1MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_fsync cluster_flush_q=[p:151 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p6 lag=35267 q=6 imm=6 dirty=10MB trend=5024 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0 commit_ms=[wal:430025 mem:45176 wait:39108 manifest:0] wal_ms=[queue:0 group:22263 append:16788 fsync:41799 batches:1160 txns:1508 batch_mb:8 avg_txn_ms=q0 g14 a11 f27]
social_media_100m: hot node0:p6 backlog=5MB wal=184MB dirty=10MB imm=6 lag=35267 q=6 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=131 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5024 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node2:p3 backlog=5MB wal=184MB dirty=10MB imm=9 lag=35271 q=8 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=129 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1479 emerg=2 budget=2MB hot_table=14927820125186777592 fg_tokens=0
social_media_100m: hot node1:p6 backlog=1MB wal=184MB dirty=11MB imm=8 lag=35272 q=7 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=129 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=2013 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=22 low_wps_windows=22 wal_group_wait_windows=0 low_cached_read_windows=0 p99=2610 writes/s=1792 reads/s=493 wal_group_avg_ms=13 bottleneck=wal_fsync
social_media_100m: progress docs=9030912 txns=35279 phase=mixed writes/s=1792 avg_writes/s=6924 roll_wps=921/6041/7731 reads/s=493 avg_reads/s=16839 roll_rps=0/677/2168 total_ops/s=2285 avg_total_ops/s=23763 roll_ops=1910/6553/8209 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=8ms ops=[id:452/0-8ms, first:32/0-9ms, f1:0/0-0ms, fm:0/0-0ms, txt:8/0-32ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=2496ms p99=2610ms band=no slo=fail mode=cleanup assist=yes target_wps=10000 active_hot=14/32 backlog=83MB wal=5915MB wal_growth=1MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_fsync cluster_flush_q=[p:139 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p6 lag=35298 q=4 imm=4 dirty=10MB trend=5256 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0 commit_ms=[wal:5629507 mem:74752 wait:67396 manifest:0] wal_ms=[queue:0 group:13671 append:6799 fsync:206208 batches:817 txns:1002 batch_mb:6 avg_txn_ms=q0 g13 a6 f205]
social_media_100m: hot node0:p6 backlog=3MB wal=184MB dirty=10MB imm=4 lag=35298 q=4 durable=0 truncate=0 primary_q=4 secondary_q=0 primary_done=135 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5256 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node2:p2 backlog=4MB wal=185MB dirty=9MB imm=9 lag=35295 q=10 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=131 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=893 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node2:p6 backlog=4MB wal=185MB dirty=10MB imm=11 lag=35301 q=11 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=137 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=668 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=23 low_wps_windows=23 wal_group_wait_windows=0 low_cached_read_windows=0 p99=2674 writes/s=6860 reads/s=812 wal_group_avg_ms=12 bottleneck=wal_fsync
social_media_100m: progress docs=9065216 txns=35412 phase=mixed writes/s=6860 avg_writes/s=6923 roll_wps=921/6041/7731 reads/s=812 avg_reads/s=16775 roll_rps=0/727/2168 total_ops/s=7672 avg_total_ops/s=23698 roll_ops=1910/6553/8209 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=9ms ops=[id:742/0-9ms, first:53/0-10ms, f1:0/0-0ms, fm:0/0-0ms, txt:15/6-33ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=986ms p99=2674ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=8/32 backlog=74MB wal=5939MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_fsync cluster_flush_q=[p:131 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p6 lag=35435 q=2 imm=2 dirty=11MB trend=5514 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0 commit_ms=[wal:1013496 mem:234416 wait:207969 manifest:0] wal_ms=[queue:0 group:54367 append:27909 fsync:115500 batches:3091 txns:4373 batch_mb:24 avg_txn_ms=q0 g12 a6 f26]
social_media_100m: hot node0:p6 backlog=3MB wal=185MB dirty=11MB imm=2 lag=35435 q=2 durable=0 truncate=0 primary_q=2 secondary_q=0 primary_done=138 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5514 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node1:p7 backlog=3MB wal=185MB dirty=10MB imm=10 lag=35438 q=10 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=132 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=254 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=0
social_media_100m: hot node0:p2 backlog=2MB wal=185MB dirty=11MB imm=9 lag=35440 q=8 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=131 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=217 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=0
social_media_100m: RED partition_stuck node0:p5 windows=3 reason="queue/immutable plateau while checkpoint lag grows and truncate_lsn is not advancing" imm=7 q=5 lag=35556 durable_delta=0 truncate_delta=0 primary_flush_delta=1 secondary_flush_delta=0 primary_q=5 secondary_q=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=24 low_wps_windows=24 wal_group_wait_windows=0 low_cached_read_windows=0 p99=2771 writes/s=6348 reads/s=6301 wal_group_avg_ms=14 bottleneck=checkpoint_advance
social_media_100m: progress docs=9096960 txns=35537 phase=mixed writes/s=6348 avg_writes/s=6920 roll_wps=921/6041/6912 reads/s=6301 avg_reads/s=16734 roll_rps=0/757/6301 total_ops/s=12649 avg_total_ops/s=23654 roll_ops=1910/6553/12649 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=10ms ops=[id:5778/0-1ms, first:391/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:132/11-35ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=806ms p99=2771ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=4/32 backlog=92MB wal=5959MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:137 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p6 lag=35552 q=2 imm=2 dirty=11MB trend=5809 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0 commit_ms=[wal:354215 mem:205691 wait:186117 manifest:0] wal_ms=[queue:4 group:52557 append:24944 fsync:65859 batches:2952 txns:3744 batch_mb:20 avg_txn_ms=q0 g14 a6 f17]
social_media_100m: hot node0:p6 backlog=3MB wal=186MB dirty=11MB imm=2 lag=35552 q=2 durable=0 truncate=0 primary_q=2 secondary_q=0 primary_done=141 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=5809 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node1:p7 backlog=4MB wal=186MB dirty=11MB imm=11 lag=35555 q=9 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=134 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=557 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=0
social_media_100m: hot node1:p3 backlog=3MB wal=185MB dirty=11MB imm=9 lag=35558 q=7 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=140 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=346 emerg=2 budget=2MB hot_table=14124715428051363376 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=25 low_wps_windows=25 wal_group_wait_windows=1 low_cached_read_windows=0 p99=2843 writes/s=6297 reads/s=951 wal_group_avg_ms=12 bottleneck=checkpoint_advance
social_media_100m: progress docs=9128448 txns=35659 phase=mixed writes/s=6297 avg_writes/s=6917 roll_wps=921/6041/6912 reads/s=951 avg_reads/s=16672 roll_rps=0/812/6301 total_ops/s=7248 avg_total_ops/s=23589 roll_ops=1910/6960/12649 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=11ms ops=[id:880/0-2ms, first:53/0-2ms, f1:0/0-0ms, fm:0/0-0ms, txt:18/10-36ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=691ms p99=2843ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=3/32 backlog=71MB wal=5981MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:125 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p6 lag=35680 q=2 imm=2 dirty=11MB trend=6137 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0 commit_ms=[wal:97771 mem:213497 wait:193800 manifest:0] wal_ms=[queue:4 group:50840 append:23050 fsync:6543 batches:2982 txns:4108 batch_mb:23 avg_txn_ms=q0 g12 a5 f1]
social_media_100m: hot node0:p6 backlog=3MB wal=186MB dirty=11MB imm=2 lag=35680 q=2 durable=0 truncate=0 primary_q=2 secondary_q=0 primary_done=141 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6137 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node1:p7 backlog=4MB wal=186MB dirty=11MB imm=11 lag=35683 q=9 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=134 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=862 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=0
social_media_100m: hot node2:p6 backlog=3MB wal=187MB dirty=11MB imm=9 lag=35683 q=9 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=143 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=491 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=26 low_wps_windows=26 wal_group_wait_windows=2 low_cached_read_windows=0 p99=2930 writes/s=6297 reads/s=396 wal_group_avg_ms=13 bottleneck=checkpoint_advance
social_media_100m: progress docs=9159936 txns=35782 phase=mixed writes/s=6297 avg_writes/s=6914 roll_wps=921/6297/6912 reads/s=396 avg_reads/s=16609 roll_rps=253/812/6301 total_ops/s=6693 avg_total_ops/s=23523 roll_ops=1910/6960/12649 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=12ms ops=[id:364/0-5ms, first:23/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:9/7-37ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=704ms p99=2930ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=39MB wal=6001MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2759MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:98 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p6 lag=35799 q=1 imm=1 dirty=11MB trend=6473 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0 commit_ms=[wal:91936 mem:198639 wait:179605 manifest:0] wal_ms=[queue:4 group:50021 append:21452 fsync:6230 batches:2884 txns:3783 batch_mb:21 avg_txn_ms=q0 g13 a5 f1]
social_media_100m: hot node0:p6 backlog=1MB wal=187MB dirty=11MB imm=1 lag=35799 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=143 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=6473 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node0:p2 backlog=2MB wal=187MB dirty=11MB imm=6 lag=35804 q=5 durable=0 truncate=0 primary_q=5 secondary_q=0 primary_done=137 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1103 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=0
social_media_100m: hot node1:p5 backlog=2MB wal=187MB dirty=0MB imm=4 lag=35798 q=4 durable=0 truncate=0 primary_q=4 secondary_q=0 primary_done=138 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=985 emerg=2 budget=2MB hot_table=14124715428051363376 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=27 low_wps_windows=27 wal_group_wait_windows=3 low_cached_read_windows=0 p99=3030 writes/s=6860 reads/s=882 wal_group_avg_ms=12 bottleneck=checkpoint_advance
social_media_100m: progress docs=9194240 txns=35916 phase=mixed writes/s=6860 avg_writes/s=6913 roll_wps=1792/6297/6912 reads/s=882 avg_reads/s=16549 roll_rps=253/812/6301 total_ops/s=7742 avg_total_ops/s=23462 roll_ops=2285/7248/12649 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=13ms ops=[id:803/0-11ms, first:55/0-11ms, f1:0/0-0ms, fm:0/0-0ms, txt:23/7-38ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=561ms p99=3030ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=11MB wal=6023MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2780MB/11s lsn_delta=[durable:1394 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:13 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p2 lag=35937 q=2 imm=2 dirty=12MB trend=1434 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=0 commit_ms=[wal:97935 mem:176871 wait:156477 manifest:0] wal_ms=[queue:5 group:52259 append:22666 fsync:6658 batches:3110 txns:4295 batch_mb:24 avg_txn_ms=q0 g12 a5 f1]
social_media_100m: hot node0:p2 backlog=1MB wal=188MB dirty=12MB imm=2 lag=35937 q=2 durable=0 truncate=0 primary_q=2 secondary_q=0 primary_done=141 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1434 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=0
social_media_100m: hot node0:p5 backlog=1MB wal=187MB dirty=4MB imm=1 lag=35936 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=149 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1104 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=0
social_media_100m: hot node1:p0 backlog=1MB wal=188MB dirty=1MB imm=1 lag=35942 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=150 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=489 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=28 low_wps_windows=0 wal_group_wait_windows=4 low_cached_read_windows=1 p99=2389 writes/s=10752 reads/s=12779 wal_group_avg_ms=11 bottleneck=wal_group_wait
social_media_100m: progress docs=9248000 txns=36127 phase=mixed writes/s=10752 avg_writes/s=6927 roll_wps=1792/6348/10752 reads/s=12779 avg_reads/s=16533 roll_rps=253/812/12779 total_ops/s=23531 avg_total_ops/s=23460 roll_ops=2285/7248/23531 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=5ms ops=[id:11738/0-1ms, first:778/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:263/1-27ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=425ms p99=2389ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=6056MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2780MB/5s lsn_delta=[durable:3103 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p1 lag=36126 q=0 imm=0 dirty=12MB trend=404 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=0 commit_ms=[wal:139947 mem:55407 wait:28049 manifest:0] wal_ms=[queue:6 group:71666 append:27459 fsync:9154 batches:3808 txns:6145 batch_mb:34 avg_txn_ms=q0 g11 a4 f1]
social_media_100m: hot node1:p1 backlog=0MB wal=189MB dirty=12MB imm=0 lag=36126 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=149 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=404 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=0
social_media_100m: hot node2:p2 backlog=0MB wal=189MB dirty=3MB imm=1 lag=36121 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=143 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=327 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=0
social_media_100m: hot node3:p6 backlog=0MB wal=189MB dirty=12MB imm=0 lag=36128 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=148 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=296 emerg=5 budget=4MB hot_table=13720768859447704617 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=5 low_cached_read_windows=0 p99=497 writes/s=9728 reads/s=7691 wal_group_avg_ms=11 bottleneck=wal_group_wait
social_media_100m: progress docs=9296640 txns=36316 phase=mixed writes/s=9728 avg_writes/s=6937 roll_wps=1792/6707/10752 reads/s=7691 avg_reads/s=16499 roll_rps=253/812/12779 total_ops/s=17419 avg_total_ops/s=23436 roll_ops=2285/7589/23531 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=6ms ops=[id:7073/0-1ms, first:464/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:152/3-29ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=246ms p99=497ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=22MB wal=6090MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2780MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:24 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p7 lag=36331 q=0 imm=1 dirty=0MB trend=473 emerg=2 budget=2MB hot_table=16823683620448006766 fg_tokens=0 commit_ms=[wal:142968 mem:86156 wait:65882 manifest:0] wal_ms=[queue:11 group:74942 append:27554 fsync:7619 batches:3887 txns:6485 batch_mb:36 avg_txn_ms=q0 g11 a4 f1]
social_media_100m: hot node1:p7 backlog=0MB wal=190MB dirty=0MB imm=1 lag=36331 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=147 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=473 emerg=2 budget=2MB hot_table=16823683620448006766 fg_tokens=0
social_media_100m: hot node2:p5 backlog=0MB wal=190MB dirty=0MB imm=1 lag=36333 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=142 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=470 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0
social_media_100m: hot node3:p3 backlog=1MB wal=190MB dirty=0MB imm=1 lag=36329 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=147 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=455 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=1 low_wps_windows=1 wal_group_wait_windows=6 low_cached_read_windows=0 p99=2825 writes/s=7116 reads/s=118 wal_group_avg_ms=11 bottleneck=checkpoint_advance
social_media_100m: progress docs=9332224 txns=36456 phase=mixed writes/s=7116 avg_writes/s=6937 roll_wps=1792/6860/10752 reads/s=118 avg_reads/s=16437 roll_rps=118/812/12779 total_ops/s=7234 avg_total_ops/s=23374 roll_ops=2285/7589/23531 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=8ms ops=[id:106/0-2ms, first:10/0-2ms, f1:0/0-0ms, fm:0/0-0ms, txt:2/15-32ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=584ms p99=2825ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=31MB wal=6115MB wal_growth=5MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2782MB/5s lsn_delta=[durable:173 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:34 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p6 lag=36481 q=2 imm=2 dirty=1MB trend=425 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0 commit_ms=[wal:107705 mem:200299 wait:187684 manifest:0] wal_ms=[queue:14 group:56387 append:23782 fsync:6811 batches:3277 txns:4746 batch_mb:27 avg_txn_ms=q0 g11 a5 f1]
social_media_100m: hot node1:p6 backlog=2MB wal=191MB dirty=1MB imm=2 lag=36481 q=2 durable=0 truncate=0 primary_q=2 secondary_q=0 primary_done=144 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=425 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0
social_media_100m: hot node0:p3 backlog=1MB wal=191MB dirty=0MB imm=1 lag=36475 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=146 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=377 emerg=5 budget=4MB hot_table=16823683620448006766 fg_tokens=0
social_media_100m: hot node1:p3 backlog=1MB wal=190MB dirty=0MB imm=1 lag=36482 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=154 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=392 emerg=2 budget=2MB hot_table=16823683620448006766 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=2 low_wps_windows=0 wal_group_wait_windows=7 low_cached_read_windows=1 p99=2653 writes/s=10700 reads/s=13055 wal_group_avg_ms=11 bottleneck=wal_group_wait
social_media_100m: progress docs=9385728 txns=36663 phase=mixed writes/s=10700 avg_writes/s=6950 roll_wps=1792/6860/10752 reads/s=13055 avg_reads/s=16423 roll_rps=118/882/13055 total_ops/s=23755 avg_total_ops/s=23373 roll_ops=2285/7672/23755 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=2ms ops=[id:12009/0-0ms, first:769/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:276/0-23ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=392ms p99=2653ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=6145MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2783MB/5s lsn_delta=[durable:252 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p1 lag=36659 q=0 imm=0 dirty=0MB trend=508 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=0 commit_ms=[wal:126323 mem:48591 wait:35772 manifest:0] wal_ms=[queue:3 group:65156 append:23328 fsync:6825 batches:3422 txns:5773 batch_mb:31 avg_txn_ms=q0 g11 a4 f1]
social_media_100m: hot node1:p1 backlog=0MB wal=191MB dirty=0MB imm=0 lag=36659 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=152 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=508 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=0
social_media_100m: hot node0:p4 backlog=0MB wal=191MB dirty=0MB imm=0 lag=36662 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=152 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=339 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=0
social_media_100m: hot node0:p5 backlog=0MB wal=191MB dirty=0MB imm=0 lag=36661 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=156 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=333 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=0 low_wps_windows=0 wal_group_wait_windows=8 low_cached_read_windows=2 p99=465 writes/s=10086 reads/s=19386 wal_group_avg_ms=9 bottleneck=wal_group_wait
social_media_100m: progress docs=9436160 txns=36862 phase=mixed writes/s=10086 avg_writes/s=6962 roll_wps=1792/6860/10752 reads/s=19386 avg_reads/s=16433 roll_rps=118/951/19386 total_ops/s=29472 avg_total_ops/s=23395 roll_ops=2285/7742/29472 read_band=no read_target_rps=25000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=1ms ops=[id:17826/0-0ms, first:1173/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:386/0-23ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=197ms p99=465ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=6178MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2784MB/5s lsn_delta=[durable:116 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p1 lag=36859 q=0 imm=0 dirty=0MB trend=837 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=0 commit_ms=[wal:135942 mem:17600 wait:3786 manifest:0] wal_ms=[queue:9 group:63391 append:23578 fsync:6711 batches:3272 txns:6388 batch_mb:36 avg_txn_ms=q0 g9 a3 f1]
social_media_100m: hot node1:p1 backlog=0MB wal=193MB dirty=0MB imm=0 lag=36859 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=152 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=837 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=0
social_media_100m: hot node2:p2 backlog=0MB wal=193MB dirty=0MB imm=0 lag=36854 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=148 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=622 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=0
social_media_100m: hot node0:p5 backlog=0MB wal=192MB dirty=0MB imm=0 lag=36860 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=156 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=592 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=1 low_wps_windows=0 wal_group_wait_windows=9 low_cached_read_windows=3 p99=725 writes/s=9267 reads/s=22789 wal_group_avg_ms=10 bottleneck=wal_group_wait
social_media_100m: progress docs=9482496 txns=37042 phase=mixed writes/s=9267 avg_writes/s=6969 roll_wps=1792/7116/10752 reads/s=22789 avg_reads/s=16455 roll_rps=118/6301/22789 total_ops/s=32056 avg_total_ops/s=23424 roll_ops=2285/12649/32056 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=2ms ops=[id:20936/0-0ms, first:1393/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:460/1-34ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=276ms p99=725ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=1MB wal=6213MB wal_growth=7MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2785MB/5s lsn_delta=[durable:419 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:1 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p2 lag=37060 q=0 imm=0 dirty=0MB trend=959 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=0 commit_ms=[wal:146682 mem:42566 wait:27764 manifest:0] wal_ms=[queue:17 group:70315 append:24728 fsync:5526 batches:3216 txns:6620 batch_mb:37 avg_txn_ms=q0 g10 a3 f0]
social_media_100m: hot node2:p2 backlog=0MB wal=194MB dirty=0MB imm=0 lag=37060 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=148 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=959 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=0
social_media_100m: hot node0:p5 backlog=0MB wal=193MB dirty=0MB imm=0 lag=37067 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=156 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=831 emerg=5 budget=4MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node1:p5 backlog=0MB wal=193MB dirty=0MB imm=0 lag=37061 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=147 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=854 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=2 low_wps_windows=1 wal_group_wait_windows=10 low_cached_read_windows=4 p99=1408 writes/s=8448 reads/s=18073 wal_group_avg_ms=10 bottleneck=checkpoint_advance
social_media_100m: progress docs=9524736 txns=37208 phase=mixed writes/s=8448 avg_writes/s=6974 roll_wps=6297/8448/10752 reads/s=18073 avg_reads/s=16460 roll_rps=118/7691/22789 total_ops/s=26521 avg_total_ops/s=23434 roll_ops=6693/17419/32056 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=3ms ops=[id:16616/0-0ms, first:1082/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:374/4-39ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=626ms p99=1408ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=15MB wal=6240MB wal_growth=5MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2785MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:16 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p5 lag=37220 q=2 imm=1 dirty=0MB trend=1173 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=0 commit_ms=[wal:111672 mem:73830 wait:62055 manifest:0] wal_ms=[queue:3 group:54368 append:23745 fsync:5885 batches:3165 txns:5144 batch_mb:28 avg_txn_ms=q0 g10 a4 f1]
social_media_100m: hot node1:p5 backlog=0MB wal=194MB dirty=0MB imm=1 lag=37220 q=2 durable=0 truncate=0 primary_q=2 secondary_q=0 primary_done=148 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1173 emerg=2 budget=2MB hot_table=13720768859447704617 fg_tokens=0
social_media_100m: hot node3:p4 backlog=1MB wal=194MB dirty=0MB imm=1 lag=37226 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=153 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=638 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node2:p6 backlog=0MB wal=195MB dirty=0MB imm=0 lag=37225 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=158 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=461 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=3 low_wps_windows=2 wal_group_wait_windows=11 low_cached_read_windows=5 p99=2985 writes/s=5427 reads/s=14814 wal_group_avg_ms=13 bottleneck=checkpoint_advance
social_media_100m: progress docs=9551872 txns=37313 phase=mixed writes/s=5427 avg_writes/s=6967 roll_wps=5427/8448/10752 reads/s=14814 avg_reads/s=16451 roll_rps=118/12779/22789 total_ops/s=20241 avg_total_ops/s=23418 roll_ops=6693/20241/32056 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=4ms ops=[id:13634/0-1ms, first:887/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:292/13-45ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=872ms p99=2985ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=18MB wal=6258MB wal_growth=3MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2787MB/5s lsn_delta=[durable:133 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:19 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=37338 q=1 imm=1 dirty=0MB trend=710 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0 commit_ms=[wal:96879 mem:97539 wait:88602 manifest:0] wal_ms=[queue:6 group:49198 append:22708 fsync:6699 batches:2586 txns:3562 batch_mb:19 avg_txn_ms=q0 g13 a6 f1]
social_media_100m: hot node2:p6 backlog=1MB wal=195MB dirty=0MB imm=1 lag=37338 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=158 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=710 emerg=2 budget=2MB hot_table=17248333830116347718 fg_tokens=0
social_media_100m: hot node1:p1 backlog=0MB wal=195MB dirty=0MB imm=0 lag=37337 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=154 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=530 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: hot node2:p4 backlog=0MB wal=195MB dirty=0MB imm=0 lag=37343 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=152 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=444 emerg=2 budget=2MB hot_table=10212636630577541217 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=4 low_wps_windows=3 wal_group_wait_windows=12 low_cached_read_windows=6 p99=3123 writes/s=6809 reads/s=108 wal_group_avg_ms=13 bottleneck=checkpoint_advance
social_media_100m: progress docs=9585920 txns=37447 phase=mixed writes/s=6809 avg_writes/s=6966 roll_wps=5427/8448/10752 reads/s=108 avg_reads/s=16390 roll_rps=108/12779/22789 total_ops/s=6917 avg_total_ops/s=23356 roll_ops=6693/20241/32056 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=5ms ops=[id:101/0-2ms, first:5/0-2ms, f1:0/0-0ms, fm:0/0-0ms, txt:1/7-46ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=1005ms p99=3123ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=29MB wal=6280MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2787MB/5s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:37 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p5 lag=37467 q=1 imm=1 dirty=0MB trend=429 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0 commit_ms=[wal:102417 mem:134094 wait:124210 manifest:0] wal_ms=[queue:2 group:57085 append:22197 fsync:7113 batches:3040 txns:4080 batch_mb:23 avg_txn_ms=q0 g13 a5 f1]
social_media_100m: hot node2:p5 backlog=1MB wal=195MB dirty=0MB imm=1 lag=37467 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=153 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=429 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0
social_media_100m: hot node3:p2 backlog=0MB wal=196MB dirty=2MB imm=1 lag=37467 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=168 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=412 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0
social_media_100m: hot node1:p7 backlog=0MB wal=196MB dirty=0MB imm=0 lag=37465 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=155 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=311 emerg=2 budget=2MB hot_table=10150445572359035967 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=5 low_wps_windows=4 wal_group_wait_windows=13 low_cached_read_windows=0 p99=3145 writes/s=6502 reads/s=152 wal_group_avg_ms=12 bottleneck=checkpoint_advance
social_media_100m: progress docs=9618432 txns=37573 phase=mixed writes/s=6502 avg_writes/s=6964 roll_wps=5427/8448/10752 reads/s=152 avg_reads/s=16330 roll_rps=108/12779/22789 total_ops/s=6654 avg_total_ops/s=23294 roll_ops=6654/20241/32056 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=6ms ops=[id:140/0-3ms, first:8/0-3ms, f1:0/0-0ms, fm:0/0-0ms, txt:3/11-47ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=901ms p99=3145ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=16MB wal=6301MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2788MB/9s lsn_delta=[durable:352 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:16 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p6 lag=37595 q=1 imm=1 dirty=2MB trend=430 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0 commit_ms=[wal:95510 mem:145334 wait:134983 manifest:0] wal_ms=[queue:4 group:50040 append:21995 fsync:6475 batches:2970 txns:4135 batch_mb:23 avg_txn_ms=q0 g12 a5 f1]
social_media_100m: hot node2:p6 backlog=1MB wal=197MB dirty=2MB imm=1 lag=37595 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=161 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=430 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0
social_media_100m: hot node1:p0 backlog=2MB wal=197MB dirty=2MB imm=2 lag=37602 q=2 durable=0 truncate=0 primary_q=2 secondary_q=0 primary_done=161 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=423 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0
social_media_100m: hot node3:p6 backlog=1MB wal=196MB dirty=1MB imm=1 lag=37595 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=157 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=422 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=6 low_wps_windows=5 wal_group_wait_windows=14 low_cached_read_windows=1 p99=3150 writes/s=8601 reads/s=12902 wal_group_avg_ms=12 bottleneck=wal_group_wait
social_media_100m: progress docs=9661440 txns=37742 phase=mixed writes/s=8601 avg_writes/s=6969 roll_wps=5427/8601/10752 reads/s=12902 avg_reads/s=16316 roll_rps=108/12902/22789 total_ops/s=21503 avg_total_ops/s=23285 roll_ops=6654/21503/32056 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=3ms ops=[id:11888/0-0ms, first:768/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:244/2-29ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=727ms p99=3150ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=6329MB wal_growth=5MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2789MB/5s lsn_delta=[durable:108 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p3 lag=37760 q=0 imm=0 dirty=0MB trend=521 emerg=6 budget=4MB hot_table=16049385931360479616 fg_tokens=0 commit_ms=[wal:125191 mem:56067 wait:44138 manifest:0] wal_ms=[queue:6 group:63882 append:25584 fsync:9791 batches:3462 txns:5250 batch_mb:29 avg_txn_ms=q0 g12 a4 f1]
social_media_100m: hot node1:p3 backlog=0MB wal=197MB dirty=0MB imm=0 lag=37760 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=164 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=521 emerg=6 budget=4MB hot_table=16049385931360479616 fg_tokens=0
social_media_100m: hot node2:p2 backlog=0MB wal=198MB dirty=0MB imm=0 lag=37752 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=153 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=561 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=0
social_media_100m: hot node1:p5 backlog=0MB wal=197MB dirty=0MB imm=0 lag=37753 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=153 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=335 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=7 low_wps_windows=0 wal_group_wait_windows=15 low_cached_read_windows=2 p99=1792 writes/s=9062 reads/s=12834 wal_group_avg_ms=11 bottleneck=wal_group_wait
social_media_100m: progress docs=9706752 txns=37919 phase=mixed writes/s=9062 avg_writes/s=6976 roll_wps=5427/9062/10752 reads/s=12834 avg_reads/s=16302 roll_rps=108/12902/22789 total_ops/s=21896 avg_total_ops/s=23278 roll_ops=6654/21896/32056 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=5ms ops=[id:11794/0-1ms, first:779/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:261/2-31ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=681ms p99=1792ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=8MB wal=6360MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2790MB/7s lsn_delta=[durable:363 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:8 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p2 lag=37942 q=0 imm=0 dirty=0MB trend=337 emerg=5 budget=4MB hot_table=16049385931360479616 fg_tokens=0 commit_ms=[wal:131023 mem:61265 wait:48185 manifest:0] wal_ms=[queue:0 group:67895 append:26049 fsync:8027 batches:3665 txns:5878 batch_mb:33 avg_txn_ms=q0 g11 a4 f1]
social_media_100m: hot node1:p2 backlog=0MB wal=198MB dirty=0MB imm=0 lag=37942 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=157 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=337 emerg=5 budget=4MB hot_table=16049385931360479616 fg_tokens=0
social_media_100m: hot node2:p5 backlog=0MB wal=198MB dirty=0MB imm=0 lag=37944 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=157 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=337 emerg=2 budget=2MB hot_table=17095084847128333938 fg_tokens=0
social_media_100m: hot node1:p7 backlog=0MB wal=198MB dirty=0MB imm=0 lag=37942 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=159 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=259 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=8 low_wps_windows=0 wal_group_wait_windows=16 low_cached_read_windows=3 p99=1761 writes/s=9369 reads/s=12761 wal_group_avg_ms=10 bottleneck=wal_group_wait
social_media_100m: progress docs=9753600 txns=38101 phase=mixed writes/s=9369 avg_writes/s=6984 roll_wps=5427/9062/10700 reads/s=12761 avg_reads/s=16288 roll_rps=108/12902/22789 total_ops/s=22130 avg_total_ops/s=23272 roll_ops=6654/21896/32056 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=5ms ops=[id:11749/0-1ms, first:760/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:251/2-33ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=666ms p99=1761ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=13MB wal=6390MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2790MB/10s lsn_delta=[durable:69 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:14 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p0 lag=38123 q=1 imm=1 dirty=0MB trend=422 emerg=3 budget=2MB hot_table=16049385931360479616 fg_tokens=0 commit_ms=[wal:126581 mem:56716 wait:43879 manifest:0] wal_ms=[queue:0 group:59806 append:24387 fsync:7865 batches:3289 txns:5690 batch_mb:31 avg_txn_ms=q0 g10 a4 f1]
social_media_100m: hot node0:p0 backlog=1MB wal=199MB dirty=0MB imm=1 lag=38123 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=158 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=422 emerg=3 budget=2MB hot_table=16049385931360479616 fg_tokens=0
social_media_100m: hot node1:p0 backlog=1MB wal=199MB dirty=0MB imm=1 lag=38127 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=164 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=415 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=0
social_media_100m: hot node3:p7 backlog=1MB wal=200MB dirty=0MB imm=1 lag=38123 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=157 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=378 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=9 low_wps_windows=1 wal_group_wait_windows=17 low_cached_read_windows=0 p99=2506 writes/s=5990 reads/s=16914 wal_group_avg_ms=12 bottleneck=checkpoint_advance
social_media_100m: progress docs=9783552 txns=38218 phase=mixed writes/s=5990 avg_writes/s=6980 roll_wps=5427/8601/10700 reads/s=16914 avg_reads/s=16289 roll_rps=108/13055/22789 total_ops/s=22904 avg_total_ops/s=23269 roll_ops=6654/22130/32056 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=6ms ops=[id:15595/0-1ms, first:988/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:330/8-49ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=779ms p99=2506ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=46MB wal=6411MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2790MB/10s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:50 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node0:p2 lag=38245 q=1 imm=1 dirty=0MB trend=521 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=0 commit_ms=[wal:98478 mem:149507 wait:139244 manifest:0] wal_ms=[queue:4 group:49429 append:23066 fsync:7129 batches:2815 txns:3939 batch_mb:23 avg_txn_ms=q0 g12 a5 f1]
social_media_100m: hot node0:p2 backlog=1MB wal=200MB dirty=0MB imm=1 lag=38245 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=158 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=521 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=0
social_media_100m: hot node2:p1 backlog=3MB wal=200MB dirty=0MB imm=3 lag=38243 q=3 durable=0 truncate=0 primary_q=3 secondary_q=0 primary_done=161 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=359 emerg=2 budget=2MB hot_table=16049385931360479616 fg_tokens=0
social_media_100m: hot node1:p1 backlog=3MB wal=200MB dirty=0MB imm=3 lag=38243 q=3 durable=0 truncate=0 primary_q=3 secondary_q=0 primary_done=160 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=304 emerg=2 budget=2MB hot_table=16823683620448006766 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=10 low_wps_windows=2 wal_group_wait_windows=18 low_cached_read_windows=0 p99=3163 writes/s=6092 reads/s=15106 wal_group_avg_ms=13 bottleneck=checkpoint_advance
social_media_100m: progress docs=9814016 txns=38341 phase=mixed writes/s=6092 avg_writes/s=6976 roll_wps=5427/8601/10700 reads/s=15106 avg_reads/s=16283 roll_rps=108/14814/22789 total_ops/s=21198 avg_total_ops/s=23259 roll_ops=6654/22130/32056 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=7ms ops=[id:13907/0-1ms, first:906/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:292/8-51ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=879ms p99=3163ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=65MB wal=6431MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2790MB/10s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:74 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p6 lag=38361 q=1 imm=1 dirty=1MB trend=488 emerg=2 budget=2MB hot_table=16823683620448006766 fg_tokens=0 commit_ms=[wal:97414 mem:184937 wait:174189 manifest:0] wal_ms=[queue:10 group:52192 append:24216 fsync:8105 batches:2981 txns:3754 batch_mb:21 avg_txn_ms=q0 g13 a6 f2]
social_media_100m: hot node3:p6 backlog=1MB wal=200MB dirty=1MB imm=1 lag=38361 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=166 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=488 emerg=2 budget=2MB hot_table=16823683620448006766 fg_tokens=0
social_media_100m: hot node1:p3 backlog=1MB wal=200MB dirty=1MB imm=1 lag=38363 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=170 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=478 emerg=2 budget=2MB hot_table=16823683620448006766 fg_tokens=0
social_media_100m: hot node3:p1 backlog=2MB wal=201MB dirty=0MB imm=2 lag=38370 q=2 durable=0 truncate=0 primary_q=2 secondary_q=0 primary_done=162 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=462 emerg=2 budget=2MB hot_table=16823683620448006766 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=11 low_wps_windows=3 wal_group_wait_windows=19 low_cached_read_windows=0 p99=3217 writes/s=6707 reads/s=15046 wal_group_avg_ms=13 bottleneck=checkpoint_advance
social_media_100m: progress docs=9847552 txns=38468 phase=mixed writes/s=6707 avg_writes/s=6973 roll_wps=5427/8448/10086 reads/s=15046 avg_reads/s=16273 roll_rps=108/15046/22789 total_ops/s=21753 avg_total_ops/s=23246 roll_ops=6654/21896/32056 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=8ms ops=[id:13812/0-1ms, first:933/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:301/8-52ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=717ms p99=3217ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=57MB wal=6453MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2790MB/10s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:63 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p1 lag=38504 q=1 imm=1 dirty=2MB trend=836 emerg=2 budget=2MB hot_table=16823683620448006766 fg_tokens=0 commit_ms=[wal:108028 mem:208184 wait:196107 manifest:0] wal_ms=[queue:4 group:56422 append:24714 fsync:6663 batches:3057 txns:4264 batch_mb:23 avg_txn_ms=q0 g13 a5 f1]
social_media_100m: hot node3:p1 backlog=1MB wal=202MB dirty=2MB imm=1 lag=38504 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=163 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=836 emerg=2 budget=2MB hot_table=16823683620448006766 fg_tokens=0
social_media_100m: hot node0:p6 backlog=3MB wal=201MB dirty=1MB imm=3 lag=38490 q=3 durable=0 truncate=0 primary_q=3 secondary_q=0 primary_done=161 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=662 emerg=2 budget=2MB hot_table=13494870204566247797 fg_tokens=0
social_media_100m: hot node1:p1 backlog=3MB wal=201MB dirty=0MB imm=3 lag=38493 q=3 durable=0 truncate=0 primary_q=3 secondary_q=0 primary_done=165 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=564 emerg=2 budget=2MB hot_table=14124715428051363376 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=12 low_wps_windows=4 wal_group_wait_windows=20 low_cached_read_windows=0 p99=3235 writes/s=6604 reads/s=9691 wal_group_avg_ms=12 bottleneck=checkpoint_advance
social_media_100m: progress docs=9880576 txns=38597 phase=mixed writes/s=6604 avg_writes/s=6971 roll_wps=5427/6809/9369 reads/s=9691 avg_reads/s=16249 roll_rps=108/14814/22789 total_ops/s=16295 avg_total_ops/s=23220 roll_ops=6654/21753/32056 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=9ms ops=[id:8937/0-1ms, first:567/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:186/10-54ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=552ms p99=3235ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=29MB wal=6474MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2793MB/12s lsn_delta=[durable:390 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:32 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p1 lag=38616 q=2 imm=2 dirty=1MB trend=861 emerg=2 budget=2MB hot_table=14124715428051363376 fg_tokens=0 commit_ms=[wal:94792 mem:171590 wait:160355 manifest:0] wal_ms=[queue:4 group:48503 append:21192 fsync:5878 batches:2652 txns:3926 batch_mb:22 avg_txn_ms=q0 g12 a5 f1]
social_media_100m: hot node1:p1 backlog=2MB wal=202MB dirty=1MB imm=2 lag=38616 q=2 durable=0 truncate=0 primary_q=2 secondary_q=0 primary_done=166 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=861 emerg=2 budget=2MB hot_table=14124715428051363376 fg_tokens=0
social_media_100m: hot node0:p5 backlog=2MB wal=201MB dirty=4MB imm=2 lag=38618 q=2 durable=0 truncate=0 primary_q=2 secondary_q=0 primary_done=171 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=523 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=0
social_media_100m: hot node3:p2 backlog=2MB wal=202MB dirty=4MB imm=2 lag=38619 q=2 durable=0 truncate=0 primary_q=2 secondary_q=0 primary_done=177 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=479 emerg=2 budget=2MB hot_table=11445065271220102742 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=13 low_wps_windows=5 wal_group_wait_windows=21 low_cached_read_windows=0 p99=3299 writes/s=6656 reads/s=108 wal_group_avg_ms=12 bottleneck=checkpoint_advance
social_media_100m: progress docs=9913856 txns=38728 phase=mixed writes/s=6656 avg_writes/s=6970 roll_wps=5427/6707/9369 reads/s=108 avg_reads/s=16191 roll_rps=108/12902/18073 total_ops/s=6764 avg_total_ops/s=23161 roll_ops=6654/21503/26521 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=10ms ops=[id:99/0-2ms, first:6/0-2ms, f1:0/0-0ms, fm:0/0-0ms, txt:2/9-55ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=570ms p99=3299ms band=no slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=15MB wal=6496MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2793MB/12s lsn_delta=[durable:0 truncate:0] bottleneck=checkpoint_advance cluster_flush_q=[p:15 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node1:p6 lag=38749 q=1 imm=1 dirty=1MB trend=505 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0 commit_ms=[wal:99961 mem:155716 wait:144466 manifest:0] wal_ms=[queue:8 group:52753 append:21933 fsync:5771 batches:2887 txns:4209 batch_mb:23 avg_txn_ms=q0 g12 a5 f1]
social_media_100m: hot node1:p6 backlog=1MB wal=203MB dirty=1MB imm=1 lag=38749 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=165 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=505 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0
social_media_100m: hot node0:p6 backlog=1MB wal=202MB dirty=1MB imm=1 lag=38744 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=164 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=369 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0
social_media_100m: hot node3:p7 backlog=1MB wal=203MB dirty=1MB imm=1 lag=38749 q=1 durable=0 truncate=0 primary_q=1 secondary_q=0 primary_done=163 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=368 emerg=2 budget=2MB hot_table=11207436792058616795 fg_tokens=0
social_media_100m: RED acceptance_gate_failed high_p99_windows=14 low_wps_windows=0 wal_group_wait_windows=22 low_cached_read_windows=1 p99=2787 writes/s=9779 reads/s=13638 wal_group_avg_ms=11 bottleneck=wal_group_wait
social_media_100m: progress docs=9962752 txns=38919 phase=mixed writes/s=9779 avg_writes/s=6979 roll_wps=5427/6707/9779 reads/s=13638 avg_reads/s=16181 roll_rps=108/12902/16914 total_ops/s=23417 avg_total_ops/s=23160 roll_ops=6654/21503/23417 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=2ms ops=[id:12556/0-0ms, first:821/0-1ms, f1:0/0-0ms, fm:0/0-0ms, txt:260/1-31ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=450ms p99=2787ms band=yes slo=fail mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=6525MB wal_growth=6MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2794MB/12s lsn_delta=[durable:429 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node2:p4 lag=38929 q=0 imm=0 dirty=0MB trend=505 emerg=2 budget=2MB hot_table=9607800770234264530 fg_tokens=0 commit_ms=[wal:128246 mem:34981 wait:22481 manifest:0] wal_ms=[queue:10 group:67434 append:25967 fsync:9608 batches:3667 txns:5629 batch_mb:31 avg_txn_ms=q0 g11 a4 f1]
social_media_100m: hot node2:p4 backlog=0MB wal=203MB dirty=0MB imm=0 lag=38929 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=166 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=505 emerg=2 budget=2MB hot_table=9607800770234264530 fg_tokens=0
social_media_100m: hot node2:p2 backlog=0MB wal=204MB dirty=0MB imm=0 lag=38918 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=162 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=498 emerg=2 budget=2MB hot_table=9607800770234264530 fg_tokens=0
social_media_100m: hot node1:p5 backlog=0MB wal=203MB dirty=0MB imm=0 lag=38919 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=164 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=481 emerg=2 budget=2MB hot_table=9607800770234264530 fg_tokens=0
social_media_100m: progress docs=10000000 txns=39063 phase=mixed writes/s=7449 avg_writes/s=6980 roll_wps=5990/6809/9779 reads/s=12868 avg_reads/s=16168 roll_rps=108/12868/16914 total_ops/s=20317 avg_total_ops/s=23148 roll_ops=6654/21503/23417 read_band=no read_target_rps=20000 read_errors=0 total_read_errors=0 read_p50=0ms read_p99=2ms ops=[id:11838/0-0ms, first:783/0-0ms, f1:0/0-0ms, fm:0/0-0ms, txt:246/1-26ms, rng:0/0-0ms, page:0/0-0ms, upd:0/0-0ms] p50=214ms p99=532ms band=final slo=pass mode=cleanup assist=no target_wps=10000 active_hot=0/32 backlog=0MB wal=6548MB wal_growth=4MB/s wal_reclaim=0MB/s cleanup_target=256MB/s cleanup_debt=2794MB/12s lsn_delta=[durable:0 truncate:0] bottleneck=wal_group_wait cluster_flush_q=[p:0 s:0] secondary_rescue=0 secondary_oldest=0ms secondary_done_delta=0 secondary_throttle_delta=0 worst=node3:p7 lag=39060 q=0 imm=0 dirty=0MB trend=1191 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=0 commit_ms=[wal:95592 mem:10450 wait:1122 manifest:0] wal_ms=[queue:2 group:49177 append:17721 fsync:6279 batches:2558 txns:4300 batch_mb:24 avg_txn_ms=q0 g11 a4 f1]
social_media_100m: hot node3:p7 backlog=0MB wal=205MB dirty=0MB imm=0 lag=39060 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=164 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=1191 emerg=2 budget=2MB hot_table=14184631258625183492 fg_tokens=0
social_media_100m: hot node2:p2 backlog=0MB wal=205MB dirty=0MB imm=0 lag=39053 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=162 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=811 emerg=2 budget=2MB hot_table=9607800770234264530 fg_tokens=0
social_media_100m: hot node1:p5 backlog=0MB wal=204MB dirty=0MB imm=0 lag=39054 q=0 durable=0 truncate=0 primary_q=0 secondary_q=0 primary_done=164 secondary_done=0 secondary_oldest=0ms rescue=0 rescue_done=0 secondary_throttle=0 trend=785 emerg=2 budget=2MB hot_table=9607800770234264530 fg_tokens=0
social_media_100m: milestone_docs=10000000 writes/s=7449 reads/s=12868 txn_p99_ms=532 read_p99_ms=2 total_mb=20040 wal_mb=6548 data_mb=19 meta_mb=0
done: docs=10000000 txns=39063 elapsed=1435.2149067s avg_writes/s=6968 avg_reads/s=16140 avg_total_ops/s=23108 read_ops=23163709 reader_errors=0 txn_p50_ms=173 txn_p99_ms=2404 read_p50_ms=0 read_p99_ms=0 stable_write_band=no read_band=no slo_violations=387 monotonic_growth_failures=3 total_mb=20075 wal_mb=6548 data_mb=19 meta_mb=0
social_media_100m: completed with 387 post-warmup SLO violations