writes_test: elapsed=5s elapsed_hms=0:00:05 docs=116224 txns=456 writes/s=23244 avg_writes/s=23244 p50=36ms p99=274ms reason=wal_group_commit_wait p99_root_cause_exclusive=wal_group_commit_wait serialize=16ms dbms_insert=28436ms batch_coord=127218ms wal_commit=343666ms wal_fsync=34069ms wal_fsync_wait=136690ms wal_fsync_actual=34069ms wal_fsync_avg_ms=4 wal_fsync_p95_ms=16 wal_fsync_p99_ms=128 wal_fsync_max_ms=256 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=1468 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=2196 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1416 fsyncs/s=1416 wal_fsync_q=64 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=16181 wal_batches/s=1506 wal_bytes/s=16938463 avg_writers_per_fsync=2 avg_bytes_per_fsync=11957 wal_writer_in_flight=32 pending_waiters=0 pending_wal_bytes=186490 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=85205 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=7083 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=468 durable_lsn_after=470 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=11ms waiter_broadcasts=7083 fsync_batch_reason=timeout last_waiters_released_in_fsync=64 durable_lsn=470 durable_lsn_advance=14746 durable_lsn_advance_bytes=84692316 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=16MB/s wal_io=16MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=29708255232 background_io_paused=0ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=79730620 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=235ms wait=0ms wal=76MB wal_live_bytes=80134346 wal_truncatable_bytes=403726 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=70MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=52427 sec_apply/s=26085 text_apply/s=26341 idx_flush/s=0 idx_busy=0ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2964 wal_priority_yield_count=2196 background_io_yield_count=485 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=219 foreground_admission_delay_total_ms=219 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=115 foreground_pressure_transitions_total=115 foreground_pressure_state_elapsed_ms=1371 foreground_pressure_time_normal_window_ms=217357 foreground_pressure_time_warm_window_ms=27074 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=10s elapsed_hms=0:00:10 docs=241664 txns=945 writes/s=25088 avg_writes/s=24166 p50=32ms p99=92ms reason=healthy p99_root_cause_exclusive=healthy serialize=18ms dbms_insert=16127ms batch_coord=143080ms wal_commit=258593ms wal_fsync=20506ms wal_fsync_wait=1015597ms wal_fsync_actual=20506ms wal_fsync_avg_ms=0 wal_fsync_p95_ms=4 wal_fsync_p99_ms=8 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=189 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1744 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=8529 fsyncs/s=8529 wal_fsync_q=96 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=21757 wal_batches/s=1594 wal_bytes/s=18347868 avg_writers_per_fsync=0 avg_bytes_per_fsync=14663 wal_writer_in_flight=64 pending_waiters=0 pending_wal_bytes=327754 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=88969 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=15021 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=956 durable_lsn_after=959 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=16ms waiter_broadcasts=15021 fsync_batch_reason=timeout last_waiters_released_in_fsync=96 durable_lsn=959 durable_lsn_advance=15647 durable_lsn_advance_bytes=625386258 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=33298579456 background_io_paused=0ms background_pause_reason=none emergency_wal_only_count=2 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=166570594 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=16ms wait=0ms wal=156MB wal_live_bytes=166807921 wal_truncatable_bytes=237327 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=147MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49385 sec_apply/s=24692 text_apply/s=24692 idx_flush/s=0 idx_busy=0ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2195 wal_priority_yield_count=1744 background_io_yield_count=0 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=16 foreground_admission_delay_total_ms=235 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=35 foreground_pressure_transitions_total=150 foreground_pressure_state_elapsed_ms=6383 foreground_pressure_time_normal_window_ms=152030 foreground_pressure_time_warm_window_ms=8137 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=15s elapsed_hms=0:00:15 docs=366592 txns=1432 writes/s=24985 avg_writes/s=24439 p50=33ms p99=81ms reason=healthy p99_root_cause_exclusive=healthy serialize=44ms dbms_insert=16855ms batch_coord=142412ms wal_commit=283861ms wal_fsync=21109ms wal_fsync_wait=138914ms wal_fsync_actual=21109ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=289 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1752 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=21 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1583 fsyncs/s=1581 wal_fsync_q=95 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=16506 wal_batches/s=1585 wal_bytes/s=18084868 avg_writers_per_fsync=1 avg_bytes_per_fsync=11418 wal_writer_in_flight=64 pending_waiters=24 pending_wal_bytes=59139 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=92346 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=22928 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=1445 durable_lsn_after=1446 oldest_waiter_age=20ms avg_waiter_age=19ms max_waiter_age=14ms waiter_broadcasts=22928 fsync_batch_reason=timeout last_waiters_released_in_fsync=47 durable_lsn=1446 durable_lsn_advance=15611 durable_lsn_advance_bytes=90284749 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=33214693376 background_io_paused=0ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=251868072 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=4ms wait=0ms wal=238MB wal_live_bytes=252232020 wal_truncatable_bytes=363948 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=223MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=51353 sec_apply/s=25804 text_apply/s=25548 idx_flush/s=0 idx_busy=0ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2197 wal_priority_yield_count=1752 background_io_yield_count=0 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=4 foreground_admission_delay_total_ms=239 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=8 foreground_pressure_transitions_total=158 foreground_pressure_state_elapsed_ms=11386 foreground_pressure_time_normal_window_ms=158536 foreground_pressure_time_warm_window_ms=2035 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=20s elapsed_hms=0:00:20 docs=491776 txns=1922 writes/s=25036 avg_writes/s=24588 p50=33ms p99=90ms reason=healthy p99_root_cause_exclusive=healthy serialize=41ms dbms_insert=17849ms batch_coord=142956ms wal_commit=256787ms wal_fsync=24373ms wal_fsync_wait=126343ms wal_fsync_actual=24373ms wal_fsync_avg_ms=3 wal_fsync_p95_ms=8 wal_fsync_p99_ms=32 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=672 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1911 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1540 fsyncs/s=1543 wal_fsync_q=64 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=16369 wal_batches/s=1555 wal_bytes/s=18120333 avg_writers_per_fsync=2 avg_bytes_per_fsync=11761 wal_writer_in_flight=33 pending_waiters=0 pending_wal_bytes=200039 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=85712 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=30643 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=1935 durable_lsn_after=1937 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=13ms waiter_broadcasts=30643 fsync_batch_reason=timeout last_waiters_released_in_fsync=64 durable_lsn=1937 durable_lsn_advance=15677 durable_lsn_advance_bytes=90741261 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=32308723712 background_io_paused=0ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=337782627 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=30ms wait=0ms wal=320MB wal_live_bytes=338093073 wal_truncatable_bytes=310446 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=299MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=51456 sec_apply/s=25856 text_apply/s=25600 idx_flush/s=0 idx_busy=0ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2438 wal_priority_yield_count=1911 background_io_yield_count=0 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=none foreground_admission_delay_window_ms=27 foreground_admission_delay_total_ms=266 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=47 foreground_pressure_transitions_total=205 foreground_pressure_state_elapsed_ms=16389 foreground_pressure_time_normal_window_ms=152498 foreground_pressure_time_warm_window_ms=7715 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=25s elapsed_hms=0:00:25 docs=610816 txns=2389 writes/s=23808 avg_writes/s=24432 p50=38ms p99=99ms reason=durable_lsn_wait p99_root_cause_exclusive=durable_lsn_wait serialize=42ms dbms_insert=20161ms batch_coord=139296ms wal_commit=308675ms wal_fsync=29884ms wal_fsync_wait=130444ms wal_fsync_actual=29884ms wal_fsync_avg_ms=4 wal_fsync_p95_ms=16 wal_fsync_p99_ms=32 wal_fsync_max_ms=256 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=1197 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=2137 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=30 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1486 fsyncs/s=1481 wal_fsync_q=121 wal_group_size=18 wal_group_fill=35 wal_bytes_per_fsync=107940 wal_batches/s=1545 wal_bytes/s=18272653 avg_writers_per_fsync=2 avg_bytes_per_fsync=12175 wal_writer_in_flight=233 pending_waiters=191 pending_wal_bytes=374972 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=85834 pending_waiters_before_fsync=18 waiters_selected_for_batch=18 eligible_waiters=16 released_waiters=16 wakeup_batches=38051 lost_wakeup_count=0 durable_lsn_gap=1 durable_lsn_before=2424 durable_lsn_after=2426 oldest_waiter_age=189ms avg_waiter_age=101ms max_waiter_age=15ms waiter_broadcasts=38051 fsync_batch_reason=none last_waiters_released_in_fsync=90 durable_lsn=2426 durable_lsn_advance=15505 durable_lsn_advance_bytes=90198185 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=31117541376 background_io_paused=0ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=423292733 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=74ms wait=0ms wal=402MB wal_live_bytes=423698728 wal_truncatable_bytes=405995 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=372MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49945 sec_apply/s=24847 text_apply/s=25098 idx_flush/s=0 idx_busy=1ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2818 wal_priority_yield_count=2245 background_io_yield_count=0 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=wal_waiter_age foreground_admission_delay_window_ms=71 foreground_admission_delay_total_ms=337 foreground_last_admission_delay_ms=2 foreground_pressure_transitions_window=49 foreground_pressure_transitions_total=254 foreground_pressure_state_elapsed_ms=21404 foreground_pressure_time_normal_window_ms=150740 foreground_pressure_time_warm_window_ms=8553 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=30s elapsed_hms=0:00:30 docs=741888 txns=2898 writes/s=26214 avg_writes/s=24729 p50=105ms p99=334ms reason=durable_lsn_wait p99_root_cause_exclusive=durable_lsn_wait serialize=91ms dbms_insert=62840ms batch_coord=96676ms wal_commit=497887ms wal_fsync=74376ms wal_fsync_wait=139456ms wal_fsync_actual=74376ms wal_fsync_avg_ms=11 wal_fsync_p95_ms=64 wal_fsync_p99_ms=128 wal_fsync_max_ms=256 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=4908 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=4389 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=22 wal_fsync_active_elapsed_ms=2 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1318 fsyncs/s=1321 wal_fsync_q=62 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=15241 wal_batches/s=1487 wal_bytes/s=18182120 avg_writers_per_fsync=2 avg_bytes_per_fsync=13919 wal_writer_in_flight=39 pending_waiters=14 pending_wal_bytes=135142 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=82954 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=44659 lost_wakeup_count=0 durable_lsn_gap=1 durable_lsn_before=2915 durable_lsn_after=2916 oldest_waiter_age=32ms avg_waiter_age=23ms max_waiter_age=12ms waiter_broadcasts=44659 fsync_batch_reason=none last_waiters_released_in_fsync=45 durable_lsn=2916 durable_lsn_advance=15836 durable_lsn_advance_bytes=91979160 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=27694989312 background_io_paused=1972ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=510051311 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=397ms wait=0ms wal=486MB wal_live_bytes=510544424 wal_truncatable_bytes=493113 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=452MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49358 sec_apply/s=24551 text_apply/s=24807 idx_flush/s=0 idx_busy=8ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=6277 wal_priority_yield_count=4365 background_io_yield_count=112 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=346 foreground_admission_delay_total_ms=683 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=287 foreground_pressure_transitions_total=541 foreground_pressure_state_elapsed_ms=1941 foreground_pressure_time_normal_window_ms=92532 foreground_pressure_time_warm_window_ms=69413 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=35s elapsed_hms=0:00:35 docs=866304 txns=3385 writes/s=24883 avg_writes/s=24751 p50=41ms p99=407ms reason=wal_group_commit_wait p99_root_cause_exclusive=wal_group_commit_wait serialize=55ms dbms_insert=34432ms batch_coord=125737ms wal_commit=350648ms wal_fsync=35596ms wal_fsync_wait=119127ms wal_fsync_actual=35596ms wal_fsync_avg_ms=5 wal_fsync_p95_ms=16 wal_fsync_p99_ms=64 wal_fsync_max_ms=512 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=1751 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=2347 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=14 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1326 fsyncs/s=1327 wal_fsync_q=159 wal_group_size=5 wal_group_fill=9 wal_bytes_per_fsync=38751 wal_batches/s=1386 wal_bytes/s=18192217 avg_writers_per_fsync=2 avg_bytes_per_fsync=13707 wal_writer_in_flight=46 pending_waiters=21 pending_wal_bytes=174435 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=72628 pending_waiters_before_fsync=5 waiters_selected_for_batch=5 eligible_waiters=5 released_waiters=5 wakeup_batches=51294 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=3400 durable_lsn_after=3405 oldest_waiter_age=25ms avg_waiter_age=16ms max_waiter_age=14ms waiter_broadcasts=51294 fsync_batch_reason=timeout last_waiters_released_in_fsync=146 durable_lsn=3405 durable_lsn_advance=15608 durable_lsn_advance_bytes=90946345 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=27833401344 background_io_paused=7489ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=595606968 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=161ms wait=0ms wal=567MB wal_live_bytes=596506758 wal_truncatable_bytes=899790 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=528MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=48095 sec_apply/s=24174 text_apply/s=23920 idx_flush/s=0 idx_busy=9ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=3179 wal_priority_yield_count=2610 background_io_yield_count=628 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=156 foreground_admission_delay_total_ms=839 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=90 foreground_pressure_transitions_total=631 foreground_pressure_state_elapsed_ms=6861 foreground_pressure_time_normal_window_ms=138287 foreground_pressure_time_warm_window_ms=22284 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=40s elapsed_hms=0:00:40 docs=992512 txns=3878 writes/s=25241 avg_writes/s=24812 p50=42ms p99=84ms reason=healthy p99_root_cause_exclusive=healthy serialize=46ms dbms_insert=21932ms batch_coord=137377ms wal_commit=245010ms wal_fsync=21856ms wal_fsync_wait=114386ms wal_fsync_actual=21856ms wal_fsync_avg_ms=3 wal_fsync_p95_ms=16 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=701 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1852 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=10 wal_fsync_active_elapsed_ms=2 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1386 fsyncs/s=1387 wal_fsync_q=50 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=14393 wal_batches/s=1386 wal_bytes/s=18108349 avg_writers_per_fsync=2 avg_bytes_per_fsync=13069 wal_writer_in_flight=12 pending_waiters=3 pending_wal_bytes=50707 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=77662 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=58228 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=3891 durable_lsn_after=3892 oldest_waiter_age=15ms avg_waiter_age=15ms max_waiter_age=14ms waiter_broadcasts=58228 fsync_batch_reason=timeout last_waiters_released_in_fsync=52 durable_lsn=3892 durable_lsn_advance=15622 durable_lsn_advance_bytes=90633824 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=29091692544 background_io_paused=9372ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=681969230 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=3ms wait=0ms wal=649MB wal_live_bytes=682331991 wal_truncatable_bytes=362761 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=605MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49770 sec_apply/s=25012 text_apply/s=24757 idx_flush/s=0 idx_busy=9ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2354 wal_priority_yield_count=2184 background_io_yield_count=868 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=839 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=631 foreground_pressure_state_elapsed_ms=11872 foreground_pressure_time_normal_window_ms=160343 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=45s elapsed_hms=0:00:45 docs=1117440 txns=4367 writes/s=24985 avg_writes/s=24832 p50=43ms p99=78ms reason=healthy p99_root_cause_exclusive=healthy serialize=49ms dbms_insert=21600ms batch_coord=137922ms wal_commit=247318ms wal_fsync=18319ms wal_fsync_wait=116355ms wal_fsync_actual=18319ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=354 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1694 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=20 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1391 fsyncs/s=1389 wal_fsync_q=71 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=18203 wal_batches/s=1391 wal_bytes/s=18228679 avg_writers_per_fsync=2 avg_bytes_per_fsync=13104 wal_writer_in_flight=60 pending_waiters=16 pending_wal_bytes=302524 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=78353 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=65178 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=4381 durable_lsn_after=4383 oldest_waiter_age=18ms avg_waiter_age=15ms max_waiter_age=14ms waiter_broadcasts=65178 fsync_batch_reason=timeout last_waiters_released_in_fsync=66 durable_lsn=4383 durable_lsn_advance=15692 durable_lsn_advance_bytes=91065484 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=29162995712 background_io_paused=9389ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=767842828 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=3ms wait=0ms wal=731MB wal_live_bytes=768467166 wal_truncatable_bytes=624338 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=682MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50432 sec_apply/s=25344 text_apply/s=25088 idx_flush/s=0 idx_busy=9ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2106 wal_priority_yield_count=2028 background_io_yield_count=888 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=839 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=631 foreground_pressure_state_elapsed_ms=16884 foreground_pressure_time_normal_window_ms=160327 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=50s elapsed_hms=0:00:50 docs=1242368 txns=4853 writes/s=24985 avg_writes/s=24847 p50=40ms p99=111ms reason=healthy p99_root_cause_exclusive=healthy serialize=37ms dbms_insert=20816ms batch_coord=137608ms wal_commit=261249ms wal_fsync=17576ms wal_fsync_wait=502573ms wal_fsync_actual=17576ms wal_fsync_avg_ms=0 wal_fsync_p95_ms=8 wal_fsync_p99_ms=8 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=355 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1605 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=14 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=4171 fsyncs/s=4172 wal_fsync_q=64 wal_group_size=4 wal_group_fill=7 wal_bytes_per_fsync=26192 wal_batches/s=1332 wal_bytes/s=18165516 avg_writers_per_fsync=0 avg_bytes_per_fsync=11178 wal_writer_in_flight=11 pending_waiters=3 pending_wal_bytes=24453 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=74351 pending_waiters_before_fsync=4 waiters_selected_for_batch=4 eligible_waiters=4 released_waiters=4 wakeup_batches=71844 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=4870 durable_lsn_after=4871 oldest_waiter_age=13ms avg_waiter_age=13ms max_waiter_age=13ms waiter_broadcasts=71844 fsync_batch_reason=timeout last_waiters_released_in_fsync=63 durable_lsn=4871 durable_lsn_advance=15598 durable_lsn_advance_bytes=233234506 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=27925676032 background_io_paused=9229ms background_pause_reason=none emergency_wal_only_count=1 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=853882961 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=27ms wait=0ms wal=813MB wal_live_bytes=854379604 wal_truncatable_bytes=496643 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=758MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50141 sec_apply/s=25196 text_apply/s=24944 idx_flush/s=0 idx_busy=10ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2012 wal_priority_yield_count=1931 background_io_yield_count=831 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=23 foreground_admission_delay_total_ms=862 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=45 foreground_pressure_transitions_total=676 foreground_pressure_state_elapsed_ms=19194 foreground_pressure_time_normal_window_ms=149801 foreground_pressure_time_warm_window_ms=10897 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=55s elapsed_hms=0:00:55 docs=1368064 txns=5345 writes/s=25139 avg_writes/s=24873 p50=39ms p99=79ms reason=healthy p99_root_cause_exclusive=healthy serialize=33ms dbms_insert=20276ms batch_coord=140077ms wal_commit=233121ms wal_fsync=16484ms wal_fsync_wait=109677ms wal_fsync_actual=16484ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=137 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1575 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=3 wal_fsync_active_elapsed_ms=4 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1348 fsyncs/s=1348 wal_fsync_q=93 wal_group_size=2 wal_group_fill=4 wal_bytes_per_fsync=18687 wal_batches/s=1348 wal_bytes/s=18147557 avg_writers_per_fsync=2 avg_bytes_per_fsync=13457 wal_writer_in_flight=32 pending_waiters=0 pending_wal_bytes=19967 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=73169 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=78588 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=5359 durable_lsn_after=5361 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=17ms waiter_broadcasts=78588 fsync_batch_reason=timeout last_waiters_released_in_fsync=64 durable_lsn=5361 durable_lsn_advance=15715 durable_lsn_advance_bytes=90758741 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=28269608960 background_io_paused=9148ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=940053269 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=1ms wait=0ms wal=895MB wal_live_bytes=940356497 wal_truncatable_bytes=303228 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=835MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50893 sec_apply/s=25318 text_apply/s=25574 idx_flush/s=0 idx_busy=10ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1956 wal_priority_yield_count=1887 background_io_yield_count=986 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=862 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=676 foreground_pressure_state_elapsed_ms=24201 foreground_pressure_time_normal_window_ms=160249 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=60s elapsed_hms=0:01:00 docs=1493504 txns=5835 writes/s=25088 avg_writes/s=24891 p50=38ms p99=70ms reason=healthy p99_root_cause_exclusive=healthy serialize=40ms dbms_insert=19171ms batch_coord=140665ms wal_commit=228920ms wal_fsync=14130ms wal_fsync_wait=102465ms wal_fsync_actual=14130ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=8 wal_fsync_max_ms=16 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=150 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1370 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=12 wal_fsync_active_elapsed_ms=1 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1249 fsyncs/s=1248 wal_fsync_q=97 wal_group_size=4 wal_group_fill=7 wal_bytes_per_fsync=28395 wal_batches/s=1249 wal_bytes/s=17902709 avg_writers_per_fsync=2 avg_bytes_per_fsync=14331 wal_writer_in_flight=37 pending_waiters=8 pending_wal_bytes=210498 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=69861 pending_waiters_before_fsync=4 waiters_selected_for_batch=4 eligible_waiters=4 released_waiters=4 wakeup_batches=84831 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=5845 durable_lsn_after=5849 oldest_waiter_age=18ms avg_waiter_age=10ms max_waiter_age=15ms waiter_broadcasts=84831 fsync_batch_reason=timeout last_waiters_released_in_fsync=105 durable_lsn=5849 durable_lsn_advance=15625 durable_lsn_advance_bytes=89470705 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=26231177216 background_io_paused=8974ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=1024824668 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=0ms wait=0ms wal=975MB wal_live_bytes=1025207100 wal_truncatable_bytes=382432 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=index_backlog backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=912MB pending_idx=512 pending_sec_apply=256 pending_sec_flush=0 oldest_idx_lsn=2987521 idx_lag_lsn=512 idx_lag=6ms sec_lag=6ms text_lag=6ms idx_apply/s=50893 sec_apply/s=25574 text_apply/s=25318 idx_flush/s=0 idx_busy=10ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1689 wal_priority_yield_count=1682 background_io_yield_count=945 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=862 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=676 foreground_pressure_state_elapsed_ms=29209 foreground_pressure_time_normal_window_ms=159826 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=65s elapsed_hms=0:01:05 docs=1618688 txns=6324 writes/s=25036 avg_writes/s=24902 p50=41ms p99=110ms reason=healthy p99_root_cause_exclusive=healthy serialize=43ms dbms_insert=21425ms batch_coord=137608ms wal_commit=254963ms wal_fsync=18223ms wal_fsync_wait=110160ms wal_fsync_actual=18223ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=381 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1645 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1319 fsyncs/s=1320 wal_fsync_q=132 wal_group_size=5 wal_group_fill=9 wal_bytes_per_fsync=40612 wal_batches/s=1327 wal_bytes/s=18041688 avg_writers_per_fsync=2 avg_bytes_per_fsync=13668 wal_writer_in_flight=0 pending_waiters=0 pending_wal_bytes=0 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=68921 pending_waiters_before_fsync=5 waiters_selected_for_batch=5 eligible_waiters=5 released_waiters=5 wakeup_batches=91434 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=6333 durable_lsn_after=6338 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=19ms waiter_broadcasts=91434 fsync_batch_reason=timeout last_waiters_released_in_fsync=132 durable_lsn=6338 durable_lsn_advance=15635 durable_lsn_advance_bytes=90251284 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=27648851968 background_io_paused=8986ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=1110616248 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=38ms wait=0ms wal=1058MB wal_live_bytes=1110681260 wal_truncatable_bytes=65012 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=988MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=48773 sec_apply/s=24510 text_apply/s=24263 idx_flush/s=0 idx_busy=14ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2064 wal_priority_yield_count=1943 background_io_yield_count=1020 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=none foreground_admission_delay_window_ms=28 foreground_admission_delay_total_ms=890 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=57 foreground_pressure_transitions_total=733 foreground_pressure_state_elapsed_ms=34171 foreground_pressure_time_normal_window_ms=146298 foreground_pressure_time_warm_window_ms=13313 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=70s elapsed_hms=0:01:10 docs=1743616 txns=6812 writes/s=24985 avg_writes/s=24908 p50=38ms p99=66ms reason=healthy p99_root_cause_exclusive=healthy serialize=32ms dbms_insert=19221ms batch_coord=140765ms wal_commit=245701ms wal_fsync=17397ms wal_fsync_wait=112722ms wal_fsync_actual=17397ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=199 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1608 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=11 wal_fsync_active_elapsed_ms=3 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1367 fsyncs/s=1365 wal_fsync_q=86 wal_group_size=5 wal_group_fill=9 wal_bytes_per_fsync=34814 wal_batches/s=1367 wal_bytes/s=18310078 avg_writers_per_fsync=2 avg_bytes_per_fsync=13395 wal_writer_in_flight=49 pending_waiters=14 pending_wal_bytes=256164 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=71809 pending_waiters_before_fsync=5 waiters_selected_for_batch=5 eligible_waiters=5 released_waiters=5 wakeup_batches=98262 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=6824 durable_lsn_after=6828 oldest_waiter_age=15ms avg_waiter_age=12ms max_waiter_age=14ms waiter_broadcasts=98262 fsync_batch_reason=timeout last_waiters_released_in_fsync=101 durable_lsn=6828 durable_lsn_advance=15715 durable_lsn_advance_bytes=91461563 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=28697427968 background_io_paused=9096ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=1196396941 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=4ms wait=0ms wal=1139MB wal_live_bytes=1197094214 wal_truncatable_bytes=697273 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=1065MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=51456 sec_apply/s=25600 text_apply/s=25856 idx_flush/s=0 idx_busy=14ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2004 wal_priority_yield_count=1904 background_io_yield_count=1144 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=890 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=1 foreground_pressure_transitions_total=734 foreground_pressure_state_elapsed_ms=39186 foreground_pressure_time_normal_window_ms=161020 foreground_pressure_time_warm_window_ms=446 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=75s elapsed_hms=0:01:15 docs=1869568 txns=7303 writes/s=25190 avg_writes/s=24927 p50=38ms p99=86ms reason=healthy p99_root_cause_exclusive=healthy serialize=34ms dbms_insert=19466ms batch_coord=140060ms wal_commit=245520ms wal_fsync=18641ms wal_fsync_wait=119368ms wal_fsync_actual=18641ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=248 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1685 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=15 wal_fsync_active_elapsed_ms=3 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1422 fsyncs/s=1422 wal_fsync_q=64 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=15482 wal_batches/s=1432 wal_bytes/s=17925843 avg_writers_per_fsync=2 avg_bytes_per_fsync=12607 wal_writer_in_flight=76 pending_waiters=16 pending_wal_bytes=252921 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=76598 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=105372 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=7316 durable_lsn_after=7318 oldest_waiter_age=16ms avg_waiter_age=14ms max_waiter_age=14ms waiter_broadcasts=105372 fsync_batch_reason=timeout last_waiters_released_in_fsync=64 durable_lsn=7318 durable_lsn_advance=15619 durable_lsn_advance_bytes=89637147 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=29796335616 background_io_paused=9057ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=1281608293 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=20ms wait=0ms wal=1221MB wal_live_bytes=1282073062 wal_truncatable_bytes=464769 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=1141MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50741 sec_apply/s=25243 text_apply/s=25498 idx_flush/s=0 idx_busy=14ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2117 wal_priority_yield_count=1990 background_io_yield_count=1021 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=17 foreground_admission_delay_total_ms=907 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=32 foreground_pressure_transitions_total=766 foreground_pressure_state_elapsed_ms=44194 foreground_pressure_time_normal_window_ms=152205 foreground_pressure_time_warm_window_ms=8153 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=80s elapsed_hms=0:01:20 docs=1994240 txns=7793 writes/s=24934 avg_writes/s=24928 p50=44ms p99=112ms reason=healthy p99_root_cause_exclusive=healthy serialize=81ms dbms_insert=22664ms batch_coord=137273ms wal_commit=250135ms wal_fsync=19581ms wal_fsync_wait=116280ms wal_fsync_actual=19581ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=563 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1729 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=16 wal_fsync_active_elapsed_ms=1 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1351 fsyncs/s=1351 wal_fsync_q=109 wal_group_size=4 wal_group_fill=7 wal_bytes_per_fsync=24937 wal_batches/s=1360 wal_bytes/s=18255544 avg_writers_per_fsync=2 avg_bytes_per_fsync=13511 wal_writer_in_flight=64 pending_waiters=12 pending_wal_bytes=268179 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=78255 pending_waiters_before_fsync=4 waiters_selected_for_batch=4 eligible_waiters=4 released_waiters=4 wakeup_batches=112129 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=7805 durable_lsn_after=7808 oldest_waiter_age=26ms avg_waiter_age=23ms max_waiter_age=19ms waiter_broadcasts=112129 fsync_batch_reason=timeout last_waiters_released_in_fsync=99 durable_lsn=7808 durable_lsn_advance=15692 durable_lsn_advance_bytes=91296737 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=28374466560 background_io_paused=9335ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=1367867842 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=34ms wait=0ms wal=1303MB wal_live_bytes=1368429769 wal_truncatable_bytes=561927 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=1218MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50690 sec_apply/s=25217 text_apply/s=25472 idx_flush/s=0 idx_busy=18ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2210 wal_priority_yield_count=2060 background_io_yield_count=930 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=26 foreground_admission_delay_total_ms=933 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=40 foreground_pressure_transitions_total=806 foreground_pressure_state_elapsed_ms=49211 foreground_pressure_time_normal_window_ms=150359 foreground_pressure_time_warm_window_ms=9856 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=85s elapsed_hms=0:01:25 docs=2119424 txns=8280 writes/s=25036 avg_writes/s=24934 p50=42ms p99=76ms reason=healthy p99_root_cause_exclusive=healthy serialize=52ms dbms_insert=21191ms batch_coord=138035ms wal_commit=259921ms wal_fsync=19052ms wal_fsync_wait=115918ms wal_fsync_actual=19052ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=519 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1658 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=18 wal_fsync_active_elapsed_ms=4 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1319 fsyncs/s=1319 wal_fsync_q=52 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=24851 wal_batches/s=1322 wal_bytes/s=18029492 avg_writers_per_fsync=2 avg_bytes_per_fsync=13669 wal_writer_in_flight=41 pending_waiters=6 pending_wal_bytes=148434 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=77871 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=118726 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=8291 durable_lsn_after=8294 oldest_waiter_age=26ms avg_waiter_age=26ms max_waiter_age=18ms waiter_broadcasts=118726 fsync_batch_reason=timeout last_waiters_released_in_fsync=58 durable_lsn=8294 durable_lsn_advance=15566 durable_lsn_advance_bytes=90179041 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=27636269056 background_io_paused=9304ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=1453370898 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=1ms wait=0ms wal=1384MB wal_live_bytes=1453752157 wal_truncatable_bytes=381259 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=index_backlog backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=1294MB pending_idx=512 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=4239361 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=52415 sec_apply/s=26334 text_apply/s=26081 idx_flush/s=0 idx_busy=20ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2104 wal_priority_yield_count=1972 background_io_yield_count=972 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=933 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=806 foreground_pressure_state_elapsed_ms=54228 foreground_pressure_time_normal_window_ms=160780 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=90s elapsed_hms=0:01:30 docs=2245120 txns=8771 writes/s=25139 avg_writes/s=24945 p50=44ms p99=103ms reason=healthy p99_root_cause_exclusive=healthy serialize=42ms dbms_insert=23109ms batch_coord=136572ms wal_commit=252796ms wal_fsync=16706ms wal_fsync_wait=114404ms wal_fsync_actual=16706ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=314 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1529 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=16 wal_fsync_active_elapsed_ms=7 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1320 fsyncs/s=1320 wal_fsync_q=64 wal_group_size=2 wal_group_fill=4 wal_bytes_per_fsync=16902 wal_batches/s=1322 wal_bytes/s=18203629 avg_writers_per_fsync=2 avg_bytes_per_fsync=13781 wal_writer_in_flight=40 pending_waiters=16 pending_wal_bytes=196208 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=75570 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=125326 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=8785 durable_lsn_after=8787 oldest_waiter_age=21ms avg_waiter_age=18ms max_waiter_age=14ms waiter_broadcasts=125326 fsync_batch_reason=timeout last_waiters_released_in_fsync=56 durable_lsn=8787 durable_lsn_advance=15739 durable_lsn_advance_bytes=90958450 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=27678212096 background_io_paused=9222ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=1539449742 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=10ms wait=0ms wal=1467MB wal_live_bytes=1539857547 wal_truncatable_bytes=407805 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=1371MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=48989 sec_apply/s=24620 text_apply/s=24368 idx_flush/s=0 idx_busy=25ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1924 wal_priority_yield_count=1837 background_io_yield_count=1069 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=9 foreground_admission_delay_total_ms=942 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=16 foreground_pressure_transitions_total=822 foreground_pressure_state_elapsed_ms=59236 foreground_pressure_time_normal_window_ms=156014 foreground_pressure_time_warm_window_ms=4352 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=95s elapsed_hms=0:01:35 docs=2370816 txns=9261 writes/s=25139 avg_writes/s=24955 p50=41ms p99=84ms reason=healthy p99_root_cause_exclusive=healthy serialize=42ms dbms_insert=21584ms batch_coord=138356ms wal_commit=236688ms wal_fsync=16101ms wal_fsync_wait=110855ms wal_fsync_actual=16101ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=293 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1599 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=19 wal_fsync_active_elapsed_ms=6 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1322 fsyncs/s=1321 wal_fsync_q=125 wal_group_size=4 wal_group_fill=8 wal_bytes_per_fsync=34310 wal_batches/s=1323 wal_bytes/s=18069126 avg_writers_per_fsync=2 avg_bytes_per_fsync=13634 wal_writer_in_flight=76 pending_waiters=64 pending_wal_bytes=357954 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=73845 pending_waiters_before_fsync=4 waiters_selected_for_batch=4 eligible_waiters=4 released_waiters=4 wakeup_batches=131931 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=9274 durable_lsn_after=9278 oldest_waiter_age=16ms avg_waiter_age=12ms max_waiter_age=11ms waiter_broadcasts=131931 fsync_batch_reason=timeout last_waiters_released_in_fsync=83 durable_lsn=9278 durable_lsn_advance=15657 durable_lsn_advance_bytes=90058966 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=27732738048 background_io_paused=9352ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=1624877545 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=1ms wait=0ms wal=1548MB wal_live_bytes=1625224098 wal_truncatable_bytes=346553 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=1448MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49870 sec_apply/s=25062 text_apply/s=24807 idx_flush/s=0 idx_busy=26ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1998 wal_priority_yield_count=1921 background_io_yield_count=973 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=942 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=822 foreground_pressure_state_elapsed_ms=64249 foreground_pressure_time_normal_window_ms=159964 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=100s elapsed_hms=0:01:40 docs=2494208 txns=9744 writes/s=24678 avg_writes/s=24942 p50=59ms p99=163ms reason=healthy p99_root_cause_exclusive=healthy serialize=217ms dbms_insert=33348ms batch_coord=126365ms wal_commit=241945ms wal_fsync=17265ms wal_fsync_wait=110040ms wal_fsync_actual=17265ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=32 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=675 wal_fsync_blocked_by_doc_store_sync_ms=82 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1723 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=5 wal_fsync_active_elapsed_ms=3 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1345 fsyncs/s=1348 wal_fsync_q=57 wal_group_size=5 wal_group_fill=9 wal_bytes_per_fsync=32911 wal_batches/s=1364 wal_bytes/s=18120170 avg_writers_per_fsync=2 avg_bytes_per_fsync=13489 wal_writer_in_flight=3 pending_waiters=2 pending_wal_bytes=27908 wal_append_lock_wait_ms=8 wal_writer_sleep_ms=78190 pending_waiters_before_fsync=5 waiters_selected_for_batch=5 eligible_waiters=5 released_waiters=5 wakeup_batches=138674 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=9761 durable_lsn_after=9763 oldest_waiter_age=28ms avg_waiter_age=25ms max_waiter_age=24ms waiter_broadcasts=138674 fsync_batch_reason=timeout last_waiters_released_in_fsync=57 durable_lsn=9763 durable_lsn_advance=15581 durable_lsn_advance_bytes=90957152 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=64MB/s wal_io=17MB/s primary_flush_io=47MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=28227665920 background_io_paused=11722ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=348ms doc_store_sync_segments_window=260 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=10 doc_store_sync_segment_bytes=4066248 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=1 doc_store_sync_pending_segments=2 doc_store_sync_pending_bytes=2922009 doc_store_sync_pending_lsn=9634 doc_store_durable_lsn=9634 doc_store_sync_worker_active=1 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=14 doc_store_sync_debt_bytes=2922009 doc_store_sync_jobs_merged=2 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=2 doc_store_sync_queue_wait_ms=67 doc_store_sync_physical_ms=348 doc_store_sync_durable_lsn_advance=303604 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=1710845478 checkpoint_sync=0ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=1665ms wait=0ms wal=1629MB wal_live_bytes=1711175593 wal_truncatable_bytes=330115 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=index_backlog backlog=30MB flush_q=15 primary_q=15 secondary_q=0 primary_io=26154ms primary_pages=0 primary_flush_batch_bytes=249628253 primary_flush_batch_entries=365704 avg_flush_batch_bytes=1559028 coalesced_tables=0 coalesced_batches=0 small_flush_count=7 small_flush_reason=pressure active_primary_flush_workers=15 per_table_flush_inflight=1 flush_worker_io_wait=0ms flush_worker_cpu=26154ms primary_slice_max=392ms primary_blocked=27974ms primary_flush_admission_wait_ms=27974 primary_flush_queued_age_ms=302 primary_flush_oldest_queued_age_ms=2 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=431ms index_log=1524MB pending_idx=512 pending_sec_apply=256 pending_sec_flush=0 oldest_idx_lsn=4988929 idx_lag_lsn=512 idx_lag=6ms sec_lag=6ms text_lag=6ms idx_apply/s=50432 sec_apply/s=25344 text_apply/s=25088 idx_flush/s=0 idx_busy=70ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=27974 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2210 wal_priority_yield_count=2141 background_io_yield_count=4046 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=36 foreground_admission_delay_total_ms=978 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=48 foreground_pressure_transitions_total=870 foreground_pressure_state_elapsed_ms=69264 foreground_pressure_time_normal_window_ms=148631 foreground_pressure_time_warm_window_ms=11813 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=105s elapsed_hms=0:01:45 docs=2619648 txns=10234 writes/s=25088 avg_writes/s=24949 p50=46ms p99=114ms reason=healthy p99_root_cause_exclusive=healthy serialize=114ms dbms_insert=25810ms batch_coord=133680ms wal_commit=263006ms wal_fsync=21241ms wal_fsync_wait=119538ms wal_fsync_actual=21241ms wal_fsync_avg_ms=3 wal_fsync_p95_ms=16 wal_fsync_p99_ms=32 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=721 wal_fsync_blocked_by_doc_store_sync_ms=17 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1873 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=16 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1376 fsyncs/s=1373 wal_fsync_q=84 wal_group_size=8 wal_group_fill=15 wal_bytes_per_fsync=36522 wal_batches/s=1384 wal_bytes/s=18390344 avg_writers_per_fsync=2 avg_bytes_per_fsync=13381 wal_writer_in_flight=79 pending_waiters=20 pending_wal_bytes=242441 wal_append_lock_wait_ms=4 wal_writer_sleep_ms=78433 pending_waiters_before_fsync=8 waiters_selected_for_batch=8 eligible_waiters=8 released_waiters=8 wakeup_batches=145538 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=10249 durable_lsn_after=10255 oldest_waiter_age=19ms avg_waiter_age=19ms max_waiter_age=11ms waiter_broadcasts=145538 fsync_batch_reason=timeout last_waiters_released_in_fsync=142 durable_lsn=10255 durable_lsn_advance=15759 durable_lsn_advance_bytes=91872330 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=20MB/s wal_io=17MB/s primary_flush_io=3MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=28865200128 background_io_paused=9677ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=53ms doc_store_sync_segments_window=39 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=2 doc_store_sync_segment_bytes=3937523 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=9634 doc_store_durable_lsn=9634 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=2 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=68 doc_store_sync_physical_ms=401 doc_store_sync_durable_lsn_advance=303604 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=1797473407 checkpoint_sync=6ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=6 checkpoint_sync_ms_per_node=1ms checkpoint_sync_blocked_window_ms=6 checkpoint_sync_blocked_total_ms=6 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=6ms checkpoint_slice_avg_ms=6 checkpoint_slice_last_ms=6 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=737280 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=15ms wait=0ms wal=1713MB wal_live_bytes=1798110986 wal_truncatable_bytes=637579 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=30652ms primary_pages=0 primary_flush_batch_bytes=18911450 primary_flush_batch_entries=27654 avg_flush_batch_bytes=1398567 coalesced_tables=0 coalesced_batches=0 small_flush_count=10 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=4498ms primary_slice_max=392ms primary_blocked=30026ms primary_flush_admission_wait_ms=2052 primary_flush_queued_age_ms=73 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=431ms index_log=1600MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50181 sec_apply/s=24963 text_apply/s=25217 idx_flush/s=0 idx_busy=79ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=6 checkpoint_yield_count=2 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=2052 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2395 wal_priority_yield_count=2218 background_io_yield_count=1382 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=5 foreground_admission_delay_total_ms=983 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=8 foreground_pressure_transitions_total=878 foreground_pressure_state_elapsed_ms=74282 foreground_pressure_time_normal_window_ms=159409 foreground_pressure_time_warm_window_ms=1528 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=110s elapsed_hms=0:01:50 docs=2746624 txns=10730 writes/s=25395 avg_writes/s=24969 p50=40ms p99=97ms reason=healthy p99_root_cause_exclusive=healthy serialize=38ms dbms_insert=21636ms batch_coord=138163ms wal_commit=249335ms wal_fsync=16680ms wal_fsync_wait=119686ms wal_fsync_actual=16680ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=188 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1636 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=17 wal_fsync_active_elapsed_ms=6 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1393 fsyncs/s=1395 wal_fsync_q=95 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=24580 wal_batches/s=1394 wal_bytes/s=18152899 avg_writers_per_fsync=2 avg_bytes_per_fsync=13011 wal_writer_in_flight=49 pending_waiters=17 pending_wal_bytes=165038 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=80894 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=152516 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=10743 durable_lsn_after=10745 oldest_waiter_age=21ms avg_waiter_age=20ms max_waiter_age=14ms waiter_broadcasts=152516 fsync_batch_reason=timeout last_waiters_released_in_fsync=72 durable_lsn=10745 durable_lsn_advance=15693 durable_lsn_advance_bytes=90785260 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=29263659008 background_io_paused=9391ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=2 doc_store_sync_segment_bytes=3937523 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=9634 doc_store_durable_lsn=9634 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=2 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=68 doc_store_sync_physical_ms=401 doc_store_sync_durable_lsn_advance=303604 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=1883392493 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=6 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=6 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=737280 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=2ms wait=0ms wal=1795MB wal_live_bytes=1883993341 wal_truncatable_bytes=600848 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=30652ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=1398567 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=392ms primary_blocked=30026ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=431ms index_log=1678MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=52992 sec_apply/s=26624 text_apply/s=26368 idx_flush/s=0 idx_busy=80ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2009 wal_priority_yield_count=1983 background_io_yield_count=705 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=983 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=878 foreground_pressure_state_elapsed_ms=79301 foreground_pressure_time_normal_window_ms=160578 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=115s elapsed_hms=0:01:55 docs=2872576 txns=11222 writes/s=25190 avg_writes/s=24978 p50=40ms p99=76ms reason=healthy p99_root_cause_exclusive=healthy serialize=29ms dbms_insert=20573ms batch_coord=138292ms wal_commit=241678ms wal_fsync=17837ms wal_fsync_wait=116651ms wal_fsync_actual=17837ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=249 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1682 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1420 fsyncs/s=1422 wal_fsync_q=64 wal_group_size=2 wal_group_fill=4 wal_bytes_per_fsync=17999 wal_batches/s=1421 wal_bytes/s=18140344 avg_writers_per_fsync=2 avg_bytes_per_fsync=12769 wal_writer_in_flight=0 pending_waiters=0 pending_wal_bytes=0 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=75294 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=159626 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=11233 durable_lsn_after=11235 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=12ms waiter_broadcasts=159626 fsync_batch_reason=timeout last_waiters_released_in_fsync=64 durable_lsn=11235 durable_lsn_advance=15646 durable_lsn_advance_bytes=90793267 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=29750198272 background_io_paused=9295ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=2 doc_store_sync_segment_bytes=3937523 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=9634 doc_store_durable_lsn=9634 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=2 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=68 doc_store_sync_physical_ms=401 doc_store_sync_durable_lsn_advance=303604 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=1969586973 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=6 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=6 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=737280 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=12ms wait=0ms wal=1877MB wal_live_bytes=1969861100 wal_truncatable_bytes=274127 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=30652ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=1398567 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=392ms primary_blocked=30026ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=431ms index_log=1755MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49870 sec_apply/s=24807 text_apply/s=25062 idx_flush/s=0 idx_busy=82ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2114 wal_priority_yield_count=2013 background_io_yield_count=841 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=5 foreground_admission_delay_total_ms=988 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=12 foreground_pressure_transitions_total=890 foreground_pressure_state_elapsed_ms=84311 foreground_pressure_time_normal_window_ms=157555 foreground_pressure_time_warm_window_ms=2542 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=120s elapsed_hms=0:02:00 docs=2996736 txns=11708 writes/s=24832 avg_writes/s=24972 p50=42ms p99=95ms reason=healthy p99_root_cause_exclusive=healthy serialize=76ms dbms_insert=22097ms batch_coord=138116ms wal_commit=259851ms wal_fsync=20886ms wal_fsync_wait=2419953ms wal_fsync_actual=20886ms wal_fsync_avg_ms=0 wal_fsync_p95_ms=1 wal_fsync_p99_ms=8 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=539 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1815 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=12 wal_fsync_active_elapsed_ms=6 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=16697 fsyncs/s=16695 wal_fsync_q=157 wal_group_size=5 wal_group_fill=10 wal_bytes_per_fsync=42714 wal_batches/s=1455 wal_bytes/s=18405098 avg_writers_per_fsync=0 avg_bytes_per_fsync=10240 wal_writer_in_flight=78 pending_waiters=50 pending_wal_bytes=295589 wal_append_lock_wait_ms=3 wal_writer_sleep_ms=79283 pending_waiters_before_fsync=5 waiters_selected_for_batch=5 eligible_waiters=5 released_waiters=5 wakeup_batches=166870 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=11720 durable_lsn_after=11725 oldest_waiter_age=17ms avg_waiter_age=12ms max_waiter_age=12ms waiter_broadcasts=166870 fsync_batch_reason=timeout last_waiters_released_in_fsync=127 durable_lsn=11725 durable_lsn_advance=15645 durable_lsn_advance_bytes=854863759 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=30421286912 background_io_paused=9416ms background_pause_reason=wal_protected emergency_wal_only_count=3 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=2 doc_store_sync_segment_bytes=3937523 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=9634 doc_store_durable_lsn=9634 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=2 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=68 doc_store_sync_physical_ms=401 doc_store_sync_durable_lsn_advance=303604 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=2055929707 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=6 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=6 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=737280 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=18ms wait=0ms wal=1959MB wal_live_bytes=2056625980 wal_truncatable_bytes=696273 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=30652ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=1398567 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=392ms primary_blocked=30026ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=431ms index_log=1831MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49182 sec_apply/s=24717 text_apply/s=24465 idx_flush/s=0 idx_busy=85ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2306 wal_priority_yield_count=2155 background_io_yield_count=798 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=9 foreground_admission_delay_total_ms=997 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=11 foreground_pressure_transitions_total=901 foreground_pressure_state_elapsed_ms=89324 foreground_pressure_time_normal_window_ms=158156 foreground_pressure_time_warm_window_ms=2565 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=125s elapsed_hms=0:02:05 docs=3122432 txns=12200 writes/s=25139 avg_writes/s=24979 p50=42ms p99=83ms reason=healthy p99_root_cause_exclusive=healthy serialize=50ms dbms_insert=22049ms batch_coord=138803ms wal_commit=237094ms wal_fsync=17077ms wal_fsync_wait=112962ms wal_fsync_actual=17077ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=321 wal_fsync_blocked_by_doc_store_sync_ms=6 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1637 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=13 wal_fsync_active_elapsed_ms=4 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1397 fsyncs/s=1398 wal_fsync_q=67 wal_group_size=4 wal_group_fill=7 wal_bytes_per_fsync=27017 wal_batches/s=1398 wal_bytes/s=18170742 avg_writers_per_fsync=2 avg_bytes_per_fsync=13021 wal_writer_in_flight=24 pending_waiters=12 pending_wal_bytes=140865 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=72984 pending_waiters_before_fsync=4 waiters_selected_for_batch=4 eligible_waiters=4 released_waiters=4 wakeup_batches=173864 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=12212 durable_lsn_after=12214 oldest_waiter_age=18ms avg_waiter_age=13ms max_waiter_age=15ms waiter_broadcasts=173864 fsync_batch_reason=timeout last_waiters_released_in_fsync=65 durable_lsn=12214 durable_lsn_advance=15703 durable_lsn_advance_bytes=91072105 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=26MB/s wal_io=17MB/s primary_flush_io=9MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=29406265344 background_io_paused=9611ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=56ms doc_store_sync_segments_window=61 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=3 doc_store_sync_segment_bytes=3937523 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=12189 doc_store_durable_lsn=12189 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=3 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=83 doc_store_sync_physical_ms=457 doc_store_sync_durable_lsn_advance=327339 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=2142327632 checkpoint_sync=0ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=6 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=6 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=737280 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=321ms wait=0ms wal=2041MB wal_live_bytes=2142716684 wal_truncatable_bytes=389052 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=37MB flush_q=6 primary_q=6 secondary_q=0 primary_io=32663ms primary_pages=0 primary_flush_batch_bytes=51389267 primary_flush_batch_entries=78406 avg_flush_batch_bytes=1404430 coalesced_tables=0 coalesced_batches=0 small_flush_count=2 small_flush_reason=pressure active_primary_flush_workers=6 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=2011ms primary_slice_max=392ms primary_blocked=30026ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=37 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=431ms index_log=1908MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49820 sec_apply/s=25037 text_apply/s=24782 idx_flush/s=0 idx_busy=90ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2036 wal_priority_yield_count=1975 background_io_yield_count=1183 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=1 foreground_admission_delay_total_ms=998 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=2 foreground_pressure_transitions_total=903 foreground_pressure_state_elapsed_ms=94360 foreground_pressure_time_normal_window_ms=160102 foreground_pressure_time_warm_window_ms=522 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=130s elapsed_hms=0:02:10 docs=3248896 txns=12693 writes/s=25292 avg_writes/s=24991 p50=61ms p99=135ms reason=healthy p99_root_cause_exclusive=healthy serialize=215ms dbms_insert=31871ms batch_coord=127561ms wal_commit=256049ms wal_fsync=16840ms wal_fsync_wait=111904ms wal_fsync_actual=16840ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=522 wal_fsync_blocked_by_doc_store_sync_ms=117 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1660 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1360 fsyncs/s=1361 wal_fsync_q=42 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=15195 wal_batches/s=1367 wal_bytes/s=18260942 avg_writers_per_fsync=2 avg_bytes_per_fsync=13419 wal_writer_in_flight=16 pending_waiters=0 pending_wal_bytes=99194 wal_append_lock_wait_ms=4 wal_writer_sleep_ms=75032 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=180673 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=12705 durable_lsn_after=12706 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=16ms waiter_broadcasts=180673 fsync_batch_reason=timeout last_waiters_released_in_fsync=42 durable_lsn=12706 durable_lsn_advance=15745 durable_lsn_advance_bytes=91370250 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=58MB/s wal_io=17MB/s primary_flush_io=41MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=28449964032 background_io_paused=11541ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=323ms doc_store_sync_segments_window=258 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=4 doc_store_sync_segment_bytes=3672816 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=12455 doc_store_durable_lsn=12455 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=3 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=98 doc_store_sync_physical_ms=780 doc_store_sync_durable_lsn_advance=392539 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=2228838869 checkpoint_sync=1ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=1 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=1 checkpoint_sync_blocked_total_ms=7 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=1ms checkpoint_slice_avg_ms=1 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=770048 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=68335684 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=1383ms wait=0ms wal=2124MB wal_live_bytes=2229234720 wal_truncatable_bytes=395851 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=70030ms primary_pages=0 primary_flush_batch_bytes=217132959 primary_flush_batch_entries=331163 avg_flush_batch_bytes=1398612 coalesced_tables=0 coalesced_batches=0 small_flush_count=11 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=37367ms primary_slice_max=478ms primary_blocked=61798ms primary_flush_admission_wait_ms=31772 primary_flush_queued_age_ms=248 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=479ms index_log=1985MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50542 sec_apply/s=25524 text_apply/s=25018 idx_flush/s=0 idx_busy=134ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=1 checkpoint_yield_count=53 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=31772 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2224 wal_priority_yield_count=2120 background_io_yield_count=3036 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=12 foreground_admission_delay_total_ms=1010 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=44 foreground_pressure_transitions_total=947 foreground_pressure_state_elapsed_ms=53610 foreground_pressure_time_normal_window_ms=152208 foreground_pressure_time_warm_window_ms=8711 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=135s elapsed_hms=0:02:15 docs=3373312 txns=13178 writes/s=24883 avg_writes/s=24987 p50=40ms p99=67ms reason=healthy p99_root_cause_exclusive=healthy serialize=28ms dbms_insert=19960ms batch_coord=138609ms wal_commit=256614ms wal_fsync=17437ms wal_fsync_wait=115909ms wal_fsync_actual=17437ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=312 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1512 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=12 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1310 fsyncs/s=1309 wal_fsync_q=47 wal_group_size=4 wal_group_fill=7 wal_bytes_per_fsync=25828 wal_batches/s=1311 wal_bytes/s=17993007 avg_writers_per_fsync=2 avg_bytes_per_fsync=13740 wal_writer_in_flight=68 pending_waiters=5 pending_wal_bytes=432147 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=78611 pending_waiters_before_fsync=4 waiters_selected_for_batch=4 eligible_waiters=4 released_waiters=4 wakeup_batches=187218 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=13187 durable_lsn_after=13191 oldest_waiter_age=25ms avg_waiter_age=25ms max_waiter_age=21ms waiter_broadcasts=187218 fsync_batch_reason=timeout last_waiters_released_in_fsync=57 durable_lsn=13191 durable_lsn_advance=15512 durable_lsn_advance_bytes=89932826 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=27472691200 background_io_paused=9199ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=4 doc_store_sync_segment_bytes=3672816 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=12455 doc_store_durable_lsn=12455 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=3 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=98 doc_store_sync_physical_ms=780 doc_store_sync_durable_lsn_advance=392539 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=2314001785 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=7 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=770048 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=68335684 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=6ms wait=0ms wal=2205MB wal_live_bytes=2314326785 wal_truncatable_bytes=325000 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=70030ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=1398612 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=478ms primary_blocked=61798ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=479ms index_log=2061MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50791 sec_apply/s=25523 text_apply/s=25268 idx_flush/s=0 idx_busy=141ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1923 wal_priority_yield_count=1829 background_io_yield_count=954 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=1010 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=947 foreground_pressure_state_elapsed_ms=58618 foreground_pressure_time_normal_window_ms=160210 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=140s elapsed_hms=0:02:20 docs=3499520 txns=13671 writes/s=25241 avg_writes/s=24996 p50=39ms p99=72ms reason=healthy p99_root_cause_exclusive=healthy serialize=29ms dbms_insert=20086ms batch_coord=139586ms wal_commit=234093ms wal_fsync=17271ms wal_fsync_wait=108883ms wal_fsync_actual=17271ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=16 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=288 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1517 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=8 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1335 fsyncs/s=1336 wal_fsync_q=50 wal_group_size=2 wal_group_fill=4 wal_bytes_per_fsync=19575 wal_batches/s=1335 wal_bytes/s=18333579 avg_writers_per_fsync=2 avg_bytes_per_fsync=13720 wal_writer_in_flight=14 pending_waiters=1 pending_wal_bytes=36864 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=71167 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=193901 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=13682 durable_lsn_after=13683 oldest_waiter_age=27ms avg_waiter_age=27ms max_waiter_age=24ms waiter_broadcasts=193901 fsync_batch_reason=timeout last_waiters_released_in_fsync=46 durable_lsn=13683 durable_lsn_advance=15764 durable_lsn_advance_bytes=91694496 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=28030533632 background_io_paused=8967ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=4 doc_store_sync_segment_bytes=3672816 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=12455 doc_store_durable_lsn=12455 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=3 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=98 doc_store_sync_physical_ms=780 doc_store_sync_durable_lsn_advance=392539 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=2400682935 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=7 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=770048 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=68335684 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=5ms wait=0ms wal=2288MB wal_live_bytes=2401010117 wal_truncatable_bytes=327182 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=70030ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=1398612 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=478ms primary_blocked=61798ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=479ms index_log=2138MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=24 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49945 sec_apply/s=24847 text_apply/s=25098 idx_flush/s=0 idx_busy=112090ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=1 sec_segments=1 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=20 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=1 l1=0 l2=0 l0_oldest_age_ms=20 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=20ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1926 wal_priority_yield_count=1822 background_io_yield_count=999 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=2 foreground_admission_delay_total_ms=1012 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=2 foreground_pressure_transitions_total=949 foreground_pressure_state_elapsed_ms=63625 foreground_pressure_time_normal_window_ms=159649 foreground_pressure_time_warm_window_ms=507 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=145s elapsed_hms=0:02:25 docs=3624448 txns=14158 writes/s=24985 avg_writes/s=24996 p50=39ms p99=91ms reason=healthy p99_root_cause_exclusive=healthy serialize=42ms dbms_insert=19898ms batch_coord=140102ms wal_commit=237715ms wal_fsync=16967ms wal_fsync_wait=108309ms wal_fsync_actual=16967ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=289 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1552 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=22 wal_fsync_active_elapsed_ms=10 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1362 fsyncs/s=1358 wal_fsync_q=104 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=22286 wal_batches/s=1362 wal_bytes/s=18197845 avg_writers_per_fsync=2 avg_bytes_per_fsync=13362 wal_writer_in_flight=103 pending_waiters=50 pending_wal_bytes=226959 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=69935 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=200691 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=14169 durable_lsn_after=14170 oldest_waiter_age=34ms avg_waiter_age=34ms max_waiter_age=11ms waiter_broadcasts=200691 fsync_batch_reason=none last_waiters_released_in_fsync=56 durable_lsn=14170 durable_lsn_advance=15609 durable_lsn_advance_bytes=90728110 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=28550627328 background_io_paused=8895ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=4 doc_store_sync_segment_bytes=3672816 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=12455 doc_store_durable_lsn=12455 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=3 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=98 doc_store_sync_physical_ms=780 doc_store_sync_durable_lsn_advance=392539 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=2486577382 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=7 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=770048 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=68335684 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=6ms wait=0ms wal=2369MB wal_live_bytes=2486815661 wal_truncatable_bytes=238279 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=70030ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=1398612 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=478ms primary_blocked=61798ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=479ms index_log=2214MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=51200 sec_apply/s=25727 text_apply/s=25472 idx_flush/s=0 idx_busy=222104ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1957 wal_priority_yield_count=1851 background_io_yield_count=986 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=1012 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=949 foreground_pressure_state_elapsed_ms=68642 foreground_pressure_time_normal_window_ms=160237 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=150s elapsed_hms=0:02:30 docs=3749632 txns=14647 writes/s=25036 avg_writes/s=24997 p50=41ms p99=108ms reason=healthy p99_root_cause_exclusive=healthy serialize=50ms dbms_insert=21429ms batch_coord=137739ms wal_commit=247099ms wal_fsync=20502ms wal_fsync_wait=109954ms wal_fsync_actual=20502ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=567 wal_fsync_blocked_by_doc_store_sync_ms=6 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1742 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=16 wal_fsync_active_elapsed_ms=1 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1375 fsyncs/s=1379 wal_fsync_q=83 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=19788 wal_batches/s=1385 wal_bytes/s=18116492 avg_writers_per_fsync=2 avg_bytes_per_fsync=13168 wal_writer_in_flight=31 pending_waiters=0 pending_wal_bytes=151095 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=72431 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=207590 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=14655 durable_lsn_after=14658 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=22ms waiter_broadcasts=207590 fsync_batch_reason=timeout last_waiters_released_in_fsync=83 durable_lsn=14658 durable_lsn_advance=15613 durable_lsn_advance_bytes=90849186 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=25MB/s wal_io=17MB/s primary_flush_io=8MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=28890365952 background_io_paused=9342ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=54ms doc_store_sync_segments_window=48 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=4 doc_store_sync_segment_bytes=3672816 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=14638 doc_store_durable_lsn=14638 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=3 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=99 doc_store_sync_physical_ms=834 doc_store_sync_durable_lsn_advance=411280 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=2572656203 checkpoint_sync=0ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=7 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=770048 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=80516331 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=245ms wait=0ms wal=2452MB wal_live_bytes=2572745230 wal_truncatable_bytes=89027 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=31MB flush_q=6 primary_q=6 secondary_q=0 primary_io=71390ms primary_pages=0 primary_flush_batch_bytes=45096932 primary_flush_batch_entries=67674 avg_flush_batch_bytes=1411296 coalesced_tables=0 coalesced_batches=0 small_flush_count=1 small_flush_reason=pressure active_primary_flush_workers=6 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=1360ms primary_slice_max=478ms primary_blocked=61798ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=24 primary_flush_oldest_queued_age_ms=7 primary_flush_coalesce_wait_ms=2 primary_flush_defer_reason=coalesce_wait primary_doc_rotate_bytes=8388608 flush_defer=1 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=479ms index_log=2291MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49671 sec_apply/s=24963 text_apply/s=24708 idx_flush/s=0 idx_busy=222112ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=5 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2202 wal_priority_yield_count=2068 background_io_yield_count=1198 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=23 foreground_admission_delay_total_ms=1035 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=46 foreground_pressure_transitions_total=995 foreground_pressure_state_elapsed_ms=51485 foreground_pressure_time_normal_window_ms=150020 foreground_pressure_time_warm_window_ms=9633 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=155s elapsed_hms=0:02:35 docs=3874816 txns=15137 writes/s=25036 avg_writes/s=24998 p50=53ms p99=147ms reason=healthy p99_root_cause_exclusive=healthy serialize=147ms dbms_insert=27507ms batch_coord=132866ms wal_commit=233258ms wal_fsync=15388ms wal_fsync_wait=108717ms wal_fsync_actual=15388ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=274 wal_fsync_blocked_by_doc_store_sync_ms=130 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1624 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=7 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1381 fsyncs/s=1380 wal_fsync_q=65 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=15263 wal_batches/s=1393 wal_bytes/s=18511364 avg_writers_per_fsync=2 avg_bytes_per_fsync=13407 wal_writer_in_flight=42 pending_waiters=10 pending_wal_bytes=145367 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=74829 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=214490 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=15151 durable_lsn_after=15153 oldest_waiter_age=18ms avg_waiter_age=17ms max_waiter_age=12ms waiter_broadcasts=214490 fsync_batch_reason=timeout last_waiters_released_in_fsync=61 durable_lsn=15153 durable_lsn_advance=15810 durable_lsn_advance_bytes=92510920 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=58MB/s wal_io=17MB/s primary_flush_io=41MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=28923920384 background_io_paused=11249ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=337ms doc_store_sync_segments_window=258 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=35 doc_store_sync_segment_bytes=3476442 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=15061 doc_store_durable_lsn=15061 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=7 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=169 doc_store_sync_physical_ms=1171 doc_store_sync_durable_lsn_advance=473218 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=2659663729 checkpoint_sync=1ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=1 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=1 checkpoint_sync_blocked_total_ms=8 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=1ms checkpoint_slice_avg_ms=1 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=753664 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=81260371 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=1142ms wait=0ms wal=2535MB wal_live_bytes=2660111040 wal_truncatable_bytes=447311 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=11MB flush_q=4 primary_q=4 secondary_q=0 primary_io=108960ms primary_pages=0 primary_flush_batch_bytes=218182062 primary_flush_batch_entries=327280 avg_flush_batch_bytes=1428355 coalesced_tables=0 coalesced_batches=0 small_flush_count=9 small_flush_reason=pressure active_primary_flush_workers=4 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=37570ms primary_slice_max=535ms primary_blocked=69221ms primary_flush_admission_wait_ms=7423 primary_flush_queued_age_ms=249 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=5 primary_flush_defer_reason=coalesce_wait primary_doc_rotate_bytes=8388608 flush_defer=2 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=479ms index_log=2368MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=51404 sec_apply/s=25830 text_apply/s=25574 idx_flush/s=0 idx_busy=222176ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=2ms rejects=0 checkpoint_sync_blocked_estimate_ms=1 checkpoint_yield_count=29 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=7423 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2142 wal_priority_yield_count=2086 background_io_yield_count=2939 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=27 foreground_admission_delay_total_ms=1062 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=50 foreground_pressure_transitions_total=1045 foreground_pressure_state_elapsed_ms=28146 foreground_pressure_time_normal_window_ms=151771 foreground_pressure_time_warm_window_ms=9974 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=160s elapsed_hms=0:02:40 docs=4000768 txns=15628 writes/s=25190 avg_writes/s=25004 p50=45ms p99=189ms reason=healthy p99_root_cause_exclusive=healthy serialize=82ms dbms_insert=25848ms batch_coord=133652ms wal_commit=257790ms wal_fsync=18615ms wal_fsync_wait=1337671ms wal_fsync_actual=18615ms wal_fsync_avg_ms=0 wal_fsync_p95_ms=2 wal_fsync_p99_ms=8 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=406 wal_fsync_blocked_by_doc_store_sync_ms=2 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1638 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=10 wal_fsync_active_elapsed_ms=6 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=13577 fsyncs/s=13578 wal_fsync_q=94 wal_group_size=3 wal_group_fill=6 wal_bytes_per_fsync=25621 wal_batches/s=1350 wal_bytes/s=18113974 avg_writers_per_fsync=0 avg_bytes_per_fsync=11113 wal_writer_in_flight=9 pending_waiters=0 pending_wal_bytes=54269 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=76671 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=221163 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=15639 durable_lsn_after=15642 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=17ms waiter_broadcasts=221163 fsync_batch_reason=timeout last_waiters_released_in_fsync=93 durable_lsn=15642 durable_lsn_advance=15656 durable_lsn_advance_bytes=754557520 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=18MB/s wal_io=17MB/s primary_flush_io=1MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=27963424768 background_io_paused=9103ms background_pause_reason=wal_protected emergency_wal_only_count=1 doc_store_sync=6ms doc_store_sync_segments_window=9 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=35 doc_store_sync_segment_bytes=3416544 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=15061 doc_store_durable_lsn=15061 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=7 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=170 doc_store_sync_physical_ms=1177 doc_store_sync_durable_lsn_advance=473218 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=2745176175 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=8 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=753664 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=81260371 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=70ms wait=0ms wal=2617MB wal_live_bytes=2745893031 wal_truncatable_bytes=716856 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=110198ms primary_pages=0 primary_flush_batch_bytes=5250339 primary_flush_batch_entries=7869 avg_flush_batch_bytes=1418315 coalesced_tables=0 coalesced_batches=0 small_flush_count=3 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=1238ms primary_slice_max=535ms primary_blocked=69221ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=12 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=coalesce_wait primary_doc_rotate_bytes=8388608 flush_defer=2 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=479ms index_log=2444MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50381 sec_apply/s=25318 text_apply/s=25062 idx_flush/s=0 idx_busy=222184ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2090 wal_priority_yield_count=1938 background_io_yield_count=1164 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=none foreground_admission_delay_window_ms=45 foreground_admission_delay_total_ms=1107 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=25 foreground_pressure_transitions_total=1070 foreground_pressure_state_elapsed_ms=33155 foreground_pressure_time_normal_window_ms=151024 foreground_pressure_time_warm_window_ms=9452 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=165s elapsed_hms=0:02:45 docs=4125952 txns=16118 writes/s=25036 avg_writes/s=25005 p50=41ms p99=100ms reason=healthy p99_root_cause_exclusive=healthy serialize=59ms dbms_insert=22057ms batch_coord=136667ms wal_commit=240624ms wal_fsync=17530ms wal_fsync_wait=119784ms wal_fsync_actual=17530ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=180 wal_fsync_blocked_by_doc_store_sync_ms=11 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1640 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1470 fsyncs/s=1470 wal_fsync_q=96 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=24974 wal_batches/s=1472 wal_bytes/s=18192284 avg_writers_per_fsync=2 avg_bytes_per_fsync=12370 wal_writer_in_flight=0 pending_waiters=0 pending_wal_bytes=0 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=78357 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=228516 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=16127 durable_lsn_after=16130 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=12ms waiter_broadcasts=228516 fsync_batch_reason=timeout last_waiters_released_in_fsync=96 durable_lsn=16130 durable_lsn_advance=15615 durable_lsn_advance_bytes=90961421 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=25MB/s wal_io=17MB/s primary_flush_io=8MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=30836523008 background_io_paused=9431ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=54ms doc_store_sync_segments_window=48 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=35 doc_store_sync_segment_bytes=3414700 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=16087 doc_store_durable_lsn=16087 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=8 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=181 doc_store_sync_physical_ms=1231 doc_store_sync_durable_lsn_advance=479680 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=2831470883 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=8 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=753664 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=81260371 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=152ms wait=0ms wal=2699MB wal_live_bytes=2831883454 wal_truncatable_bytes=412571 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=111878ms primary_pages=0 primary_flush_batch_bytes=41955364 primary_flush_batch_entries=57952 avg_flush_batch_bytes=1417504 coalesced_tables=0 coalesced_batches=0 small_flush_count=1 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=1680ms primary_slice_max=535ms primary_blocked=69221ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=30 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=coalesce_wait primary_doc_rotate_bytes=8388608 flush_defer=2 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=479ms index_log=2521MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49358 sec_apply/s=24807 text_apply/s=24551 idx_flush/s=0 idx_busy=222185ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=2ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2051 wal_priority_yield_count=1963 background_io_yield_count=1245 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=7 foreground_admission_delay_total_ms=1114 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=32 foreground_pressure_transitions_total=1102 foreground_pressure_state_elapsed_ms=38163 foreground_pressure_time_normal_window_ms=155667 foreground_pressure_time_warm_window_ms=4005 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=170s elapsed_hms=0:02:50 docs=4241664 txns=16571 writes/s=23142 avg_writes/s=24950 p50=127ms p99=692ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=203ms dbms_insert=77404ms batch_coord=84324ms wal_commit=227948ms wal_fsync=16442ms wal_fsync_wait=33516117ms wal_fsync_actual=16442ms wal_fsync_avg_ms=0 wal_fsync_p95_ms=1 wal_fsync_p99_ms=4 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=1 wal_fsync_submit_to_complete_ms=620 wal_fsync_blocked_by_doc_store_sync_ms=144 wal_fsync_blocked_by_checkpoint_ms=2 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1689 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=10 wal_fsync_active_elapsed_ms=2 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=44674 fsyncs/s=44673 wal_fsync_q=79 wal_group_size=4 wal_group_fill=7 wal_bytes_per_fsync=24151 wal_batches/s=1367 wal_bytes/s=17288738 avg_writers_per_fsync=0 avg_bytes_per_fsync=9043 wal_writer_in_flight=48 pending_waiters=13 pending_wal_bytes=227196 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=72046 pending_waiters_before_fsync=4 waiters_selected_for_batch=4 eligible_waiters=5 released_waiters=5 wakeup_batches=235346 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=16593 durable_lsn_after=16597 oldest_waiter_age=21ms avg_waiter_age=14ms max_waiter_age=18ms waiter_broadcasts=235346 fsync_batch_reason=timeout last_waiters_released_in_fsync=78 durable_lsn=16597 durable_lsn_advance=14931 durable_lsn_advance_bytes=2020052749 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=68MB/s wal_io=16MB/s primary_flush_io=52MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=28718399488 background_io_paused=12127ms background_pause_reason=wal_protected emergency_wal_only_count=4 doc_store_sync=425ms doc_store_sync_segments_window=307 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=4 doc_store_sync_segment_bytes=4105074 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=6 doc_store_sync_pending_segments=7 doc_store_sync_pending_bytes=10149326 doc_store_sync_pending_lsn=16401 doc_store_durable_lsn=16401 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=103 doc_store_sync_debt_bytes=10149326 doc_store_sync_jobs_merged=9 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=224 doc_store_sync_physical_ms=1656 doc_store_sync_durable_lsn_advance=524504 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=2912978702 checkpoint_sync=2ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=2 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=2 checkpoint_sync_blocked_total_ms=10 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=2ms checkpoint_slice_avg_ms=2 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=24576 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=90257717 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=38948ms wait=0ms wal=2777MB wal_live_bytes=2913704451 wal_truncatable_bytes=725749 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=771MB flush_q=174 primary_q=64 secondary_q=0 primary_io=146666ms primary_pages=516 primary_flush_batch_bytes=277864915 primary_flush_batch_entries=554423 avg_flush_batch_bytes=1428009 coalesced_tables=0 coalesced_batches=0 small_flush_count=6 small_flush_reason=pressure active_primary_flush_workers=61 per_table_flush_inflight=64 flush_worker_io_wait=3ms flush_worker_cpu=34785ms primary_slice_max=786ms primary_blocked=157958ms primary_flush_admission_wait_ms=88737 primary_flush_queued_age_ms=25915 primary_flush_oldest_queued_age_ms=2409 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=5782 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=2183ms index_log=2592MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=55275 sec_apply/s=27765 text_apply/s=27510 idx_flush/s=0 idx_busy=222260ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=2 checkpoint_yield_count=10 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=88737 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2276 wal_priority_yield_count=2174 background_io_yield_count=4683 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=10733 foreground_admission_delay_total_ms=11847 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=113 foreground_pressure_transitions_total=1215 foreground_pressure_state_elapsed_ms=1187 foreground_pressure_time_normal_window_ms=96669 foreground_pressure_time_warm_window_ms=27031 foreground_pressure_time_constrained_window_ms=34258 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=3084
writes_test: elapsed=175s elapsed_hms=0:02:55 docs=4364544 txns=17051 writes/s=24576 avg_writes/s=24940 p50=209ms p99=445ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=173ms dbms_insert=103104ms batch_coord=55632ms wal_commit=219505ms wal_fsync=14651ms wal_fsync_wait=104667ms wal_fsync_actual=14651ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=379 wal_fsync_blocked_by_doc_store_sync_ms=89 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1699 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1344 fsyncs/s=1345 wal_fsync_q=74 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=19371 wal_batches/s=1345 wal_bytes/s=17784261 avg_writers_per_fsync=2 avg_bytes_per_fsync=13230 wal_writer_in_flight=47 pending_waiters=0 pending_wal_bytes=245957 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=70528 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=242072 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=17076 durable_lsn_after=17078 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=23ms waiter_broadcasts=242072 fsync_batch_reason=timeout last_waiters_released_in_fsync=74 durable_lsn=17078 durable_lsn_advance=15400 durable_lsn_advance_bytes=88987658 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=55MB/s wal_io=16MB/s primary_flush_io=39MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=28143779840 background_io_paused=10624ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=292ms doc_store_sync_segments_window=233 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=4 doc_store_sync_segment_bytes=4098852 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=16401 doc_store_durable_lsn=16401 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=9 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=235 doc_store_sync_physical_ms=1948 doc_store_sync_durable_lsn_advance=524590 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=2997980502 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=10 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=24576 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=90257717 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=38986ms wait=0ms wal=2857MB wal_live_bytes=2997990397 wal_truncatable_bytes=9895 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=599MB flush_q=177 primary_q=64 secondary_q=0 primary_io=185876ms primary_pages=516 primary_flush_batch_bytes=207278996 primary_flush_batch_entries=370085 avg_flush_batch_bytes=1469527 coalesced_tables=0 coalesced_batches=0 small_flush_count=1 small_flush_reason=pressure active_primary_flush_workers=63 per_table_flush_inflight=64 flush_worker_io_wait=0ms flush_worker_cpu=39210ms primary_slice_max=786ms primary_blocked=379289ms primary_flush_admission_wait_ms=221331 primary_flush_queued_age_ms=135543 primary_flush_oldest_queued_age_ms=7429 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=18861 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=5779ms index_log=2667MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=51353 sec_apply/s=25804 text_apply/s=25548 idx_flush/s=0 idx_busy=222365ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=221331 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2164 wal_priority_yield_count=2098 background_io_yield_count=2899 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=38398 foreground_admission_delay_total_ms=50245 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=12 foreground_pressure_transitions_total=1227 foreground_pressure_state_elapsed_ms=5426 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=1828 foreground_pressure_time_recovery_window_ms=2534 foreground_pressure_time_emergency_window_ms=154823
writes_test: elapsed=180s elapsed_hms=0:03:00 docs=4491264 txns=17546 writes/s=25344 avg_writes/s=24951 p50=115ms p99=242ms reason=healthy p99_root_cause_exclusive=healthy serialize=236ms dbms_insert=60138ms batch_coord=99200ms wal_commit=228895ms wal_fsync=13156ms wal_fsync_wait=95412ms wal_fsync_actual=13156ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=389 wal_fsync_blocked_by_doc_store_sync_ms=83 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1486 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=21 wal_fsync_active_elapsed_ms=1 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1191 fsyncs/s=1191 wal_fsync_q=32 wal_group_size=1 wal_group_fill=2 wal_bytes_per_fsync=9592 wal_batches/s=1191 wal_bytes/s=18004060 avg_writers_per_fsync=2 avg_bytes_per_fsync=15110 wal_writer_in_flight=33 pending_waiters=1 pending_wal_bytes=155106 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=63576 pending_waiters_before_fsync=1 waiters_selected_for_batch=1 eligible_waiters=1 released_waiters=1 wakeup_batches=248029 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=17560 durable_lsn_after=17561 oldest_waiter_age=13ms avg_waiter_age=13ms max_waiter_age=15ms waiter_broadcasts=248029 fsync_batch_reason=timeout last_waiters_released_in_fsync=32 durable_lsn=17561 durable_lsn_advance=15460 durable_lsn_advance_bytes=90012755 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=41MB/s wal_io=17MB/s primary_flush_io=24MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=25014829056 background_io_paused=9625ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=192ms doc_store_sync_segments_window=139 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=8 doc_store_sync_segment_bytes=4094906 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=17223 doc_store_durable_lsn=17223 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=9 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=244 doc_store_sync_physical_ms=2140 doc_store_sync_durable_lsn_advance=528000 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=3083117705 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=10 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=24576 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=90257717 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=39289ms wait=0ms wal=2938MB wal_live_bytes=3083165903 wal_truncatable_bytes=48198 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=587MB flush_q=204 primary_q=64 secondary_q=0 primary_io=224247ms primary_pages=516 primary_flush_batch_bytes=126291960 primary_flush_batch_entries=207127 avg_flush_batch_bytes=1482016 coalesced_tables=0 coalesced_batches=0 small_flush_count=1 small_flush_reason=pressure active_primary_flush_workers=64 per_table_flush_inflight=64 flush_worker_io_wait=0ms flush_worker_cpu=38371ms primary_slice_max=923ms primary_blocked=607676ms primary_flush_admission_wait_ms=228387 primary_flush_queued_age_ms=186413 primary_flush_oldest_queued_age_ms=12435 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=31686 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=10293ms index_log=2745MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49671 sec_apply/s=24708 text_apply/s=24963 idx_flush/s=0 idx_busy=222511ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=2ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=228387 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1909 wal_priority_yield_count=1851 background_io_yield_count=1947 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=38660 foreground_admission_delay_total_ms=88905 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=2 foreground_pressure_transitions_total=1229 foreground_pressure_state_elapsed_ms=10436 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=102 foreground_pressure_time_emergency_window_ms=160738
writes_test: elapsed=185s elapsed_hms=0:03:05 docs=4617216 txns=18038 writes/s=25190 avg_writes/s=24957 p50=113ms p99=252ms reason=healthy p99_root_cause_exclusive=healthy serialize=185ms dbms_insert=56701ms batch_coord=104241ms wal_commit=233315ms wal_fsync=15220ms wal_fsync_wait=99551ms wal_fsync_actual=15220ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=476 wal_fsync_blocked_by_doc_store_sync_ms=110 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1631 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=12 wal_fsync_active_elapsed_ms=1 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1253 fsyncs/s=1253 wal_fsync_q=124 wal_group_size=5 wal_group_fill=9 wal_bytes_per_fsync=35281 wal_batches/s=1256 wal_bytes/s=18668736 avg_writers_per_fsync=2 avg_bytes_per_fsync=14893 wal_writer_in_flight=63 pending_waiters=3 pending_wal_bytes=307111 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=65149 pending_waiters_before_fsync=5 waiters_selected_for_batch=5 eligible_waiters=5 released_waiters=5 wakeup_batches=254296 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=18055 durable_lsn_after=18059 oldest_waiter_age=10ms avg_waiter_age=6ms max_waiter_age=15ms waiter_broadcasts=254296 fsync_batch_reason=timeout last_waiters_released_in_fsync=122 durable_lsn=18059 durable_lsn_advance=15964 durable_lsn_advance_bytes=93338936 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=45MB/s wal_io=17MB/s primary_flush_io=28MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=26340229120 background_io_paused=9874ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=237ms doc_store_sync_segments_window=156 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=5 doc_store_sync_segment_bytes=3904315 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=17821 doc_store_durable_lsn=17821 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=9 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=250 doc_store_sync_physical_ms=2377 doc_store_sync_durable_lsn_advance=545470 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=3170759138 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=10 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=24576 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=90257717 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=40362ms wait=0ms wal=3022MB wal_live_bytes=3171317659 wal_truncatable_bytes=558521 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=549MB flush_q=228 primary_q=64 secondary_q=0 primary_io=263314ms primary_pages=516 primary_flush_batch_bytes=149836540 primary_flush_batch_entries=228818 avg_flush_batch_bytes=1508942 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=64 per_table_flush_inflight=64 flush_worker_io_wait=0ms flush_worker_cpu=39067ms primary_slice_max=931ms primary_blocked=893350ms primary_flush_admission_wait_ms=285674 primary_flush_queued_age_ms=260425 primary_flush_oldest_queued_age_ms=17460 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=44589 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=14323ms index_log=2822MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49515 sec_apply/s=24757 text_apply/s=24757 idx_flush/s=0 idx_busy=222644ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=2ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=285674 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2135 wal_priority_yield_count=2015 background_io_yield_count=2185 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=40020 foreground_admission_delay_total_ms=128925 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=1229 foreground_pressure_state_elapsed_ms=15456 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=161646
writes_test: elapsed=190s elapsed_hms=0:03:10 docs=4742400 txns=18526 writes/s=25036 avg_writes/s=24960 p50=108ms p99=202ms reason=healthy p99_root_cause_exclusive=healthy serialize=193ms dbms_insert=52572ms batch_coord=105937ms wal_commit=217652ms wal_fsync=14695ms wal_fsync_wait=95825ms wal_fsync_actual=14695ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=409 wal_fsync_blocked_by_doc_store_sync_ms=83 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1608 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1271 fsyncs/s=1271 wal_fsync_q=75 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=24518 wal_batches/s=1271 wal_bytes/s=18247794 avg_writers_per_fsync=2 avg_bytes_per_fsync=14352 wal_writer_in_flight=0 pending_waiters=0 pending_wal_bytes=0 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=63352 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=260654 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=18547 durable_lsn_after=18550 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=25ms waiter_broadcasts=260654 fsync_batch_reason=timeout last_waiters_released_in_fsync=75 durable_lsn=18550 durable_lsn_advance=15674 durable_lsn_advance_bytes=91251266 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=51MB/s wal_io=17MB/s primary_flush_io=34MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=26583498752 background_io_paused=10218ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=279ms doc_store_sync_segments_window=199 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=3 doc_store_sync_segment_bytes=3765223 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=18372 doc_store_durable_lsn=18372 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=9 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=254 doc_store_sync_physical_ms=2656 doc_store_sync_durable_lsn_advance=576901 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=3257479975 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=10 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=24576 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=90257717 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=40136ms wait=0ms wal=3105MB wal_live_bytes=3257632609 wal_truncatable_bytes=152634 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=525MB flush_q=258 primary_q=63 secondary_q=0 primary_io=304721ms primary_pages=516 primary_flush_batch_bytes=183276515 primary_flush_batch_entries=278569 avg_flush_batch_bytes=1519727 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=63 per_table_flush_inflight=64 flush_worker_io_wait=0ms flush_worker_cpu=41407ms primary_slice_max=931ms primary_blocked=1202276ms primary_flush_admission_wait_ms=308926 primary_flush_queued_age_ms=387541 primary_flush_oldest_queued_age_ms=22477 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=57456 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=15969ms index_log=2898MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=51302 sec_apply/s=25778 text_apply/s=25523 idx_flush/s=0 idx_busy=222774ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=308926 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2051 wal_priority_yield_count=1981 background_io_yield_count=2504 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=39080 foreground_admission_delay_total_ms=168005 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=1229 foreground_pressure_state_elapsed_ms=20475 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=159863
writes_test: elapsed=195s elapsed_hms=0:03:15 docs=4868608 txns=19020 writes/s=25241 avg_writes/s=24967 p50=113ms p99=422ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=177ms dbms_insert=65755ms batch_coord=92904ms wal_commit=220392ms wal_fsync=15721ms wal_fsync_wait=108796ms wal_fsync_actual=15721ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=521 wal_fsync_blocked_by_doc_store_sync_ms=66 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1690 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1402 fsyncs/s=1402 wal_fsync_q=32 wal_group_size=1 wal_group_fill=2 wal_bytes_per_fsync=11499 wal_batches/s=1403 wal_bytes/s=17987602 avg_writers_per_fsync=2 avg_bytes_per_fsync=12828 wal_writer_in_flight=0 pending_waiters=0 pending_wal_bytes=0 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=72765 pending_waiters_before_fsync=1 waiters_selected_for_batch=1 eligible_waiters=1 released_waiters=1 wakeup_batches=267665 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=19031 durable_lsn_after=19032 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=24ms waiter_broadcasts=267665 fsync_batch_reason=timeout last_waiters_released_in_fsync=32 durable_lsn=19032 durable_lsn_advance=15420 durable_lsn_advance_bytes=89938010 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=44MB/s wal_io=17MB/s primary_flush_io=27MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=29406265344 background_io_paused=9918ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=234ms doc_store_sync_segments_window=161 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=4 doc_store_sync_segment_bytes=3638758 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=1 doc_store_sync_pending_segments=2 doc_store_sync_pending_bytes=2729824 doc_store_sync_pending_lsn=18372 doc_store_durable_lsn=18372 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=2 doc_store_sync_debt_bytes=2729824 doc_store_sync_jobs_merged=9 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=260 doc_store_sync_physical_ms=2890 doc_store_sync_durable_lsn_advance=581734 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=3342508512 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=10 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=24576 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=90257717 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=38738ms wait=0ms wal=3186MB wal_live_bytes=3342643082 wal_truncatable_bytes=134570 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=384MB flush_q=251 primary_q=64 secondary_q=0 primary_io=344590ms primary_pages=516 primary_flush_batch_bytes=145841414 primary_flush_batch_entries=223180 avg_flush_batch_bytes=1522298 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=64 per_table_flush_inflight=64 flush_worker_io_wait=0ms flush_worker_cpu=39869ms primary_slice_max=960ms primary_blocked=1463281ms primary_flush_admission_wait_ms=261005 primary_flush_queued_age_ms=432025 primary_flush_oldest_queued_age_ms=27492 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=70206 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=24828ms index_log=2976MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50893 sec_apply/s=25574 text_apply/s=25318 idx_flush/s=0 idx_busy=222913ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=2ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=261005 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2128 wal_priority_yield_count=2069 background_io_yield_count=1918 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=38560 foreground_admission_delay_total_ms=206565 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=1229 foreground_pressure_state_elapsed_ms=25487 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=160931
writes_test: elapsed=200s elapsed_hms=0:03:20 docs=4992512 txns=19504 writes/s=24780 avg_writes/s=24962 p50=127ms p99=336ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=167ms dbms_insert=68635ms batch_coord=92444ms wal_commit=233498ms wal_fsync=17249ms wal_fsync_wait=113917ms wal_fsync_actual=17249ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=32 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=641 wal_fsync_blocked_by_doc_store_sync_ms=65 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1866 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=11 wal_fsync_active_elapsed_ms=2 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1426 fsyncs/s=1424 wal_fsync_q=111 wal_group_size=4 wal_group_fill=7 wal_bytes_per_fsync=32971 wal_batches/s=1442 wal_bytes/s=18246702 avg_writers_per_fsync=2 avg_bytes_per_fsync=12786 wal_writer_in_flight=45 pending_waiters=24 pending_wal_bytes=155078 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=73198 pending_waiters_before_fsync=4 waiters_selected_for_batch=4 eligible_waiters=4 released_waiters=4 wakeup_batches=274789 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=19519 durable_lsn_after=19521 oldest_waiter_age=18ms avg_waiter_age=12ms max_waiter_age=13ms waiter_broadcasts=274789 fsync_batch_reason=timeout last_waiters_released_in_fsync=108 durable_lsn=19521 durable_lsn_advance=15690 durable_lsn_advance_bytes=91092554 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=39MB/s wal_io=17MB/s primary_flush_io=22MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=29926359040 background_io_paused=10070ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=198ms doc_store_sync_segments_window=139 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=7 doc_store_sync_segment_bytes=4090154 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=18372 doc_store_durable_lsn=18372 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=9 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=261 doc_store_sync_physical_ms=3088 doc_store_sync_durable_lsn_advance=581734 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=3428217477 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=10 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=24576 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=90257717 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=39164ms wait=0ms wal=3268MB wal_live_bytes=3428887501 wal_truncatable_bytes=670024 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=273MB flush_q=227 primary_q=64 secondary_q=0 primary_io=382982ms primary_pages=516 primary_flush_batch_bytes=117869290 primary_flush_batch_entries=181664 avg_flush_batch_bytes=1512653 coalesced_tables=0 coalesced_batches=0 small_flush_count=1 small_flush_reason=pressure active_primary_flush_workers=64 per_table_flush_inflight=64 flush_worker_io_wait=0ms flush_worker_cpu=38392ms primary_slice_max=960ms primary_blocked=1720250ms primary_flush_admission_wait_ms=256969 primary_flush_queued_age_ms=623012 primary_flush_oldest_queued_age_ms=32495 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=82986 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=24828ms index_log=3052MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=47360 sec_apply/s=23552 text_apply/s=23808 idx_flush/s=0 idx_busy=223059ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=256969 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2390 wal_priority_yield_count=2259 background_io_yield_count=1670 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=39320 foreground_admission_delay_total_ms=245885 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=1229 foreground_pressure_state_elapsed_ms=30499 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=160484
writes_test: elapsed=205s elapsed_hms=0:03:25 docs=5118720 txns=19997 writes/s=25241 avg_writes/s=24969 p50=114ms p99=264ms reason=healthy p99_root_cause_exclusive=healthy serialize=205ms dbms_insert=60319ms batch_coord=99181ms wal_commit=224936ms wal_fsync=15528ms wal_fsync_wait=110329ms wal_fsync_actual=15528ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=463 wal_fsync_blocked_by_doc_store_sync_ms=54 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1702 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=14 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1408 fsyncs/s=1408 wal_fsync_q=111 wal_group_size=4 wal_group_fill=8 wal_bytes_per_fsync=33883 wal_batches/s=1409 wal_bytes/s=18492305 avg_writers_per_fsync=2 avg_bytes_per_fsync=13111 wal_writer_in_flight=91 pending_waiters=44 pending_wal_bytes=551007 wal_append_lock_wait_ms=3 wal_writer_sleep_ms=73626 pending_waiters_before_fsync=4 waiters_selected_for_batch=4 eligible_waiters=4 released_waiters=4 wakeup_batches=281830 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=20013 durable_lsn_after=20016 oldest_waiter_age=16ms avg_waiter_age=11ms max_waiter_age=20ms waiter_broadcasts=281830 fsync_batch_reason=timeout last_waiters_released_in_fsync=85 durable_lsn=20016 durable_lsn_advance=15797 durable_lsn_advance_bytes=92318296 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=38MB/s wal_io=17MB/s primary_flush_io=21MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=29565648896 background_io_paused=9897ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=176ms doc_store_sync_segments_window=142 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=4 doc_store_sync_segment_bytes=3905963 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=18372 doc_store_durable_lsn=18372 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=9 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=273 doc_store_sync_physical_ms=3264 doc_store_sync_durable_lsn_advance=581734 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=3515442657 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=10 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=24576 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=90257717 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=39850ms wait=0ms wal=3350MB wal_live_bytes=3516192108 wal_truncatable_bytes=749451 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=160MB flush_q=182 primary_q=62 secondary_q=0 primary_io=424116ms primary_pages=516 primary_flush_batch_bytes=113686689 primary_flush_batch_entries=175145 avg_flush_batch_bytes=1487467 coalesced_tables=0 coalesced_batches=0 small_flush_count=6 small_flush_reason=pressure active_primary_flush_workers=61 per_table_flush_inflight=62 flush_worker_io_wait=0ms flush_worker_cpu=41134ms primary_slice_max=969ms primary_blocked=2044794ms primary_flush_admission_wait_ms=324544 primary_flush_queued_age_ms=848045 primary_flush_oldest_queued_age_ms=37522 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=95419 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=33987ms index_log=3129MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=55165 sec_apply/s=27455 text_apply/s=27710 idx_flush/s=0 idx_busy=223205ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=2ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=324544 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2149 wal_priority_yield_count=2082 background_io_yield_count=1519 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=39560 foreground_admission_delay_total_ms=285445 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=1229 foreground_pressure_state_elapsed_ms=35521 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=160716
writes_test: elapsed=210s elapsed_hms=0:03:30 docs=5241856 txns=20477 writes/s=24627 avg_writes/s=24961 p50=125ms p99=307ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=180ms dbms_insert=63397ms batch_coord=95115ms wal_commit=223026ms wal_fsync=14595ms wal_fsync_wait=105803ms wal_fsync_actual=14595ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=429 wal_fsync_blocked_by_doc_store_sync_ms=28 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1689 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=6 wal_fsync_active_elapsed_ms=1 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1358 fsyncs/s=1360 wal_fsync_q=53 wal_group_size=2 wal_group_fill=4 wal_bytes_per_fsync=17240 wal_batches/s=1359 wal_bytes/s=18226261 avg_writers_per_fsync=2 avg_bytes_per_fsync=13437 wal_writer_in_flight=5 pending_waiters=3 pending_wal_bytes=15598 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=70088 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=288632 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=20502 durable_lsn_after=20504 oldest_waiter_age=16ms avg_waiter_age=16ms max_waiter_age=23ms waiter_broadcasts=288632 fsync_batch_reason=timeout last_waiters_released_in_fsync=51 durable_lsn=20504 durable_lsn_advance=15603 durable_lsn_advance_bytes=91399900 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=38MB/s wal_io=17MB/s primary_flush_io=21MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=28470935552 background_io_paused=10760ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=172ms doc_store_sync_segments_window=174 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=2 doc_store_sync_segment_bytes=3076014 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=18372 doc_store_durable_lsn=18372 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=9 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=279 doc_store_sync_physical_ms=3436 doc_store_sync_durable_lsn_advance=581734 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=3602188218 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=10 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=6 checkpoint_bytes_per_slice=24576 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=90257717 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=39203ms wait=0ms wal=3434MB wal_live_bytes=3602466641 wal_truncatable_bytes=278423 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=index_backlog backlog=53MB flush_q=84 primary_q=44 secondary_q=0 primary_io=463849ms primary_pages=873 primary_flush_batch_bytes=110544737 primary_flush_batch_entries=299263 avg_flush_batch_bytes=1425869 coalesced_tables=0 coalesced_batches=0 small_flush_count=31 small_flush_reason=pressure active_primary_flush_workers=37 per_table_flush_inflight=42 flush_worker_io_wait=0ms flush_worker_cpu=39733ms primary_slice_max=971ms primary_blocked=2355579ms primary_flush_admission_wait_ms=310785 primary_flush_queued_age_ms=1751658 primary_flush_oldest_queued_age_ms=42353 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=104961 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=40232ms index_log=3205MB pending_idx=256 pending_sec_apply=256 pending_sec_flush=23 oldest_idx_lsn=10484481 idx_lag_lsn=256 idx_lag=7ms sec_lag=7ms text_lag=0ms idx_apply/s=50005 sec_apply/s=25127 text_apply/s=24878 idx_flush/s=0 idx_busy=223387ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=3ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=310785 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2090 wal_priority_yield_count=2030 background_io_yield_count=2659 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=38880 foreground_admission_delay_total_ms=324325 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=13 foreground_pressure_transitions_total=1242 foreground_pressure_state_elapsed_ms=40332 foreground_pressure_time_normal_window_ms=3560 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=6993 foreground_pressure_time_emergency_window_ms=149972
writes_test: elapsed=215s elapsed_hms=0:03:35 docs=5368832 txns=20974 writes/s=25395 avg_writes/s=24971 p50=108ms p99=304ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=142ms dbms_insert=61825ms batch_coord=98670ms wal_commit=230583ms wal_fsync=14690ms wal_fsync_wait=99870ms wal_fsync_actual=14690ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=364 wal_fsync_blocked_by_doc_store_sync_ms=56 wal_fsync_blocked_by_checkpoint_ms=2 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1512 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=13 wal_fsync_active_elapsed_ms=3 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1260 fsyncs/s=1260 wal_fsync_q=125 wal_group_size=4 wal_group_fill=7 wal_bytes_per_fsync=32171 wal_batches/s=1263 wal_bytes/s=18117932 avg_writers_per_fsync=2 avg_bytes_per_fsync=14367 wal_writer_in_flight=73 pending_waiters=12 pending_wal_bytes=357082 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=66595 pending_waiters_before_fsync=4 waiters_selected_for_batch=4 eligible_waiters=4 released_waiters=4 wakeup_batches=294933 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=20988 durable_lsn_after=20992 oldest_waiter_age=19ms avg_waiter_age=15ms max_waiter_age=12ms waiter_broadcasts=294933 fsync_batch_reason=timeout last_waiters_released_in_fsync=123 durable_lsn=20992 durable_lsn_advance=15589 durable_lsn_advance_bytes=90532255 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=39MB/s wal_io=17MB/s primary_flush_io=22MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=26445086720 background_io_paused=10630ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=165ms doc_store_sync_segments_window=159 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=8 doc_store_sync_segment_bytes=3538105 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=2 doc_store_sync_pending_segments=4 doc_store_sync_pending_bytes=4248617 doc_store_sync_pending_lsn=20977 doc_store_durable_lsn=20956 doc_store_sync_worker_active=1 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=19 doc_store_sync_debt_bytes=4248617 doc_store_sync_jobs_merged=15 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=2 doc_store_sync_queue_wait_ms=390 doc_store_sync_physical_ms=3601 doc_store_sync_durable_lsn_advance=630574 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=114994881 wal_pinned_by_checkpoint_bytes=3687452968 checkpoint_sync=9ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=9 checkpoint_sync_ms_per_node=2ms checkpoint_sync_blocked_window_ms=9 checkpoint_sync_blocked_total_ms=19 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=2 checkpoint_slice_count_window=2 checkpoint_slices_per_node=0 checkpoint_sync_slice=9ms checkpoint_slice_avg_ms=4 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=2097152 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=115208148 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=39276ms wait=0ms wal=3514MB wal_live_bytes=3688146473 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=24MB flush_q=10 primary_q=10 secondary_q=0 primary_io=489301ms primary_pages=6017 primary_flush_batch_bytes=119277002 primary_flush_batch_entries=486764 avg_flush_batch_bytes=1392149 coalesced_tables=0 coalesced_batches=0 small_flush_count=60 small_flush_reason=pressure active_primary_flush_workers=10 per_table_flush_inflight=9 flush_worker_io_wait=1ms flush_worker_cpu=25451ms primary_slice_max=971ms primary_blocked=2453243ms primary_flush_admission_wait_ms=97664 primary_flush_queued_age_ms=1468981 primary_flush_oldest_queued_age_ms=43108 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=105996 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=3282MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49920 sec_apply/s=25088 text_apply/s=24832 idx_flush/s=0 idx_busy=223467ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=9 checkpoint_yield_count=18 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=97664 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1936 wal_priority_yield_count=1871 background_io_yield_count=2772 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=38960 foreground_admission_delay_total_ms=363285 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=50 foreground_pressure_transitions_total=1292 foreground_pressure_state_elapsed_ms=45290 foreground_pressure_time_normal_window_ms=49015 foreground_pressure_time_warm_window_ms=1071 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=25208 foreground_pressure_time_emergency_window_ms=85219
writes_test: elapsed=220s elapsed_hms=0:03:40 docs=5484544 txns=21426 writes/s=23142 avg_writes/s=24929 p50=186ms p99=755ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=178ms dbms_insert=96270ms batch_coord=62866ms wal_commit=217051ms wal_fsync=13408ms wal_fsync_wait=568492ms wal_fsync_actual=13408ms wal_fsync_avg_ms=0 wal_fsync_p95_ms=4 wal_fsync_p99_ms=8 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=342 wal_fsync_blocked_by_doc_store_sync_ms=60 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1521 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=10 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=6546 fsyncs/s=6544 wal_fsync_q=62 wal_group_size=2 wal_group_fill=5 wal_bytes_per_fsync=22654 wal_batches/s=1263 wal_bytes/s=17241005 avg_writers_per_fsync=0 avg_bytes_per_fsync=11502 wal_writer_in_flight=45 pending_waiters=28 pending_wal_bytes=235970 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=65783 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=301237 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=21454 durable_lsn_after=21456 oldest_waiter_age=19ms avg_waiter_age=15ms max_waiter_age=17ms waiter_broadcasts=301237 fsync_batch_reason=timeout last_waiters_released_in_fsync=62 durable_lsn=21456 durable_lsn_advance=14903 durable_lsn_advance_bytes=376357537 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=42MB/s wal_io=16MB/s primary_flush_io=26MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=26478641152 background_io_paused=11518ms background_pause_reason=wal_protected emergency_wal_only_count=2 doc_store_sync=214ms doc_store_sync_segments_window=161 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=4 doc_store_sync_segment_bytes=4101634 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=21164 doc_store_durable_lsn=21164 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=15 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=397 doc_store_sync_physical_ms=3815 doc_store_sync_durable_lsn_advance=673725 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=3769345504 checkpoint_sync=0ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=19 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=2097152 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=117606276 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=3294ms wait=0ms wal=3593MB wal_live_bytes=3769698764 wal_truncatable_bytes=353260 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=73MB flush_q=42 primary_q=42 secondary_q=0 primary_io=529078ms primary_pages=6017 primary_flush_batch_bytes=136363313 primary_flush_batch_entries=202808 avg_flush_batch_bytes=1406461 coalesced_tables=0 coalesced_batches=0 small_flush_count=5 small_flush_reason=pressure active_primary_flush_workers=42 per_table_flush_inflight=15 flush_worker_io_wait=2ms flush_worker_cpu=39775ms primary_slice_max=971ms primary_blocked=2569056ms primary_flush_admission_wait_ms=115813 primary_flush_queued_age_ms=99 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=105996 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=3353MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49152 sec_apply/s=24576 text_apply/s=24576 idx_flush/s=0 idx_busy=223655ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=90 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=115813 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1933 wal_priority_yield_count=1927 background_io_yield_count=2467 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=2300 foreground_admission_delay_total_ms=365585 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=23 foreground_pressure_transitions_total=1315 foreground_pressure_state_elapsed_ms=13032 foreground_pressure_time_normal_window_ms=142055 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=16426 foreground_pressure_time_emergency_window_ms=1841
writes_test: elapsed=225s elapsed_hms=0:03:45 docs=5599232 txns=21873 writes/s=22937 avg_writes/s=24885 p50=198ms p99=803ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=127ms dbms_insert=103945ms batch_coord=56190ms wal_commit=214559ms wal_fsync=14136ms wal_fsync_wait=95342ms wal_fsync_actual=14136ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=330 wal_fsync_blocked_by_doc_store_sync_ms=40 wal_fsync_blocked_by_checkpoint_ms=1 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1458 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=4 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1229 fsyncs/s=1232 wal_fsync_q=32 wal_group_size=1 wal_group_fill=2 wal_bytes_per_fsync=9495 wal_batches/s=1231 wal_bytes/s=16737232 avg_writers_per_fsync=2 avg_bytes_per_fsync=13609 wal_writer_in_flight=35 pending_waiters=3 pending_wal_bytes=141030 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=62721 pending_waiters_before_fsync=1 waiters_selected_for_batch=1 eligible_waiters=3 released_waiters=3 wakeup_batches=307397 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=21906 durable_lsn_after=21907 oldest_waiter_age=28ms avg_waiter_age=28ms max_waiter_age=24ms waiter_broadcasts=307397 fsync_batch_reason=timeout last_waiters_released_in_fsync=38 durable_lsn=21907 durable_lsn_advance=14395 durable_lsn_advance_bytes=83833957 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=42MB/s wal_io=15MB/s primary_flush_io=27MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=25773998080 background_io_paused=11211ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=198ms doc_store_sync_segments_window=144 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=3 doc_store_sync_segment_bytes=4087849 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=3 doc_store_sync_pending_segments=4 doc_store_sync_pending_bytes=7109236 doc_store_sync_pending_lsn=21895 doc_store_durable_lsn=21895 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=19 doc_store_sync_debt_bytes=7109236 doc_store_sync_jobs_merged=19 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=450 doc_store_sync_physical_ms=4013 doc_store_sync_durable_lsn_advance=681922 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=240595624 wal_pinned_by_checkpoint_bytes=3848694684 checkpoint_sync=2ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=2 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=2 checkpoint_sync_blocked_total_ms=21 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=2ms checkpoint_slice_avg_ms=2 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=1490944 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=120107547 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=3028ms wait=0ms wal=3669MB wal_live_bytes=3848977804 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=10MB flush_q=4 primary_q=4 secondary_q=0 primary_io=561205ms primary_pages=7379 primary_flush_batch_bytes=142771633 primary_flush_batch_entries=605105 avg_flush_batch_bytes=1358345 coalesced_tables=0 coalesced_batches=0 small_flush_count=45 small_flush_reason=pressure active_primary_flush_workers=3 per_table_flush_inflight=3 flush_worker_io_wait=1ms flush_worker_cpu=32126ms primary_slice_max=971ms primary_blocked=2690094ms primary_flush_admission_wait_ms=121038 primary_flush_queued_age_ms=1377 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106080 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=3424MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=47124 sec_apply/s=23434 text_apply/s=23689 idx_flush/s=0 idx_busy=223760ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=2 checkpoint_yield_count=96 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=121038 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1849 wal_priority_yield_count=1815 background_io_yield_count=2715 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=none foreground_admission_delay_window_ms=1252 foreground_admission_delay_total_ms=366837 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=10 foreground_pressure_transitions_total=1325 foreground_pressure_state_elapsed_ms=18045 foreground_pressure_time_normal_window_ms=158561 foreground_pressure_time_warm_window_ms=906 foreground_pressure_time_constrained_window_ms=1021 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=230s elapsed_hms=0:03:50 docs=5713408 txns=22319 writes/s=22835 avg_writes/s=24840 p50=205ms p99=718ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=128ms dbms_insert=107287ms batch_coord=51315ms wal_commit=201345ms wal_fsync=17139ms wal_fsync_wait=89767ms wal_fsync_actual=17139ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=16 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=724 wal_fsync_blocked_by_doc_store_sync_ms=98 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1654 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1197 fsyncs/s=1198 wal_fsync_q=91 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=22711 wal_batches/s=1198 wal_bytes/s=16196853 avg_writers_per_fsync=2 avg_bytes_per_fsync=13522 wal_writer_in_flight=48 pending_waiters=0 pending_wal_bytes=298375 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=59770 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=313387 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=22343 durable_lsn_after=22346 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=19ms waiter_broadcasts=313387 fsync_batch_reason=timeout last_waiters_released_in_fsync=91 durable_lsn=22346 durable_lsn_advance=14051 durable_lsn_advance_bytes=81000545 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=49MB/s wal_io=15MB/s primary_flush_io=34MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=25111298048 background_io_paused=12257ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=302ms doc_store_sync_segments_window=227 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=2 doc_store_sync_segment_bytes=3906768 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=22327 doc_store_durable_lsn=22327 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=24 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=640 doc_store_sync_physical_ms=4315 doc_store_sync_durable_lsn_advance=707760 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=3925357801 checkpoint_sync=5ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=5 checkpoint_sync_ms_per_node=1ms checkpoint_sync_blocked_window_ms=5 checkpoint_sync_blocked_total_ms=26 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=3 checkpoint_slice_count_window=3 checkpoint_slices_per_node=0 checkpoint_sync_slice=5ms checkpoint_slice_avg_ms=1 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=360448 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=122649892 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=1034ms wait=0ms wal=3741MB wal_live_bytes=3925671377 wal_truncatable_bytes=313576 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=5MB flush_q=2 primary_q=2 secondary_q=0 primary_io=588103ms primary_pages=7429 primary_flush_batch_bytes=179427979 primary_flush_batch_entries=323988 avg_flush_batch_bytes=1344703 coalesced_tables=0 coalesced_batches=0 small_flush_count=21 small_flush_reason=pressure active_primary_flush_workers=2 per_table_flush_inflight=1 flush_worker_io_wait=0ms flush_worker_cpu=26898ms primary_slice_max=971ms primary_blocked=2703112ms primary_flush_admission_wait_ms=13018 primary_flush_queued_age_ms=326 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106081 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=3494MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=45266 sec_apply/s=22761 text_apply/s=22505 idx_flush/s=0 idx_busy=223836ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=3ms rejects=0 checkpoint_sync_blocked_estimate_ms=5 checkpoint_yield_count=95 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=13018 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2216 wal_priority_yield_count=2061 background_io_yield_count=4174 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=366837 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=2 foreground_pressure_transitions_total=1327 foreground_pressure_state_elapsed_ms=23058 foreground_pressure_time_normal_window_ms=159232 foreground_pressure_time_warm_window_ms=1044 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=235s elapsed_hms=0:03:55 docs=5839872 txns=22813 writes/s=25292 avg_writes/s=24850 p50=109ms p99=442ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=79ms dbms_insert=62930ms batch_coord=97589ms wal_commit=238900ms wal_fsync=17848ms wal_fsync_wait=100706ms wal_fsync_actual=17848ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=16 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=511 wal_fsync_blocked_by_doc_store_sync_ms=2 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1649 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=13 wal_fsync_active_elapsed_ms=2 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1253 fsyncs/s=1252 wal_fsync_q=124 wal_group_size=4 wal_group_fill=7 wal_bytes_per_fsync=31035 wal_batches/s=1254 wal_bytes/s=18169081 avg_writers_per_fsync=2 avg_bytes_per_fsync=14500 wal_writer_in_flight=35 pending_waiters=4 pending_wal_bytes=102915 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=64411 pending_waiters_before_fsync=4 waiters_selected_for_batch=4 eligible_waiters=4 released_waiters=4 wakeup_batches=319651 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=22830 durable_lsn_after=22834 oldest_waiter_age=16ms avg_waiter_age=12ms max_waiter_age=10ms waiter_broadcasts=319651 fsync_batch_reason=timeout last_waiters_released_in_fsync=124 durable_lsn=22834 durable_lsn_advance=15609 durable_lsn_advance_bytes=90830645 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=24MB/s wal_io=17MB/s primary_flush_io=7MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=26277314560 background_io_paused=10361ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=64ms doc_store_sync_segments_window=51 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=5 doc_store_sync_segment_bytes=3808933 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=22773 doc_store_durable_lsn=22773 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=24 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=640 doc_store_sync_physical_ms=4379 doc_store_sync_durable_lsn_advance=712274 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=4010793463 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=26 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=360448 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=125268610 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=134ms wait=0ms wal=3824MB wal_live_bytes=4011528710 wal_truncatable_bytes=735247 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=589949ms primary_pages=7429 primary_flush_batch_bytes=38820406 primary_flush_batch_entries=57538 avg_flush_batch_bytes=1343127 coalesced_tables=0 coalesced_batches=0 small_flush_count=4 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=1 flush_worker_io_wait=0ms flush_worker_cpu=1846ms primary_slice_max=971ms primary_blocked=2703112ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=29 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106081 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=3571MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=48956 sec_apply/s=24733 text_apply/s=24223 idx_flush/s=0 idx_busy=223849ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=3ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=97 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2081 wal_priority_yield_count=1964 background_io_yield_count=1692 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=366837 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=1327 foreground_pressure_state_elapsed_ms=28068 foreground_pressure_time_normal_window_ms=160628 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=240s elapsed_hms=0:04:00 docs=5965568 txns=23304 writes/s=25139 avg_writes/s=24856 p50=67ms p99=139ms reason=healthy p99_root_cause_exclusive=healthy serialize=93ms dbms_insert=35098ms batch_coord=124529ms wal_commit=259412ms wal_fsync=18377ms wal_fsync_wait=107379ms wal_fsync_actual=18377ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=32 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=489 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1625 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=14 wal_fsync_active_elapsed_ms=3 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1267 fsyncs/s=1265 wal_fsync_q=165 wal_group_size=9 wal_group_fill=17 wal_bytes_per_fsync=49841 wal_batches/s=1270 wal_bytes/s=18143921 avg_writers_per_fsync=2 avg_bytes_per_fsync=14297 wal_writer_in_flight=82 pending_waiters=50 pending_wal_bytes=130771 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=70563 pending_waiters_before_fsync=9 waiters_selected_for_batch=9 eligible_waiters=9 released_waiters=9 wakeup_batches=325978 lost_wakeup_count=0 durable_lsn_gap=1 durable_lsn_before=23320 durable_lsn_after=23324 oldest_waiter_age=32ms avg_waiter_age=28ms max_waiter_age=11ms waiter_broadcasts=325978 fsync_batch_reason=none last_waiters_released_in_fsync=131 durable_lsn=23324 durable_lsn_advance=15727 durable_lsn_advance_bytes=90462580 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=26566721536 background_io_paused=10160ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=4ms doc_store_sync_segments_window=5 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=5 doc_store_sync_segment_bytes=3808933 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=23084 doc_store_durable_lsn=23084 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=24 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=641 doc_store_sync_physical_ms=4383 doc_store_sync_durable_lsn_advance=713338 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=4096716475 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=26 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=360448 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=127962632 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=27ms wait=0ms wal=3905MB wal_live_bytes=4097320699 wal_truncatable_bytes=604224 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=590074ms primary_pages=7429 primary_flush_batch_bytes=4204622 primary_flush_batch_entries=6230 avg_flush_batch_bytes=1343127 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=1 flush_worker_io_wait=0ms flush_worker_cpu=125ms primary_slice_max=971ms primary_blocked=2703112ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=2 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106081 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=3648MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49820 sec_apply/s=24782 text_apply/s=25037 idx_flush/s=0 idx_busy=223862ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=3ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=95 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2083 wal_priority_yield_count=1951 background_io_yield_count=1334 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=none foreground_admission_delay_window_ms=3 foreground_admission_delay_total_ms=366840 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=2 foreground_pressure_transitions_total=1329 foreground_pressure_state_elapsed_ms=33077 foreground_pressure_time_normal_window_ms=159925 foreground_pressure_time_warm_window_ms=1 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=245s elapsed_hms=0:04:05 docs=6092032 txns=23797 writes/s=25292 avg_writes/s=24865 p50=73ms p99=236ms reason=healthy p99_root_cause_exclusive=healthy serialize=64ms dbms_insert=40820ms batch_coord=117524ms wal_commit=281142ms wal_fsync=24279ms wal_fsync_wait=109789ms wal_fsync_actual=24279ms wal_fsync_avg_ms=3 wal_fsync_p95_ms=16 wal_fsync_p99_ms=64 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=995 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1824 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=17 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1265 fsyncs/s=1266 wal_fsync_q=32 wal_group_size=1 wal_group_fill=2 wal_bytes_per_fsync=11066 wal_batches/s=1276 wal_bytes/s=17836038 avg_writers_per_fsync=2 avg_bytes_per_fsync=14126 wal_writer_in_flight=35 pending_waiters=6 pending_wal_bytes=147404 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=69524 pending_waiters_before_fsync=1 waiters_selected_for_batch=1 eligible_waiters=4 released_waiters=4 wakeup_batches=332308 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=23808 durable_lsn_after=23809 oldest_waiter_age=18ms avg_waiter_age=18ms max_waiter_age=11ms waiter_broadcasts=332308 fsync_batch_reason=timeout last_waiters_released_in_fsync=35 durable_lsn=23809 durable_lsn_advance=15458 durable_lsn_advance_bytes=89418371 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=26562527232 background_io_paused=10181ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=5 doc_store_sync_segment_bytes=3808933 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=23084 doc_store_durable_lsn=23084 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=24 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=641 doc_store_sync_physical_ms=4383 doc_store_sync_durable_lsn_advance=713338 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=4181808665 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=26 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=360448 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=130652554 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=54ms wait=0ms wal=3986MB wal_live_bytes=4181933605 wal_truncatable_bytes=124940 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=590074ms primary_pages=7429 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=1343127 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=1 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=971ms primary_blocked=2703112ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106081 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=3726MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50842 sec_apply/s=25548 text_apply/s=25293 idx_flush/s=0 idx_busy=223870ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=95 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2393 wal_priority_yield_count=2155 background_io_yield_count=1284 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=32 foreground_admission_delay_total_ms=366872 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=56 foreground_pressure_transitions_total=1385 foreground_pressure_state_elapsed_ms=38091 foreground_pressure_time_normal_window_ms=150589 foreground_pressure_time_warm_window_ms=9459 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=250s elapsed_hms=0:04:10 docs=6216448 txns=24284 writes/s=24883 avg_writes/s=24865 p50=65ms p99=141ms reason=healthy p99_root_cause_exclusive=healthy serialize=108ms dbms_insert=34363ms batch_coord=125709ms wal_commit=231337ms wal_fsync=17949ms wal_fsync_wait=106586ms wal_fsync_actual=17949ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=544 wal_fsync_blocked_by_doc_store_sync_ms=15 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1695 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=8 wal_fsync_active_elapsed_ms=2 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1369 fsyncs/s=1370 wal_fsync_q=68 wal_group_size=3 wal_group_fill=6 wal_bytes_per_fsync=27025 wal_batches/s=1372 wal_bytes/s=17983456 avg_writers_per_fsync=2 avg_bytes_per_fsync=13122 wal_writer_in_flight=29 pending_waiters=2 pending_wal_bytes=138173 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=70928 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=339162 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=24297 durable_lsn_after=24298 oldest_waiter_age=10ms avg_waiter_age=5ms max_waiter_age=15ms waiter_broadcasts=339162 fsync_batch_reason=timeout last_waiters_released_in_fsync=69 durable_lsn=24298 durable_lsn_advance=15666 durable_lsn_advance_bytes=89942027 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=28MB/s wal_io=17MB/s primary_flush_io=11MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=28726788096 background_io_paused=10926ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=81ms doc_store_sync_segments_window=70 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=3 doc_store_sync_segment_bytes=3927111 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=2 doc_store_sync_pending_segments=4 doc_store_sync_pending_bytes=5420822 doc_store_sync_pending_lsn=24241 doc_store_durable_lsn=24241 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=30 doc_store_sync_debt_bytes=5420822 doc_store_sync_jobs_merged=24 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=643 doc_store_sync_physical_ms=4464 doc_store_sync_durable_lsn_advance=747605 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=4266616028 checkpoint_sync=0ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=26 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=360448 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=133304881 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=235ms wait=0ms wal=4067MB wal_live_bytes=4267239885 wal_truncatable_bytes=623857 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=index_backlog backlog=30MB flush_q=11 primary_q=11 secondary_q=0 primary_io=604598ms primary_pages=7429 primary_flush_batch_bytes=61889747 primary_flush_batch_entries=94418 avg_flush_batch_bytes=1345947 coalesced_tables=0 coalesced_batches=0 small_flush_count=2 small_flush_reason=pressure active_primary_flush_workers=11 per_table_flush_inflight=1 flush_worker_io_wait=0ms flush_worker_cpu=14524ms primary_slice_max=971ms primary_blocked=2707358ms primary_flush_admission_wait_ms=4246 primary_flush_queued_age_ms=24 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106081 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=3802MB pending_idx=512 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=12433409 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=49820 sec_apply/s=25037 text_apply/s=24782 idx_flush/s=0 idx_busy=223916ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=91 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=4246 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2144 wal_priority_yield_count=2058 background_io_yield_count=1856 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=16 foreground_admission_delay_total_ms=366888 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=23 foreground_pressure_transitions_total=1408 foreground_pressure_state_elapsed_ms=43102 foreground_pressure_time_normal_window_ms=156439 foreground_pressure_time_warm_window_ms=4589 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=255s elapsed_hms=0:04:15 docs=6340608 txns=24769 writes/s=24832 avg_writes/s=24865 p50=99ms p99=213ms reason=healthy p99_root_cause_exclusive=healthy serialize=154ms dbms_insert=50486ms batch_coord=110120ms wal_commit=235312ms wal_fsync=16089ms wal_fsync_wait=105829ms wal_fsync_actual=16089ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=464 wal_fsync_blocked_by_doc_store_sync_ms=47 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1699 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=16 wal_fsync_active_elapsed_ms=4 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1355 fsyncs/s=1353 wal_fsync_q=98 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=24188 wal_batches/s=1359 wal_bytes/s=18464559 avg_writers_per_fsync=2 avg_bytes_per_fsync=13616 wal_writer_in_flight=111 pending_waiters=33 pending_wal_bytes=411681 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=70864 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=345929 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=24790 durable_lsn_after=24793 oldest_waiter_age=16ms avg_waiter_age=14ms max_waiter_age=11ms waiter_broadcasts=345929 fsync_batch_reason=timeout last_waiters_released_in_fsync=91 durable_lsn=24793 durable_lsn_advance=15825 durable_lsn_advance_bytes=92143313 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=38MB/s wal_io=17MB/s primary_flush_io=21MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=28437381120 background_io_paused=11366ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=163ms doc_store_sync_segments_window=139 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=4 doc_store_sync_segment_bytes=3806581 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=24681 doc_store_durable_lsn=24681 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=24 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=646 doc_store_sync_physical_ms=4627 doc_store_sync_durable_lsn_advance=781162 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=4353874694 checkpoint_sync=2ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=2 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=2 checkpoint_sync_blocked_total_ms=28 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=2ms checkpoint_slice_avg_ms=2 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=1171456 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=136001757 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=592ms wait=0ms wal=4150MB wal_live_bytes=4354422204 wal_truncatable_bytes=547510 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=76MB flush_q=34 primary_q=34 secondary_q=0 primary_io=645008ms primary_pages=7429 primary_flush_batch_bytes=111213423 primary_flush_batch_entries=162614 avg_flush_batch_bytes=1350014 coalesced_tables=0 coalesced_batches=0 small_flush_count=8 small_flush_reason=pressure active_primary_flush_workers=34 per_table_flush_inflight=8 flush_worker_io_wait=0ms flush_worker_cpu=40410ms primary_slice_max=1114ms primary_blocked=2775635ms primary_flush_admission_wait_ms=68277 primary_flush_queued_age_ms=53 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106082 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=3879MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50893 sec_apply/s=25318 text_apply/s=25574 idx_flush/s=0 idx_busy=224089ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=3ms rejects=0 checkpoint_sync_blocked_estimate_ms=2 checkpoint_yield_count=94 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=68277 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2158 wal_priority_yield_count=2103 background_io_yield_count=2198 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=10 foreground_admission_delay_total_ms=366898 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=22 foreground_pressure_transitions_total=1430 foreground_pressure_state_elapsed_ms=44838 foreground_pressure_time_normal_window_ms=156338 foreground_pressure_time_warm_window_ms=4085 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=260s elapsed_hms=0:04:20 docs=6464512 txns=25253 writes/s=24780 avg_writes/s=24863 p50=121ms p99=460ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=132ms dbms_insert=74316ms batch_coord=84757ms wal_commit=264572ms wal_fsync=17518ms wal_fsync_wait=646658ms wal_fsync_actual=17518ms wal_fsync_avg_ms=0 wal_fsync_p95_ms=4 wal_fsync_p99_ms=8 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=565 wal_fsync_blocked_by_doc_store_sync_ms=68 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1608 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=18 wal_fsync_active_elapsed_ms=6 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=6353 fsyncs/s=6353 wal_fsync_q=57 wal_group_size=1 wal_group_fill=2 wal_bytes_per_fsync=9602 wal_batches/s=1269 wal_bytes/s=17899128 avg_writers_per_fsync=0 avg_bytes_per_fsync=12262 wal_writer_in_flight=41 pending_waiters=9 pending_wal_bytes=69873 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=67897 pending_waiters_before_fsync=1 waiters_selected_for_batch=1 eligible_waiters=2 released_waiters=2 wakeup_batches=352250 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=25273 durable_lsn_after=25275 oldest_waiter_age=29ms avg_waiter_age=29ms max_waiter_age=22ms waiter_broadcasts=352250 fsync_batch_reason=timeout last_waiters_released_in_fsync=41 durable_lsn=25275 durable_lsn_advance=15447 durable_lsn_advance_bytes=389548877 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=35MB/s wal_io=17MB/s primary_flush_io=18MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=26470252544 background_io_paused=11343ms background_pause_reason=none emergency_wal_only_count=1 doc_store_sync=176ms doc_store_sync_segments_window=130 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=3 doc_store_sync_segment_bytes=1825243 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=25085 doc_store_durable_lsn=25085 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=1 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=24 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=648 doc_store_sync_physical_ms=4803 doc_store_sync_durable_lsn_advance=788617 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=4439028110 checkpoint_sync=2ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=2 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=2 checkpoint_sync_blocked_total_ms=30 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=2ms checkpoint_slice_avg_ms=2 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=753664 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=138721895 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=147ms wait=0ms wal=4232MB wal_live_bytes=4439227007 wal_truncatable_bytes=198897 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=669195ms primary_pages=7429 primary_flush_batch_bytes=95509652 primary_flush_batch_entries=133335 avg_flush_batch_bytes=1336332 coalesced_tables=0 coalesced_batches=0 small_flush_count=19 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=1 flush_worker_io_wait=0ms flush_worker_cpu=24187ms primary_slice_max=1114ms primary_blocked=2844683ms primary_flush_admission_wait_ms=69048 primary_flush_queued_age_ms=119 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106083 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=3955MB pending_idx=512 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=12929537 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=48217 sec_apply/s=23860 text_apply/s=24357 idx_flush/s=0 idx_busy=224189ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=2 checkpoint_yield_count=93 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=69048 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2118 wal_priority_yield_count=2012 background_io_yield_count=2452 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=30 foreground_admission_delay_total_ms=366928 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=91 foreground_pressure_transitions_total=1521 foreground_pressure_state_elapsed_ms=46681 foreground_pressure_time_normal_window_ms=146860 foreground_pressure_time_warm_window_ms=13700 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=265s elapsed_hms=0:04:25 docs=6591232 txns=25747 writes/s=25344 avg_writes/s=24872 p50=86ms p99=191ms reason=healthy p99_root_cause_exclusive=healthy serialize=102ms dbms_insert=43852ms batch_coord=114474ms wal_commit=240982ms wal_fsync=15901ms wal_fsync_wait=1514996ms wal_fsync_actual=15901ms wal_fsync_avg_ms=0 wal_fsync_p95_ms=2 wal_fsync_p99_ms=8 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=387 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1502 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=18 wal_fsync_active_elapsed_ms=3 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=11144 fsyncs/s=11144 wal_fsync_q=48 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=15816 wal_batches/s=1237 wal_bytes/s=17863634 avg_writers_per_fsync=0 avg_bytes_per_fsync=8284 wal_writer_in_flight=44 pending_waiters=20 pending_wal_bytes=237753 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=66326 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=358438 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=25757 durable_lsn_after=25758 oldest_waiter_age=20ms avg_waiter_age=12ms max_waiter_age=19ms waiter_broadcasts=358438 fsync_batch_reason=timeout last_waiters_released_in_fsync=38 durable_lsn=25758 durable_lsn_advance=15440 durable_lsn_advance_bytes=461649369 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=25996296192 background_io_paused=10198ms background_pause_reason=wal_protected emergency_wal_only_count=1 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=3 doc_store_sync_segment_bytes=1825243 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=25085 doc_store_durable_lsn=25085 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=1 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=24 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=648 doc_store_sync_physical_ms=4803 doc_store_sync_durable_lsn_advance=788617 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=4523595877 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=30 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=753664 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=141376859 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=14ms wait=0ms wal=4312MB wal_live_bytes=4523707435 wal_truncatable_bytes=111558 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=669195ms primary_pages=7429 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=1336332 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=1 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=1114ms primary_blocked=2844683ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106083 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4032MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=51404 sec_apply/s=25830 text_apply/s=25574 idx_flush/s=0 idx_busy=224203ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=3ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=92 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1885 wal_priority_yield_count=1822 background_io_yield_count=1465 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=366928 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=1521 foreground_pressure_state_elapsed_ms=51702 foreground_pressure_time_normal_window_ms=160212 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=270s elapsed_hms=0:04:30 docs=6715392 txns=26233 writes/s=24832 avg_writes/s=24871 p50=99ms p99=250ms reason=healthy p99_root_cause_exclusive=healthy serialize=138ms dbms_insert=50044ms batch_coord=110835ms wal_commit=250375ms wal_fsync=17801ms wal_fsync_wait=101936ms wal_fsync_actual=17801ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=32 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=535 wal_fsync_blocked_by_doc_store_sync_ms=57 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1667 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=1 wal_fsync_active_elapsed_ms=1 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1253 fsyncs/s=1255 wal_fsync_q=151 wal_group_size=5 wal_group_fill=9 wal_bytes_per_fsync=39532 wal_batches/s=1263 wal_bytes/s=18372858 avg_writers_per_fsync=2 avg_bytes_per_fsync=14651 wal_writer_in_flight=35 pending_waiters=0 pending_wal_bytes=169202 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=64784 pending_waiters_before_fsync=5 waiters_selected_for_batch=5 eligible_waiters=5 released_waiters=5 wakeup_batches=364715 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=26247 durable_lsn_after=26252 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=19ms waiter_broadcasts=364715 fsync_batch_reason=timeout last_waiters_released_in_fsync=151 durable_lsn=26252 durable_lsn_advance=15790 durable_lsn_advance_bytes=91970343 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=42MB/s wal_io=17MB/s primary_flush_io=25MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=26252148736 background_io_paused=11756ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=200ms doc_store_sync_segments_window=170 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=2 doc_store_sync_segment_bytes=3539455 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=26063 doc_store_durable_lsn=26063 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=24 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=659 doc_store_sync_physical_ms=5003 doc_store_sync_durable_lsn_advance=830403 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=4610033813 checkpoint_sync=3ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=3 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=3 checkpoint_sync_blocked_total_ms=33 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=3ms checkpoint_slice_avg_ms=3 checkpoint_slice_last_ms=3 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=1540096 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=144064866 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=544ms wait=0ms wal=4396MB wal_live_bytes=4610665143 wal_truncatable_bytes=631330 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=688080ms primary_pages=7429 primary_flush_batch_bytes=134308045 primary_flush_batch_entries=196734 avg_flush_batch_bytes=1330154 coalesced_tables=0 coalesced_batches=0 small_flush_count=17 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=18885ms primary_slice_max=1114ms primary_blocked=2861051ms primary_flush_admission_wait_ms=16368 primary_flush_queued_age_ms=94 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106083 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4108MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=51916 sec_apply/s=26085 text_apply/s=25830 idx_flush/s=0 idx_busy=224290ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=3 checkpoint_yield_count=94 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=16368 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2193 wal_priority_yield_count=2051 background_io_yield_count=3074 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=23 foreground_admission_delay_total_ms=366951 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=40 foreground_pressure_transitions_total=1561 foreground_pressure_state_elapsed_ms=56714 foreground_pressure_time_normal_window_ms=152735 foreground_pressure_time_warm_window_ms=8168 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=275s elapsed_hms=0:04:35 docs=6829824 txns=26681 writes/s=22886 avg_writes/s=24835 p50=148ms p99=551ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=106ms dbms_insert=79311ms batch_coord=76399ms wal_commit=321123ms wal_fsync=37610ms wal_fsync_wait=550627ms wal_fsync_actual=37610ms wal_fsync_avg_ms=1 wal_fsync_p95_ms=8 wal_fsync_p99_ms=32 wal_fsync_max_ms=256 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=2455 wal_fsync_blocked_by_doc_store_sync_ms=21 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=2628 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=20 wal_fsync_active_elapsed_ms=3 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=5047 fsyncs/s=5046 wal_fsync_q=67 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=16204 wal_batches/s=1198 wal_bytes/s=17197007 avg_writers_per_fsync=0 avg_bytes_per_fsync=12720 wal_writer_in_flight=40 pending_waiters=16 pending_wal_bytes=220402 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=63556 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=370516 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=26716 durable_lsn_after=26718 oldest_waiter_age=20ms avg_waiter_age=14ms max_waiter_age=22ms waiter_broadcasts=370516 fsync_batch_reason=none last_waiters_released_in_fsync=51 durable_lsn=26718 durable_lsn_advance=14910 durable_lsn_advance_bytes=320935780 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=19MB/s wal_io=16MB/s primary_flush_io=3MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=24402460672 background_io_paused=11205ms background_pause_reason=wal_protected emergency_wal_only_count=1 doc_store_sync=47ms doc_store_sync_segments_window=22 doc_store_sync_segment_avg_ms=2 doc_store_sync_segment_max_ms=5 doc_store_sync_segment_bytes=3534751 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=3 doc_store_sync_pending_segments=2 doc_store_sync_pending_bytes=2278672 doc_store_sync_pending_lsn=26711 doc_store_durable_lsn=26711 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=188 doc_store_sync_debt_bytes=2278672 doc_store_sync_jobs_merged=28 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=993 doc_store_sync_physical_ms=5050 doc_store_sync_durable_lsn_advance=835472 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=4692020816 checkpoint_sync=0ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=33 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=3 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=1540096 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=146659660 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=216ms wait=0ms wal=4472MB wal_live_bytes=4692020816 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=16MB flush_q=4 primary_q=4 secondary_q=0 primary_io=689460ms primary_pages=7429 primary_flush_batch_bytes=17832597 primary_flush_batch_entries=31731 avg_flush_batch_bytes=1327910 coalesced_tables=0 coalesced_batches=0 small_flush_count=2 small_flush_reason=pressure active_primary_flush_workers=4 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=1380ms primary_slice_max=1114ms primary_blocked=2861051ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=13 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106083 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4179MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=43342 sec_apply/s=21798 text_apply/s=21544 idx_flush/s=0 idx_busy=224338ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=90 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=3595 wal_priority_yield_count=3014 background_io_yield_count=1931 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=none foreground_admission_delay_window_ms=81 foreground_admission_delay_total_ms=367032 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=74 foreground_pressure_transitions_total=1635 foreground_pressure_state_elapsed_ms=33684 foreground_pressure_time_normal_window_ms=145144 foreground_pressure_time_warm_window_ms=13569 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=280s elapsed_hms=0:04:40 docs=6930432 txns=27073 writes/s=20121 avg_writes/s=24751 p50=294ms p99=1351ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=64ms dbms_insert=133457ms batch_coord=29068ms wal_commit=289474ms wal_fsync=44716ms wal_fsync_wait=185917ms wal_fsync_actual=44716ms wal_fsync_avg_ms=5 wal_fsync_p95_ms=32 wal_fsync_p99_ms=64 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=3030 wal_fsync_blocked_by_doc_store_sync_ms=136 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=2930 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=2 wal_fsync_active_elapsed_ms=2 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1737 fsyncs/s=1738 wal_fsync_q=62 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=16074 wal_batches/s=1253 wal_bytes/s=14496781 avg_writers_per_fsync=1 avg_bytes_per_fsync=11180 wal_writer_in_flight=36 pending_waiters=1 pending_wal_bytes=197901 wal_append_lock_wait_ms=3 wal_writer_sleep_ms=66322 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=376508 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=27109 durable_lsn_after=27111 oldest_waiter_age=12ms avg_waiter_age=12ms max_waiter_age=14ms waiter_broadcasts=376508 fsync_batch_reason=timeout last_waiters_released_in_fsync=61 durable_lsn=27111 durable_lsn_advance=12578 durable_lsn_advance_bytes=97195889 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=38MB/s wal_io=13MB/s primary_flush_io=25MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=25065160704 background_io_paused=14871ms background_pause_reason=none emergency_wal_only_count=1 doc_store_sync=335ms doc_store_sync_segments_window=184 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=6 doc_store_sync_segment_bytes=3442925 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=27078 doc_store_durable_lsn=27078 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=44 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=2097 doc_store_sync_physical_ms=5385 doc_store_sync_durable_lsn_advance=860480 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=4760366365 checkpoint_sync=3ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=3 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=3 checkpoint_sync_blocked_total_ms=36 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=3ms checkpoint_slice_avg_ms=3 checkpoint_slice_last_ms=3 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=835584 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=148851754 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=793ms wait=0ms wal=4538MB wal_live_bytes=4760817467 wal_truncatable_bytes=451102 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=20MB flush_q=8 primary_q=8 secondary_q=0 primary_io=709804ms primary_pages=7429 primary_flush_batch_bytes=134970252 primary_flush_batch_entries=245420 avg_flush_batch_bytes=1312579 coalesced_tables=0 coalesced_batches=0 small_flush_count=20 small_flush_reason=pressure active_primary_flush_workers=7 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=20344ms primary_slice_max=1114ms primary_blocked=2861051ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=123 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106083 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4240MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=42113 sec_apply/s=21184 text_apply/s=20929 idx_flush/s=0 idx_busy=224401ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=3 checkpoint_yield_count=86 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=4266 wal_priority_yield_count=3428 background_io_yield_count=6308 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=none foreground_admission_delay_window_ms=84 foreground_admission_delay_total_ms=367116 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=148 foreground_pressure_transitions_total=1783 foreground_pressure_state_elapsed_ms=38710 foreground_pressure_time_normal_window_ms=128671 foreground_pressure_time_warm_window_ms=33988 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=285s elapsed_hms=0:04:45 docs=7028480 txns=27456 writes/s=19609 avg_writes/s=24661 p50=264ms p99=1237ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=72ms dbms_insert=127079ms batch_coord=33026ms wal_commit=241372ms wal_fsync=23935ms wal_fsync_wait=102589ms wal_fsync_actual=23935ms wal_fsync_avg_ms=4 wal_fsync_p95_ms=16 wal_fsync_p99_ms=64 wal_fsync_max_ms=512 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=1294 wal_fsync_blocked_by_doc_store_sync_ms=63 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1899 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=6 wal_fsync_active_elapsed_ms=2 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1191 fsyncs/s=1191 wal_fsync_q=115 wal_group_size=4 wal_group_fill=7 wal_bytes_per_fsync=31859 wal_batches/s=1226 wal_bytes/s=14071474 avg_writers_per_fsync=2 avg_bytes_per_fsync=11808 wal_writer_in_flight=50 pending_waiters=10 pending_wal_bytes=233269 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=64947 pending_waiters_before_fsync=4 waiters_selected_for_batch=4 eligible_waiters=4 released_waiters=4 wakeup_batches=382462 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=27489 durable_lsn_after=27491 oldest_waiter_age=18ms avg_waiter_age=13ms max_waiter_age=13ms waiter_broadcasts=382462 fsync_batch_reason=timeout last_waiters_released_in_fsync=111 durable_lsn=27491 durable_lsn_advance=12142 durable_lsn_advance_bytes=70300690 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=43MB/s wal_io=13MB/s primary_flush_io=30MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=24985468928 background_io_paused=12132ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=218ms doc_store_sync_segments_window=178 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=7 doc_store_sync_segment_bytes=3447560 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=27445 doc_store_durable_lsn=27445 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=45 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=2103 doc_store_sync_physical_ms=5603 doc_store_sync_durable_lsn_advance=871065 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=4826631261 checkpoint_sync=3ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=3 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=3 checkpoint_sync_blocked_total_ms=39 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=2 checkpoint_slice_count_window=2 checkpoint_slices_per_node=0 checkpoint_sync_slice=3ms checkpoint_slice_avg_ms=1 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=253952 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=150536872 checkpoint_reentry_after_cooldown_ms=243 compaction_pause_reason=none mem=2050ms wait=0ms wal=4602MB wal_live_bytes=4827462730 wal_truncatable_bytes=831469 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=27MB flush_q=14 primary_q=13 secondary_q=0 primary_io=749494ms primary_pages=7945 primary_flush_batch_bytes=157381879 primary_flush_batch_entries=471762 avg_flush_batch_bytes=1302403 coalesced_tables=0 coalesced_batches=0 small_flush_count=18 small_flush_reason=pressure active_primary_flush_workers=10 per_table_flush_inflight=8 flush_worker_io_wait=4ms flush_worker_cpu=39686ms primary_slice_max=1114ms primary_blocked=2891312ms primary_flush_admission_wait_ms=30261 primary_flush_queued_age_ms=138 primary_flush_oldest_queued_age_ms=1200 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106167 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4300MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=37803 sec_apply/s=19028 text_apply/s=18775 idx_flush/s=0 idx_busy=224515ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=3 checkpoint_yield_count=75 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=30261 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2576 wal_priority_yield_count=2310 background_io_yield_count=3484 foreground_latency_guard_trigger_count=0 foreground_pressure_state=constrained foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=849 foreground_admission_delay_total_ms=367965 foreground_last_admission_delay_ms=16 foreground_pressure_transitions_window=106 foreground_pressure_transitions_total=1889 foreground_pressure_state_elapsed_ms=43726 foreground_pressure_time_normal_window_ms=138557 foreground_pressure_time_warm_window_ms=21737 foreground_pressure_time_constrained_window_ms=246 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=290s elapsed_hms=0:04:50 docs=7129344 txns=27850 writes/s=20172 avg_writes/s=24583 p50=286ms p99=954ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=64ms dbms_insert=127550ms batch_coord=32048ms wal_commit=196238ms wal_fsync=18744ms wal_fsync_wait=94209ms wal_fsync_actual=18744ms wal_fsync_avg_ms=3 wal_fsync_p95_ms=16 wal_fsync_p99_ms=64 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=904 wal_fsync_blocked_by_doc_store_sync_ms=57 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1664 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1170 fsyncs/s=1170 wal_fsync_q=48 wal_group_size=2 wal_group_fill=4 wal_bytes_per_fsync=18702 wal_batches/s=1183 wal_bytes/s=14671931 avg_writers_per_fsync=2 avg_bytes_per_fsync=12543 wal_writer_in_flight=0 pending_waiters=0 pending_wal_bytes=0 wal_append_lock_wait_ms=5 wal_writer_sleep_ms=60905 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=388315 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=27884 durable_lsn_after=27886 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=20ms waiter_broadcasts=388315 fsync_batch_reason=timeout last_waiters_released_in_fsync=48 durable_lsn=27886 durable_lsn_advance=12645 durable_lsn_advance_bytes=73424351 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=36MB/s wal_io=13MB/s primary_flush_io=23MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=24528289792 background_io_paused=11098ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=178ms doc_store_sync_segments_window=135 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=4 doc_store_sync_segment_bytes=3101846 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=27749 doc_store_durable_lsn=27749 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=45 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=2106 doc_store_sync_physical_ms=5781 doc_store_sync_durable_lsn_advance=881357 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=4896579266 checkpoint_sync=6ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=6 checkpoint_sync_ms_per_node=1ms checkpoint_sync_blocked_window_ms=6 checkpoint_sync_blocked_total_ms=45 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=5 checkpoint_slice_count_window=5 checkpoint_slices_per_node=1 checkpoint_sync_slice=6ms checkpoint_slice_avg_ms=1 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=24576 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=153210584 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=1793ms wait=0ms wal=4668MB wal_live_bytes=4896875824 wal_truncatable_bytes=296558 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=787280ms primary_pages=9875 primary_flush_batch_bytes=122115775 primary_flush_batch_entries=511471 avg_flush_batch_bytes=1293426 coalesced_tables=0 coalesced_batches=0 small_flush_count=45 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=2 flush_worker_io_wait=2ms flush_worker_cpu=37784ms primary_slice_max=1114ms primary_blocked=2917729ms primary_flush_admission_wait_ms=26417 primary_flush_queued_age_ms=1610 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106182 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4362MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=41988 sec_apply/s=21121 text_apply/s=20866 idx_flush/s=0 idx_busy=224632ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=6 checkpoint_yield_count=93 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=26417 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2229 wal_priority_yield_count=2040 background_io_yield_count=2522 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=335 foreground_admission_delay_total_ms=368300 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=65 foreground_pressure_transitions_total=1954 foreground_pressure_state_elapsed_ms=5879 foreground_pressure_time_normal_window_ms=146052 foreground_pressure_time_warm_window_ms=13378 foreground_pressure_time_constrained_window_ms=761 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=295s elapsed_hms=0:04:55 docs=7247104 txns=28310 writes/s=23552 avg_writes/s=24566 p50=195ms p99=625ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=63ms dbms_insert=106537ms batch_coord=54064ms wal_commit=212689ms wal_fsync=12844ms wal_fsync_wait=88911ms wal_fsync_actual=12844ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=129 wal_fsync_blocked_by_doc_store_sync_ms=2 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1318 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=16 wal_fsync_active_elapsed_ms=7 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1104 fsyncs/s=1103 wal_fsync_q=64 wal_group_size=2 wal_group_fill=4 wal_bytes_per_fsync=17594 wal_batches/s=1104 wal_bytes/s=16729662 avg_writers_per_fsync=2 avg_bytes_per_fsync=15150 wal_writer_in_flight=67 pending_waiters=8 pending_wal_bytes=291652 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=55803 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=393833 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=28332 durable_lsn_after=28334 oldest_waiter_age=30ms avg_waiter_age=27ms max_waiter_age=22ms waiter_broadcasts=393833 fsync_batch_reason=timeout last_waiters_released_in_fsync=60 durable_lsn=28334 durable_lsn_advance=14324 durable_lsn_advance_bytes=83598395 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=16MB/s wal_io=15MB/s primary_flush_io=1MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=23160946688 background_io_paused=10099ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=9ms doc_store_sync_segments_window=10 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=4 doc_store_sync_segment_bytes=3101846 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=28143 doc_store_durable_lsn=28143 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=46 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=2110 doc_store_sync_physical_ms=5790 doc_store_sync_durable_lsn_advance=882970 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=4975685421 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=45 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=24576 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=155610965 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=23ms wait=0ms wal=4744MB wal_live_bytes=4975955095 wal_truncatable_bytes=269674 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=787616ms primary_pages=9875 primary_flush_batch_bytes=8390980 primary_flush_batch_entries=12890 avg_flush_batch_bytes=1293426 coalesced_tables=0 coalesced_batches=0 small_flush_count=1 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=1 flush_worker_io_wait=0ms flush_worker_cpu=336ms primary_slice_max=1114ms primary_blocked=2917729ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=3 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106182 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4434MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=47539 sec_apply/s=23896 text_apply/s=23642 idx_flush/s=0 idx_busy=224655ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=93 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1623 wal_priority_yield_count=1602 background_io_yield_count=1841 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=368300 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=1954 foreground_pressure_state_elapsed_ms=10889 foreground_pressure_time_normal_window_ms=160269 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=300s elapsed_hms=0:05:00 docs=7358720 txns=28746 writes/s=22323 avg_writes/s=24529 p50=250ms p99=850ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=72ms dbms_insert=117237ms batch_coord=41461ms wal_commit=208879ms wal_fsync=14112ms wal_fsync_wait=106685ms wal_fsync_actual=14112ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=154 wal_fsync_blocked_by_doc_store_sync_ms=16 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1493 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=2 wal_fsync_active_elapsed_ms=2 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1349 fsyncs/s=1350 wal_fsync_q=88 wal_group_size=4 wal_group_fill=7 wal_bytes_per_fsync=24796 wal_batches/s=1350 wal_bytes/s=16518361 avg_writers_per_fsync=2 avg_bytes_per_fsync=12239 wal_writer_in_flight=46 pending_waiters=0 pending_wal_bytes=233011 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=71751 pending_waiters_before_fsync=4 waiters_selected_for_batch=4 eligible_waiters=4 released_waiters=4 wakeup_batches=400585 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=28777 durable_lsn_after=28780 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=14ms waiter_broadcasts=400585 fsync_batch_reason=timeout last_waiters_released_in_fsync=86 durable_lsn=28780 durable_lsn_advance=14284 durable_lsn_advance_bytes=82641726 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=28MB/s wal_io=15MB/s primary_flush_io=13MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=28340912128 background_io_paused=10762ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=81ms doc_store_sync_segments_window=84 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=3 doc_store_sync_segment_bytes=4052153 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=28719 doc_store_durable_lsn=28719 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=46 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=2112 doc_store_sync_physical_ms=5871 doc_store_sync_durable_lsn_advance=904176 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=5053769031 checkpoint_sync=1ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=1 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=1 checkpoint_sync_blocked_total_ms=46 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=1ms checkpoint_slice_avg_ms=1 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=1114112 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=158049500 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=371ms wait=0ms wal=4818MB wal_live_bytes=5054241180 wal_truncatable_bytes=472149 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=45MB flush_q=16 primary_q=16 secondary_q=0 primary_io=802601ms primary_pages=9875 primary_flush_batch_bytes=72392114 primary_flush_batch_entries=108613 avg_flush_batch_bytes=1290979 coalesced_tables=0 coalesced_batches=0 small_flush_count=5 small_flush_reason=pressure active_primary_flush_workers=16 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=14985ms primary_slice_max=1114ms primary_blocked=2919530ms primary_flush_admission_wait_ms=1801 primary_flush_queued_age_ms=26 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106182 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4503MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=44288 sec_apply/s=22016 text_apply/s=22272 idx_flush/s=0 idx_busy=224707ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=1 checkpoint_yield_count=93 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=1801 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1850 wal_priority_yield_count=1859 background_io_yield_count=1590 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=2 foreground_admission_delay_total_ms=368302 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=8 foreground_pressure_transitions_total=1962 foreground_pressure_state_elapsed_ms=15902 foreground_pressure_time_normal_window_ms=159196 foreground_pressure_time_warm_window_ms=1552 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=305s elapsed_hms=0:05:05 docs=7460352 txns=29143 writes/s=20326 avg_writes/s=24460 p50=273ms p99=1271ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=58ms dbms_insert=124169ms batch_coord=32780ms wal_commit=184793ms wal_fsync=12233ms wal_fsync_wait=95329ms wal_fsync_actual=12233ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=216 wal_fsync_blocked_by_doc_store_sync_ms=24 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1340 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1197 fsyncs/s=1197 wal_fsync_q=32 wal_group_size=1 wal_group_fill=2 wal_bytes_per_fsync=10317 wal_batches/s=1197 wal_bytes/s=14668244 avg_writers_per_fsync=2 avg_bytes_per_fsync=12252 wal_writer_in_flight=0 pending_waiters=0 pending_wal_bytes=0 wal_append_lock_wait_ms=3 wal_writer_sleep_ms=63651 pending_waiters_before_fsync=1 waiters_selected_for_batch=1 eligible_waiters=1 released_waiters=1 wakeup_batches=406571 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=29175 durable_lsn_after=29176 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=23ms waiter_broadcasts=406571 fsync_batch_reason=timeout last_waiters_released_in_fsync=32 durable_lsn=29176 durable_lsn_advance=12661 durable_lsn_advance_bytes=73341223 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=34MB/s wal_io=13MB/s primary_flush_io=21MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=25069355008 background_io_paused=10899ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=127ms doc_store_sync_segments_window=125 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=3 doc_store_sync_segment_bytes=3811532 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=1 doc_store_sync_pending_segments=2 doc_store_sync_pending_bytes=2836369 doc_store_sync_pending_lsn=29103 doc_store_durable_lsn=29103 doc_store_sync_worker_active=1 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=4 doc_store_sync_debt_bytes=2836369 doc_store_sync_jobs_merged=46 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=2 doc_store_sync_queue_wait_ms=2114 doc_store_sync_physical_ms=5998 doc_store_sync_durable_lsn_advance=919878 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=5123547084 checkpoint_sync=1ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=1 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=1 checkpoint_sync_blocked_total_ms=47 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=1ms checkpoint_slice_avg_ms=1 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=376832 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=160325094 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=395ms wait=0ms wal=4885MB wal_live_bytes=5123662712 wal_truncatable_bytes=115628 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=51MB flush_q=25 primary_q=25 secondary_q=0 primary_io=844446ms primary_pages=9875 primary_flush_batch_bytes=111219917 primary_flush_batch_entries=169111 avg_flush_batch_bytes=1295680 coalesced_tables=0 coalesced_batches=0 small_flush_count=10 small_flush_reason=pressure active_primary_flush_workers=25 per_table_flush_inflight=5 flush_worker_io_wait=0ms flush_worker_cpu=41845ms primary_slice_max=1295ms primary_blocked=2983562ms primary_flush_admission_wait_ms=64032 primary_flush_queued_age_ms=89 primary_flush_oldest_queued_age_ms=5 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106185 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4565MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=43471 sec_apply/s=21608 text_apply/s=21862 idx_flush/s=0 idx_busy=224864ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=1 checkpoint_yield_count=95 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=64032 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1687 wal_priority_yield_count=1698 background_io_yield_count=1993 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=368302 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=1962 foreground_pressure_state_elapsed_ms=20923 foreground_pressure_time_normal_window_ms=160275 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=310s elapsed_hms=0:05:10 docs=7564800 txns=29551 writes/s=20889 avg_writes/s=24402 p50=263ms p99=1146ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=63ms dbms_insert=127497ms batch_coord=36549ms wal_commit=209842ms wal_fsync=14968ms wal_fsync_wait=102396ms wal_fsync_actual=14968ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=359 wal_fsync_blocked_by_doc_store_sync_ms=19 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1518 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=16 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1233 fsyncs/s=1232 wal_fsync_q=96 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=22422 wal_batches/s=1241 wal_bytes/s=15054400 avg_writers_per_fsync=2 avg_bytes_per_fsync=12194 wal_writer_in_flight=28 pending_waiters=27 pending_wal_bytes=156366 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=64684 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=412731 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=29579 durable_lsn_after=29582 oldest_waiter_age=17ms avg_waiter_age=11ms max_waiter_age=12ms waiter_broadcasts=412731 fsync_batch_reason=timeout last_waiters_released_in_fsync=77 durable_lsn=29582 durable_lsn_advance=12961 durable_lsn_advance_bytes=75115635 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=27MB/s wal_io=14MB/s primary_flush_io=13MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=25874661376 background_io_paused=10914ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=101ms doc_store_sync_segments_window=107 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=5 doc_store_sync_segment_bytes=3003039 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=29540 doc_store_durable_lsn=29540 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=46 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=2114 doc_store_sync_physical_ms=6099 doc_store_sync_durable_lsn_advance=929907 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=5194416509 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=47 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=376832 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=162569497 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=142ms wait=0ms wal=4952MB wal_live_bytes=5194800758 wal_truncatable_bytes=384249 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=869069ms primary_pages=9875 primary_flush_batch_bytes=72455798 primary_flush_batch_entries=111455 avg_flush_batch_bytes=1291656 coalesced_tables=0 coalesced_batches=0 small_flush_count=22 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=3 flush_worker_io_wait=0ms flush_worker_cpu=24623ms primary_slice_max=1295ms primary_blocked=3009368ms primary_flush_admission_wait_ms=25806 primary_flush_queued_age_ms=79 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106185 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4629MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=43428 sec_apply/s=21587 text_apply/s=21841 idx_flush/s=0 idx_busy=224988ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=93 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=25806 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1916 wal_priority_yield_count=1873 background_io_yield_count=1980 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=29 foreground_admission_delay_total_ms=368331 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=75 foreground_pressure_transitions_total=2037 foreground_pressure_state_elapsed_ms=21541 foreground_pressure_time_normal_window_ms=145864 foreground_pressure_time_warm_window_ms=14906 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=315s elapsed_hms=0:05:15 docs=7673600 txns=29976 writes/s=21760 avg_writes/s=24360 p50=221ms p99=673ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=52ms dbms_insert=105274ms batch_coord=49246ms wal_commit=198663ms wal_fsync=13950ms wal_fsync_wait=99154ms wal_fsync_actual=13950ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=413 wal_fsync_blocked_by_doc_store_sync_ms=17 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1439 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=32 wal_fsync_active_elapsed_ms=8 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1265 fsyncs/s=1261 wal_fsync_q=63 wal_group_size=3 wal_group_fill=6 wal_bytes_per_fsync=25231 wal_batches/s=1288 wal_bytes/s=16366200 avg_writers_per_fsync=2 avg_bytes_per_fsync=12785 wal_writer_in_flight=226 pending_waiters=226 pending_wal_bytes=1371649 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=65734 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=419036 lost_wakeup_count=0 durable_lsn_gap=1 durable_lsn_before=30008 durable_lsn_after=30009 oldest_waiter_age=138ms avg_waiter_age=92ms max_waiter_age=9ms waiter_broadcasts=419036 fsync_batch_reason=none last_waiters_released_in_fsync=36 durable_lsn=30009 durable_lsn_advance=13717 durable_lsn_advance_bytes=80615720 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=31MB/s wal_io=15MB/s primary_flush_io=16MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=26415726592 background_io_paused=11044ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=99ms doc_store_sync_segments_window=100 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=3 doc_store_sync_segment_bytes=3819343 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=3 doc_store_sync_pending_segments=5 doc_store_sync_pending_bytes=6809246 doc_store_sync_pending_lsn=29979 doc_store_durable_lsn=29965 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=63 doc_store_sync_debt_bytes=6809246 doc_store_sync_jobs_merged=47 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=2128 doc_store_sync_physical_ms=6198 doc_store_sync_durable_lsn_advance=942757 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=164472734 wal_pinned_by_checkpoint_bytes=5271018963 checkpoint_sync=0ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=47 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=7 checkpoint_bytes_per_slice=376832 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=164909899 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=292ms wait=0ms wal=5025MB wal_live_bytes=5271069316 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=15MB flush_q=10 primary_q=10 secondary_q=0 primary_io=878590ms primary_pages=9875 primary_flush_batch_bytes=87074089 primary_flush_batch_entries=134217 avg_flush_batch_bytes=1297734 coalesced_tables=0 coalesced_batches=0 small_flush_count=7 small_flush_reason=pressure active_primary_flush_workers=10 per_table_flush_inflight=4 flush_worker_io_wait=0ms flush_worker_cpu=9521ms primary_slice_max=1295ms primary_blocked=3011643ms primary_flush_admission_wait_ms=2275 primary_flush_queued_age_ms=44 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106185 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4696MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=44444 sec_apply/s=22349 text_apply/s=22095 idx_flush/s=0 idx_busy=225050ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=96 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=2275 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1790 wal_priority_yield_count=1995 background_io_yield_count=2146 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=wal_waiter_age foreground_admission_delay_window_ms=5 foreground_admission_delay_total_ms=368336 foreground_last_admission_delay_ms=1 foreground_pressure_transitions_window=3 foreground_pressure_transitions_total=2040 foreground_pressure_state_elapsed_ms=26551 foreground_pressure_time_normal_window_ms=155375 foreground_pressure_time_warm_window_ms=74 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=320s elapsed_hms=0:05:20 docs=7752704 txns=30284 writes/s=15820 avg_writes/s=24227 p50=428ms p99=1292ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=9ms dbms_insert=145330ms batch_coord=13305ms wal_commit=908882ms wal_fsync=106581ms wal_fsync_wait=143618ms wal_fsync_actual=106581ms wal_fsync_avg_ms=30 wal_fsync_p95_ms=256 wal_fsync_p99_ms=256 wal_fsync_max_ms=1000 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=5888 wal_fsync_blocked_by_doc_store_sync_ms=608 wal_fsync_blocked_by_checkpoint_ms=2 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=4533 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=32 wal_fsync_active_elapsed_ms=15 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=689 fsyncs/s=689 wal_fsync_q=85 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=21773 wal_batches/s=1097 wal_bytes/s=11410959 avg_writers_per_fsync=2 avg_bytes_per_fsync=16549 wal_writer_in_flight=274 pending_waiters=244 pending_wal_bytes=614230 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=55008 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=422484 lost_wakeup_count=0 durable_lsn_gap=1 durable_lsn_before=30314 durable_lsn_after=30317 oldest_waiter_age=133ms avg_waiter_age=78ms max_waiter_age=37ms waiter_broadcasts=422484 fsync_batch_reason=timeout last_waiters_released_in_fsync=71 durable_lsn=30317 durable_lsn_advance=9791 durable_lsn_advance_bytes=57061881 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=29MB/s wal_io=10MB/s primary_flush_io=19MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=14466154496 background_io_paused=18966ms background_pause_reason=wal_protected emergency_wal_only_count=52 doc_store_sync=471ms doc_store_sync_segments_window=140 doc_store_sync_segment_avg_ms=3 doc_store_sync_segment_max_ms=56 doc_store_sync_segment_bytes=3674347 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=6 doc_store_sync_pending_segments=7 doc_store_sync_pending_bytes=11209297 doc_store_sync_pending_lsn=30300 doc_store_durable_lsn=30289 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=142 doc_store_sync_debt_bytes=11209297 doc_store_sync_jobs_merged=100 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=7385 doc_store_sync_physical_ms=6669 doc_store_sync_durable_lsn_advance=958578 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=333454692 wal_pinned_by_checkpoint_bytes=5324987809 checkpoint_sync=172ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=172 checkpoint_sync_ms_per_node=43ms checkpoint_sync_blocked_window_ms=172 checkpoint_sync_blocked_total_ms=219 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=172ms checkpoint_slice_avg_ms=172 checkpoint_slice_last_ms=172 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1581056 checkpoint_yielded_due_to_wal=100ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=166630467 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=1373ms wait=0ms wal=5076MB wal_live_bytes=5325022547 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=2MB flush_q=2 primary_q=2 secondary_q=0 primary_io=890003ms primary_pages=9875 primary_flush_batch_bytes=101791904 primary_flush_batch_entries=155677 avg_flush_batch_bytes=1286491 coalesced_tables=0 coalesced_batches=0 small_flush_count=17 small_flush_reason=pressure active_primary_flush_workers=2 per_table_flush_inflight=1 flush_worker_io_wait=0ms flush_worker_cpu=11413ms primary_slice_max=1295ms primary_blocked=3012007ms primary_flush_admission_wait_ms=364 primary_flush_queued_age_ms=85 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106185 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4745MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=32318 sec_apply/s=16286 text_apply/s=16031 idx_flush/s=0 idx_busy=225070ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=172 checkpoint_yield_count=79 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=364 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=7509 wal_priority_yield_count=5539 background_io_yield_count=10601 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=wal_waiter_age foreground_admission_delay_window_ms=880 foreground_admission_delay_total_ms=369216 foreground_last_admission_delay_ms=1 foreground_pressure_transitions_window=447 foreground_pressure_transitions_total=2487 foreground_pressure_state_elapsed_ms=590 foreground_pressure_time_normal_window_ms=60834 foreground_pressure_time_warm_window_ms=100806 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=325s elapsed_hms=0:05:25 docs=7833600 txns=30602 writes/s=16179 avg_writes/s=24103 p50=363ms p99=1425ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=16ms dbms_insert=145265ms batch_coord=16085ms wal_commit=876355ms wal_fsync=105163ms wal_fsync_wait=141317ms wal_fsync_actual=105163ms wal_fsync_avg_ms=30 wal_fsync_p95_ms=256 wal_fsync_p99_ms=256 wal_fsync_max_ms=512 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=5536 wal_fsync_blocked_by_doc_store_sync_ms=894 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=4290 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=32 wal_fsync_active_elapsed_ms=1 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=687 fsyncs/s=691 wal_fsync_q=94 wal_group_size=3 wal_group_fill=6 wal_bytes_per_fsync=25592 wal_batches/s=1031 wal_bytes/s=11692943 avg_writers_per_fsync=2 avg_bytes_per_fsync=17251 wal_writer_in_flight=80 pending_waiters=45 pending_wal_bytes=468826 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=50903 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=425940 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=30633 durable_lsn_after=30636 oldest_waiter_age=17ms avg_waiter_age=10ms max_waiter_age=22ms waiter_broadcasts=425940 fsync_batch_reason=timeout last_waiters_released_in_fsync=94 durable_lsn=30636 durable_lsn_advance=10273 durable_lsn_advance_bytes=59621373 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=23MB/s wal_io=11MB/s primary_flush_io=12MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=14491320320 background_io_paused=14167ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=601ms doc_store_sync_segments_window=96 doc_store_sync_segment_avg_ms=6 doc_store_sync_segment_max_ms=138 doc_store_sync_segment_bytes=2863195 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=30570 doc_store_durable_lsn=30570 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=135 doc_store_sync_jobs_suppressed=1 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=11641 doc_store_sync_physical_ms=7270 doc_store_sync_durable_lsn_advance=968755 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=5381401799 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=219 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=172 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1581056 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=168437578 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=1236ms wait=0ms wal=5130MB wal_live_bytes=5381401875 wal_truncatable_bytes=76 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=895722ms primary_pages=9875 primary_flush_batch_bytes=62969107 primary_flush_batch_entries=96446 avg_flush_batch_bytes=1277755 coalesced_tables=0 coalesced_batches=0 small_flush_count=11 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=5719ms primary_slice_max=1295ms primary_blocked=3012007ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=49 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106185 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4795MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=28614 sec_apply/s=14307 text_apply/s=14307 idx_flush/s=0 idx_busy=225086ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=80 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=7604 wal_priority_yield_count=5430 background_io_yield_count=5713 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=none foreground_admission_delay_window_ms=906 foreground_admission_delay_total_ms=370122 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=432 foreground_pressure_transitions_total=2919 foreground_pressure_state_elapsed_ms=526 foreground_pressure_time_normal_window_ms=51673 foreground_pressure_time_warm_window_ms=110768 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=330s elapsed_hms=0:05:30 docs=7905792 txns=30885 writes/s=14438 avg_writes/s=23956 p50=376ms p99=2244ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=24ms dbms_insert=140048ms batch_coord=16533ms wal_commit=574370ms wal_fsync=77576ms wal_fsync_wait=119660ms wal_fsync_actual=77576ms wal_fsync_avg_ms=23 wal_fsync_p95_ms=256 wal_fsync_p99_ms=256 wal_fsync_max_ms=512 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=5605 wal_fsync_blocked_by_doc_store_sync_ms=247 wal_fsync_blocked_by_checkpoint_ms=1 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=4055 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=32 wal_fsync_active_elapsed_ms=25 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=668 fsyncs/s=664 wal_fsync_q=191 wal_group_size=20 wal_group_fill=39 wal_bytes_per_fsync=120101 wal_batches/s=1008 wal_bytes/s=10779064 avg_writers_per_fsync=2 avg_bytes_per_fsync=15056 wal_writer_in_flight=693 pending_waiters=681 pending_wal_bytes=3715022 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=49337 pending_waiters_before_fsync=20 waiters_selected_for_batch=20 eligible_waiters=13 released_waiters=13 wakeup_batches=429263 lost_wakeup_count=0 durable_lsn_gap=1 durable_lsn_before=30910 durable_lsn_after=30911 oldest_waiter_age=606ms avg_waiter_age=357ms max_waiter_age=53ms waiter_broadcasts=429263 fsync_batch_reason=none last_waiters_released_in_fsync=65 durable_lsn=30911 durable_lsn_advance=8581 durable_lsn_advance_bytes=50033420 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=30MB/s wal_io=10MB/s primary_flush_io=20MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=13954449408 background_io_paused=18104ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=331ms doc_store_sync_segments_window=119 doc_store_sync_segment_avg_ms=2 doc_store_sync_segment_max_ms=22 doc_store_sync_segment_bytes=3652611 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=15 doc_store_sync_pending_segments=23 doc_store_sync_pending_bytes=29439424 doc_store_sync_pending_lsn=30873 doc_store_durable_lsn=30846 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=576 doc_store_sync_debt_bytes=29439424 doc_store_sync_jobs_merged=144 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=11978 doc_store_sync_physical_ms=7601 doc_store_sync_durable_lsn_advance=981656 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=678758204 wal_pinned_by_checkpoint_bytes=5429265625 checkpoint_sync=124ms checkpoint_pause_reason=wal_fsync_wait checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=124 checkpoint_sync_ms_per_node=31ms checkpoint_sync_blocked_window_ms=124 checkpoint_sync_blocked_total_ms=343 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=124ms checkpoint_slice_avg_ms=124 checkpoint_slice_last_ms=124 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=376832 checkpoint_yielded_due_to_wal=125ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=169917895 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=1411ms wait=0ms wal=5176MB wal_live_bytes=5429295030 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=48MB flush_q=22 primary_q=22 secondary_q=0 primary_io=928205ms primary_pages=9875 primary_flush_batch_bytes=109095666 primary_flush_batch_entries=162047 avg_flush_batch_bytes=1273906 coalesced_tables=0 coalesced_batches=0 small_flush_count=9 small_flush_reason=pressure active_primary_flush_workers=22 per_table_flush_inflight=2 flush_worker_io_wait=0ms flush_worker_cpu=32483ms primary_slice_max=1295ms primary_blocked=3049705ms primary_flush_admission_wait_ms=37698 primary_flush_queued_age_ms=82 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106186 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4839MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=29293 sec_apply/s=14774 text_apply/s=14519 idx_flush/s=0 idx_busy=225203ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=3ms rejects=0 checkpoint_sync_blocked_estimate_ms=124 checkpoint_yield_count=82 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=37698 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=6176 wal_priority_yield_count=4965 background_io_yield_count=8561 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=wal_waiter_age foreground_admission_delay_window_ms=644 foreground_admission_delay_total_ms=370766 foreground_last_admission_delay_ms=5 foreground_pressure_transitions_window=434 foreground_pressure_transitions_total=3353 foreground_pressure_state_elapsed_ms=1196 foreground_pressure_time_normal_window_ms=44766 foreground_pressure_time_warm_window_ms=104925 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=336s elapsed_hms=0:05:36 docs=7942400 txns=31025 writes/s=7321 avg_writes/s=23638 p50=459ms p99=2787ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=4ms dbms_insert=99323ms batch_coord=5880ms wal_commit=543656ms wal_fsync=72085ms wal_fsync_wait=90667ms wal_fsync_actual=72085ms wal_fsync_avg_ms=33 wal_fsync_p95_ms=256 wal_fsync_p99_ms=1000 wal_fsync_max_ms=5000 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=6495 wal_fsync_blocked_by_doc_store_sync_ms=558 wal_fsync_blocked_by_checkpoint_ms=4 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=3451 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=32 wal_fsync_active_elapsed_ms=305 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=429 fsyncs/s=429 wal_fsync_q=113 wal_group_size=10 wal_group_fill=19 wal_bytes_per_fsync=71511 wal_batches/s=564 wal_bytes/s=5198619 avg_writers_per_fsync=2 avg_bytes_per_fsync=13039 wal_writer_in_flight=358 pending_waiters=358 pending_wal_bytes=2093615 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=31480 pending_waiters_before_fsync=10 waiters_selected_for_batch=10 eligible_waiters=3 released_waiters=3 wakeup_batches=431408 lost_wakeup_count=0 durable_lsn_gap=1 durable_lsn_before=31052 durable_lsn_after=31055 oldest_waiter_age=2003ms avg_waiter_age=1893ms max_waiter_age=1210ms waiter_broadcasts=431408 fsync_batch_reason=none last_waiters_released_in_fsync=72 durable_lsn=31055 durable_lsn_advance=4852 durable_lsn_advance_bytes=27969290 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=13MB/s wal_io=4MB/s primary_flush_io=9MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=8980004864 background_io_paused=16451ms background_pause_reason=wal_protected emergency_wal_only_count=56 doc_store_sync=427ms doc_store_sync_segments_window=114 doc_store_sync_segment_avg_ms=3 doc_store_sync_segment_max_ms=97 doc_store_sync_segment_bytes=2735870 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=30916 doc_store_durable_lsn=30916 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=147 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=12272 doc_store_sync_physical_ms=8028 doc_store_sync_durable_lsn_advance=984768 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=5455923015 checkpoint_sync=4ms checkpoint_pause_reason=wal_fsync_wait checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=4 checkpoint_sync_ms_per_node=1ms checkpoint_sync_blocked_window_ms=4 checkpoint_sync_blocked_total_ms=347 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=4ms checkpoint_slice_avg_ms=4 checkpoint_slice_last_ms=4 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1597440 checkpoint_yielded_due_to_wal=75ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=170797326 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=524ms wait=0ms wal=5201MB wal_live_bytes=5455992958 wal_truncatable_bytes=69943 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=945842ms primary_pages=9875 primary_flush_batch_bytes=50401297 primary_flush_batch_entries=74017 avg_flush_batch_bytes=1269344 coalesced_tables=0 coalesced_batches=0 small_flush_count=17 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=1 flush_worker_io_wait=0ms flush_worker_cpu=17637ms primary_slice_max=1295ms primary_blocked=3075894ms primary_flush_admission_wait_ms=26189 primary_flush_queued_age_ms=68 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106186 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4861MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=23784 sec_apply/s=11764 text_apply/s=12019 idx_flush/s=0 idx_busy=225227ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=3ms rejects=0 checkpoint_sync_blocked_estimate_ms=4 checkpoint_yield_count=72 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=26189 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=5860 wal_priority_yield_count=6252 background_io_yield_count=9049 foreground_latency_guard_trigger_count=0 foreground_pressure_state=constrained foreground_pressure_source=wal_waiter_age foreground_admission_delay_window_ms=336 foreground_admission_delay_total_ms=371102 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=281 foreground_pressure_transitions_total=3634 foreground_pressure_state_elapsed_ms=2701 foreground_pressure_time_normal_window_ms=44699 foreground_pressure_time_warm_window_ms=71042 foreground_pressure_time_constrained_window_ms=36 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=341s elapsed_hms=0:05:41 docs=7947264 txns=31044 writes/s=972 avg_writes/s=23305 p50=3566ms p99=5175ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=0ms dbms_insert=69987ms batch_coord=0ms wal_commit=3156512ms wal_fsync=162688ms wal_fsync_wait=163791ms wal_fsync_actual=162688ms wal_fsync_avg_ms=3253 wal_fsync_p95_ms=10000 wal_fsync_p99_ms=10000 wal_fsync_max_ms=10000 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=10972 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=8089 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=10 wal_fsync_active_elapsed_ms=35 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=5 fsyncs/s=10 wal_fsync_q=695 wal_group_size=30 wal_group_fill=58 wal_bytes_per_fsync=188397 wal_batches/s=73 wal_bytes/s=692907 avg_writers_per_fsync=13 avg_bytes_per_fsync=75801 wal_writer_in_flight=307 pending_waiters=307 pending_wal_bytes=1768100 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=4150 pending_waiters_before_fsync=30 waiters_selected_for_batch=30 eligible_waiters=30 released_waiters=30 wakeup_batches=431458 lost_wakeup_count=0 durable_lsn_gap=1 durable_lsn_before=31060 durable_lsn_after=31079 oldest_waiter_age=7306ms avg_waiter_age=5119ms max_waiter_age=2695ms waiter_broadcasts=431458 fsync_batch_reason=none last_waiters_released_in_fsync=595 durable_lsn=31079 durable_lsn_advance=659 durable_lsn_advance_bytes=3790050 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=0MB/s wal_io=0MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=209715200 background_io_paused=11379ms background_pause_reason=wal_protected emergency_wal_only_count=55 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=97 doc_store_sync_segment_bytes=2735870 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=30916 doc_store_durable_lsn=30916 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=147 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=12272 doc_store_sync_physical_ms=8028 doc_store_sync_durable_lsn_advance=984768 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=5459261974 checkpoint_sync=0ms checkpoint_pause_reason=wal_fsync_wait checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=347 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=4 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1597440 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=170797326 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=2079ms wait=0ms wal=5205MB wal_live_bytes=5459261974 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=945842ms primary_pages=9875 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=1269344 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=1 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=1295ms primary_blocked=3075894ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106186 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4864MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=4636 sec_apply/s=2353 text_apply/s=2282 idx_flush/s=0 idx_busy=225227ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=4ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=74 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=12648 wal_priority_yield_count=6748 background_io_yield_count=627 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=wal_waiter_age foreground_admission_delay_window_ms=2080 foreground_admission_delay_total_ms=373182 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=55 foreground_pressure_transitions_total=3689 foreground_pressure_state_elapsed_ms=7710 foreground_pressure_time_normal_window_ms=22011 foreground_pressure_time_warm_window_ms=120758 foreground_pressure_time_constrained_window_ms=14020 foreground_pressure_time_recovery_window_ms=176 foreground_pressure_time_emergency_window_ms=6693
writes_test: elapsed=346s elapsed_hms=0:05:46 docs=7995136 txns=31231 writes/s=9574 avg_writes/s=23107 p50=744ms p99=8133ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=17ms dbms_insert=311765ms batch_coord=6672ms wal_commit=3114896ms wal_fsync=162390ms wal_fsync_wait=194928ms wal_fsync_actual=162390ms wal_fsync_avg_ms=58 wal_fsync_p95_ms=128 wal_fsync_p99_ms=2000 wal_fsync_max_ms=10000 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=6689 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=4743 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=11 wal_fsync_active_elapsed_ms=1 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=555 fsyncs/s=556 wal_fsync_q=47 wal_group_size=2 wal_group_fill=4 wal_bytes_per_fsync=19272 wal_batches/s=852 wal_bytes/s=6913777 avg_writers_per_fsync=2 avg_bytes_per_fsync=13053 wal_writer_in_flight=30 pending_waiters=2 pending_wal_bytes=130129 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=50055 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=434241 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=31270 durable_lsn_after=31272 oldest_waiter_age=11ms avg_waiter_age=11ms max_waiter_age=15ms waiter_broadcasts=434241 fsync_batch_reason=timeout last_waiters_released_in_fsync=49 durable_lsn=31272 durable_lsn_advance=6260 durable_lsn_advance_bytes=36328039 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=6MB/s wal_io=6MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=11681136640 background_io_paused=11116ms background_pause_reason=wal_protected emergency_wal_only_count=137 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=97 doc_store_sync_segment_bytes=2735870 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=30916 doc_store_durable_lsn=30916 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=147 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=12272 doc_store_sync_physical_ms=8028 doc_store_sync_durable_lsn_advance=984768 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=5492996138 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=347 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=4 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1597440 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=171956925 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=1833ms wait=0ms wal=5237MB wal_live_bytes=5493283866 wal_truncatable_bytes=287728 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=1MB flush_q=2 primary_q=2 secondary_q=0 primary_io=946704ms primary_pages=10650 primary_flush_batch_bytes=4195631 primary_flush_batch_entries=77130 avg_flush_batch_bytes=1269344 coalesced_tables=0 coalesced_batches=0 small_flush_count=4 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=1 flush_worker_io_wait=0ms flush_worker_cpu=862ms primary_slice_max=1295ms primary_blocked=3075894ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=4 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106186 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4894MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=30791 sec_apply/s=15522 text_apply/s=15268 idx_flush/s=0 idx_busy=225230ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=5ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=70 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=7126 wal_priority_yield_count=5163 background_io_yield_count=751 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=1414 foreground_admission_delay_total_ms=374596 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=157 foreground_pressure_transitions_total=3846 foreground_pressure_state_elapsed_ms=1875 foreground_pressure_time_normal_window_ms=66480 foreground_pressure_time_warm_window_ms=76269 foreground_pressure_time_constrained_window_ms=25345 foreground_pressure_time_recovery_window_ms=57152 foreground_pressure_time_emergency_window_ms=124
writes_test: elapsed=351s elapsed_hms=0:05:51 docs=8070656 txns=31527 writes/s=15104 avg_writes/s=22993 p50=325ms p99=2882ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=52ms dbms_insert=138589ms batch_coord=21577ms wal_commit=172640ms wal_fsync=24572ms wal_fsync_wait=92244ms wal_fsync_actual=24572ms wal_fsync_avg_ms=4 wal_fsync_p95_ms=16 wal_fsync_p99_ms=32 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=1214 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1766 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=16 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1075 fsyncs/s=1074 wal_fsync_q=121 wal_group_size=3 wal_group_fill=6 wal_bytes_per_fsync=26615 wal_batches/s=1078 wal_bytes/s=10775460 avg_writers_per_fsync=1 avg_bytes_per_fsync=10008 wal_writer_in_flight=102 pending_waiters=15 pending_wal_bytes=313720 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=60760 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=439615 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=31559 durable_lsn_after=31562 oldest_waiter_age=20ms avg_waiter_age=14ms max_waiter_age=16ms waiter_broadcasts=439615 fsync_batch_reason=timeout last_waiters_released_in_fsync=96 durable_lsn=31562 durable_lsn_advance=9263 durable_lsn_advance_bytes=53783971 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=16MB/s wal_io=10MB/s primary_flush_io=6MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=22544384000 background_io_paused=10193ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=97 doc_store_sync_segment_bytes=2735870 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=30916 doc_store_durable_lsn=30916 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=147 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=12272 doc_store_sync_physical_ms=8028 doc_store_sync_durable_lsn_advance=984768 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=5543643996 checkpoint_sync=3ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=3 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=3 checkpoint_sync_blocked_total_ms=350 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=3 checkpoint_slice_count_window=3 checkpoint_slices_per_node=0 checkpoint_sync_slice=3ms checkpoint_slice_avg_ms=1 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1376256 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=173479717 checkpoint_reentry_after_cooldown_ms=135 compaction_pause_reason=none mem=1001ms wait=0ms wal=5285MB wal_live_bytes=5544186320 wal_truncatable_bytes=542324 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=959739ms primary_pages=19272 primary_flush_batch_bytes=27270365 primary_flush_batch_entries=501310 avg_flush_batch_bytes=1266137 coalesced_tables=0 coalesced_batches=0 small_flush_count=28 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=1ms flush_worker_cpu=13034ms primary_slice_max=1295ms primary_blocked=3077813ms primary_flush_admission_wait_ms=1919 primary_flush_queued_age_ms=59 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106186 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4940MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=31145 sec_apply/s=15446 text_apply/s=15699 idx_flush/s=0 idx_busy=225244ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=4ms rejects=0 checkpoint_sync_blocked_estimate_ms=3 checkpoint_yield_count=87 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=1919 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2369 wal_priority_yield_count=2069 background_io_yield_count=1589 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=2 foreground_admission_delay_total_ms=374598 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=2 foreground_pressure_transitions_total=3848 foreground_pressure_state_elapsed_ms=6889 foreground_pressure_time_normal_window_ms=160018 foreground_pressure_time_warm_window_ms=536 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=356s elapsed_hms=0:05:56 docs=8160512 txns=31877 writes/s=17971 avg_writes/s=22922 p50=273ms p99=1589ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=28ms dbms_insert=129233ms batch_coord=30259ms wal_commit=185010ms wal_fsync=14826ms wal_fsync_wait=241112ms wal_fsync_actual=14826ms wal_fsync_avg_ms=1 wal_fsync_p95_ms=8 wal_fsync_p99_ms=8 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=212 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1412 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=2301 fsyncs/s=2302 wal_fsync_q=64 wal_group_size=2 wal_group_fill=4 wal_bytes_per_fsync=17927 wal_batches/s=1174 wal_bytes/s=13014584 avg_writers_per_fsync=0 avg_bytes_per_fsync=11620 wal_writer_in_flight=0 pending_waiters=0 pending_wal_bytes=0 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=64721 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=445492 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=31908 durable_lsn_after=31910 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=12ms waiter_broadcasts=445492 fsync_batch_reason=timeout last_waiters_released_in_fsync=64 durable_lsn=31910 durable_lsn_advance=11151 durable_lsn_advance_bytes=133756982 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=12MB/s wal_io=12MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=24633147392 background_io_paused=10030ms background_pause_reason=none emergency_wal_only_count=1 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=97 doc_store_sync_segment_bytes=2735870 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=30916 doc_store_durable_lsn=30916 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=147 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=12272 doc_store_sync_physical_ms=8028 doc_store_sync_durable_lsn_advance=984768 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=5605547354 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=350 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1376256 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=175395617 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=59ms wait=0ms wal=5344MB wal_live_bytes=5605808306 wal_truncatable_bytes=260952 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=index_backlog backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=960368ms primary_pages=19871 primary_flush_batch_bytes=2097588 primary_flush_batch_entries=38557 avg_flush_batch_bytes=1265889 coalesced_tables=0 coalesced_batches=0 small_flush_count=4 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=629ms primary_slice_max=1295ms primary_blocked=3077813ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=4 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106186 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=4995MB pending_idx=256 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=16321025 idx_lag_lsn=512 idx_lag=5ms sec_lag=0ms text_lag=5ms idx_apply/s=35072 sec_apply/s=17408 text_apply/s=17664 idx_flush/s=0 idx_busy=225251ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=4ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=94 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1776 wal_priority_yield_count=1720 background_io_yield_count=1473 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=1 foreground_admission_delay_total_ms=374599 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=3 foreground_pressure_transitions_total=3851 foreground_pressure_state_elapsed_ms=11897 foreground_pressure_time_normal_window_ms=159231 foreground_pressure_time_warm_window_ms=515 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=361s elapsed_hms=0:06:01 docs=8268288 txns=32298 writes/s=21555 avg_writes/s=22903 p50=252ms p99=1248ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=31ms dbms_insert=126484ms batch_coord=40689ms wal_commit=206963ms wal_fsync=15217ms wal_fsync_wait=636177ms wal_fsync_actual=15217ms wal_fsync_avg_ms=0 wal_fsync_p95_ms=8 wal_fsync_p99_ms=8 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=154 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1468 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=15 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=4977 fsyncs/s=4976 wal_fsync_q=115 wal_group_size=4 wal_group_fill=7 wal_bytes_per_fsync=27256 wal_batches/s=1268 wal_bytes/s=15822914 avg_writers_per_fsync=0 avg_bytes_per_fsync=9712 wal_writer_in_flight=99 pending_waiters=35 pending_wal_bytes=232300 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=66783 pending_waiters_before_fsync=4 waiters_selected_for_batch=4 eligible_waiters=4 released_waiters=4 wakeup_batches=451825 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=32330 durable_lsn_after=32333 oldest_waiter_age=26ms avg_waiter_age=18ms max_waiter_age=15ms waiter_broadcasts=451825 fsync_batch_reason=timeout last_waiters_released_in_fsync=100 durable_lsn=32333 durable_lsn_advance=13500 durable_lsn_advance_bytes=241645730 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=15MB/s wal_io=15MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=26617053184 background_io_paused=10207ms background_pause_reason=wal_protected emergency_wal_only_count=1 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=97 doc_store_sync_segment_bytes=2735870 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=30916 doc_store_durable_lsn=30916 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=147 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=12272 doc_store_sync_physical_ms=8028 doc_store_sync_durable_lsn_advance=984768 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=5679434796 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=350 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1376256 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=177729529 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=0ms wait=0ms wal=5416MB wal_live_bytes=5680446053 wal_truncatable_bytes=1011257 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=960368ms primary_pages=19871 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=1265889 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=1295ms primary_blocked=3077813ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106186 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=5061MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=43428 sec_apply/s=21841 text_apply/s=21587 idx_flush/s=0 idx_busy=225275ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=97 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1820 wal_priority_yield_count=1793 background_io_yield_count=1366 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=374599 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=3851 foreground_pressure_state_elapsed_ms=16912 foreground_pressure_time_normal_window_ms=161054 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=366s elapsed_hms=0:06:06 docs=8381184 txns=32739 writes/s=22579 avg_writes/s=22899 p50=228ms p99=742ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=52ms dbms_insert=107847ms batch_coord=47648ms wal_commit=214252ms wal_fsync=17147ms wal_fsync_wait=593499ms wal_fsync_actual=17147ms wal_fsync_avg_ms=0 wal_fsync_p95_ms=8 wal_fsync_p99_ms=8 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=356 wal_fsync_blocked_by_doc_store_sync_ms=16 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1571 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=1 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=4933 fsyncs/s=4935 wal_fsync_q=63 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=14893 wal_batches/s=1323 wal_bytes/s=16519164 avg_writers_per_fsync=0 avg_bytes_per_fsync=9862 wal_writer_in_flight=16 pending_waiters=0 pending_wal_bytes=102193 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=67693 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=458411 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=32770 durable_lsn_after=32772 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=11ms waiter_broadcasts=458411 fsync_batch_reason=timeout last_waiters_released_in_fsync=63 durable_lsn=32772 durable_lsn_advance=14081 durable_lsn_advance_bytes=243380632 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=24MB/s wal_io=15MB/s primary_flush_io=9MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=27577548800 background_io_paused=10961ms background_pause_reason=none emergency_wal_only_count=2 doc_store_sync=54ms doc_store_sync_segments_window=53 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=44 doc_store_sync_segment_bytes=3408057 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=32714 doc_store_durable_lsn=32714 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=147 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=12274 doc_store_sync_physical_ms=8082 doc_store_sync_durable_lsn_advance=1012689 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=5758260924 checkpoint_sync=0ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=350 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1376256 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=180120792 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=217ms wait=0ms wal=5489MB wal_live_bytes=5758594926 wal_truncatable_bytes=334002 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=68MB flush_q=27 primary_q=15 secondary_q=0 primary_io=968047ms primary_pages=19871 primary_flush_batch_bytes=47202939 primary_flush_batch_entries=65248 avg_flush_batch_bytes=1268472 coalesced_tables=0 coalesced_batches=0 small_flush_count=2 small_flush_reason=pressure active_primary_flush_workers=15 per_table_flush_inflight=2 flush_worker_io_wait=0ms flush_worker_cpu=7679ms primary_slice_max=1295ms primary_blocked=3079946ms primary_flush_admission_wait_ms=2133 primary_flush_queued_age_ms=12 primary_flush_oldest_queued_age_ms=53 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=106201 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=5130MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=44755 sec_apply/s=22249 text_apply/s=22505 idx_flush/s=0 idx_busy=225325ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=90 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=2133 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1995 wal_priority_yield_count=1940 background_io_yield_count=1615 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=none foreground_admission_delay_window_ms=12 foreground_admission_delay_total_ms=374611 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=38 foreground_pressure_transitions_total=3889 foreground_pressure_state_elapsed_ms=21925 foreground_pressure_time_normal_window_ms=154293 foreground_pressure_time_warm_window_ms=5985 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=371s elapsed_hms=0:06:11 docs=8477696 txns=33117 writes/s=19302 avg_writes/s=22850 p50=308ms p99=1239ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=57ms dbms_insert=139671ms batch_coord=21134ms wal_commit=194206ms wal_fsync=12493ms wal_fsync_wait=2244244ms wal_fsync_actual=12493ms wal_fsync_avg_ms=0 wal_fsync_p95_ms=1 wal_fsync_p99_ms=8 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=447 wal_fsync_blocked_by_doc_store_sync_ms=17 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1429 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=13770 fsyncs/s=13770 wal_fsync_q=68 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=20192 wal_batches/s=1207 wal_bytes/s=14078849 avg_writers_per_fsync=0 avg_bytes_per_fsync=8543 wal_writer_in_flight=3 pending_waiters=0 pending_wal_bytes=14842 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=67778 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=464449 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=33150 durable_lsn_after=33153 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=22ms waiter_broadcasts=464449 fsync_batch_reason=timeout last_waiters_released_in_fsync=68 durable_lsn=33153 durable_lsn_advance=12187 durable_lsn_advance_bytes=588257596 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=34MB/s wal_io=13MB/s primary_flush_io=21MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=25325207552 background_io_paused=10572ms background_pause_reason=none emergency_wal_only_count=2 doc_store_sync=128ms doc_store_sync_segments_window=116 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=3 doc_store_sync_segment_bytes=3866852 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=32766 doc_store_durable_lsn=32766 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=147 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=12280 doc_store_sync_physical_ms=8210 doc_store_sync_durable_lsn_advance=1048325 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=5825249953 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=350 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1376256 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=182185556 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=924420ms wait=867213ms wal=5553MB wal_live_bytes=5825249953 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=997MB flush_q=439 primary_q=64 secondary_q=0 primary_io=1009286ms primary_pages=20413 primary_flush_batch_bytes=110835804 primary_flush_batch_entries=163160 avg_flush_batch_bytes=1274567 coalesced_tables=0 coalesced_batches=0 small_flush_count=1 small_flush_reason=pressure active_primary_flush_workers=64 per_table_flush_inflight=64 flush_worker_io_wait=0ms flush_worker_cpu=41239ms primary_slice_max=1353ms primary_blocked=3218983ms primary_flush_admission_wait_ms=139037 primary_flush_queued_age_ms=27049 primary_flush_oldest_queued_age_ms=5069 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=119468 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=5190MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=41133 sec_apply/s=20694 text_apply/s=20439 idx_flush/s=0 idx_busy=225471ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=4ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=89 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=139037 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1665 wal_priority_yield_count=1798 background_io_yield_count=1499 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=25991 foreground_admission_delay_total_ms=400602 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=114 foreground_pressure_transitions_total=4003 foreground_pressure_state_elapsed_ms=3061 foreground_pressure_time_normal_window_ms=9368 foreground_pressure_time_warm_window_ms=27540 foreground_pressure_time_constrained_window_ms=33990 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=88836
writes_test: elapsed=376s elapsed_hms=0:06:16 docs=8577280 txns=33506 writes/s=19916 avg_writes/s=22811 p50=296ms p99=1258ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=42ms dbms_insert=137742ms batch_coord=21604ms wal_commit=185521ms wal_fsync=12043ms wal_fsync_wait=99070ms wal_fsync_actual=12043ms wal_fsync_avg_ms=1 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=301 wal_fsync_blocked_by_doc_store_sync_ms=31 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1502 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=17 wal_fsync_active_elapsed_ms=8 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1233 fsyncs/s=1230 wal_fsync_q=32 wal_group_size=1 wal_group_fill=2 wal_bytes_per_fsync=9499 wal_batches/s=1235 wal_bytes/s=14415983 avg_writers_per_fsync=2 avg_bytes_per_fsync=11702 wal_writer_in_flight=21 pending_waiters=13 pending_wal_bytes=103395 wal_append_lock_wait_ms=3 wal_writer_sleep_ms=68582 pending_waiters_before_fsync=1 waiters_selected_for_batch=1 eligible_waiters=2 released_waiters=2 wakeup_batches=470603 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=33538 durable_lsn_after=33540 oldest_waiter_age=16ms avg_waiter_age=16ms max_waiter_age=10ms waiter_broadcasts=470603 fsync_batch_reason=timeout last_waiters_released_in_fsync=44 durable_lsn=33540 durable_lsn_advance=12397 durable_lsn_advance_bytes=72016332 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=49MB/s wal_io=13MB/s primary_flush_io=36MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=25866272768 background_io_paused=12061ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=251ms doc_store_sync_segments_window=208 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=3 doc_store_sync_segment_bytes=3176973 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=32767 doc_store_durable_lsn=32767 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=147 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=12285 doc_store_sync_physical_ms=8461 doc_store_sync_durable_lsn_advance=1048409 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=5893239039 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=350 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1376256 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=184335642 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=961023ms wait=896900ms wal=5617MB wal_live_bytes=5893363268 wal_truncatable_bytes=124229 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=index_backlog backlog=813MB flush_q=422 primary_q=64 secondary_q=0 primary_io=1048587ms primary_pages=20413 primary_flush_batch_bytes=191132531 primary_flush_batch_entries=284998 avg_flush_batch_bytes=1284534 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=64 per_table_flush_inflight=64 flush_worker_io_wait=0ms flush_worker_cpu=39301ms primary_slice_max=1353ms primary_blocked=3524041ms primary_flush_admission_wait_ms=305058 primary_flush_queued_age_ms=376071 primary_flush_oldest_queued_age_ms=10080 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=133124 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=5251MB pending_idx=512 pending_sec_apply=256 pending_sec_flush=23 oldest_idx_lsn=17155073 idx_lag_lsn=512 idx_lag=11ms sec_lag=11ms text_lag=11ms idx_apply/s=37444 sec_apply/s=18595 text_apply/s=18849 idx_flush/s=0 idx_busy=225592ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=4ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=92 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=305058 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1710 wal_priority_yield_count=1887 background_io_yield_count=3067 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=31020 foreground_admission_delay_total_ms=431622 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=4003 foreground_pressure_state_elapsed_ms=8077 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=160837
writes_test: elapsed=381s elapsed_hms=0:06:21 docs=8675840 txns=33891 writes/s=19712 avg_writes/s=22771 p50=320ms p99=1006ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=63ms dbms_insert=142974ms batch_coord=19109ms wal_commit=186273ms wal_fsync=12687ms wal_fsync_wait=103532ms wal_fsync_actual=12687ms wal_fsync_avg_ms=1 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=379 wal_fsync_blocked_by_doc_store_sync_ms=73 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1487 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=7 wal_fsync_active_elapsed_ms=4 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1288 fsyncs/s=1290 wal_fsync_q=74 wal_group_size=2 wal_group_fill=4 wal_bytes_per_fsync=20724 wal_batches/s=1288 wal_bytes/s=14456591 avg_writers_per_fsync=1 avg_bytes_per_fsync=11211 wal_writer_in_flight=39 pending_waiters=2 pending_wal_bytes=165594 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=71026 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=477055 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=33927 durable_lsn_after=33929 oldest_waiter_age=16ms avg_waiter_age=8ms max_waiter_age=16ms waiter_broadcasts=477055 fsync_batch_reason=timeout last_waiters_released_in_fsync=61 durable_lsn=33929 durable_lsn_advance=12430 durable_lsn_advance_bytes=72336155 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=43MB/s wal_io=13MB/s primary_flush_io=30MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=27028094976 background_io_paused=11483ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=222ms doc_store_sync_segments_window=178 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=3 doc_store_sync_segment_bytes=3878626 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=1 doc_store_sync_pending_segments=2 doc_store_sync_pending_bytes=3902079 doc_store_sync_pending_lsn=32768 doc_store_durable_lsn=32768 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=1 doc_store_sync_debt_bytes=3902079 doc_store_sync_jobs_merged=147 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=12296 doc_store_sync_physical_ms=8683 doc_store_sync_durable_lsn_advance=1048450 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=5961403661 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=350 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1376256 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=186429116 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=996772ms wait=932717ms wal=5684MB wal_live_bytes=5961844571 wal_truncatable_bytes=440910 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=684MB flush_q=424 primary_q=64 secondary_q=0 primary_io=1088915ms primary_pages=20413 primary_flush_batch_bytes=157644980 primary_flush_batch_entries=243075 avg_flush_batch_bytes=1284194 coalesced_tables=0 coalesced_batches=0 small_flush_count=2 small_flush_reason=pressure active_primary_flush_workers=64 per_table_flush_inflight=64 flush_worker_io_wait=1ms flush_worker_cpu=40327ms primary_slice_max=1353ms primary_blocked=3791771ms primary_flush_admission_wait_ms=267730 primary_flush_queued_age_ms=578136 primary_flush_oldest_queued_age_ms=15096 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=146863 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=43909ms index_log=5311MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=41479 sec_apply/s=20612 text_apply/s=20866 idx_flush/s=0 idx_busy=225763ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=4ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=91 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=267730 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1791 wal_priority_yield_count=1933 background_io_yield_count=2050 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=31100 foreground_admission_delay_total_ms=462722 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=4003 foreground_pressure_state_elapsed_ms=13092 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=160872
writes_test: elapsed=386s elapsed_hms=0:06:26 docs=8768512 txns=34253 writes/s=18534 avg_writes/s=22716 p50=294ms p99=1317ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=61ms dbms_insert=139219ms batch_coord=18765ms wal_commit=185829ms wal_fsync=11491ms wal_fsync_wait=98536ms wal_fsync_actual=11491ms wal_fsync_avg_ms=1 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=429 wal_fsync_blocked_by_doc_store_sync_ms=45 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1435 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=7 wal_fsync_active_elapsed_ms=2 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1151 fsyncs/s=1151 wal_fsync_q=67 wal_group_size=5 wal_group_fill=9 wal_bytes_per_fsync=32813 wal_batches/s=1151 wal_bytes/s=13561620 avg_writers_per_fsync=2 avg_bytes_per_fsync=11776 wal_writer_in_flight=15 pending_waiters=2 pending_wal_bytes=77911 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=66188 pending_waiters_before_fsync=5 waiters_selected_for_batch=5 eligible_waiters=5 released_waiters=5 wakeup_batches=482813 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=34288 durable_lsn_after=34292 oldest_waiter_age=26ms avg_waiter_age=26ms max_waiter_age=22ms waiter_broadcasts=482813 fsync_batch_reason=timeout last_waiters_released_in_fsync=71 durable_lsn=34292 durable_lsn_advance=11631 durable_lsn_advance_bytes=67806783 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=30MB/s wal_io=12MB/s primary_flush_io=18MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=24171773952 background_io_paused=11081ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=129ms doc_store_sync_segments_window=99 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=4 doc_store_sync_segment_bytes=5050860 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=33934 doc_store_durable_lsn=33934 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=3 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=147 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=12299 doc_store_sync_physical_ms=8812 doc_store_sync_durable_lsn_advance=1061881 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=6025663054 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=350 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1376256 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=188442424 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=901650ms wait=840538ms wal=5745MB wal_live_bytes=6025935964 wal_truncatable_bytes=272910 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=index_backlog backlog=610MB flush_q=416 primary_q=64 secondary_q=0 primary_io=1127862ms primary_pages=20413 primary_flush_batch_bytes=98029318 primary_flush_batch_entries=149297 avg_flush_batch_bytes=1294601 coalesced_tables=12 coalesced_batches=24 small_flush_count=1 small_flush_reason=pressure active_primary_flush_workers=64 per_table_flush_inflight=64 flush_worker_io_wait=0ms flush_worker_cpu=38947ms primary_slice_max=1667ms primary_blocked=3957683ms primary_flush_admission_wait_ms=165912 primary_flush_queued_age_ms=486339 primary_flush_oldest_queued_age_ms=20108 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=160209 flush_retry=0 flush_coalesced=12 flush_dedup=0 max_flush_wait=43909ms index_log=5368MB pending_idx=512 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=17537537 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=36245 sec_apply/s=17996 text_apply/s=18249 idx_flush/s=0 idx_busy=225918ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=99 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=165912 primary_flush_worker_slots=64 primary_flush_coalesced_count=12 wal_fsync_blocked_ms=1714 wal_priority_yield_count=1844 background_io_yield_count=1745 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=29080 foreground_admission_delay_total_ms=491802 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=4003 foreground_pressure_state_elapsed_ms=18108 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=160286
writes_test: elapsed=391s elapsed_hms=0:06:31 docs=8842752 txns=34544 writes/s=14848 avg_writes/s=22615 p50=376ms p99=1584ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=24ms dbms_insert=141835ms batch_coord=9357ms wal_commit=633342ms wal_fsync=77397ms wal_fsync_wait=138938ms wal_fsync_actual=77397ms wal_fsync_avg_ms=18 wal_fsync_p95_ms=256 wal_fsync_p99_ms=256 wal_fsync_max_ms=1000 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=4537 wal_fsync_blocked_by_doc_store_sync_ms=423 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=3675 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=17 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=833 fsyncs/s=831 wal_fsync_q=48 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=15635 wal_batches/s=1107 wal_bytes/s=10755360 avg_writers_per_fsync=2 avg_bytes_per_fsync=12924 wal_writer_in_flight=47 pending_waiters=17 pending_wal_bytes=162017 wal_append_lock_wait_ms=3 wal_writer_sleep_ms=61754 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=486968 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=34582 durable_lsn_after=34583 oldest_waiter_age=18ms avg_waiter_age=18ms max_waiter_age=15ms waiter_broadcasts=486968 fsync_batch_reason=timeout last_waiters_released_in_fsync=38 durable_lsn=34583 durable_lsn_advance=9294 durable_lsn_advance_bytes=53701196 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=28MB/s wal_io=10MB/s primary_flush_io=18MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=17502830592 background_io_paused=15328ms background_pause_reason=wal_protected emergency_wal_only_count=56 doc_store_sync=396ms doc_store_sync_segments_window=130 doc_store_sync_segment_avg_ms=3 doc_store_sync_segment_max_ms=2 doc_store_sync_segment_bytes=3087928 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=33977 doc_store_durable_lsn=33977 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=147 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=12306 doc_store_sync_physical_ms=9208 doc_store_sync_durable_lsn_advance=1081399 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=6076531442 checkpoint_sync=0ms checkpoint_pause_reason=wal_fsync_wait checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=350 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1376256 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=190046219 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=664176ms wait=612592ms wal=5793MB wal_live_bytes=6076807689 wal_truncatable_bytes=276247 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=508MB flush_q=385 primary_q=63 secondary_q=0 primary_io=1169248ms primary_pages=20413 primary_flush_batch_bytes=99492519 primary_flush_batch_entries=151826 avg_flush_batch_bytes=1294840 coalesced_tables=8 coalesced_batches=16 small_flush_count=1 small_flush_reason=pressure active_primary_flush_workers=63 per_table_flush_inflight=64 flush_worker_io_wait=0ms flush_worker_cpu=41386ms primary_slice_max=1694ms primary_blocked=4358112ms primary_flush_admission_wait_ms=400429 primary_flush_queued_age_ms=960273 primary_flush_oldest_queued_age_ms=25132 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=173634 flush_retry=0 flush_coalesced=20 flush_dedup=5 max_flush_wait=43909ms index_log=5414MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=25750 sec_apply/s=13000 text_apply/s=12750 idx_flush/s=0 idx_busy=226015ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=2ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=83 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=400429 primary_flush_worker_slots=64 primary_flush_coalesced_count=8 wal_fsync_blocked_ms=5934 wal_priority_yield_count=4517 background_io_yield_count=5354 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=23220 foreground_admission_delay_total_ms=515022 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=4003 foreground_pressure_state_elapsed_ms=23134 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=161000
writes_test: elapsed=396s elapsed_hms=0:06:36 docs=8907520 txns=34797 writes/s=12953 avg_writes/s=22493 p50=441ms p99=2806ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=29ms dbms_insert=162945ms batch_coord=3683ms wal_commit=597773ms wal_fsync=95627ms wal_fsync_wait=135050ms wal_fsync_actual=95627ms wal_fsync_avg_ms=28 wal_fsync_p95_ms=256 wal_fsync_p99_ms=256 wal_fsync_max_ms=256 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=5739 wal_fsync_blocked_by_doc_store_sync_ms=554 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=4125 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=32 wal_fsync_active_elapsed_ms=15 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=677 fsyncs/s=674 wal_fsync_q=38 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=13827 wal_batches/s=1025 wal_bytes/s=9348579 avg_writers_per_fsync=2 avg_bytes_per_fsync=13696 wal_writer_in_flight=154 pending_waiters=120 pending_wal_bytes=456014 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=56707 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=4 released_waiters=4 wakeup_batches=490339 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=34827 durable_lsn_after=34831 oldest_waiter_age=128ms avg_waiter_age=62ms max_waiter_age=23ms waiter_broadcasts=490339 fsync_batch_reason=timeout last_waiters_released_in_fsync=87 durable_lsn=34831 durable_lsn_advance=7906 durable_lsn_advance_bytes=46170651 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=32MB/s wal_io=8MB/s primary_flush_io=24MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=14038335488 background_io_paused=17467ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=520ms doc_store_sync_segments_window=168 doc_store_sync_segment_avg_ms=3 doc_store_sync_segment_max_ms=59 doc_store_sync_segment_bytes=3863281 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=2 doc_store_sync_pending_segments=2 doc_store_sync_pending_bytes=2833154 doc_store_sync_pending_lsn=34715 doc_store_durable_lsn=34715 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=45 doc_store_sync_debt_bytes=2833154 doc_store_sync_jobs_merged=147 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=12315 doc_store_sync_physical_ms=9728 doc_store_sync_durable_lsn_advance=1096671 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=6120443380 checkpoint_sync=0ms checkpoint_pause_reason=wal_fsync_wait checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=350 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1376256 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=191499373 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=485859ms wait=441714ms wal=5835MB wal_live_bytes=6120456144 wal_truncatable_bytes=12764 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=473MB flush_q=447 primary_q=64 secondary_q=0 primary_io=1209760ms primary_pages=20413 primary_flush_batch_bytes=127813826 primary_flush_batch_entries=196526 avg_flush_batch_bytes=1289714 coalesced_tables=7 coalesced_batches=14 small_flush_count=4 small_flush_reason=pressure active_primary_flush_workers=64 per_table_flush_inflight=64 flush_worker_io_wait=0ms flush_worker_cpu=40512ms primary_slice_max=1694ms primary_blocked=4638719ms primary_flush_admission_wait_ms=280607 primary_flush_queued_age_ms=1545719 primary_flush_oldest_queued_age_ms=30170 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=187229 flush_retry=0 flush_coalesced=27 flush_dedup=11 max_flush_wait=43909ms index_log=5453MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=28079 sec_apply/s=13913 text_apply/s=14166 idx_flush/s=0 idx_busy=226132ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=4ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=81 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=280607 primary_flush_worker_slots=64 primary_flush_coalesced_count=7 wal_fsync_blocked_ms=6871 wal_priority_yield_count=5253 background_io_yield_count=7952 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=19820 foreground_admission_delay_total_ms=534842 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=4003 foreground_pressure_state_elapsed_ms=28159 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=160292
writes_test: elapsed=401s elapsed_hms=0:06:41 docs=8933120 txns=34896 writes/s=5120 avg_writes/s=22277 p50=491ms p99=1843ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=7ms dbms_insert=69493ms batch_coord=1184ms wal_commit=250889ms wal_fsync=55017ms wal_fsync_wait=71632ms wal_fsync_actual=55017ms wal_fsync_avg_ms=41 wal_fsync_p95_ms=256 wal_fsync_p99_ms=256 wal_fsync_max_ms=5000 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=13035 wal_fsync_blocked_by_doc_store_sync_ms=148 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=3015 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=32 wal_fsync_active_elapsed_ms=967 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=263 fsyncs/s=263 wal_fsync_q=151 wal_group_size=15 wal_group_fill=29 wal_bytes_per_fsync=99320 wal_batches/s=453 wal_bytes/s=3826414 avg_writers_per_fsync=2 avg_bytes_per_fsync=12712 wal_writer_in_flight=518 pending_waiters=517 pending_wal_bytes=2931732 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=25607 pending_waiters_before_fsync=15 waiters_selected_for_batch=15 eligible_waiters=3 released_waiters=3 wakeup_batches=491658 lost_wakeup_count=0 durable_lsn_gap=1 durable_lsn_before=34920 durable_lsn_after=34921 oldest_waiter_age=4888ms avg_waiter_age=4542ms max_waiter_age=479ms waiter_broadcasts=491658 fsync_batch_reason=none last_waiters_released_in_fsync=36 durable_lsn=34921 durable_lsn_advance=2915 durable_lsn_advance_bytes=16767567 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=25MB/s wal_io=3MB/s primary_flush_io=22MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=5544869888 background_io_paused=121227ms background_pause_reason=wal_protected emergency_wal_only_count=16 doc_store_sync=193ms doc_store_sync_segments_window=78 doc_store_sync_segment_avg_ms=2 doc_store_sync_segment_max_ms=22 doc_store_sync_segment_bytes=3684819 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=55 doc_store_sync_pending_segments=57 doc_store_sync_pending_bytes=88730446 doc_store_sync_pending_lsn=34828 doc_store_durable_lsn=34790 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=5582 doc_store_sync_debt_bytes=88730446 doc_store_sync_jobs_merged=170 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=12325 doc_store_sync_physical_ms=9921 doc_store_sync_durable_lsn_advance=1103371 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=3068541016 wal_pinned_by_checkpoint_bytes=6136809234 checkpoint_sync=0ms checkpoint_pause_reason=wal_fsync_wait checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=350 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1376256 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=191990531 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=176634ms wait=160540ms wal=5850MB wal_live_bytes=6136902843 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=364MB flush_q=420 primary_q=64 secondary_q=0 primary_io=1244767ms primary_pages=20413 primary_flush_batch_bytes=119218537 primary_flush_batch_entries=179523 avg_flush_batch_bytes=1281625 coalesced_tables=4 coalesced_batches=8 small_flush_count=3 small_flush_reason=pressure active_primary_flush_workers=64 per_table_flush_inflight=64 flush_worker_io_wait=0ms flush_worker_cpu=35007ms primary_slice_max=1694ms primary_blocked=4906501ms primary_flush_admission_wait_ms=267782 primary_flush_queued_age_ms=1388239 primary_flush_oldest_queued_age_ms=37550 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=202384 flush_retry=0 flush_coalesced=31 flush_dedup=28 max_flush_wait=43909ms index_log=5469MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=3539 sec_apply/s=1799 text_apply/s=1739 idx_flush/s=0 idx_busy=226157ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=2ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=99 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=267782 primary_flush_worker_slots=64 primary_flush_coalesced_count=4 wal_fsync_blocked_ms=4642 wal_priority_yield_count=10260 background_io_yield_count=105869 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=7360 foreground_admission_delay_total_ms=542202 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=4003 foreground_pressure_state_elapsed_ms=35570 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=78663
writes_test: elapsed=409s elapsed_hms=0:06:49 docs=8963328 txns=35014 writes/s=6041 avg_writes/s=21915 p50=740ms p99=8898ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=2ms dbms_insert=325218ms batch_coord=1785ms wal_commit=4073898ms wal_fsync=277259ms wal_fsync_wait=287938ms wal_fsync_actual=277259ms wal_fsync_avg_ms=165 wal_fsync_p95_ms=256 wal_fsync_p99_ms=10000 wal_fsync_max_ms=10000 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=8102 wal_fsync_blocked_by_doc_store_sync_ms=943 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=11516 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=9 wal_fsync_active_elapsed_ms=4 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=328 fsyncs/s=335 wal_fsync_q=62 wal_group_size=2 wal_group_fill=4 wal_bytes_per_fsync=19554 wal_batches/s=527 wal_bytes/s=4267057 avg_writers_per_fsync=2 avg_bytes_per_fsync=14542 wal_writer_in_flight=23 pending_waiters=0 pending_wal_bytes=137434 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=30078 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=493333 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=35053 durable_lsn_after=35054 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=16ms waiter_broadcasts=493333 fsync_batch_reason=timeout last_waiters_released_in_fsync=60 durable_lsn=35054 durable_lsn_advance=4270 durable_lsn_advance_bytes=24359351 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=4MB/s primary_flush_io=13MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=7054819328 background_io_paused=80298ms background_pause_reason=wal_protected emergency_wal_only_count=112 doc_store_sync=758ms doc_store_sync_segments_window=121 doc_store_sync_segment_avg_ms=6 doc_store_sync_segment_max_ms=121 doc_store_sync_segment_bytes=4876743 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=36 doc_store_sync_pending_segments=37 doc_store_sync_pending_bytes=55491316 doc_store_sync_pending_lsn=34830 doc_store_durable_lsn=34828 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=9276 doc_store_sync_debt_bytes=55491316 doc_store_sync_jobs_merged=233 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=181027 doc_store_sync_physical_ms=10679 doc_store_sync_durable_lsn_advance=1108728 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=962302890 wal_pinned_by_checkpoint_bytes=6159181459 checkpoint_sync=0ms checkpoint_pause_reason=wal_fsync_wait checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=350 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1376256 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=192733405 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=230528ms wait=208375ms wal=5873MB wal_live_bytes=6159472947 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=305MB flush_q=406 primary_q=64 secondary_q=0 primary_io=1308052ms primary_pages=20413 primary_flush_batch_bytes=72471916 primary_flush_batch_entries=109373 avg_flush_batch_bytes=1276927 coalesced_tables=1 coalesced_batches=2 small_flush_count=5 small_flush_reason=pressure active_primary_flush_workers=64 per_table_flush_inflight=64 flush_worker_io_wait=0ms flush_worker_cpu=63285ms primary_slice_max=5727ms primary_blocked=5314259ms primary_flush_admission_wait_ms=407758 primary_flush_queued_age_ms=1269183 primary_flush_oldest_queued_age_ms=42697 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=217023 flush_retry=0 flush_coalesced=33 flush_dedup=38 max_flush_wait=43909ms index_log=5488MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=23249 sec_apply/s=11752 text_apply/s=11497 idx_flush/s=0 idx_busy=226208ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=5ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=86 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=407758 primary_flush_worker_slots=64 primary_flush_coalesced_count=2 wal_fsync_blocked_ms=17915 wal_priority_yield_count=6656 background_io_yield_count=100040 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=10600 foreground_admission_delay_total_ms=552802 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=4003 foreground_pressure_state_elapsed_ms=40693 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=323004
writes_test: elapsed=414s elapsed_hms=0:06:54 docs=9018368 txns=35229 writes/s=11008 avg_writes/s=21783 p50=534ms p99=3164ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=7ms dbms_insert=155918ms batch_coord=2273ms wal_commit=371322ms wal_fsync=89116ms wal_fsync_wait=122157ms wal_fsync_actual=89116ms wal_fsync_avg_ms=22 wal_fsync_p95_ms=256 wal_fsync_p99_ms=256 wal_fsync_max_ms=512 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=5883 wal_fsync_blocked_by_doc_store_sync_ms=516 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=4444 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=16 wal_fsync_active_elapsed_ms=12 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=800 fsyncs/s=799 wal_fsync_q=61 wal_group_size=2 wal_group_fill=4 wal_bytes_per_fsync=18570 wal_batches/s=965 wal_bytes/s=7876860 avg_writers_per_fsync=1 avg_bytes_per_fsync=9831 wal_writer_in_flight=17 pending_waiters=14 pending_wal_bytes=95910 wal_append_lock_wait_ms=3 wal_writer_sleep_ms=60283 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=497330 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=35266 durable_lsn_after=35268 oldest_waiter_age=18ms avg_waiter_age=11ms max_waiter_age=13ms waiter_broadcasts=497330 fsync_batch_reason=timeout last_waiters_released_in_fsync=60 durable_lsn=35268 durable_lsn_advance=6867 durable_lsn_advance_bytes=39295175 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=27MB/s wal_io=7MB/s primary_flush_io=20MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=16739467264 background_io_paused=23461ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=550ms doc_store_sync_segments_window=197 doc_store_sync_segment_avg_ms=2 doc_store_sync_segment_max_ms=11 doc_store_sync_segment_bytes=2979065 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=7 doc_store_sync_pending_segments=9 doc_store_sync_pending_bytes=11873023 doc_store_sync_pending_lsn=35171 doc_store_durable_lsn=35171 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=280 doc_store_sync_debt_bytes=11873023 doc_store_sync_jobs_merged=233 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=199128 doc_store_sync_physical_ms=11229 doc_store_sync_durable_lsn_advance=1114878 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=193272437 wal_pinned_by_checkpoint_bytes=6196481867 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=350 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1376256 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=193890344 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=457962ms wait=420784ms wal=5907MB wal_live_bytes=6196757230 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=255MB flush_q=419 primary_q=62 secondary_q=0 primary_io=1349195ms primary_pages=20413 primary_flush_batch_bytes=107767956 primary_flush_batch_entries=169587 avg_flush_batch_bytes=1269710 coalesced_tables=3 coalesced_batches=6 small_flush_count=12 small_flush_reason=pressure active_primary_flush_workers=62 per_table_flush_inflight=63 flush_worker_io_wait=0ms flush_worker_cpu=41143ms primary_slice_max=5727ms primary_blocked=5608027ms primary_flush_admission_wait_ms=293768 primary_flush_queued_age_ms=2152775 primary_flush_oldest_queued_age_ms=47718 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=230108 flush_retry=0 flush_coalesced=39 flush_dedup=51 max_flush_wait=45725ms index_log=5521MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=21523 sec_apply/s=10635 text_apply/s=10888 idx_flush/s=0 idx_busy=226281ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=5ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=78 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=293768 primary_flush_worker_slots=64 primary_flush_coalesced_count=6 wal_fsync_blocked_ms=6869 wal_priority_yield_count=5400 background_io_yield_count=18367 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=17140 foreground_admission_delay_total_ms=569942 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=4003 foreground_pressure_state_elapsed_ms=45713 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=160378
writes_test: elapsed=419s elapsed_hms=0:06:59 docs=9055488 txns=35374 writes/s=7424 avg_writes/s=21612 p50=1025ms p99=2074ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=2ms dbms_insert=153784ms batch_coord=597ms wal_commit=2120615ms wal_fsync=135140ms wal_fsync_wait=147003ms wal_fsync_actual=135140ms wal_fsync_avg_ms=127 wal_fsync_p95_ms=1000 wal_fsync_p99_ms=2000 wal_fsync_max_ms=2000 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=8730 wal_fsync_blocked_by_doc_store_sync_ms=82 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=6204 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=32 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=217 fsyncs/s=212 wal_fsync_q=52 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=15173 wal_batches/s=776 wal_bytes/s=5369800 avg_writers_per_fsync=4 avg_bytes_per_fsync=24517 wal_writer_in_flight=148 pending_waiters=141 pending_wal_bytes=711018 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=44007 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=498394 lost_wakeup_count=0 durable_lsn_gap=1 durable_lsn_before=35408 durable_lsn_after=35410 oldest_waiter_age=79ms avg_waiter_age=45ms max_waiter_age=13ms waiter_broadcasts=498394 fsync_batch_reason=none last_waiters_released_in_fsync=51 durable_lsn=35410 durable_lsn_advance=4516 durable_lsn_advance_bytes=26086375 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=28MB/s wal_io=5MB/s primary_flush_io=23MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=4441767936 background_io_paused=55724ms background_pause_reason=wal_protected emergency_wal_only_count=220 doc_store_sync=246ms doc_store_sync_segments_window=113 doc_store_sync_segment_avg_ms=2 doc_store_sync_segment_max_ms=8 doc_store_sync_segment_bytes=4468214 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=85 doc_store_sync_pending_segments=57 doc_store_sync_pending_bytes=89482842 doc_store_sync_pending_lsn=35342 doc_store_durable_lsn=35209 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=3 doc_store_sync_oldest_age_ms=3241 doc_store_sync_debt_bytes=89482842 doc_store_sync_jobs_merged=336 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=240551 doc_store_sync_physical_ms=11475 doc_store_sync_durable_lsn_advance=1119411 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=2526157302 wal_pinned_by_checkpoint_bytes=6221524966 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=350 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1376256 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=194640556 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=265874ms wait=236704ms wal=5931MB wal_live_bytes=6221575455 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=143MB flush_q=349 primary_q=64 secondary_q=0 primary_io=1388700ms primary_pages=20413 primary_flush_batch_bytes=125762093 primary_flush_batch_entries=192235 avg_flush_batch_bytes=1251113 coalesced_tables=1 coalesced_batches=2 small_flush_count=43 small_flush_reason=pressure active_primary_flush_workers=64 per_table_flush_inflight=64 flush_worker_io_wait=0ms flush_worker_cpu=39505ms primary_slice_max=5727ms primary_blocked=5951533ms primary_flush_admission_wait_ms=343506 primary_flush_queued_age_ms=2721011 primary_flush_oldest_queued_age_ms=48656 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=243646 flush_retry=0 flush_coalesced=44 flush_dedup=71 max_flush_wait=51410ms index_log=5544MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=17578 sec_apply/s=8665 text_apply/s=8912 idx_flush/s=0 idx_busy=226329ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=3ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=79 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=343506 primary_flush_worker_slots=64 primary_flush_coalesced_count=5 wal_fsync_blocked_ms=8769 wal_priority_yield_count=6837 background_io_yield_count=82698 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=11380 foreground_admission_delay_total_ms=581322 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=4003 foreground_pressure_state_elapsed_ms=50729 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=158927
writes_test: elapsed=424s elapsed_hms=0:07:04 docs=9060096 txns=35400 writes/s=921 avg_writes/s=21368 p50=1422ms p99=3985ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=2ms dbms_insert=79525ms batch_coord=0ms wal_commit=109618ms wal_fsync=21338ms wal_fsync_wait=21562ms wal_fsync_actual=21338ms wal_fsync_avg_ms=260 wal_fsync_p95_ms=2000 wal_fsync_p99_ms=5000 wal_fsync_max_ms=5000 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=15427 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=534 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=32 wal_fsync_active_elapsed_ms=1948 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=16 fsyncs/s=16 wal_fsync_q=97 wal_group_size=7 wal_group_fill=13 wal_bytes_per_fsync=47724 wal_batches/s=99 wal_bytes/s=795551 avg_writers_per_fsync=2 avg_bytes_per_fsync=16050 wal_writer_in_flight=617 pending_waiters=590 pending_wal_bytes=3412401 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=5297 pending_waiters_before_fsync=7 waiters_selected_for_batch=7 eligible_waiters=8 released_waiters=8 wakeup_batches=498476 lost_wakeup_count=0 durable_lsn_gap=1 durable_lsn_before=35421 durable_lsn_after=35423 oldest_waiter_age=7504ms avg_waiter_age=5319ms max_waiter_age=2572ms waiter_broadcasts=498476 fsync_batch_reason=none last_waiters_released_in_fsync=121 durable_lsn=35423 durable_lsn_advance=226 durable_lsn_advance_bytes=1316137 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=0MB/s primary_flush_io=17MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=352321536 background_io_paused=56758ms background_pause_reason=wal_protected emergency_wal_only_count=20 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=8 doc_store_sync_segment_bytes=4468214 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=231 doc_store_sync_pending_segments=97 doc_store_sync_pending_bytes=174772696 doc_store_sync_pending_lsn=35407 doc_store_durable_lsn=35209 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=3 doc_store_sync_oldest_age_ms=10971 doc_store_sync_debt_bytes=174772696 doc_store_sync_jobs_merged=479 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=240552 doc_store_sync_physical_ms=11475 doc_store_sync_durable_lsn_advance=1119411 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=4668172909 wal_pinned_by_checkpoint_bytes=6222946172 checkpoint_sync=0ms checkpoint_pause_reason=wal_fsync_wait checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=350 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1376256 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=194688861 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=14344ms wait=12441ms wal=5933MB wal_live_bytes=6222968663 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=61MB flush_q=239 primary_q=62 secondary_q=0 primary_io=1431159ms primary_pages=20413 primary_flush_batch_bytes=90286772 primary_flush_batch_entries=137270 avg_flush_batch_bytes=1232437 coalesced_tables=0 coalesced_batches=0 small_flush_count=62 small_flush_reason=pressure active_primary_flush_workers=62 per_table_flush_inflight=61 flush_worker_io_wait=0ms flush_worker_cpu=42459ms primary_slice_max=5727ms primary_blocked=6267995ms primary_flush_admission_wait_ms=316462 primary_flush_queued_age_ms=2524957 primary_flush_oldest_queued_age_ms=53073 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=257298 flush_retry=0 flush_coalesced=46 flush_dedup=82 max_flush_wait=53519ms index_log=5548MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=653 sec_apply/s=373 text_apply/s=280 idx_flush/s=0 idx_busy=226332ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=5ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=64 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=316462 primary_flush_worker_slots=64 primary_flush_coalesced_count=2 wal_fsync_blocked_ms=750 wal_priority_yield_count=11668 background_io_yield_count=167012 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=1060 foreground_admission_delay_total_ms=582382 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=4003 foreground_pressure_state_elapsed_ms=58354 foreground_pressure_time_normal_window_ms=0 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=22802
writes_test: elapsed=431s elapsed_hms=0:07:11 docs=9107456 txns=35576 writes/s=9472 avg_writes/s=21130 p50=576ms p99=9458ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=6ms dbms_insert=330454ms batch_coord=4554ms wal_commit=5058191ms wal_fsync=322497ms wal_fsync_wait=358133ms wal_fsync_actual=322497ms wal_fsync_avg_ms=116 wal_fsync_p95_ms=256 wal_fsync_p99_ms=10000 wal_fsync_max_ms=10000 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=6022 wal_fsync_blocked_by_doc_store_sync_ms=1733 wal_fsync_blocked_by_checkpoint_ms=16 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=13997 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=22 wal_fsync_active_elapsed_ms=6 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=550 fsyncs/s=553 wal_fsync_q=53 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=12400 wal_batches/s=775 wal_bytes/s=6564230 avg_writers_per_fsync=2 avg_bytes_per_fsync=13098 wal_writer_in_flight=53 pending_waiters=20 pending_wal_bytes=153129 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=48084 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=501242 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=35610 durable_lsn_after=35612 oldest_waiter_age=20ms avg_waiter_age=20ms max_waiter_age=13ms waiter_broadcasts=501242 fsync_batch_reason=timeout last_waiters_released_in_fsync=53 durable_lsn=35612 durable_lsn_advance=6229 durable_lsn_advance_bytes=36230366 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=20MB/s wal_io=6MB/s primary_flush_io=14MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=11626610688 background_io_paused=31400ms background_pause_reason=wal_protected emergency_wal_only_count=66 doc_store_sync=1332ms doc_store_sync_segments_window=221 doc_store_sync_segment_avg_ms=6 doc_store_sync_segment_max_ms=26 doc_store_sync_segment_bytes=8011316 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=35570 doc_store_durable_lsn=35570 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=4 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=742 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=680265 doc_store_sync_physical_ms=12807 doc_store_sync_durable_lsn_advance=1133528 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=6256847963 checkpoint_sync=72ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=72 checkpoint_sync_ms_per_node=18ms checkpoint_sync_blocked_window_ms=72 checkpoint_sync_blocked_total_ms=422 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=72ms checkpoint_slice_avg_ms=72 checkpoint_slice_last_ms=72 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=2097152 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=195737539 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=136891ms wait=108199ms wal=5965MB wal_live_bytes=6257048975 wal_truncatable_bytes=201012 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=1MB flush_q=0 primary_q=0 secondary_q=0 primary_io=1473622ms primary_pages=22216 primary_flush_batch_bytes=78483074 primary_flush_batch_entries=119620 avg_flush_batch_bytes=1177998 coalesced_tables=2 coalesced_batches=4 small_flush_count=185 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=18 flush_worker_io_wait=0ms flush_worker_cpu=42463ms primary_slice_max=5727ms primary_blocked=6551132ms primary_flush_admission_wait_ms=283137 primary_flush_queued_age_ms=4625768 primary_flush_oldest_queued_age_ms=29073 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=265189 flush_retry=0 flush_coalesced=63 flush_dedup=98 max_flush_wait=54341ms index_log=5576MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=25217 sec_apply/s=12736 text_apply/s=12481 idx_flush/s=0 idx_busy=226369ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=5ms rejects=0 checkpoint_sync_blocked_estimate_ms=72 checkpoint_yield_count=80 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=283137 primary_flush_worker_slots=64 primary_flush_coalesced_count=17 wal_fsync_blocked_ms=22109 wal_priority_yield_count=5869 background_io_yield_count=89314 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=13440 foreground_admission_delay_total_ms=595822 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=59 foreground_pressure_transitions_total=4062 foreground_pressure_state_elapsed_ms=63388 foreground_pressure_time_normal_window_ms=1969 foreground_pressure_time_warm_window_ms=6564 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=23519 foreground_pressure_time_emergency_window_ms=355255
writes_test: elapsed=436s elapsed_hms=0:07:16 docs=9171968 txns=35829 writes/s=12902 avg_writes/s=21036 p50=466ms p99=2384ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=11ms dbms_insert=152374ms batch_coord=10646ms wal_commit=548085ms wal_fsync=99271ms wal_fsync_wait=133959ms wal_fsync_actual=99271ms wal_fsync_avg_ms=24 wal_fsync_p95_ms=256 wal_fsync_p99_ms=256 wal_fsync_max_ms=512 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=6296 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=5143 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=27 wal_fsync_active_elapsed_ms=4 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=794 fsyncs/s=795 wal_fsync_q=56 wal_group_size=1 wal_group_fill=1 wal_bytes_per_fsync=8237 wal_batches/s=1138 wal_bytes/s=9512372 avg_writers_per_fsync=2 avg_bytes_per_fsync=11961 wal_writer_in_flight=51 pending_waiters=19 pending_wal_bytes=116244 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=63205 pending_waiters_before_fsync=1 waiters_selected_for_batch=1 eligible_waiters=2 released_waiters=2 wakeup_batches=505218 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=35869 durable_lsn_after=35870 oldest_waiter_age=32ms avg_waiter_age=24ms max_waiter_age=10ms waiter_broadcasts=505218 fsync_batch_reason=none last_waiters_released_in_fsync=34 durable_lsn=35870 durable_lsn_advance=8220 durable_lsn_advance_bytes=47557087 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=9MB/s wal_io=9MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=16659775488 background_io_paused=11432ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=1ms doc_store_sync_segments_window=1 doc_store_sync_segment_avg_ms=1 doc_store_sync_segment_max_ms=26 doc_store_sync_segment_bytes=8011316 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=35607 doc_store_durable_lsn=35607 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=4 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=742 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=680265 doc_store_sync_physical_ms=12808 doc_store_sync_durable_lsn_advance=1133931 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=6301880432 checkpoint_sync=0ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=422 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=72 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=2097152 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=197141473 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=35548ms wait=392ms wal=6008MB wal_live_bytes=6302168519 wal_truncatable_bytes=288087 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=1MB flush_q=0 primary_q=0 secondary_q=0 primary_io=1473687ms primary_pages=22216 primary_flush_batch_bytes=1048578 primary_flush_batch_entries=1603 avg_flush_batch_bytes=1177998 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=17 flush_worker_io_wait=0ms flush_worker_cpu=65ms primary_slice_max=5727ms primary_blocked=6551132ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=29073 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=265189 flush_retry=0 flush_coalesced=63 flush_dedup=98 max_flush_wait=54341ms index_log=5615MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=24271 sec_apply/s=12263 text_apply/s=12007 idx_flush/s=0 idx_busy=226373ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=4ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=75 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=7148 wal_priority_yield_count=5581 background_io_yield_count=820 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=17470 foreground_admission_delay_total_ms=613292 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=216 foreground_pressure_transitions_total=4278 foreground_pressure_state_elapsed_ms=68406 foreground_pressure_time_normal_window_ms=30634 foreground_pressure_time_warm_window_ms=54357 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=10249 foreground_pressure_time_emergency_window_ms=65548
writes_test: elapsed=441s elapsed_hms=0:07:21 docs=9239808 txns=36096 writes/s=13568 avg_writes/s=20951 p50=471ms p99=2498ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=10ms dbms_insert=147757ms batch_coord=9997ms wal_commit=399369ms wal_fsync=85598ms wal_fsync_wait=129207ms wal_fsync_actual=85598ms wal_fsync_avg_ms=19 wal_fsync_p95_ms=256 wal_fsync_p99_ms=256 wal_fsync_max_ms=512 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=5836 wal_fsync_blocked_by_doc_store_sync_ms=400 wal_fsync_blocked_by_checkpoint_ms=13 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=4454 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=32 wal_fsync_active_elapsed_ms=12 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=887 fsyncs/s=884 wal_fsync_q=65 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=14563 wal_batches/s=1079 wal_bytes/s=9834927 avg_writers_per_fsync=1 avg_bytes_per_fsync=10895 wal_writer_in_flight=227 pending_waiters=191 pending_wal_bytes=477732 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=56700 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=2 released_waiters=2 wakeup_batches=509640 lost_wakeup_count=0 durable_lsn_gap=1 durable_lsn_before=36129 durable_lsn_after=36130 oldest_waiter_age=355ms avg_waiter_age=132ms max_waiter_age=58ms waiter_broadcasts=509640 fsync_batch_reason=none last_waiters_released_in_fsync=45 durable_lsn=36130 durable_lsn_advance=8331 durable_lsn_advance_bytes=48177930 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=20MB/s wal_io=9MB/s primary_flush_io=11MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=18538823680 background_io_paused=21963ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=315ms doc_store_sync_segments_window=110 doc_store_sync_segment_avg_ms=2 doc_store_sync_segment_max_ms=2 doc_store_sync_segment_bytes=2629806 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=11 doc_store_sync_pending_segments=7 doc_store_sync_pending_bytes=10273964 doc_store_sync_pending_lsn=36111 doc_store_durable_lsn=36107 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=406 doc_store_sync_debt_bytes=10273964 doc_store_sync_jobs_merged=780 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=686864 doc_store_sync_physical_ms=13123 doc_store_sync_durable_lsn_advance=1153911 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=595364674 wal_pinned_by_checkpoint_bytes=6347924312 checkpoint_sync=10ms checkpoint_pause_reason=wal_fsync_wait checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=10 checkpoint_sync_ms_per_node=2ms checkpoint_sync_blocked_window_ms=10 checkpoint_sync_blocked_total_ms=432 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=2 checkpoint_slice_count_window=2 checkpoint_slices_per_node=0 checkpoint_sync_slice=10ms checkpoint_slice_avg_ms=5 checkpoint_slice_last_ms=6 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1638400 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=198590389 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=75344ms wait=43538ms wal=6051MB wal_live_bytes=6347929182 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=1MB flush_q=0 primary_q=0 secondary_q=0 primary_io=1485662ms primary_pages=22216 primary_flush_batch_bytes=58936198 primary_flush_batch_entries=90065 avg_flush_batch_bytes=1166507 coalesced_tables=1 coalesced_batches=2 small_flush_count=47 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=4ms flush_worker_cpu=11971ms primary_slice_max=5727ms primary_blocked=6554269ms primary_flush_admission_wait_ms=3137 primary_flush_queued_age_ms=50 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=265191 flush_retry=0 flush_coalesced=64 flush_dedup=98 max_flush_wait=54341ms index_log=5657MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=21716 sec_apply/s=10986 text_apply/s=10730 idx_flush/s=0 idx_busy=226402ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=3ms rejects=0 checkpoint_sync_blocked_estimate_ms=10 checkpoint_yield_count=76 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=3137 primary_flush_worker_slots=64 primary_flush_coalesced_count=1 wal_fsync_blocked_ms=6592 wal_priority_yield_count=5561 background_io_yield_count=17328 foreground_latency_guard_trigger_count=0 foreground_pressure_state=recovery foreground_pressure_source=wal_waiter_age foreground_admission_delay_window_ms=13591 foreground_admission_delay_total_ms=626883 foreground_last_admission_delay_ms=2 foreground_pressure_transitions_window=295 foreground_pressure_transitions_total=4573 foreground_pressure_state_elapsed_ms=1387 foreground_pressure_time_normal_window_ms=38944 foreground_pressure_time_warm_window_ms=62101 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=18816 foreground_pressure_time_emergency_window_ms=36466
writes_test: elapsed=446s elapsed_hms=0:07:26 docs=9310976 txns=36372 writes/s=14233 avg_writes/s=20876 p50=464ms p99=2458ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=12ms dbms_insert=157340ms batch_coord=11467ms wal_commit=562541ms wal_fsync=101133ms wal_fsync_wait=145400ms wal_fsync_actual=101133ms wal_fsync_avg_ms=21 wal_fsync_p95_ms=256 wal_fsync_p99_ms=256 wal_fsync_max_ms=512 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=5970 wal_fsync_blocked_by_doc_store_sync_ms=95 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=5259 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=29 wal_fsync_active_elapsed_ms=3 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=929 fsyncs/s=931 wal_fsync_q=53 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=9649 wal_batches/s=1169 wal_bytes/s=10391914 avg_writers_per_fsync=1 avg_bytes_per_fsync=11358 wal_writer_in_flight=63 pending_waiters=35 pending_wal_bytes=173697 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=61429 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=514297 lost_wakeup_count=0 durable_lsn_gap=1 durable_lsn_before=36410 durable_lsn_after=36411 oldest_waiter_age=35ms avg_waiter_age=29ms max_waiter_age=15ms waiter_broadcasts=514297 fsync_batch_reason=none last_waiters_released_in_fsync=37 durable_lsn=36411 durable_lsn_advance=9023 durable_lsn_advance_bytes=52895807 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=14MB/s wal_io=9MB/s primary_flush_io=5MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=19583205376 background_io_paused=17423ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=181ms doc_store_sync_segments_window=70 doc_store_sync_segment_avg_ms=2 doc_store_sync_segment_max_ms=2 doc_store_sync_segment_bytes=2629806 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=2 doc_store_sync_pending_segments=4 doc_store_sync_pending_bytes=5154875 doc_store_sync_pending_lsn=36398 doc_store_durable_lsn=36369 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=45 doc_store_sync_debt_bytes=5154875 doc_store_sync_jobs_merged=795 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=690584 doc_store_sync_physical_ms=13304 doc_store_sync_durable_lsn_advance=1158735 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=399822248 wal_pinned_by_checkpoint_bytes=6397634051 checkpoint_sync=0ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=432 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=6 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1638400 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=200129638 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=29751ms wait=10360ms wal=6099MB wal_live_bytes=6397879690 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=2MB flush_q=2 primary_q=2 secondary_q=0 primary_io=1489675ms primary_pages=22216 primary_flush_batch_bytes=27355036 primary_flush_batch_entries=40677 avg_flush_batch_bytes=1160508 coalesced_tables=0 coalesced_batches=0 small_flush_count=23 small_flush_reason=pressure active_primary_flush_workers=2 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=4013ms primary_slice_max=5727ms primary_blocked=6554269ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=6 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=265191 flush_retry=0 flush_coalesced=64 flush_dedup=98 max_flush_wait=54341ms index_log=5701MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=29381 sec_apply/s=14562 text_apply/s=14818 idx_flush/s=0 idx_busy=226410ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=3ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=78 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=7334 wal_priority_yield_count=5562 background_io_yield_count=8024 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=none foreground_admission_delay_window_ms=540 foreground_admission_delay_total_ms=627423 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=424 foreground_pressure_transitions_total=4997 foreground_pressure_state_elapsed_ms=659 foreground_pressure_time_normal_window_ms=65761 foreground_pressure_time_warm_window_ms=99061 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=2655 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=451s elapsed_hms=0:07:31 docs=9373440 txns=36616 writes/s=12492 avg_writes/s=20783 p50=508ms p99=2158ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=11ms dbms_insert=149693ms batch_coord=6153ms wal_commit=823335ms wal_fsync=109587ms wal_fsync_wait=137788ms wal_fsync_actual=109587ms wal_fsync_avg_ms=34 wal_fsync_p95_ms=256 wal_fsync_p99_ms=512 wal_fsync_max_ms=1000 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=6991 wal_fsync_blocked_by_doc_store_sync_ms=322 wal_fsync_blocked_by_checkpoint_ms=23 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=5383 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=1 wal_fsync_active_elapsed_ms=8 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=625 fsyncs/s=630 wal_fsync_q=85 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=23622 wal_batches/s=1012 wal_bytes/s=9085240 avg_writers_per_fsync=2 avg_bytes_per_fsync=14465 wal_writer_in_flight=0 pending_waiters=0 pending_wal_bytes=0 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=54643 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=517449 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=36651 durable_lsn_after=36654 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=21ms waiter_broadcasts=517449 fsync_batch_reason=timeout last_waiters_released_in_fsync=85 durable_lsn=36654 durable_lsn_advance=7752 durable_lsn_advance_bytes=45595616 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=19MB/s wal_io=8MB/s primary_flush_io=11MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=13149143040 background_io_paused=27550ms background_pause_reason=wal_protected emergency_wal_only_count=37 doc_store_sync=359ms doc_store_sync_segments_window=113 doc_store_sync_segment_avg_ms=3 doc_store_sync_segment_max_ms=53 doc_store_sync_segment_bytes=2623540 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=12 doc_store_sync_pending_segments=8 doc_store_sync_pending_bytes=8823788 doc_store_sync_pending_lsn=36634 doc_store_durable_lsn=36634 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=570 doc_store_sync_debt_bytes=8823788 doc_store_sync_jobs_merged=828 doc_store_sync_jobs_suppressed=2 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=699673 doc_store_sync_physical_ms=13663 doc_store_sync_durable_lsn_advance=1168532 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=402944704 wal_pinned_by_checkpoint_bytes=6440741877 checkpoint_sync=41ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=41 checkpoint_sync_ms_per_node=10ms checkpoint_sync_blocked_window_ms=41 checkpoint_sync_blocked_total_ms=473 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=2 checkpoint_slice_count_window=2 checkpoint_slices_per_node=0 checkpoint_sync_slice=41ms checkpoint_slice_avg_ms=20 checkpoint_slice_last_ms=7 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=425984 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=201271864 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=34111ms wait=16392ms wal=6141MB wal_live_bytes=6440971137 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=2MB flush_q=2 primary_q=2 secondary_q=0 primary_io=1497985ms primary_pages=22216 primary_flush_batch_bytes=61020281 primary_flush_batch_entries=90689 avg_flush_batch_bytes=1145703 coalesced_tables=1 coalesced_batches=2 small_flush_count=46 small_flush_reason=pressure active_primary_flush_workers=2 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=8310ms primary_slice_max=5727ms primary_blocked=6554401ms primary_flush_admission_wait_ms=132 primary_flush_queued_age_ms=101 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=265196 flush_retry=0 flush_coalesced=65 flush_dedup=98 max_flush_wait=54341ms index_log=5739MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=22528 sec_apply/s=11264 text_apply/s=11264 idx_flush/s=0 idx_busy=226422ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=3ms rejects=0 checkpoint_sync_blocked_estimate_ms=41 checkpoint_yield_count=67 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=132 primary_flush_worker_slots=64 primary_flush_coalesced_count=1 wal_fsync_blocked_ms=7957 wal_priority_yield_count=6138 background_io_yield_count=22668 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=none foreground_admission_delay_window_ms=1164 foreground_admission_delay_total_ms=628587 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=406 foreground_pressure_transitions_total=5403 foreground_pressure_state_elapsed_ms=445 foreground_pressure_time_normal_window_ms=60092 foreground_pressure_time_warm_window_ms=100564 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=456s elapsed_hms=0:07:36 docs=9434368 txns=36853 writes/s=12185 avg_writes/s=20689 p50=423ms p99=2483ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=14ms dbms_insert=140413ms batch_coord=13322ms wal_commit=415370ms wal_fsync=75117ms wal_fsync_wait=108020ms wal_fsync_actual=75117ms wal_fsync_avg_ms=22 wal_fsync_p95_ms=256 wal_fsync_p99_ms=256 wal_fsync_max_ms=1000 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=6172 wal_fsync_blocked_by_doc_store_sync_ms=279 wal_fsync_blocked_by_checkpoint_ms=3 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=3899 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=32 wal_fsync_active_elapsed_ms=17 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=671 fsyncs/s=664 wal_fsync_q=79 wal_group_size=7 wal_group_fill=13 wal_bytes_per_fsync=38111 wal_batches/s=941 wal_bytes/s=8968788 avg_writers_per_fsync=2 avg_bytes_per_fsync=12204 wal_writer_in_flight=771 pending_waiters=758 pending_wal_bytes=4222385 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=52384 pending_waiters_before_fsync=7 waiters_selected_for_batch=7 eligible_waiters=6 released_waiters=6 wakeup_batches=520773 lost_wakeup_count=0 durable_lsn_gap=1 durable_lsn_before=36868 durable_lsn_after=36871 oldest_waiter_age=877ms avg_waiter_age=391ms max_waiter_age=288ms waiter_broadcasts=520773 fsync_batch_reason=timeout last_waiters_released_in_fsync=68 durable_lsn=36871 durable_lsn_advance=6937 durable_lsn_advance_bytes=40567220 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=19MB/s wal_io=8MB/s primary_flush_io=11MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=13950255104 background_io_paused=18402ms background_pause_reason=wal_protected emergency_wal_only_count=2 doc_store_sync=372ms doc_store_sync_segments_window=133 doc_store_sync_segment_avg_ms=2 doc_store_sync_segment_max_ms=21 doc_store_sync_segment_bytes=2739923 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=36843 doc_store_durable_lsn=36843 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=868 doc_store_sync_jobs_suppressed=3 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=705009 doc_store_sync_physical_ms=14035 doc_store_sync_durable_lsn_advance=1175625 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=6479598995 checkpoint_sync=7ms checkpoint_pause_reason=wal_fsync_wait checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=7 checkpoint_sync_ms_per_node=1ms checkpoint_sync_blocked_window_ms=7 checkpoint_sync_blocked_total_ms=480 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=2 checkpoint_slice_count_window=2 checkpoint_slices_per_node=0 checkpoint_sync_slice=7ms checkpoint_slice_avg_ms=3 checkpoint_slice_last_ms=3 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1425408 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=202656223 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=26957ms wait=12075ms wal=6177MB wal_live_bytes=6479598995 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=1503556ms primary_pages=22216 primary_flush_batch_bytes=57878220 primary_flush_batch_entries=84371 avg_flush_batch_bytes=1134420 coalesced_tables=1 coalesced_batches=2 small_flush_count=45 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=1 flush_worker_io_wait=0ms flush_worker_cpu=5571ms primary_slice_max=5727ms primary_blocked=6554401ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=63 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=265199 flush_retry=0 flush_coalesced=66 flush_dedup=98 max_flush_wait=54341ms index_log=5776MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=25600 sec_apply/s=12673 text_apply/s=12926 idx_flush/s=0 idx_busy=226430ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=3ms rejects=0 checkpoint_sync_blocked_estimate_ms=7 checkpoint_yield_count=60 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=1 wal_fsync_blocked_ms=5733 wal_priority_yield_count=5632 background_io_yield_count=12160 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=wal_waiter_age foreground_admission_delay_window_ms=413 foreground_admission_delay_total_ms=629000 foreground_last_admission_delay_ms=5 foreground_pressure_transitions_window=323 foreground_pressure_transitions_total=5726 foreground_pressure_state_elapsed_ms=1444 foreground_pressure_time_normal_window_ms=57057 foreground_pressure_time_warm_window_ms=76570 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=461s elapsed_hms=0:07:41 docs=9436160 txns=36860 writes/s=358 avg_writes/s=20468 p50=3651ms p99=3825ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=1ms dbms_insert=25620ms batch_coord=0ms wal_commit=270829ms wal_fsync=28751ms wal_fsync_wait=28866ms wal_fsync_actual=28751ms wal_fsync_avg_ms=1369 wal_fsync_p95_ms=5000 wal_fsync_p99_ms=5000 wal_fsync_max_ms=5000 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=9902 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=2667 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=29 wal_fsync_active_elapsed_ms=710 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=3 fsyncs/s=4 wal_fsync_q=174 wal_group_size=28 wal_group_fill=54 wal_bytes_per_fsync=146972 wal_batches/s=31 wal_bytes/s=275317 avg_writers_per_fsync=6 avg_bytes_per_fsync=34826 wal_writer_in_flight=867 pending_waiters=867 pending_wal_bytes=4921956 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=1508 pending_waiters_before_fsync=28 waiters_selected_for_batch=28 eligible_waiters=24 released_waiters=24 wakeup_batches=520794 lost_wakeup_count=0 durable_lsn_gap=1 durable_lsn_before=36889 durable_lsn_after=36895 oldest_waiter_age=4915ms avg_waiter_age=3829ms max_waiter_age=886ms waiter_broadcasts=520794 fsync_batch_reason=none last_waiters_released_in_fsync=93 durable_lsn=36895 durable_lsn_advance=128 durable_lsn_advance_bytes=731356 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=0MB/s wal_io=0MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=92274688 background_io_paused=12746ms background_pause_reason=wal_protected emergency_wal_only_count=29 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=21 doc_store_sync_segment_bytes=2739923 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=2 doc_store_sync_pending_segments=1 doc_store_sync_pending_bytes=1332154 doc_store_sync_pending_lsn=36878 doc_store_durable_lsn=36843 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=1348 doc_store_sync_debt_bytes=1332154 doc_store_sync_jobs_merged=869 doc_store_sync_jobs_suppressed=3 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=705009 doc_store_sync_physical_ms=14035 doc_store_sync_durable_lsn_advance=1175625 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=202549645 wal_pinned_by_checkpoint_bytes=6480733179 checkpoint_sync=0ms checkpoint_pause_reason=wal_fsync_wait checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=480 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=3 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1425408 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=202656223 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=813ms wait=0ms wal=6178MB wal_live_bytes=6480771744 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=1503611ms primary_pages=22216 primary_flush_batch_bytes=1049736 primary_flush_batch_entries=1599 avg_flush_batch_bytes=1134420 coalesced_tables=0 coalesced_batches=0 small_flush_count=1 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=1 flush_worker_io_wait=0ms flush_worker_cpu=55ms primary_slice_max=5727ms primary_blocked=6554401ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=265199 flush_retry=0 flush_coalesced=66 flush_dedup=98 max_flush_wait=54341ms index_log=5777MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=3743 sec_apply/s=1913 text_apply/s=1830 idx_flush/s=0 idx_busy=226430ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=4ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=71 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=4186 wal_priority_yield_count=7279 background_io_yield_count=1641 foreground_latency_guard_trigger_count=0 foreground_pressure_state=emergency foreground_pressure_source=wal_waiter_age foreground_admission_delay_window_ms=538 foreground_admission_delay_total_ms=629538 foreground_last_admission_delay_ms=20 foreground_pressure_transitions_window=15 foreground_pressure_transitions_total=5741 foreground_pressure_state_elapsed_ms=6454 foreground_pressure_time_normal_window_ms=7653 foreground_pressure_time_warm_window_ms=12559 foreground_pressure_time_constrained_window_ms=12570 foreground_pressure_time_recovery_window_ms=3 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=467s elapsed_hms=0:07:47 docs=9482496 txns=37041 writes/s=9267 avg_writes/s=20305 p50=333ms p99=8834ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=14ms dbms_insert=296304ms batch_coord=9797ms wal_commit=5875815ms wal_fsync=221625ms wal_fsync_wait=259815ms wal_fsync_actual=221625ms wal_fsync_avg_ms=67 wal_fsync_p95_ms=32 wal_fsync_p99_ms=256 wal_fsync_max_ms=10000 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=5295 wal_fsync_blocked_by_doc_store_sync_ms=37 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=8676 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=19 wal_fsync_active_elapsed_ms=6 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=656 fsyncs/s=660 wal_fsync_q=32 wal_group_size=1 wal_group_fill=2 wal_bytes_per_fsync=10285 wal_batches/s=679 wal_bytes/s=6530179 avg_writers_per_fsync=1 avg_bytes_per_fsync=11361 wal_writer_in_flight=36 pending_waiters=12 pending_wal_bytes=169271 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=42010 pending_waiters_before_fsync=1 waiters_selected_for_batch=1 eligible_waiters=3 released_waiters=3 wakeup_batches=524095 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=37074 durable_lsn_after=37075 oldest_waiter_age=20ms avg_waiter_age=20ms max_waiter_age=13ms waiter_broadcasts=524095 fsync_batch_reason=timeout last_waiters_released_in_fsync=46 durable_lsn=37075 durable_lsn_advance=6422 durable_lsn_advance_bytes=37502921 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=12MB/s wal_io=6MB/s primary_flush_io=6MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=13883146240 background_io_paused=13224ms background_pause_reason=wal_protected emergency_wal_only_count=51 doc_store_sync=137ms doc_store_sync_segments_window=63 doc_store_sync_segment_avg_ms=2 doc_store_sync_segment_max_ms=32 doc_store_sync_segment_bytes=3080174 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=1 doc_store_sync_pending_segments=1 doc_store_sync_pending_bytes=1467183 doc_store_sync_pending_lsn=37060 doc_store_durable_lsn=37040 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=3 doc_store_sync_debt_bytes=1467183 doc_store_sync_jobs_merged=872 doc_store_sync_jobs_suppressed=3 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=708436 doc_store_sync_physical_ms=14172 doc_store_sync_durable_lsn_advance=1181084 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=203615539 wal_pinned_by_checkpoint_bytes=6515220387 checkpoint_sync=0ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=480 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=3 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1425408 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=203805509 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=29426ms wait=14629ms wal=6211MB wal_live_bytes=6515556105 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=1MB flush_q=1 primary_q=1 secondary_q=0 primary_io=1508794ms primary_pages=22216 primary_flush_batch_bytes=32605161 primary_flush_batch_entries=49620 avg_flush_batch_bytes=1129696 coalesced_tables=0 coalesced_batches=0 small_flush_count=15 small_flush_reason=pressure active_primary_flush_workers=1 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=5183ms primary_slice_max=5727ms primary_blocked=6554814ms primary_flush_admission_wait_ms=413 primary_flush_queued_age_ms=24 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=265200 flush_retry=0 flush_coalesced=66 flush_dedup=98 max_flush_wait=54341ms index_log=5806MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=31114 sec_apply/s=15430 text_apply/s=15683 idx_flush/s=0 idx_busy=226452ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=5ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=85 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=413 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=13073 wal_priority_yield_count=4198 background_io_yield_count=3732 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=checkpoint_progress foreground_admission_delay_window_ms=1206 foreground_admission_delay_total_ms=630744 foreground_last_admission_delay_ms=1 foreground_pressure_transitions_window=132 foreground_pressure_transitions_total=5873 foreground_pressure_state_elapsed_ms=1763 foreground_pressure_time_normal_window_ms=89731 foreground_pressure_time_warm_window_ms=172218 foreground_pressure_time_constrained_window_ms=3550 foreground_pressure_time_recovery_window_ms=43525 foreground_pressure_time_emergency_window_ms=10194
writes_test: elapsed=472s elapsed_hms=0:07:52 docs=9559552 txns=37343 writes/s=15411 avg_writes/s=20253 p50=307ms p99=1992ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=37ms dbms_insert=134483ms batch_coord=22734ms wal_commit=157821ms wal_fsync=11750ms wal_fsync_wait=91250ms wal_fsync_actual=11750ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=199 wal_fsync_blocked_by_doc_store_sync_ms=12 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1318 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=7 wal_fsync_active_elapsed_ms=2 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1063 fsyncs/s=1064 wal_fsync_q=87 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=22347 wal_batches/s=1063 wal_bytes/s=11514187 avg_writers_per_fsync=1 avg_bytes_per_fsync=10817 wal_writer_in_flight=36 pending_waiters=9 pending_wal_bytes=204366 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=62275 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=529419 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=37379 durable_lsn_after=37382 oldest_waiter_age=27ms avg_waiter_age=24ms max_waiter_age=24ms waiter_broadcasts=529419 fsync_batch_reason=timeout last_waiters_released_in_fsync=84 durable_lsn=37382 durable_lsn_advance=9767 durable_lsn_advance_bytes=57593017 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=16MB/s wal_io=10MB/s primary_flush_io=6MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=22322085888 background_io_paused=10733ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=51ms doc_store_sync_segments_window=60 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=2 doc_store_sync_segment_bytes=3080174 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=37354 doc_store_durable_lsn=37354 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=873 doc_store_sync_jobs_suppressed=3 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=708443 doc_store_sync_physical_ms=14223 doc_store_sync_durable_lsn_advance=1189354 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=6569579521 checkpoint_sync=1ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=1 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=1 checkpoint_sync_blocked_total_ms=481 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=1ms checkpoint_slice_avg_ms=1 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=712704 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=205484335 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=51179ms wait=30744ms wal=6263MB wal_live_bytes=6569948264 wal_truncatable_bytes=368743 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=6MB flush_q=6 primary_q=6 secondary_q=0 primary_io=1514632ms primary_pages=22216 primary_flush_batch_bytes=33631841 primary_flush_batch_entries=50955 avg_flush_batch_bytes=1128520 coalesced_tables=0 coalesced_batches=0 small_flush_count=12 small_flush_reason=pressure active_primary_flush_workers=6 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=5838ms primary_slice_max=5727ms primary_blocked=6555875ms primary_flush_admission_wait_ms=1061 primary_flush_queued_age_ms=18 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=265200 flush_retry=0 flush_coalesced=66 flush_dedup=98 max_flush_wait=54341ms index_log=5853MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=30852 sec_apply/s=15298 text_apply/s=15553 idx_flush/s=0 idx_busy=226486ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=4ms rejects=0 checkpoint_sync_blocked_estimate_ms=1 checkpoint_yield_count=99 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=1061 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1463 wal_priority_yield_count=1638 background_io_yield_count=2285 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=none foreground_admission_delay_window_ms=157 foreground_admission_delay_total_ms=630901 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=20 foreground_pressure_transitions_total=5893 foreground_pressure_state_elapsed_ms=6783 foreground_pressure_time_normal_window_ms=156320 foreground_pressure_time_warm_window_ms=4274 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=477s elapsed_hms=0:07:57 docs=9650944 txns=37699 writes/s=18278 avg_writes/s=20232 p50=320ms p99=2057ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=83ms dbms_insert=147357ms batch_coord=19463ms wal_commit=174240ms wal_fsync=12064ms wal_fsync_wait=97753ms wal_fsync_actual=12064ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=261 wal_fsync_blocked_by_doc_store_sync_ms=37 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1466 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1187 fsyncs/s=1188 wal_fsync_q=59 wal_group_size=2 wal_group_fill=4 wal_bytes_per_fsync=19417 wal_batches/s=1187 wal_bytes/s=13168847 avg_writers_per_fsync=1 avg_bytes_per_fsync=11091 wal_writer_in_flight=0 pending_waiters=0 pending_wal_bytes=0 wal_append_lock_wait_ms=6 wal_writer_sleep_ms=68625 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=535360 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=37733 durable_lsn_after=37735 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=21ms waiter_broadcasts=535360 fsync_batch_reason=timeout last_waiters_released_in_fsync=59 durable_lsn=37735 durable_lsn_advance=11330 durable_lsn_advance_bytes=65892088 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=30MB/s wal_io=12MB/s primary_flush_io=18MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=24876417024 background_io_paused=11422ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=159ms doc_store_sync_segments_window=164 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=3 doc_store_sync_segment_bytes=2965801 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=1 doc_store_sync_pending_segments=1 doc_store_sync_pending_bytes=1331292 doc_store_sync_pending_lsn=37697 doc_store_durable_lsn=37697 doc_store_sync_worker_active=1 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=14 doc_store_sync_debt_bytes=1331292 doc_store_sync_jobs_merged=873 doc_store_sync_jobs_suppressed=3 doc_store_sync_active_files=1 doc_store_sync_queue_wait_ms=708447 doc_store_sync_physical_ms=14382 doc_store_sync_durable_lsn_advance=1202986 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=207447540 wal_pinned_by_checkpoint_bytes=6632339428 checkpoint_sync=3ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=3 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=3 checkpoint_sync_blocked_total_ms=484 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=3ms checkpoint_slice_avg_ms=3 checkpoint_slice_last_ms=3 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=712704 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=207533222 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=397436ms wait=370054ms wal=6323MB wal_live_bytes=6632362541 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=39MB flush_q=44 primary_q=41 secondary_q=0 primary_io=1549503ms primary_pages=22813 primary_flush_batch_bytes=95718000 primary_flush_batch_entries=143550 avg_flush_batch_bytes=1120100 coalesced_tables=0 coalesced_batches=0 small_flush_count=31 small_flush_reason=pressure active_primary_flush_workers=41 per_table_flush_inflight=23 flush_worker_io_wait=0ms flush_worker_cpu=34871ms primary_slice_max=5727ms primary_blocked=6599967ms primary_flush_admission_wait_ms=44092 primary_flush_queued_age_ms=626 primary_flush_oldest_queued_age_ms=703 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=partition_inflight_cap primary_doc_rotate_bytes=8388608 flush_defer=265412 flush_retry=0 flush_coalesced=66 flush_dedup=98 max_flush_wait=54341ms index_log=5909MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=35584 sec_apply/s=17664 text_apply/s=17920 idx_flush/s=0 idx_busy=226652ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=4ms rejects=0 checkpoint_sync_blocked_estimate_ms=3 checkpoint_yield_count=92 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=44092 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1683 wal_priority_yield_count=1852 background_io_yield_count=2399 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=1061 foreground_admission_delay_total_ms=631962 foreground_last_admission_delay_ms=8 foreground_pressure_transitions_window=32 foreground_pressure_transitions_total=5925 foreground_pressure_state_elapsed_ms=11797 foreground_pressure_time_normal_window_ms=151857 foreground_pressure_time_warm_window_ms=8191 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=482s elapsed_hms=0:08:02 docs=9738496 txns=38043 writes/s=17510 avg_writes/s=20204 p50=308ms p99=1660ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=39ms dbms_insert=141511ms batch_coord=19152ms wal_commit=178047ms wal_fsync=11436ms wal_fsync_wait=99260ms wal_fsync_actual=11436ms wal_fsync_avg_ms=1 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=250 wal_fsync_blocked_by_doc_store_sync_ms=35 wal_fsync_blocked_by_checkpoint_ms=2 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1379 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=7 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1161 fsyncs/s=1161 wal_fsync_q=85 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=24582 wal_batches/s=1161 wal_bytes/s=12808561 avg_writers_per_fsync=1 avg_bytes_per_fsync=11029 wal_writer_in_flight=43 pending_waiters=4 pending_wal_bytes=238385 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=69779 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=541165 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=38077 durable_lsn_after=38079 oldest_waiter_age=14ms avg_waiter_age=10ms max_waiter_age=21ms waiter_broadcasts=541165 fsync_batch_reason=none last_waiters_released_in_fsync=71 durable_lsn=38079 durable_lsn_advance=10995 durable_lsn_advance_bytes=64023943 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=37MB/s wal_io=12MB/s primary_flush_io=25MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=24394072064 background_io_paused=12140ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=220ms doc_store_sync_segments_window=230 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=3 doc_store_sync_segment_bytes=3077563 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=4 doc_store_sync_pending_segments=6 doc_store_sync_pending_bytes=8275411 doc_store_sync_pending_lsn=38063 doc_store_durable_lsn=38059 doc_store_sync_worker_active=1 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=41 doc_store_sync_debt_bytes=8275411 doc_store_sync_jobs_merged=873 doc_store_sync_jobs_suppressed=3 doc_store_sync_active_files=1 doc_store_sync_queue_wait_ms=708466 doc_store_sync_physical_ms=14602 doc_store_sync_durable_lsn_advance=1214422 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=836107155 wal_pinned_by_checkpoint_bytes=6692388565 checkpoint_sync=26ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=26 checkpoint_sync_ms_per_node=6ms checkpoint_sync_blocked_window_ms=26 checkpoint_sync_blocked_total_ms=510 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=5 checkpoint_slice_count_window=5 checkpoint_slices_per_node=1 checkpoint_sync_slice=26ms checkpoint_slice_avg_ms=5 checkpoint_slice_last_ms=3 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=2097152 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=209426503 checkpoint_reentry_after_cooldown_ms=195 compaction_pause_reason=none mem=545083ms wait=508916ms wal=6382MB wal_live_bytes=6692889829 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=7MB flush_q=7 primary_q=7 secondary_q=0 primary_io=1588506ms primary_pages=26431 primary_flush_batch_bytes=133619970 primary_flush_batch_entries=211690 avg_flush_batch_bytes=1109344 coalesced_tables=1 coalesced_batches=2 small_flush_count=45 small_flush_reason=pressure active_primary_flush_workers=7 per_table_flush_inflight=11 flush_worker_io_wait=4ms flush_worker_cpu=38999ms primary_slice_max=5727ms primary_blocked=6744220ms primary_flush_admission_wait_ms=144253 primary_flush_queued_age_ms=16293 primary_flush_oldest_queued_age_ms=1024 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=266400 flush_retry=0 flush_coalesced=67 flush_dedup=98 max_flush_wait=54341ms index_log=5963MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=36462 sec_apply/s=18103 text_apply/s=18358 idx_flush/s=0 idx_busy=226726ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=2ms rejects=0 checkpoint_sync_blocked_estimate_ms=26 checkpoint_yield_count=71 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=144253 primary_flush_worker_slots=64 primary_flush_coalesced_count=1 wal_fsync_blocked_ms=1588 wal_priority_yield_count=1762 background_io_yield_count=3311 foreground_latency_guard_trigger_count=0 foreground_pressure_state=constrained foreground_pressure_source=primary_queue_age foreground_admission_delay_window_ms=8799 foreground_admission_delay_total_ms=640761 foreground_last_admission_delay_ms=14 foreground_pressure_transitions_window=49 foreground_pressure_transitions_total=5974 foreground_pressure_state_elapsed_ms=16818 foreground_pressure_time_normal_window_ms=124711 foreground_pressure_time_warm_window_ms=29112 foreground_pressure_time_constrained_window_ms=4538 foreground_pressure_time_recovery_window_ms=1504 foreground_pressure_time_emergency_window_ms=1193
writes_test: elapsed=487s elapsed_hms=0:08:07 docs=9840896 txns=38442 writes/s=20480 avg_writes/s=20207 p50=238ms p99=1214ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=45ms dbms_insert=124953ms batch_coord=40272ms wal_commit=201037ms wal_fsync=17404ms wal_fsync_wait=106340ms wal_fsync_actual=17404ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=32 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=493 wal_fsync_blocked_by_doc_store_sync_ms=15 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1730 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1275 fsyncs/s=1276 wal_fsync_q=40 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=21472 wal_batches/s=1281 wal_bytes/s=15115084 avg_writers_per_fsync=2 avg_bytes_per_fsync=11848 wal_writer_in_flight=40 pending_waiters=0 pending_wal_bytes=218056 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=66601 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=547545 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=38477 durable_lsn_after=38478 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=24ms waiter_broadcasts=547545 fsync_batch_reason=timeout last_waiters_released_in_fsync=40 durable_lsn=38478 durable_lsn_advance=12776 durable_lsn_advance_bytes=75594284 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=23MB/s wal_io=14MB/s primary_flush_io=9MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=26713522176 background_io_paused=11093ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=88ms doc_store_sync_segments_window=99 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=2 doc_store_sync_segment_bytes=2996325 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=38458 doc_store_durable_lsn=38458 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=873 doc_store_sync_jobs_suppressed=3 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=708470 doc_store_sync_physical_ms=14690 doc_store_sync_durable_lsn_advance=1225476 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=6764029055 checkpoint_sync=2ms checkpoint_pause_reason=primary_flush_queue checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=2 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=2 checkpoint_sync_blocked_total_ms=512 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=2ms checkpoint_slice_avg_ms=2 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1564672 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=211580927 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=52117ms wait=22907ms wal=6450MB wal_live_bytes=6764255682 wal_truncatable_bytes=226627 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=0MB flush_q=1 primary_q=1 secondary_q=0 primary_io=1593466ms primary_pages=26474 primary_flush_batch_bytes=49452404 primary_flush_batch_entries=73041 avg_flush_batch_bytes=1104203 coalesced_tables=0 coalesced_batches=0 small_flush_count=26 small_flush_reason=pressure active_primary_flush_workers=1 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=4960ms primary_slice_max=5727ms primary_blocked=6744224ms primary_flush_admission_wait_ms=4 primary_flush_queued_age_ms=24 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=266401 flush_retry=0 flush_coalesced=67 flush_dedup=98 max_flush_wait=54341ms index_log=6026MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=23 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=41988 sec_apply/s=21121 text_apply/s=20866 idx_flush/s=0 idx_busy=226749ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=2 checkpoint_yield_count=95 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=4 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1979 wal_priority_yield_count=2086 background_io_yield_count=2169 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=none foreground_admission_delay_window_ms=2641 foreground_admission_delay_total_ms=643402 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=79 foreground_pressure_transitions_total=6053 foreground_pressure_state_elapsed_ms=21738 foreground_pressure_time_normal_window_ms=133296 foreground_pressure_time_warm_window_ms=24898 foreground_pressure_time_constrained_window_ms=2142 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=492s elapsed_hms=0:08:12 docs=9950464 txns=38870 writes/s=21913 avg_writes/s=20224 p50=245ms p99=780ms reason=checkpoint_wal_admission_deferred p99_root_cause_exclusive=checkpoint_wal_admission_deferred serialize=59ms dbms_insert=117638ms batch_coord=39994ms wal_commit=230529ms wal_fsync=19197ms wal_fsync_wait=104485ms wal_fsync_actual=19197ms wal_fsync_avg_ms=3 wal_fsync_p95_ms=8 wal_fsync_p99_ms=32 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=653 wal_fsync_blocked_by_doc_store_sync_ms=23 wal_fsync_blocked_by_checkpoint_ms=1 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1937 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1231 fsyncs/s=1231 wal_fsync_q=32 wal_group_size=1 wal_group_fill=1 wal_bytes_per_fsync=8143 wal_batches/s=1252 wal_bytes/s=15407180 avg_writers_per_fsync=2 avg_bytes_per_fsync=12509 wal_writer_in_flight=32 pending_waiters=0 pending_wal_bytes=241106 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=69119 pending_waiters_before_fsync=1 waiters_selected_for_batch=1 eligible_waiters=1 released_waiters=1 wakeup_batches=553703 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=38894 durable_lsn_after=38895 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=11ms waiter_broadcasts=553703 fsync_batch_reason=timeout last_waiters_released_in_fsync=32 durable_lsn=38895 durable_lsn_advance=13341 durable_lsn_advance_bytes=77035901 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=25MB/s wal_io=14MB/s primary_flush_io=11MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=25828524032 background_io_paused=11413ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=87ms doc_store_sync_segments_window=98 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=2 doc_store_sync_segment_bytes=2985209 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=38867 doc_store_durable_lsn=38867 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=874 doc_store_sync_jobs_suppressed=3 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=708487 doc_store_sync_physical_ms=14777 doc_store_sync_durable_lsn_advance=1236968 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=6837117461 checkpoint_sync=5ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=5 checkpoint_sync_ms_per_node=1ms checkpoint_sync_blocked_window_ms=5 checkpoint_sync_blocked_total_ms=517 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=2 checkpoint_slice_count_window=2 checkpoint_slices_per_node=0 checkpoint_sync_slice=5ms checkpoint_slice_avg_ms=2 checkpoint_slice_last_ms=2 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=2097152 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=213911051 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=63032ms wait=35268ms wal=6518MB wal_live_bytes=6837216034 wal_truncatable_bytes=98573 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=1601950ms primary_pages=26474 primary_flush_batch_bytes=57951450 primary_flush_batch_entries=83336 avg_flush_batch_bytes=1101534 coalesced_tables=0 coalesced_batches=0 small_flush_count=15 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=8484ms primary_slice_max=5727ms primary_blocked=6744319ms primary_flush_admission_wait_ms=95 primary_flush_queued_age_ms=13 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=266401 flush_retry=0 flush_coalesced=67 flush_dedup=98 max_flush_wait=54341ms index_log=6093MB pending_idx=512 pending_sec_apply=256 pending_sec_flush=23 oldest_idx_lsn=19901441 idx_lag_lsn=512 idx_lag=8ms sec_lag=8ms text_lag=8ms idx_apply/s=45778 sec_apply/s=22761 text_apply/s=23016 idx_flush/s=0 idx_busy=226793ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=2 sec_segments=1 text_segments=1 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=2318 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=2 l1=0 l2=0 l0_oldest_age_ms=2318 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=2318ms throttle=4ms rejects=0 checkpoint_sync_blocked_estimate_ms=5 checkpoint_yield_count=95 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=95 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2224 wal_priority_yield_count=2306 background_io_yield_count=2302 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=checkpoint_progress foreground_admission_delay_window_ms=207 foreground_admission_delay_total_ms=643609 foreground_last_admission_delay_ms=1 foreground_pressure_transitions_window=97 foreground_pressure_transitions_total=6150 foreground_pressure_state_elapsed_ms=8691 foreground_pressure_time_normal_window_ms=141864 foreground_pressure_time_warm_window_ms=18277 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=497s elapsed_hms=0:08:17 docs=10000000 txns=39063 writes/s=9907 avg_writes/s=20120 p50=232ms p99=1065ms reason=secondary_compaction_debt p99_root_cause_exclusive=secondary_compaction_debt serialize=46ms dbms_insert=54004ms batch_coord=23064ms wal_commit=95580ms wal_fsync=5451ms wal_fsync_wait=701464ms wal_fsync_actual=5451ms wal_fsync_avg_ms=0 wal_fsync_p95_ms=1 wal_fsync_p99_ms=8 wal_fsync_max_ms=16 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=116 wal_fsync_blocked_by_doc_store_sync_ms=3 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=650 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=7059 fsyncs/s=7059 wal_fsync_q=64 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=13281 wal_batches/s=509 wal_bytes/s=6682860 avg_writers_per_fsync=0 avg_bytes_per_fsync=13262 wal_writer_in_flight=0 pending_waiters=0 pending_wal_bytes=0 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=31275 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=556252 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=39071 durable_lsn_after=39073 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=17ms waiter_broadcasts=556252 fsync_batch_reason=timeout last_waiters_released_in_fsync=64 durable_lsn=39073 durable_lsn_advance=5695 durable_lsn_advance_bytes=468097978 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=9MB/s wal_io=6MB/s primary_flush_io=3MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=10691280896 background_io_paused=4805ms background_pause_reason=none emergency_wal_only_count=1 doc_store_sync=23ms doc_store_sync_segments_window=27 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=3 doc_store_sync_segment_bytes=2964651 doc_store_sync_reason=async doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=39039 doc_store_durable_lsn=39039 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=2 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=874 doc_store_sync_jobs_suppressed=3 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=708488 doc_store_sync_physical_ms=14800 doc_store_sync_durable_lsn_advance=1241764 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=6868764711 checkpoint_sync=1ms checkpoint_pause_reason=none checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=1 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=1 checkpoint_sync_blocked_total_ms=518 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=1 checkpoint_slice_count_window=1 checkpoint_slices_per_node=0 checkpoint_sync_slice=1ms checkpoint_slice_avg_ms=1 checkpoint_slice_last_ms=1 checkpoint_slice_max_ms=172 checkpoint_bytes_per_slice=1138688 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=214902976 checkpoint_reentry_after_cooldown_ms=250 compaction_pause_reason=none mem=15632ms wait=3787ms wal=6548MB wal_live_bytes=6868764711 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=checkpoint_sync backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=1603020ms primary_pages=26474 primary_flush_batch_bytes=15760703 primary_flush_batch_entries=24125 avg_flush_batch_bytes=1100751 coalesced_tables=0 coalesced_batches=0 small_flush_count=5 small_flush_reason=pressure active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=1ms flush_worker_cpu=1069ms primary_slice_max=5727ms primary_blocked=6744319ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=4 primary_flush_oldest_queued_age_ms=1 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=266401 flush_retry=0 flush_coalesced=67 flush_dedup=98 max_flush_wait=54341ms index_log=6123MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=78 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=41520 sec_apply/s=20887 text_apply/s=20632 idx_flush/s=2 idx_busy=938090ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=2 sec_compact_running_bytes=8388652B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=7 sec_segments=5 text_segments=2 secondary_compaction_queue=1 secondary_compaction_oldest_age_ms=1301 secondary_compaction_bytes_pending=8388652B secondary_compaction_eligible_segments=2 secondary_compaction_eligible_bytes=8388652B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=1 secondary_compaction_scheduled_jobs=1 secondary_compaction_oldest_eligible_age_ms=1301 secondary_compaction_forced_due_to_lag=1 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=7 l1=0 l2=0 l0_oldest_age_ms=357303 read_amp=4 write_amp=1 compaction_debt=24MB compacted/s=0B oldest_segment=357303ms throttle=4ms rejects=0 checkpoint_sync_blocked_estimate_ms=1 checkpoint_yield_count=108 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=710 wal_priority_yield_count=804 background_io_yield_count=749 foreground_latency_guard_trigger_count=0 foreground_pressure_state=warm foreground_pressure_source=checkpoint_progress foreground_admission_delay_window_ms=85 foreground_admission_delay_total_ms=643694 foreground_last_admission_delay_ms=1 foreground_pressure_transitions_window=13 foreground_pressure_transitions_total=6163 foreground_pressure_state_elapsed_ms=13703 foreground_pressure_time_normal_window_ms=68374 foreground_pressure_time_warm_window_ms=1877 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: done docs=10000000 elapsed=497s elapsed_hms=0:08:17 log=../data/social_media_data_1m\writes_log.txt other_log=../data/social_media_data_1m\writes_other_log.txt