writes_test: elapsed=5s elapsed_hms=0:00:05 docs=115200 txns=450 writes/s=23040 avg_writes/s=23040 p50=35ms p99=69ms reason=wal_group_commit_wait p99_root_cause_exclusive=wal_group_commit_wait serialize=26ms dbms_insert=15862ms batch_coord=139710ms wal_commit=251089ms wal_fsync=20835ms wal_fsync_wait=135584ms wal_fsync_actual=20835ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=356 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1818 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=32 wal_fsync_active_elapsed_ms=25 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1579 fsyncs/s=1572 wal_fsync_q=93 wal_group_size=2 wal_group_fill=4 wal_bytes_per_fsync=18753 wal_batches/s=1599 wal_bytes/s=16950847 avg_writers_per_fsync=1 avg_bytes_per_fsync=10626 wal_writer_in_flight=230 pending_waiters=193 pending_wal_bytes=619035 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=89351 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=4 released_waiters=4 wakeup_batches=7863 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=464 durable_lsn_after=466 oldest_waiter_age=73ms avg_waiter_age=46ms max_waiter_age=40ms waiter_broadcasts=7863 fsync_batch_reason=timeout last_waiters_released_in_fsync=51 durable_lsn=466 durable_lsn_advance=14586 durable_lsn_advance_bytes=83553130 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=16MB/s wal_io=16MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=32988200960 background_io_paused=0ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=79059919 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=0ms wait=0ms wal=72MB wal_live_bytes=79071910 wal_truncatable_bytes=11991 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=69MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50695 sec_apply/s=25473 text_apply/s=25221 idx_flush/s=0 idx_busy=2ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2308 wal_priority_yield_count=1901 background_io_yield_count=61 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=0 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=0 foreground_pressure_state_elapsed_ms=7764 foreground_pressure_time_normal_window_ms=244587 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=10s elapsed_hms=0:00:10 docs=241664 txns=944 writes/s=25292 avg_writes/s=24166 p50=35ms p99=104ms reason=healthy p99_root_cause_exclusive=healthy serialize=17ms dbms_insert=18084ms batch_coord=141486ms wal_commit=268469ms wal_fsync=21846ms wal_fsync_wait=134436ms wal_fsync_actual=21846ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=128 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=287 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1947 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=17 wal_fsync_active_elapsed_ms=11 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1565 fsyncs/s=1568 wal_fsync_q=91 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=20682 wal_batches/s=1566 wal_bytes/s=18341516 avg_writers_per_fsync=2 avg_bytes_per_fsync=11823 wal_writer_in_flight=55 pending_waiters=34 pending_wal_bytes=197864 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=87343 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=15706 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=957 durable_lsn_after=960 oldest_waiter_age=31ms avg_waiter_age=19ms max_waiter_age=16ms waiter_broadcasts=15706 fsync_batch_reason=none last_waiters_released_in_fsync=75 durable_lsn=960 durable_lsn_advance=15840 durable_lsn_advance_bytes=92729671 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=32946257920 background_io_paused=0ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=166329606 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=34ms wait=0ms wal=156MB wal_live_bytes=166657698 wal_truncatable_bytes=328092 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=146MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50589 sec_apply/s=25167 text_apply/s=25422 idx_flush/s=0 idx_busy=2ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2426 wal_priority_yield_count=1865 background_io_yield_count=0 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=33 foreground_admission_delay_total_ms=33 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=97 foreground_pressure_transitions_total=97 foreground_pressure_state_elapsed_ms=4504 foreground_pressure_time_normal_window_ms=138927 foreground_pressure_time_warm_window_ms=23099 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=15s elapsed_hms=0:00:15 docs=367104 txns=1435 writes/s=25088 avg_writes/s=24473 p50=35ms p99=79ms reason=healthy p99_root_cause_exclusive=healthy serialize=12ms dbms_insert=17413ms batch_coord=142197ms wal_commit=263299ms wal_fsync=17154ms wal_fsync_wait=124407ms wal_fsync_actual=17154ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=142 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1537 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1397 fsyncs/s=1400 wal_fsync_q=64 wal_group_size=2 wal_group_fill=4 wal_bytes_per_fsync=17138 wal_batches/s=1399 wal_bytes/s=18021470 avg_writers_per_fsync=2 avg_bytes_per_fsync=12896 wal_writer_in_flight=63 pending_waiters=0 pending_wal_bytes=326848 wal_append_lock_wait_ms=1 wal_writer_sleep_ms=86918 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=22707 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=1447 durable_lsn_after=1449 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=15ms waiter_broadcasts=22707 fsync_batch_reason=timeout last_waiters_released_in_fsync=64 durable_lsn=1449 durable_lsn_advance=15643 durable_lsn_advance_bytes=90286373 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=29305602048 background_io_paused=0ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=251779217 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=6ms wait=0ms wal=238MB wal_live_bytes=252054355 wal_truncatable_bytes=275138 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=222MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50640 sec_apply/s=25192 text_apply/s=25447 idx_flush/s=0 idx_busy=2ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1921 wal_priority_yield_count=1536 background_io_yield_count=0 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=6 foreground_admission_delay_total_ms=39 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=12 foreground_pressure_transitions_total=109 foreground_pressure_state_elapsed_ms=9514 foreground_pressure_time_normal_window_ms=157348 foreground_pressure_time_warm_window_ms=3036 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=20s elapsed_hms=0:00:20 docs=492544 txns=1924 writes/s=25088 avg_writes/s=24627 p50=34ms p99=90ms reason=healthy p99_root_cause_exclusive=healthy serialize=17ms dbms_insert=17584ms batch_coord=142147ms wal_commit=256519ms wal_fsync=20889ms wal_fsync_wait=129523ms wal_fsync_actual=20889ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=32 wal_fsync_max_ms=64 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=336 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1811 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=20 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1506 fsyncs/s=1504 wal_fsync_q=95 wal_group_size=3 wal_group_fill=6 wal_bytes_per_fsync=25240 wal_batches/s=1518 wal_bytes/s=18156340 avg_writers_per_fsync=2 avg_bytes_per_fsync=12044 wal_writer_in_flight=56 pending_waiters=39 pending_wal_bytes=293718 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=86594 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=30227 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=1938 durable_lsn_after=1941 oldest_waiter_age=21ms avg_waiter_age=13ms max_waiter_age=15ms waiter_broadcasts=30227 fsync_batch_reason=timeout last_waiters_released_in_fsync=94 durable_lsn=1941 durable_lsn_advance=15676 durable_lsn_advance_bytes=90578180 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=31608274944 background_io_paused=0ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=337438798 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=39ms wait=0ms wal=320MB wal_live_bytes=337771688 wal_truncatable_bytes=332890 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=298MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50043 sec_apply/s=24895 text_apply/s=25147 idx_flush/s=0 idx_busy=2ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2275 wal_priority_yield_count=1811 background_io_yield_count=0 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=39 foreground_admission_delay_total_ms=78 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=53 foreground_pressure_transitions_total=162 foreground_pressure_state_elapsed_ms=14503 foreground_pressure_time_normal_window_ms=147945 foreground_pressure_time_warm_window_ms=12234 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=25s elapsed_hms=0:00:25 docs=617728 txns=2414 writes/s=25036 avg_writes/s=24709 p50=34ms p99=48ms reason=healthy p99_root_cause_exclusive=healthy serialize=19ms dbms_insert=16676ms batch_coord=143443ms wal_commit=260016ms wal_fsync=17118ms wal_fsync_wait=128666ms wal_fsync_actual=17118ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=8 wal_fsync_max_ms=16 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=95 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1486 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=19 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1452 fsyncs/s=1452 wal_fsync_q=77 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=16553 wal_batches/s=1452 wal_bytes/s=18174023 avg_writers_per_fsync=2 avg_bytes_per_fsync=12521 wal_writer_in_flight=56 pending_waiters=24 pending_wal_bytes=108856 wal_append_lock_wait_ms=2 wal_writer_sleep_ms=86399 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=37490 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=2426 durable_lsn_after=2427 oldest_waiter_age=18ms avg_waiter_age=17ms max_waiter_age=11ms waiter_broadcasts=37490 fsync_batch_reason=timeout last_waiters_released_in_fsync=52 durable_lsn=2427 durable_lsn_advance=15652 durable_lsn_advance_bytes=90942838 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=30446452736 background_io_paused=0ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=423355241 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=0ms wait=0ms wal=402MB wal_live_bytes=423765967 wal_truncatable_bytes=410726 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=374MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50589 sec_apply/s=25422 text_apply/s=25167 idx_flush/s=0 idx_busy=2ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1862 wal_priority_yield_count=1486 background_io_yield_count=0 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=78 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=162 foreground_pressure_state_elapsed_ms=19512 foreground_pressure_time_normal_window_ms=160645 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=30s elapsed_hms=0:00:30 docs=742400 txns=2901 writes/s=24934 avg_writes/s=24746 p50=36ms p99=408ms reason=wal_group_commit_wait p99_root_cause_exclusive=wal_group_commit_wait serialize=18ms dbms_insert=33794ms batch_coord=126105ms wal_commit=355069ms wal_fsync=27982ms wal_fsync_wait=135772ms wal_fsync_actual=27982ms wal_fsync_avg_ms=3 wal_fsync_p95_ms=8 wal_fsync_p99_ms=32 wal_fsync_max_ms=512 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=927 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=2154 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=17 wal_fsync_active_elapsed_ms=6 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1505 fsyncs/s=1506 wal_fsync_q=32 wal_group_size=1 wal_group_fill=2 wal_bytes_per_fsync=10689 wal_batches/s=1571 wal_bytes/s=18098253 avg_writers_per_fsync=2 avg_bytes_per_fsync=12024 wal_writer_in_flight=28 pending_waiters=4 pending_wal_bytes=85086 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=81827 pending_waiters_before_fsync=1 waiters_selected_for_batch=1 eligible_waiters=1 released_waiters=1 wakeup_batches=45024 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=2916 durable_lsn_after=2917 oldest_waiter_age=20ms avg_waiter_age=20ms max_waiter_age=12ms waiter_broadcasts=45024 fsync_batch_reason=timeout last_waiters_released_in_fsync=32 durable_lsn=2917 durable_lsn_advance=15634 durable_lsn_advance_bytes=90596181 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=31566331904 background_io_paused=2034ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=509153377 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=256ms wait=0ms wal=483MB wal_live_bytes=509513910 wal_truncatable_bytes=360533 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=450MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=33402 sec_apply/s=16573 text_apply/s=16828 idx_flush/s=0 idx_busy=5ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2796 wal_priority_yield_count=2224 background_io_yield_count=212 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=164 foreground_admission_delay_total_ms=242 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=66 foreground_pressure_transitions_total=228 foreground_pressure_state_elapsed_ms=24522 foreground_pressure_time_normal_window_ms=144339 foreground_pressure_time_warm_window_ms=16286 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=35s elapsed_hms=0:00:35 docs=867584 txns=3390 writes/s=25036 avg_writes/s=24788 p50=42ms p99=73ms reason=healthy p99_root_cause_exclusive=healthy serialize=43ms dbms_insert=21139ms batch_coord=138611ms wal_commit=261455ms wal_fsync=18692ms wal_fsync_wait=114033ms wal_fsync_actual=18692ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=280 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=0 wal_fsync_blocked_by_io_scheduler_ms=1636 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=16 wal_fsync_active_elapsed_ms=5 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1321 fsyncs/s=1319 wal_fsync_q=96 wal_group_size=3 wal_group_fill=5 wal_bytes_per_fsync=22720 wal_batches/s=1322 wal_bytes/s=18263518 avg_writers_per_fsync=2 avg_bytes_per_fsync=13815 wal_writer_in_flight=31 pending_waiters=30 pending_wal_bytes=177722 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=72507 pending_waiters_before_fsync=3 waiters_selected_for_batch=3 eligible_waiters=3 released_waiters=3 wakeup_batches=51623 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=3405 durable_lsn_after=3408 oldest_waiter_age=18ms avg_waiter_age=18ms max_waiter_age=11ms waiter_broadcasts=51623 fsync_batch_reason=timeout last_waiters_released_in_fsync=86 durable_lsn=3408 durable_lsn_advance=15685 durable_lsn_advance_bytes=91165754 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=27703377920 background_io_paused=6681ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=594879126 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=0ms wait=0ms wal=566MB wal_live_bytes=595741227 wal_truncatable_bytes=862101 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=526MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50176 sec_apply/s=25088 text_apply/s=25088 idx_flush/s=0 idx_busy=5ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=0ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=2052 wal_priority_yield_count=1859 background_io_yield_count=776 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=242 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=228 foreground_pressure_state_elapsed_ms=29530 foreground_pressure_time_normal_window_ms=160171 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=40s elapsed_hms=0:00:40 docs=991744 txns=3874 writes/s=24832 avg_writes/s=24793 p50=45ms p99=70ms reason=healthy p99_root_cause_exclusive=healthy serialize=33ms dbms_insert=22178ms batch_coord=134981ms wal_commit=253063ms wal_fsync=15931ms wal_fsync_wait=110468ms wal_fsync_actual=15931ms wal_fsync_avg_ms=2 wal_fsync_p95_ms=8 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=165 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=3 wal_fsync_blocked_by_io_scheduler_ms=1482 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=32 wal_fsync_active_elapsed_ms=4 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=1265 fsyncs/s=1261 wal_fsync_q=64 wal_group_size=2 wal_group_fill=5 wal_bytes_per_fsync=21238 wal_batches/s=1265 wal_bytes/s=17854513 avg_writers_per_fsync=2 avg_bytes_per_fsync=14119 wal_writer_in_flight=62 pending_waiters=56 pending_wal_bytes=428851 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=72084 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=57930 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=3886 durable_lsn_after=3888 oldest_waiter_age=28ms avg_waiter_age=26ms max_waiter_age=20ms waiter_broadcasts=57930 fsync_batch_reason=timeout last_waiters_released_in_fsync=64 durable_lsn=3888 durable_lsn_advance=15390 durable_lsn_advance_bytes=89054338 concurrent_compact_fsync=1 fsync_blocked_by_compaction=15931ms foreground_io=17MB/s wal_io=17MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=wal_protected wal_reserved_budget_used=26424115200 background_io_paused=8275ms background_pause_reason=wal_protected emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=680008615 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=1ms wait=0ms wal=646MB wal_live_bytes=680024461 wal_truncatable_bytes=15846 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=602MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50741 sec_apply/s=25498 text_apply/s=25243 idx_flush/s=0 idx_busy=5ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=0 sec_segments=0 text_segments=0 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=0 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=none secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=0 l1=0 l2=0 l0_oldest_age_ms=0 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=0ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=1857 wal_priority_yield_count=1774 background_io_yield_count=975 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=242 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=228 foreground_pressure_state_elapsed_ms=34540 foreground_pressure_time_normal_window_ms=157705 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: elapsed=45s elapsed_hms=0:00:45 docs=1000000 txns=3907 writes/s=1651 avg_writes/s=22222 p50=50ms p99=71ms reason=wal_group_commit_wait p99_root_cause_exclusive=wal_group_commit_wait serialize=7ms dbms_insert=1639ms batch_coord=17231ms wal_commit=18326ms wal_fsync=1511ms wal_fsync_wait=7530ms wal_fsync_actual=1511ms wal_fsync_avg_ms=3 wal_fsync_p95_ms=16 wal_fsync_p99_ms=16 wal_fsync_max_ms=32 wal_fsync_enter_to_submit_ms=0 wal_fsync_submit_to_complete_ms=9 wal_fsync_blocked_by_doc_store_sync_ms=0 wal_fsync_blocked_by_checkpoint_ms=0 wal_fsync_blocked_by_compaction_ms=10 wal_fsync_blocked_by_io_scheduler_ms=120 wal_global_sync_lock_wait_total_ms=0 wal_global_sync_lock_wait_avg_ms=0 wal_global_sync_lock_wait_max_ms=0 wal_fsync_active=0 wal_fsync_active_elapsed_ms=0 wal_only_violation_count=0 global_sync_lock_current_owner=0 global_sync_lock_current_held_ms=0 wal_waiting_for_global_sync_lock_ms=0 fsync_attempts/s=89 fsyncs/s=95 wal_fsync_q=36 wal_group_size=2 wal_group_fill=3 wal_bytes_per_fsync=13137 wal_batches/s=89 wal_bytes/s=1127814 avg_writers_per_fsync=2 avg_bytes_per_fsync=12678 wal_writer_in_flight=0 pending_waiters=0 pending_wal_bytes=0 wal_append_lock_wait_ms=0 wal_writer_sleep_ms=4506 pending_waiters_before_fsync=2 waiters_selected_for_batch=2 eligible_waiters=2 released_waiters=2 wakeup_batches=58406 lost_wakeup_count=0 durable_lsn_gap=0 durable_lsn_before=3920 durable_lsn_after=3921 oldest_waiter_age=0ms avg_waiter_age=0ms max_waiter_age=23ms waiter_broadcasts=58406 fsync_batch_reason=timeout last_waiters_released_in_fsync=36 durable_lsn=3921 durable_lsn_advance=1050 durable_lsn_advance_bytes=6035022 concurrent_compact_fsync=0 fsync_blocked_by_compaction=0ms foreground_io=1MB/s wal_io=1MB/s primary_flush_io=0MB/s compaction_io=0MB/s secondary_compaction_io=0MB/s text_compaction_io=0MB/s io_scheduler_mode=normal wal_reserved_budget_used=1983905792 background_io_paused=562ms background_pause_reason=none emergency_wal_only_count=0 doc_store_sync=0ms doc_store_sync_segments_window=0 doc_store_sync_segment_avg_ms=0 doc_store_sync_segment_max_ms=0 doc_store_sync_segment_bytes=0 doc_store_sync_reason=none doc_store_sync_blocked_foreground=0ms doc_store_sync_queue=0 doc_store_sync_pending_segments=0 doc_store_sync_pending_bytes=0 doc_store_sync_pending_lsn=0 doc_store_durable_lsn=0 doc_store_sync_worker_active=0 doc_store_sync_worker_count=1 doc_store_sync_batch_size=0 doc_store_sync_oldest_age_ms=0 doc_store_sync_debt_bytes=0 doc_store_sync_jobs_merged=0 doc_store_sync_jobs_suppressed=0 doc_store_sync_active_files=0 doc_store_sync_queue_wait_ms=0 doc_store_sync_physical_ms=0 doc_store_sync_durable_lsn_advance=0 checkpoint_pinned_by_doc_store_lsn=0 wal_pinned_by_doc_store_bytes=0 wal_pinned_by_checkpoint_bytes=685710790 checkpoint_sync=0ms checkpoint_pause_reason=safe_wal_defer checkpoint_paused=0ms checkpoint_forced_pause_ms=0 checkpoint_actual_sync_ms=0 checkpoint_sync_ms_per_node=0ms checkpoint_sync_blocked_window_ms=0 checkpoint_sync_blocked_total_ms=0 checkpoint_defer_window_ms=0 checkpoint_defer_total_ms=0 checkpoint_currently_syncing=0 checkpoint_current_sync_elapsed_ms=0 checkpoint_sync_state_owner=0 checkpoint_sync_state_held_ms=0 checkpoint_sync_no_progress_ms=0 checkpoint_sync_slices=0 checkpoint_slice_count_window=0 checkpoint_slices_per_node=0 checkpoint_sync_slice=0ms checkpoint_slice_avg_ms=0 checkpoint_slice_last_ms=0 checkpoint_slice_max_ms=0 checkpoint_bytes_per_slice=0 checkpoint_yielded_due_to_wal=0ms checkpoint_yielded_due_to_primary_flush_age=0ms checkpoint_yielded_due_to_time_budget=0ms checkpoint_blocked_due_to_primary_flush_age=0ms checkpoint_cluster_budget_limit_ms=0 checkpoint_cluster_sync_budget_used_ms=0 checkpoint_cluster_sync_budget_exhausted=0 checkpoint_cluster_budget_denied_count=0 checkpoint_cluster_budget_denied_ms=0 checkpoint_node_budget_limit_ms=0 checkpoint_node_budget_used_ms_current_window=0 checkpoint_sync_started_after_budget_exhausted_count=0 checkpoint_sync_denied_due_to_budget=0 checkpoint_budget_limited_wal_bytes=0 checkpoint_reentry_after_cooldown_ms=0 compaction_pause_reason=none mem=0ms wait=0ms wal=652MB wal_live_bytes=685710790 wal_truncatable_bytes=0 wal_pinned_by_index_bytes=0 wal_pinned_by_text_index_bytes=0 wal_pinned_by_oldest_reader_bytes=0 wal_pinned_by_recovery_or_manifest_bytes=0 wal_truncate_block_reason=reader_or_unknown backlog=0MB flush_q=0 primary_q=0 secondary_q=0 primary_io=0ms primary_pages=0 primary_flush_batch_bytes=0 primary_flush_batch_entries=0 avg_flush_batch_bytes=0 coalesced_tables=0 coalesced_batches=0 small_flush_count=0 small_flush_reason=none active_primary_flush_workers=0 per_table_flush_inflight=0 flush_worker_io_wait=0ms flush_worker_cpu=0ms primary_slice_max=0ms primary_blocked=0ms primary_flush_admission_wait_ms=0 primary_flush_queued_age_ms=0 primary_flush_oldest_queued_age_ms=0 primary_flush_coalesce_wait_ms=0 primary_flush_defer_reason=none primary_doc_rotate_bytes=8388608 flush_defer=0 flush_retry=0 flush_coalesced=0 flush_dedup=0 max_flush_wait=0ms index_log=607MB pending_idx=0 pending_sec_apply=0 pending_sec_flush=0 oldest_idx_lsn=0 idx_lag_lsn=0 idx_lag=0ms sec_lag=0ms text_lag=0ms idx_apply/s=50432 sec_apply/s=25344 text_apply/s=25088 idx_flush/s=2 idx_busy=23232ms compact_busy=0ms compact_yield=0 compact_pause=0ms compact_budget=0MB/s sec_compact_budget=0MB/s sec_compact_budget_used=0B_window sec_compact_budget_reserved=0B/s sec_compact_running_segments=0 sec_compact_running_bytes=0B sec_compact_actual_compacted_bytes_window=0B sec_compact_effective_mb_s=0MB/s text_compact_budget=0MB/s primary_flush_budget=192MB/s wal_reserved=128MB/s segments=4 sec_segments=2 text_segments=2 secondary_compaction_queue=0 secondary_compaction_oldest_age_ms=1665 secondary_compaction_bytes_pending=0B secondary_compaction_eligible_segments=0 secondary_compaction_eligible_bytes=0B secondary_compaction_not_eligible_reason=below_min_segments secondary_compaction_pending_jobs=0 secondary_compaction_scheduled_jobs=0 secondary_compaction_oldest_eligible_age_ms=0 secondary_compaction_forced_due_to_lag=0 secondary_compaction_paused_by_wal_ms=0 secondary_compaction_paused_by_p99_ms=0 l0=4 l1=0 l2=0 l0_oldest_age_ms=1665 read_amp=1 write_amp=1 compaction_debt=0MB compacted/s=0B oldest_segment=1665ms throttle=1ms rejects=0 checkpoint_sync_blocked_estimate_ms=0 checkpoint_yield_count=0 checkpoint_skipped_due_to_wal_pressure=0 primary_flush_queue_wait_ms=0 primary_flush_worker_slots=64 primary_flush_coalesced_count=0 wal_fsync_blocked_ms=163 wal_priority_yield_count=134 background_io_yield_count=74 foreground_latency_guard_trigger_count=0 foreground_pressure_state=normal foreground_pressure_source=none foreground_admission_delay_window_ms=0 foreground_admission_delay_total_ms=242 foreground_last_admission_delay_ms=0 foreground_pressure_transitions_window=0 foreground_pressure_transitions_total=228 foreground_pressure_state_elapsed_ms=39547 foreground_pressure_time_normal_window_ms=12842 foreground_pressure_time_warm_window_ms=0 foreground_pressure_time_constrained_window_ms=0 foreground_pressure_time_recovery_window_ms=0 foreground_pressure_time_emergency_window_ms=0
writes_test: done docs=1000000 elapsed=45s elapsed_hms=0:00:45 log=../data/social_media_data_1m\writes_log.txt other_log=../data/social_media_data_1m\writes_other_log.txt