{ "driver": "scheduler", "faults": [ { "command": "taskset -c 0 stress_fixture --db /home/runner/work/monokulo/monokulo/target/coverage/stress/fault-rpc.db --driver scheduler --tenants 16 --orders 2 --large-window-orders 300 --seed 24601 --background-readers 2 --background-http-readers 2 --background-writers 1 --warmup 0 --measured 2 --drain 6 --write-lock-ms 25 --rpc-delay-ms 2 --rpc-fail-until-height 2 --rpc-fail-every 3", "detail": "3 injected RPC failures", "file": "fault-rpc", "fixture": { "background_admin_writers": 1, "background_http_readers": 2, "background_http_reads_completed": 28, "background_readers": 2, "background_reads_completed": 60, "background_writes_completed": 12, "checkpoint_busy": 0, "custody_delay_ms": 0, "custody_max_wait_us": 0, "custody_scans_completed": 0, "custody_slots": 0, "database_bytes": 471040, "db_jobs_completed": 149, "db_write_query_max_us": 33519, "db_write_queue_wait_max_us": 32319, "drain_ticks": 6, "driver": "scheduler", "http_max_latency_us": 3990, "large_window_orders": 300, "measured_duration_ms": 62, "measured_ticks": 2, "orders_per_tenant": 2, "points": [ { "duration_ms": 35, "error": "Daemon(Request(\"scheduled fixture RPC failure\"))", "highwater": 0, "lagging_tenants": 0, "min_tenant_cursor": 0, "ok": false, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 14970880, "resident_bytes": 14970880 }, "progressed_tenants": 0, "tick": 0 }, { "duration_ms": 27, "error": null, "highwater": 0, "lagging_tenants": 0, "min_tenant_cursor": 0, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 15298560, "resident_bytes": 15298560 }, "progressed_tenants": 0, "tick": 1 }, { "duration_ms": 46, "error": null, "highwater": 3, "lagging_tenants": 0, "min_tenant_cursor": 3, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 15884288, "resident_bytes": 15884288 }, "progressed_tenants": 16, "tick": 2 }, { "duration_ms": 48, "error": null, "highwater": 4, "lagging_tenants": 0, "min_tenant_cursor": 4, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 15974400, "resident_bytes": 15974400 }, "progressed_tenants": 16, "tick": 3 }, { "duration_ms": 35, "error": null, "highwater": 5, "lagging_tenants": 0, "min_tenant_cursor": 5, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16019456, "resident_bytes": 16019456 }, "progressed_tenants": 16, "tick": 4 }, { "duration_ms": 47, "error": null, "highwater": 6, "lagging_tenants": 0, "min_tenant_cursor": 6, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16052224, "resident_bytes": 16052224 }, "progressed_tenants": 16, "tick": 5 }, { "duration_ms": 35, "error": null, "highwater": 7, "lagging_tenants": 0, "min_tenant_cursor": 7, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16076800, "resident_bytes": 16076800 }, "progressed_tenants": 16, "tick": 6 }, { "duration_ms": 49, "error": null, "highwater": 8, "lagging_tenants": 0, "min_tenant_cursor": 8, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16084992, "resident_bytes": 16084992 }, "progressed_tenants": 16, "tick": 7 } ], "read_max_latency_us": 805, "rpc_calls": 57, "rpc_delay_ms": 2, "rpc_fail_every": 3, "rpc_fail_until_height": 2, "rpc_failures": 3, "schema_version": 3, "seed": 24601, "sqlite_version": "3.53.2", "tenants": 16, "timer_max_delay_us": 2002, "wal_bytes": 4185952, "wal_checkpointed_pages": 279, "wal_log_pages": 279, "warmup_ticks": 0, "write_lock_hold_ms_per_tick": 25, "write_lock_until_tick": 18446744073709551615, "write_max_latency_us": 33773 }, "min_tenant_cursor": 8, "name": "transient daemon RPC failures under SQLite contention", "scenario_checksum": "d61357425730fcd865c51b5b530f3211f9aae8002453aaafb8f1f4a15bad3f27", "scenario_version": 3, "status": "recovered", "tenants": 16 }, { "command": "taskset -c 0 stress_fixture --db /home/runner/work/monokulo/monokulo/target/coverage/stress/fault-sqlite-lock.db --driver scheduler --tenants 16 --orders 2 --large-window-orders 300 --seed 24601 --background-readers 2 --background-http-readers 2 --background-writers 1 --warmup 0 --measured 2 --drain 6 --write-lock-ms 1500 --write-lock-until-tick 2", "detail": "1500 ms per tick for two ticks", "file": "fault-sqlite-lock", "fixture": { "background_admin_writers": 1, "background_http_readers": 2, "background_http_reads_completed": 292, "background_readers": 2, "background_reads_completed": 563, "background_writes_completed": 6, "checkpoint_busy": 0, "custody_delay_ms": 0, "custody_max_wait_us": 0, "custody_scans_completed": 0, "custody_slots": 0, "database_bytes": 471040, "db_jobs_completed": 169, "db_write_query_max_us": 1530164, "db_write_queue_wait_max_us": 1529690, "drain_ticks": 6, "driver": "scheduler", "http_max_latency_us": 1209, "large_window_orders": 300, "measured_duration_ms": 3083, "measured_ticks": 2, "orders_per_tenant": 2, "points": [ { "duration_ms": 1548, "error": null, "highwater": 1, "lagging_tenants": 0, "min_tenant_cursor": 1, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 15843328, "resident_bytes": 15843328 }, "progressed_tenants": 0, "tick": 0 }, { "duration_ms": 1535, "error": null, "highwater": 2, "lagging_tenants": 0, "min_tenant_cursor": 2, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16015360, "resident_bytes": 16015360 }, "progressed_tenants": 16, "tick": 1 }, { "duration_ms": 4, "error": null, "highwater": 3, "lagging_tenants": 0, "min_tenant_cursor": 3, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16039936, "resident_bytes": 16039936 }, "progressed_tenants": 16, "tick": 2 }, { "duration_ms": 4, "error": null, "highwater": 4, "lagging_tenants": 0, "min_tenant_cursor": 4, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16056320, "resident_bytes": 16056320 }, "progressed_tenants": 16, "tick": 3 }, { "duration_ms": 4, "error": null, "highwater": 5, "lagging_tenants": 0, "min_tenant_cursor": 5, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16072704, "resident_bytes": 16072704 }, "progressed_tenants": 16, "tick": 4 }, { "duration_ms": 5, "error": null, "highwater": 6, "lagging_tenants": 0, "min_tenant_cursor": 6, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16076800, "resident_bytes": 16076800 }, "progressed_tenants": 16, "tick": 5 }, { "duration_ms": 4, "error": null, "highwater": 7, "lagging_tenants": 0, "min_tenant_cursor": 7, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16080896, "resident_bytes": 16080896 }, "progressed_tenants": 16, "tick": 6 }, { "duration_ms": 4, "error": null, "highwater": 8, "lagging_tenants": 0, "min_tenant_cursor": 8, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16093184, "resident_bytes": 16093184 }, "progressed_tenants": 16, "tick": 7 } ], "read_max_latency_us": 999, "rpc_calls": 324, "rpc_delay_ms": 0, "rpc_fail_every": 0, "rpc_fail_until_height": 0, "rpc_failures": 0, "schema_version": 3, "seed": 24601, "sqlite_version": "3.53.2", "tenants": 16, "timer_max_delay_us": 1813, "wal_bytes": 4185952, "wal_checkpointed_pages": 316, "wal_log_pages": 316, "warmup_ticks": 0, "write_lock_hold_ms_per_tick": 1500, "write_lock_until_tick": 2, "write_max_latency_us": 1530523 }, "min_tenant_cursor": 8, "name": "blocked SQLite writer, then recovery", "scenario_checksum": "d61357425730fcd865c51b5b530f3211f9aae8002453aaafb8f1f4a15bad3f27", "scenario_version": 3, "status": "recovered", "tenants": 16 }, { "command": "taskset -c 0 stress_fixture --db /home/runner/work/monokulo/monokulo/target/coverage/stress/fault-custody.db --driver scheduler --tenants 16 --orders 2 --large-window-orders 300 --seed 24601 --background-readers 2 --background-http-readers 2 --background-writers 1 --warmup 0 --measured 2 --drain 6 --write-lock-ms 25 --custody-slots 1 --custody-delay-ms 5", "detail": "128 scans; max slot wait 102507 µs", "file": "fault-custody", "fixture": { "background_admin_writers": 1, "background_http_readers": 2, "background_http_reads_completed": 104, "background_readers": 2, "background_reads_completed": 202, "background_writes_completed": 81, "checkpoint_busy": 0, "custody_delay_ms": 5, "custody_max_wait_us": 102507, "custody_scans_completed": 128, "custody_slots": 1, "database_bytes": 471040, "db_jobs_completed": 244, "db_write_query_max_us": 33561, "db_write_queue_wait_max_us": 32886, "drain_ticks": 6, "driver": "scheduler", "http_max_latency_us": 1183, "large_window_orders": 300, "measured_duration_ms": 287, "measured_ticks": 2, "orders_per_tenant": 2, "points": [ { "duration_ms": 148, "error": null, "highwater": 1, "lagging_tenants": 0, "min_tenant_cursor": 1, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 15917056, "resident_bytes": 15917056 }, "progressed_tenants": 0, "tick": 0 }, { "duration_ms": 139, "error": null, "highwater": 2, "lagging_tenants": 0, "min_tenant_cursor": 2, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16093184, "resident_bytes": 16093184 }, "progressed_tenants": 16, "tick": 1 }, { "duration_ms": 135, "error": null, "highwater": 3, "lagging_tenants": 0, "min_tenant_cursor": 3, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16121856, "resident_bytes": 16121856 }, "progressed_tenants": 16, "tick": 2 }, { "duration_ms": 138, "error": null, "highwater": 4, "lagging_tenants": 0, "min_tenant_cursor": 4, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16150528, "resident_bytes": 16150528 }, "progressed_tenants": 16, "tick": 3 }, { "duration_ms": 138, "error": null, "highwater": 5, "lagging_tenants": 0, "min_tenant_cursor": 5, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16166912, "resident_bytes": 16166912 }, "progressed_tenants": 16, "tick": 4 }, { "duration_ms": 138, "error": null, "highwater": 6, "lagging_tenants": 0, "min_tenant_cursor": 6, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16183296, "resident_bytes": 16183296 }, "progressed_tenants": 16, "tick": 5 }, { "duration_ms": 136, "error": null, "highwater": 7, "lagging_tenants": 0, "min_tenant_cursor": 7, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16203776, "resident_bytes": 16203776 }, "progressed_tenants": 16, "tick": 6 }, { "duration_ms": 135, "error": null, "highwater": 8, "lagging_tenants": 0, "min_tenant_cursor": 8, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16220160, "resident_bytes": 16220160 }, "progressed_tenants": 16, "tick": 7 } ], "read_max_latency_us": 812, "rpc_calls": 136, "rpc_delay_ms": 0, "rpc_fail_every": 0, "rpc_fail_until_height": 0, "rpc_failures": 0, "schema_version": 3, "seed": 24601, "sqlite_version": "3.53.2", "tenants": 16, "timer_max_delay_us": 2301, "wal_bytes": 4185952, "wal_checkpointed_pages": 382, "wal_log_pages": 382, "warmup_ticks": 0, "write_lock_hold_ms_per_tick": 25, "write_lock_until_tick": 18446744073709551615, "write_max_latency_us": 33754 }, "min_tenant_cursor": 8, "name": "single custody scan slot under concurrent tenant scans", "scenario_checksum": "d61357425730fcd865c51b5b530f3211f9aae8002453aaafb8f1f4a15bad3f27", "scenario_version": 3, "status": "recovered", "tenants": 16 } ], "hardware": "hardware.json", "profile": "ci", "results": [ { "command": "taskset -c 0 stress_fixture --db /home/runner/work/monokulo/monokulo/target/coverage/stress/point-32.db --driver scheduler --tenants 32 --orders 2 --large-window-orders 300 --seed 24601 --background-readers 2 --background-http-readers 2 --background-writers 1 --warmup 2 --measured 4 --drain 3 --write-lock-ms 25", "exit_code": 0, "final_lagging_tenants": 0, "final_oldest_lag_blocks": 0, "fixture": { "background_admin_writers": 1, "background_http_readers": 2, "background_http_reads_completed": 38, "background_readers": 2, "background_reads_completed": 72, "background_writes_completed": 11, "checkpoint_busy": 0, "custody_delay_ms": 0, "custody_max_wait_us": 0, "custody_scans_completed": 0, "custody_slots": 0, "database_bytes": 495616, "db_jobs_completed": 203, "db_write_query_max_us": 33594, "db_write_queue_wait_max_us": 33605, "drain_ticks": 3, "driver": "scheduler", "http_max_latency_us": 2588, "large_window_orders": 300, "measured_duration_ms": 165, "measured_ticks": 4, "orders_per_tenant": 2, "points": [ { "duration_ms": 55, "error": null, "highwater": 1, "lagging_tenants": 0, "min_tenant_cursor": 1, "ok": true, "oldest_lag_blocks": 0, "phase": "warmup", "process_memory": { "peak_resident_bytes": 15892480, "resident_bytes": 15892480 }, "progressed_tenants": 0, "tick": 0 }, { "duration_ms": 41, "error": null, "highwater": 2, "lagging_tenants": 0, "min_tenant_cursor": 2, "ok": true, "oldest_lag_blocks": 0, "phase": "warmup", "process_memory": { "peak_resident_bytes": 16068608, "resident_bytes": 16068608 }, "progressed_tenants": 32, "tick": 1 }, { "duration_ms": 41, "error": null, "highwater": 3, "lagging_tenants": 0, "min_tenant_cursor": 3, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16109568, "resident_bytes": 16109568 }, "progressed_tenants": 32, "tick": 2 }, { "duration_ms": 41, "error": null, "highwater": 4, "lagging_tenants": 0, "min_tenant_cursor": 4, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16211968, "resident_bytes": 16211968 }, "progressed_tenants": 32, "tick": 3 }, { "duration_ms": 41, "error": null, "highwater": 5, "lagging_tenants": 0, "min_tenant_cursor": 5, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16236544, "resident_bytes": 16236544 }, "progressed_tenants": 32, "tick": 4 }, { "duration_ms": 42, "error": null, "highwater": 6, "lagging_tenants": 0, "min_tenant_cursor": 6, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16285696, "resident_bytes": 16285696 }, "progressed_tenants": 32, "tick": 5 }, { "duration_ms": 41, "error": null, "highwater": 7, "lagging_tenants": 0, "min_tenant_cursor": 7, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16326656, "resident_bytes": 16326656 }, "progressed_tenants": 32, "tick": 6 }, { "duration_ms": 41, "error": null, "highwater": 8, "lagging_tenants": 0, "min_tenant_cursor": 8, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16334848, "resident_bytes": 16334848 }, "progressed_tenants": 32, "tick": 7 }, { "duration_ms": 41, "error": null, "highwater": 9, "lagging_tenants": 0, "min_tenant_cursor": 9, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16338944, "resident_bytes": 16338944 }, "progressed_tenants": 32, "tick": 8 } ], "read_max_latency_us": 1169, "rpc_calls": 74, "rpc_delay_ms": 0, "rpc_fail_every": 0, "rpc_fail_until_height": 0, "rpc_failures": 0, "schema_version": 3, "seed": 24601, "sqlite_version": "3.53.2", "tenants": 32, "timer_max_delay_us": 1756, "wal_bytes": 4185952, "wal_checkpointed_pages": 413, "wal_log_pages": 413, "warmup_ticks": 2, "write_lock_hold_ms_per_tick": 25, "write_lock_until_tick": 18446744073709551615, "write_max_latency_us": 34337 }, "hardware": "hardware.json", "measured_duration_ms": 165, "min_tenant_cursor": 9, "peak_resident_bytes": 16338944, "progressed_tenants": 32, "responsive": true, "scenario_checksum": "d61357425730fcd865c51b5b530f3211f9aae8002453aaafb8f1f4a15bad3f27", "scenario_version": 3, "seed": 24601, "status": "sustainable", "tenants": 32, "timer_max_delay_us": 1756 }, { "command": "taskset -c 0 stress_fixture --db /home/runner/work/monokulo/monokulo/target/coverage/stress/point-64.db --driver scheduler --tenants 64 --orders 2 --large-window-orders 300 --seed 24601 --background-readers 2 --background-http-readers 2 --background-writers 1 --warmup 2 --measured 4 --drain 3 --write-lock-ms 25", "exit_code": 0, "final_lagging_tenants": 0, "final_oldest_lag_blocks": 0, "fixture": { "background_admin_writers": 1, "background_http_readers": 2, "background_http_reads_completed": 44, "background_readers": 2, "background_reads_completed": 82, "background_writes_completed": 19, "checkpoint_busy": 0, "custody_delay_ms": 0, "custody_max_wait_us": 0, "custody_scans_completed": 0, "custody_slots": 0, "database_bytes": 565248, "db_jobs_completed": 220, "db_write_query_max_us": 33610, "db_write_queue_wait_max_us": 32906, "drain_ticks": 3, "driver": "scheduler", "http_max_latency_us": 1323, "large_window_orders": 300, "measured_duration_ms": 186, "measured_ticks": 4, "orders_per_tenant": 2, "points": [ { "duration_ms": 64, "error": null, "highwater": 1, "lagging_tenants": 0, "min_tenant_cursor": 1, "ok": true, "oldest_lag_blocks": 0, "phase": "warmup", "process_memory": { "peak_resident_bytes": 16220160, "resident_bytes": 16220160 }, "progressed_tenants": 0, "tick": 0 }, { "duration_ms": 47, "error": null, "highwater": 2, "lagging_tenants": 0, "min_tenant_cursor": 2, "ok": true, "oldest_lag_blocks": 0, "phase": "warmup", "process_memory": { "peak_resident_bytes": 16453632, "resident_bytes": 16453632 }, "progressed_tenants": 64, "tick": 1 }, { "duration_ms": 47, "error": null, "highwater": 3, "lagging_tenants": 0, "min_tenant_cursor": 3, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16506880, "resident_bytes": 16506880 }, "progressed_tenants": 64, "tick": 2 }, { "duration_ms": 46, "error": null, "highwater": 4, "lagging_tenants": 0, "min_tenant_cursor": 4, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16523264, "resident_bytes": 16523264 }, "progressed_tenants": 64, "tick": 3 }, { "duration_ms": 46, "error": null, "highwater": 5, "lagging_tenants": 0, "min_tenant_cursor": 5, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16543744, "resident_bytes": 16543744 }, "progressed_tenants": 64, "tick": 4 }, { "duration_ms": 47, "error": null, "highwater": 6, "lagging_tenants": 0, "min_tenant_cursor": 6, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16572416, "resident_bytes": 16572416 }, "progressed_tenants": 64, "tick": 5 }, { "duration_ms": 47, "error": null, "highwater": 7, "lagging_tenants": 0, "min_tenant_cursor": 7, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16601088, "resident_bytes": 16601088 }, "progressed_tenants": 64, "tick": 6 }, { "duration_ms": 47, "error": null, "highwater": 8, "lagging_tenants": 0, "min_tenant_cursor": 8, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16629760, "resident_bytes": 16629760 }, "progressed_tenants": 64, "tick": 7 }, { "duration_ms": 47, "error": null, "highwater": 9, "lagging_tenants": 0, "min_tenant_cursor": 9, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16637952, "resident_bytes": 16637952 }, "progressed_tenants": 64, "tick": 8 } ], "read_max_latency_us": 898, "rpc_calls": 80, "rpc_delay_ms": 0, "rpc_fail_every": 0, "rpc_fail_until_height": 0, "rpc_failures": 0, "schema_version": 3, "seed": 24601, "sqlite_version": "3.53.2", "tenants": 64, "timer_max_delay_us": 1930, "wal_bytes": 4185952, "wal_checkpointed_pages": 498, "wal_log_pages": 498, "warmup_ticks": 2, "write_lock_hold_ms_per_tick": 25, "write_lock_until_tick": 18446744073709551615, "write_max_latency_us": 33861 }, "hardware": "hardware.json", "measured_duration_ms": 186, "min_tenant_cursor": 9, "peak_resident_bytes": 16637952, "progressed_tenants": 64, "responsive": true, "scenario_checksum": "d61357425730fcd865c51b5b530f3211f9aae8002453aaafb8f1f4a15bad3f27", "scenario_version": 3, "seed": 24601, "status": "sustainable", "tenants": 64, "timer_max_delay_us": 1930 }, { "command": "taskset -c 0 stress_fixture --db /home/runner/work/monokulo/monokulo/target/coverage/stress/point-128.db --driver scheduler --tenants 128 --orders 2 --large-window-orders 300 --seed 24601 --background-readers 2 --background-http-readers 2 --background-writers 1 --warmup 2 --measured 4 --drain 3 --write-lock-ms 25", "exit_code": 0, "final_lagging_tenants": 0, "final_oldest_lag_blocks": 0, "fixture": { "background_admin_writers": 1, "background_http_readers": 2, "background_http_reads_completed": 54, "background_readers": 2, "background_reads_completed": 103, "background_writes_completed": 29, "checkpoint_busy": 0, "custody_delay_ms": 0, "custody_max_wait_us": 0, "custody_scans_completed": 0, "custody_slots": 0, "database_bytes": 708608, "db_jobs_completed": 248, "db_write_query_max_us": 33562, "db_write_queue_wait_max_us": 32795, "drain_ticks": 3, "driver": "scheduler", "http_max_latency_us": 1527, "large_window_orders": 300, "measured_duration_ms": 233, "measured_ticks": 4, "orders_per_tenant": 2, "points": [ { "duration_ms": 81, "error": null, "highwater": 1, "lagging_tenants": 0, "min_tenant_cursor": 1, "ok": true, "oldest_lag_blocks": 0, "phase": "warmup", "process_memory": { "peak_resident_bytes": 16158720, "resident_bytes": 16158720 }, "progressed_tenants": 0, "tick": 0 }, { "duration_ms": 59, "error": null, "highwater": 2, "lagging_tenants": 0, "min_tenant_cursor": 2, "ok": true, "oldest_lag_blocks": 0, "phase": "warmup", "process_memory": { "peak_resident_bytes": 16551936, "resident_bytes": 16551936 }, "progressed_tenants": 128, "tick": 1 }, { "duration_ms": 59, "error": null, "highwater": 3, "lagging_tenants": 0, "min_tenant_cursor": 3, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16642048, "resident_bytes": 16642048 }, "progressed_tenants": 128, "tick": 2 }, { "duration_ms": 58, "error": null, "highwater": 4, "lagging_tenants": 0, "min_tenant_cursor": 4, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16678912, "resident_bytes": 16678912 }, "progressed_tenants": 128, "tick": 3 }, { "duration_ms": 58, "error": null, "highwater": 5, "lagging_tenants": 0, "min_tenant_cursor": 5, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16715776, "resident_bytes": 16715776 }, "progressed_tenants": 128, "tick": 4 }, { "duration_ms": 58, "error": null, "highwater": 6, "lagging_tenants": 0, "min_tenant_cursor": 6, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16748544, "resident_bytes": 16748544 }, "progressed_tenants": 128, "tick": 5 }, { "duration_ms": 58, "error": null, "highwater": 7, "lagging_tenants": 0, "min_tenant_cursor": 7, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16760832, "resident_bytes": 16760832 }, "progressed_tenants": 128, "tick": 6 }, { "duration_ms": 58, "error": null, "highwater": 8, "lagging_tenants": 0, "min_tenant_cursor": 8, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16781312, "resident_bytes": 16781312 }, "progressed_tenants": 128, "tick": 7 }, { "duration_ms": 59, "error": null, "highwater": 9, "lagging_tenants": 0, "min_tenant_cursor": 9, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16916480, "resident_bytes": 16916480 }, "progressed_tenants": 128, "tick": 8 } ], "read_max_latency_us": 2267, "rpc_calls": 90, "rpc_delay_ms": 0, "rpc_fail_every": 0, "rpc_fail_until_height": 0, "rpc_failures": 0, "schema_version": 3, "seed": 24601, "sqlite_version": "3.53.2", "tenants": 128, "timer_max_delay_us": 2052, "wal_bytes": 4198312, "wal_checkpointed_pages": 663, "wal_log_pages": 663, "warmup_ticks": 2, "write_lock_hold_ms_per_tick": 25, "write_lock_until_tick": 18446744073709551615, "write_max_latency_us": 33816 }, "hardware": "hardware.json", "measured_duration_ms": 233, "min_tenant_cursor": 9, "peak_resident_bytes": 16916480, "progressed_tenants": 128, "responsive": true, "scenario_checksum": "d61357425730fcd865c51b5b530f3211f9aae8002453aaafb8f1f4a15bad3f27", "scenario_version": 3, "seed": 24601, "status": "sustainable", "tenants": 128, "timer_max_delay_us": 2052 } ], "scenario": { "background_admin_writers": 1, "background_http_readers": 2, "background_readers": 2, "ci_tenant_points": [ 32, 64, 128 ], "drain_ticks": 3, "fault_custody_delay_ms": 5, "fault_custody_slots": 1, "fault_drain_ticks": 6, "fault_rpc_delay_ms": 2, "fault_rpc_fail_every": 3, "fault_rpc_fail_until_height": 2, "fault_sqlite_lock_ms": 1500, "fault_tenants": 16, "full_tenant_points": [ 32, 64, 128, 256, 512 ], "large_window_orders": 300, "max_http_latency_ms": 250, "max_oldest_lag_blocks": 3, "max_timer_delay_ms": 100, "measured_ticks": 4, "orders_per_tenant": 2, "poll_interval_ms": 5000, "schema_version": 3, "seed": 24601, "warmup_ticks": 2, "write_lock_hold_ms_per_tick": 25 }, "scenario_checksum": "d61357425730fcd865c51b5b530f3211f9aae8002453aaafb8f1f4a15bad3f27", "started_at_utc": 1792317499 }