fio
apt install -y fio ioping
Intenal eMMC (296MB/s-296MB/s) SCA64G 58.2 GiB
root@g9:~# fio --filename=/dev/mmcblk0p3 --rw=read --direct=1 --bs=1M --ioengine=libaio --runtime=60 --numjobs=1 --time_based --group_reporting --name=seq_read --iodepth=16
seq_read: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=16
fio-3.36
Starting 1 process
Jobs: 1 (f=1): [R(1)][100.0%][r=284MiB/s][r=284 IOPS][eta 00m:00s]
seq_read: (groupid=0, jobs=1): err= 0: pid=10358: Fri Apr 4 15:24:22 2025
read: IOPS=282, BW=283MiB/s (296MB/s)(16.6GiB/60057msec)
slat (usec): min=93, max=1862, avg=155.44, stdev=29.98
clat (msec): min=8, max=105, avg=56.43, stdev= 2.72
lat (msec): min=8, max=105, avg=56.59, stdev= 2.72
clat percentiles (msec):
| 1.00th=[ 55], 5.00th=[ 56], 10.00th=[ 56], 20.00th=[ 56],
| 30.00th=[ 57], 40.00th=[ 57], 50.00th=[ 57], 60.00th=[ 57],
| 70.00th=[ 57], 80.00th=[ 57], 90.00th=[ 57], 95.00th=[ 57],
| 99.00th=[ 64], 99.50th=[ 81], 99.90th=[ 89], 99.95th=[ 93],
| 99.99th=[ 103]
bw ( KiB/s): min=223232, max=294323, per=100.00%, avg=289521.04, stdev=8081.61, samples=119
iops : min= 218, max= 287, avg=282.44, stdev= 7.88, samples=119
lat (msec) : 10=0.01%, 20=0.02%, 50=0.05%, 100=99.91%, 250=0.01%
cpu : usr=1.31%, sys=5.58%, ctx=18129, majf=0, minf=4107
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=99.9%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=16972,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=16
Run status group 0 (all jobs):
READ: bw=283MiB/s (296MB/s), 283MiB/s-283MiB/s (296MB/s-296MB/s), io=16.6GiB (17.8GB), run=60057-60057msec
Disk stats (read/write):
mmcblk0: ios=33853/0, sectors=34665472/0, merge=0/1, ticks=1876970/0, in_queue=1876971, util=93.19%
Raid 1 of two nvme (3305MB/s-3305MB/s)
root@g9:~# fio --filename=/dev/md0p1 --rw=read --direct=1 --bs=1M --ioengine=libaio --runtime=60 --numjobs=1 --time_based --group_reporting --name=seq_read --iodepth=16
seq_read: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=16
fio-3.36
Starting 1 process
Jobs: 1 (f=1): [R(1)][100.0%][r=3155MiB/s][r=3155 IOPS][eta 00m:00s]
seq_read: (groupid=0, jobs=1): err= 0: pid=10342: Fri Apr 4 15:22:24 2025
read: IOPS=3151, BW=3152MiB/s (3305MB/s)(185GiB/60005msec)
slat (usec): min=38, max=483, avg=109.55, stdev=40.31
clat (usec): min=1404, max=20568, avg=4964.45, stdev=215.28
lat (usec): min=1619, max=20666, avg=5074.00, stdev=212.23
clat percentiles (usec):
| 1.00th=[ 4883], 5.00th=[ 4883], 10.00th=[ 4883], 20.00th=[ 4948],
| 30.00th=[ 4948], 40.00th=[ 4948], 50.00th=[ 4948], 60.00th=[ 4948],
| 70.00th=[ 4948], 80.00th=[ 5014], 90.00th=[ 5014], 95.00th=[ 5014],
| 99.00th=[ 5211], 99.50th=[ 5473], 99.90th=[ 7767], 99.95th=[ 8356],
| 99.99th=[10814]
bw ( MiB/s): min= 3045, max= 3182, per=100.00%, avg=3154.12, stdev=17.16, samples=119
iops : min= 3045, max= 3182, avg=3154.11, stdev=17.21, samples=119
lat (msec) : 2=0.01%, 4=0.01%, 10=99.96%, 20=0.02%, 50=0.01%
cpu : usr=0.83%, sys=39.16%, ctx=188785, majf=0, minf=4109
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=189121,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=16
Run status group 0 (all jobs):
READ: bw=3152MiB/s (3305MB/s), 3152MiB/s-3152MiB/s (3305MB/s-3305MB/s), io=185GiB (198GB), run=60005-60005msec
Disk stats (read/write):
md0: ios=1510146/16093, sectors=386597376/4104680, merge=0/0, ticks=5250573/34132, in_queue=5284705, util=72.00%, aggrios=756484/8065, aggsectors=193659904/2058484, aggrmerge=0/17, aggrticks=2632221/16445, aggrin_queue=2648729, aggrutil=99.84%
nvme0n1: ios=756484/12, sectors=193659904/0, merge=0/0, ticks=3409719/51, in_queue=3409821, util=99.77%
nvme1n1: ios=756484/16119, sectors=193659904/4116968, merge=0/34, ticks=1854723/32840, in_queue=1887638, util=99.84%
single nvme1 (1671MB/s-1671MB/s) ShiJi 128GB M.2-NVMe
root@g9:~# fio --filename=/dev/nvme0n1 --rw=read --direct=1 --bs=1M --ioengine=libaio --runtime=60 --numjobs=1 --time_based --group_reporting --name=seq_read --iodepth=16
seq_read: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=16
fio-3.36
Starting 1 process
Jobs: 1 (f=1): [R(1)][100.0%][r=1593MiB/s][r=1592 IOPS][eta 00m:00s]
seq_read: (groupid=0, jobs=1): err= 0: pid=10372: Fri Apr 4 15:26:48 2025
read: IOPS=1593, BW=1594MiB/s (1671MB/s)(93.4GiB/60010msec)
slat (usec): min=44, max=1228, avg=223.13, stdev=40.51
clat (usec): min=2829, max=18640, avg=9813.77, stdev=158.12
lat (usec): min=3150, max=18797, avg=10036.91, stdev=150.55
clat percentiles (usec):
| 1.00th=[ 9634], 5.00th=[ 9765], 10.00th=[ 9765], 20.00th=[ 9765],
| 30.00th=[ 9765], 40.00th=[ 9765], 50.00th=[ 9765], 60.00th=[ 9765],
| 70.00th=[ 9896], 80.00th=[ 9896], 90.00th=[ 9896], 95.00th=[ 9896],
| 99.00th=[ 9896], 99.50th=[ 9896], 99.90th=[10028], 99.95th=[11731],
| 99.99th=[14877]
bw ( MiB/s): min= 1590, max= 1602, per=100.00%, avg=1595.04, stdev= 3.40, samples=119
iops : min= 1590, max= 1602, avg=1595.04, stdev= 3.40, samples=119
lat (msec) : 4=0.01%, 10=99.90%, 20=0.08%
cpu : usr=0.83%, sys=40.70%, ctx=95629, majf=0, minf=4107
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=95638,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=16
Run status group 0 (all jobs):
READ: bw=1594MiB/s (1671MB/s), 1594MiB/s-1594MiB/s (1671MB/s-1671MB/s), io=93.4GiB (100GB), run=60010-60010msec
Disk stats (read/write):
nvme0n1: ios=763682/0, sectors=195501848/0, merge=0/0, ticks=7333711/0, in_queue=7333711, util=99.91%
single nvme2 (1686MB/s-1686MB/s) Samsung PM951 NVMe SAMSUNG 512GB
root@g9:~# fio --filename=/dev/nvme1n1 --rw=read --direct=1 --bs=1M --ioengine=libaio --runtime=60 --numjobs=1 --time_based --group_reporting --name=seq_read --iodepth=16
seq_read: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=16
fio-3.36
Starting 1 process
Jobs: 1 (f=1): [R(1)][100.0%][r=1610MiB/s][r=1609 IOPS][eta 00m:00s]
seq_read: (groupid=0, jobs=1): err= 0: pid=10381: Fri Apr 4 15:28:00 2025
read: IOPS=1607, BW=1608MiB/s (1686MB/s)(94.2GiB/60010msec)
slat (usec): min=83, max=1234, avg=225.86, stdev=13.67
clat (usec): min=3067, max=18420, avg=9722.94, stdev=142.61
lat (usec): min=3363, max=18645, avg=9948.81, stdev=139.30
clat percentiles (usec):
| 1.00th=[ 9634], 5.00th=[ 9765], 10.00th=[ 9765], 20.00th=[ 9765],
| 30.00th=[ 9765], 40.00th=[ 9765], 50.00th=[ 9765], 60.00th=[ 9765],
| 70.00th=[ 9765], 80.00th=[ 9765], 90.00th=[ 9765], 95.00th=[ 9765],
| 99.00th=[ 9765], 99.50th=[ 9765], 99.90th=[10814], 99.95th=[12125],
| 99.99th=[13829]
bw ( MiB/s): min= 1596, max= 1610, per=100.00%, avg=1609.31, stdev= 1.53, samples=119
iops : min= 1596, max= 1610, avg=1609.31, stdev= 1.53, samples=119
lat (msec) : 4=0.01%, 10=99.86%, 20=0.13%
cpu : usr=0.65%, sys=41.18%, ctx=96375, majf=0, minf=4107
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=96487,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=16
Run status group 0 (all jobs):
READ: bw=1608MiB/s (1686MB/s), 1608MiB/s-1608MiB/s (1686MB/s-1686MB/s), io=94.2GiB (101GB), run=60010-60010msec
Disk stats (read/write):
nvme1n1: ios=770390/0, sectors=197219840/0, merge=0/0, ticks=7331818/0, in_queue=7331818, util=99.92%
ioping
eMMC SCA64G 58.2 GiB - 6.22 ms
root@g9:~# ioping /dev/mmcblk0p3 4 KiB <<< /dev/mmcblk0p3 (block device 55.2 GiB): request=1 time=5.95 ms (warmup) 4 KiB <<< /dev/mmcblk0p3 (block device 55.2 GiB): request=2 time=6.14 ms 4 KiB <<< /dev/mmcblk0p3 (block device 55.2 GiB): request=3 time=6.18 ms 4 KiB <<< /dev/mmcblk0p3 (block device 55.2 GiB): request=4 time=6.22 ms 4 KiB <<< /dev/mmcblk0p3 (block device 55.2 GiB): request=5 time=6.21 ms 4 KiB <<< /dev/mmcblk0p3 (block device 55.2 GiB): request=6 time=6.27 ms 4 KiB <<< /dev/mmcblk0p3 (block device 55.2 GiB): request=7 time=6.20 ms
RAID 1 - 1.66 ms
root@g9:~# ioping /dev/md0p1 4 KiB <<< /dev/md0p1 (block device 595.9 GiB): request=1 time=840.5 us (warmup) 4 KiB <<< /dev/md0p1 (block device 595.9 GiB): request=2 time=1.65 ms 4 KiB <<< /dev/md0p1 (block device 595.9 GiB): request=3 time=1.71 ms 4 KiB <<< /dev/md0p1 (block device 595.9 GiB): request=4 time=1.66 ms 4 KiB <<< /dev/md0p1 (block device 595.9 GiB): request=5 time=1.61 ms 4 KiB <<< /dev/md0p1 (block device 595.9 GiB): request=6 time=1.55 ms 4 KiB <<< /dev/md0p1 (block device 595.9 GiB): request=7 time=1.66 ms 4 KiB <<< /dev/md0p1 (block device 595.9 GiB): request=8 time=349.7 us (fast) 4 KiB <<< /dev/md0p1 (block device 595.9 GiB): request=9 time=1.80 ms 4 KiB <<< /dev/md0p1 (block device 595.9 GiB): request=10 time=1.30 ms 4 KiB <<< /dev/md0p1 (block device 595.9 GiB): request=11 time=351.0 us (fast)
NVMe 1 - ShiJi 128GB M.2-NVMe - 0.3 ms
root@g9:~# ioping /dev/nvme0n1 4 KiB <<< /dev/nvme0n1 (block device 119.2 GiB): request=1 time=219.6 us (warmup) 4 KiB <<< /dev/nvme0n1 (block device 119.2 GiB): request=2 time=311.8 us 4 KiB <<< /dev/nvme0n1 (block device 119.2 GiB): request=3 time=360.2 us 4 KiB <<< /dev/nvme0n1 (block device 119.2 GiB): request=4 time=195.3 us 4 KiB <<< /dev/nvme0n1 (block device 119.2 GiB): request=5 time=390.5 us 4 KiB <<< /dev/nvme0n1 (block device 119.2 GiB): request=6 time=203.0 us
NVMe 2 - PM951 NVMe SAMSUNG 512GB - 1.60 ms
root@g9:~# ioping /dev/nvme1n1 4 KiB <<< /dev/nvme1n1 (block device 476.9 GiB): request=1 time=1.55 ms (warmup) 4 KiB <<< /dev/nvme1n1 (block device 476.9 GiB): request=2 time=2.05 ms 4 KiB <<< /dev/nvme1n1 (block device 476.9 GiB): request=3 time=1.44 ms 4 KiB <<< /dev/nvme1n1 (block device 476.9 GiB): request=4 time=1.99 ms 4 KiB <<< /dev/nvme1n1 (block device 476.9 GiB): request=5 time=995.8 us
Tests with parameters from https://forums.lawrencesystems.com/t/linux-benchmarking-with-fio/11122
cd /mnt
root@g9:/mnt# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=4k --size=4G --readwrite=randwrite --ramp_time=4
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
fio-3.36
Starting 1 process
test: Laying out IO file (1 file / 4096MiB)
Jobs: 1 (f=1): [w(1)][89.5%][w=128MiB/s][w=32.8k IOPS][eta 00m:04s]
test: (groupid=0, jobs=1): err= 0: pid=10777: Fri Apr 4 15:43:30 2025
write: IOPS=31.3k, BW=122MiB/s (128MB/s)(3595MiB/29401msec); 0 zone resets
slat (usec): min=5, max=1163, avg= 8.90, stdev= 2.77
clat (usec): min=7, max=19689, avg=22.39, stdev=54.45
lat (usec): min=25, max=19698, avg=31.29, stdev=54.58
clat percentiles (usec):
| 1.00th=[ 20], 5.00th=[ 21], 10.00th=[ 21], 20.00th=[ 21],
| 30.00th=[ 21], 40.00th=[ 21], 50.00th=[ 23], 60.00th=[ 23],
| 70.00th=[ 23], 80.00th=[ 23], 90.00th=[ 24], 95.00th=[ 24],
| 99.00th=[ 27], 99.50th=[ 29], 99.90th=[ 36], 99.95th=[ 67],
| 99.99th=[ 1418]
bw ( KiB/s): min=116616, max=131414, per=99.97%, avg=125179.40, stdev=2305.97, samples=58
iops : min=29154, max=32853, avg=31294.69, stdev=576.48, samples=58
lat (usec) : 10=0.01%, 20=2.94%, 50=97.01%, 100=0.01%, 250=0.01%
lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01%
lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%
cpu : usr=6.64%, sys=30.41%, ctx=920381, majf=0, minf=36
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,920350,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=122MiB/s (128MB/s), 122MiB/s-122MiB/s (128MB/s-128MB/s), io=3595MiB (3770MB), run=29401-29401msec
Disk stats (read/write):
md0: ios=1/1057412, sectors=8/8459296, merge=0/0, ticks=0/30655, in_queue=30655, util=50.42%, aggrios=0/524691, aggsectors=4/4250232, aggrmerge=0/6593, aggrticks=0/9181, aggrin_queue=9228, aggrutil=25.85%
nvme0n1: ios=0/524407, sectors=0/4195304, merge=0/12, ticks=0/8444, in_queue=8448, util=23.19%
nvme1n1: ios=1/524976, sectors=8/4305160, merge=0/13175, ticks=0/9919, in_queue=10008, util=25.85%
root@g9:/mnt# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=4k --size=4G --readwrite=randwrite --ramp_time=4
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
fio-3.36
Starting 1 process
Jobs: 1 (f=1): [w(1)][88.9%][w=126MiB/s][w=32.2k IOPS][eta 00m:04s]
test: (groupid=0, jobs=1): err= 0: pid=10788: Fri Apr 4 15:44:18 2025
write: IOPS=33.2k, BW=130MiB/s (136MB/s)(3598MiB/27709msec); 0 zone resets
slat (nsec): min=3349, max=81303, avg=4412.39, stdev=695.91
clat (usec): min=4, max=17660, avg=25.09, stdev=40.94
lat (usec): min=18, max=17665, avg=29.51, stdev=40.96
clat percentiles (usec):
| 1.00th=[ 16], 5.00th=[ 17], 10.00th=[ 19], 20.00th=[ 24],
| 30.00th=[ 25], 40.00th=[ 25], 50.00th=[ 25], 60.00th=[ 26],
| 70.00th=[ 27], 80.00th=[ 28], 90.00th=[ 28], 95.00th=[ 28],
| 99.00th=[ 30], 99.50th=[ 32], 99.90th=[ 37], 99.95th=[ 277],
| 99.99th=[ 1237]
bw ( KiB/s): min=124240, max=155206, per=100.00%, avg=133036.42, stdev=8824.40, samples=55
iops : min=31060, max=38801, avg=33259.04, stdev=2206.06, samples=55
lat (usec) : 10=0.01%, 20=13.85%, 50=86.09%, 100=0.01%, 250=0.01%
lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01%
lat (msec) : 2=0.02%, 4=0.01%, 10=0.01%, 20=0.01%
cpu : usr=5.85%, sys=17.64%, ctx=921002, majf=0, minf=36
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,920998,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=130MiB/s (136MB/s), 130MiB/s-130MiB/s (136MB/s-136MB/s), io=3598MiB (3772MB), run=27709-27709msec
Disk stats (read/write):
md0: ios=0/1041851, sectors=0/8334808, merge=0/0, ticks=0/22371, in_queue=22371, util=70.75%, aggrios=0/524300, aggsectors=0/4194376, aggrmerge=0/3, aggrticks=0/11656, aggrin_queue=11669, aggrutil=35.91%
nvme0n1: ios=0/524294, sectors=0/4194304, merge=0/0, ticks=0/11276, in_queue=11280, util=33.02%
nvme1n1: ios=0/524306, sectors=0/4194448, merge=0/6, ticks=0/12037, in_queue=12059, util=35.91%
root@g9:/mnt# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=4k --size=4G --readwrite=readwrite --ramp_time=4
test: (g=0): rw=rw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
fio-3.36
Starting 1 process
Jobs: 1 (f=1): [M(1)][94.8%][r=26.7MiB/s,w=27.2MiB/s][r=6847,w=6957 IOPS][eta 00m:04s]
test: (groupid=0, jobs=1): err= 0: pid=16795: Fri Apr 4 15:49:01 2025
read: IOPS=7234, BW=28.3MiB/s (29.6MB/s)(1946MiB/68865msec)
slat (usec): min=3, max=126, avg= 9.63, stdev= 3.41
clat (nsec): min=1259, max=7650.0k, avg=86010.39, stdev=150058.98
lat (usec): min=16, max=7672, avg=95.64, stdev=150.95
clat percentiles (usec):
| 1.00th=[ 17], 5.00th=[ 19], 10.00th=[ 20], 20.00th=[ 21],
| 30.00th=[ 22], 40.00th=[ 23], 50.00th=[ 25], 60.00th=[ 32],
| 70.00th=[ 88], 80.00th=[ 104], 90.00th=[ 310], 95.00th=[ 318],
| 99.00th=[ 644], 99.50th=[ 660], 99.90th=[ 2040], 99.95th=[ 2311],
| 99.99th=[ 2835]
bw ( KiB/s): min=19711, max=50364, per=100.00%, avg=28947.30, stdev=7447.39, samples=137
iops : min= 4927, max=12591, avg=7236.69, stdev=1861.85, samples=137
write: IOPS=7224, BW=28.2MiB/s (29.6MB/s)(1944MiB/68865msec); 0 zone resets
slat (usec): min=3, max=117, avg=10.16, stdev= 3.50
clat (usec): min=2, max=7346, avg=29.86, stdev=31.88
lat (usec): min=20, max=7357, avg=40.02, stdev=32.47
clat percentiles (usec):
| 1.00th=[ 21], 5.00th=[ 23], 10.00th=[ 24], 20.00th=[ 25],
| 30.00th=[ 26], 40.00th=[ 28], 50.00th=[ 29], 60.00th=[ 29],
| 70.00th=[ 31], 80.00th=[ 31], 90.00th=[ 34], 95.00th=[ 38],
| 99.00th=[ 92], 99.50th=[ 96], 99.90th=[ 110], 99.95th=[ 119],
| 99.99th=[ 143]
bw ( KiB/s): min=19559, max=50925, per=100.00%, avg=28903.48, stdev=7404.34, samples=137
iops : min= 4889, max=12731, avg=7225.71, stdev=1851.06, samples=137
lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=6.97%, 50=72.99%
lat (usec) : 100=9.11%, 250=4.46%, 500=5.78%, 750=0.54%, 1000=0.01%
lat (msec) : 2=0.07%, 4=0.05%, 10=0.01%
cpu : usr=5.89%, sys=19.51%, ctx=995676, majf=0, minf=36
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=498210,497541,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
READ: bw=28.3MiB/s (29.6MB/s), 28.3MiB/s-28.3MiB/s (29.6MB/s-29.6MB/s), io=1946MiB (2041MB), run=68865-68865msec
WRITE: bw=28.2MiB/s (29.6MB/s), 28.2MiB/s-28.2MiB/s (29.6MB/s-29.6MB/s), io=1944MiB (2038MB), run=68865-68865msec
Disk stats (read/write):
md0: ios=523367/522739, sectors=4186936/4181912, merge=0/0, ticks=42854/11357, in_queue=54211, util=74.38%, aggrios=262312/262003, aggsectors=2098500/2095972, aggrmerge=0/7, aggrticks=21582/6167, aggrin_queue=27779, aggrutil=48.37%
nvme0n1: ios=262272/261949, sectors=2098176/2095480, merge=0/0, ticks=30290/6357, in_queue=36654, util=48.37%
nvme1n1: ios=262353/262058, sectors=2098824/2096464, merge=0/14, ticks=12874/5977, in_queue=18905, util=24.28%
root@g9:/mnt# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=4M --size=4G --readwrite=write --ramp_time=4
test: (g=0): rw=write, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=1
fio-3.36
Starting 1 process
Jobs: 1 (f=0): [/(1)][-.-%][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=16805: Fri Apr 4 15:49:27 2025
write: IOPS=276, BW=1106MiB/s (1160MB/s)(4096MiB/3704msec); 0 zone resets
cpu : usr=8.72%, sys=29.57%, ctx=1025, majf=0, minf=9
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,1024,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=1106MiB/s (1160MB/s), 1106MiB/s-1106MiB/s (1160MB/s-1160MB/s), io=4096MiB (4295MB), run=3704-3704msec
Disk stats (read/write):
md0: ios=0/39352, sectors=0/8265984, merge=0/0, ticks=0/29449, in_queue=29449, util=73.14%, aggrios=0/19968, aggsectors=0/4194304, aggrmerge=0/0, aggrticks=0/14788, aggrin_queue=14787, aggrutil=83.01%
nvme0n1: ios=0/19456, sectors=0/4194304, merge=0/0, ticks=0/21370, in_queue=21369, util=83.01%
nvme1n1: ios=0/20480, sectors=0/4194304, merge=0/0, ticks=0/8207, in_queue=8206, util=38.09%
root@g9:/mnt# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=4M --size=4G --readwrite=read --ramp_time=4
test: (g=0): rw=read, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=1
fio-3.36
Starting 1 process
Jobs: 1 (f=0)
test: (groupid=0, jobs=1): err= 0: pid=16813: Fri Apr 4 15:49:40 2025
read: IOPS=525, BW=2103MiB/s (2205MB/s)(4096MiB/1948msec)
cpu : usr=0.15%, sys=40.42%, ctx=1025, majf=0, minf=1033
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=1024,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
READ: bw=2103MiB/s (2205MB/s), 2103MiB/s-2103MiB/s (2205MB/s-2205MB/s), io=4096MiB (4295MB), run=1948-1948msec
Disk stats (read/write):
md0: ios=33859/0, sectors=8158208/0, merge=0/0, ticks=22467/0, in_queue=22467, util=65.85%, aggrios=17408/0, aggsectors=4194304/0, aggrmerge=0/0, aggrticks=11572/0, aggrin_queue=11572, aggrutil=91.02%
nvme0n1: ios=17408/0, sectors=4194304/0, merge=0/0, ticks=7978/0, in_queue=7978, util=64.80%
nvme1n1: ios=17408/0, sectors=4194304/0, merge=0/0, ticks=15167/0, in_queue=15167, util=91.02%
4x4TB Samsung 990 PRO
single disk
root@storage3:/storage/software/benchmarks/phoronix-test-suite-master# apt install -y fio ioping
Reading package lists... Done
Building dependency tree... Done
Reading state information... Done
fio is already the newest version (3.36-1ubuntu0.1).
ioping is already the newest version (1.3-1).
0 upgraded, 0 newly installed, 0 to remove and 49 not upgraded.
root@storage3:/storage/software/benchmarks/phoronix-test-suite-master# fio --filename=/dev/nvme0n1 --rw=read --direct=1 --bs=1M --ioengine=libaio --runtime=60 --numjobs=1 --time_based --group_reporting --name=seq_read --iodepth=16
seq_read: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=16
fio-3.36
Starting 1 process
Jobs: 1 (f=1): [R(1)][100.0%][r=1726MiB/s][r=1726 IOPS][eta 00m:00s]
seq_read: (groupid=0, jobs=1): err= 0: pid=39369: Sat Apr 5 17:50:13 2025
read: IOPS=1726, BW=1726MiB/s (1810MB/s)(101GiB/60010msec)
slat (usec): min=35, max=1081, avg=132.88, stdev=11.94
clat (usec): min=1865, max=17270, avg=9133.78, stdev=98.76
lat (usec): min=2155, max=17395, avg=9266.66, stdev=95.80
clat percentiles (usec):
| 1.00th=[ 9110], 5.00th=[ 9110], 10.00th=[ 9110], 20.00th=[ 9110],
| 30.00th=[ 9110], 40.00th=[ 9110], 50.00th=[ 9110], 60.00th=[ 9110],
| 70.00th=[ 9110], 80.00th=[ 9110], 90.00th=[ 9110], 95.00th=[ 9110],
| 99.00th=[ 9241], 99.50th=[ 9241], 99.90th=[ 9241], 99.95th=[ 9241],
| 99.99th=[11469]
bw ( MiB/s): min= 1718, max= 1730, per=100.00%, avg=1728.05, stdev= 1.09, samples=119
iops : min= 1718, max= 1730, avg=1728.05, stdev= 1.09, samples=119
lat (msec) : 2=0.01%, 4=0.01%, 10=99.98%, 20=0.01%
cpu : usr=1.06%, sys=28.55%, ctx=103574, majf=0, minf=4108
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=103581,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=16
Run status group 0 (all jobs):
READ: bw=1726MiB/s (1810MB/s), 1726MiB/s-1726MiB/s (1810MB/s-1810MB/s), io=101GiB (109GB), run=60010-60010msec
Disk stats (read/write):
nvme0n1: ios=827093/0, sectors=211735808/0, merge=0/0, ticks=7347095/0, in_queue=7347096, util=99.94%
raid 0 device
root@storage3:/storage/software/benchmarks/fio# fio --filename=/dev/md0 --rw=read --direct=1 --bs=1M --ioengine=libaio --runtime=60 --numjobs=1 --time_based --group_reporting --name=seq_read --iodepth=16
seq_read: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=16
fio-3.36
Starting 1 process
Jobs: 1 (f=1): [R(1)][100.0%][r=3083MiB/s][r=3083 IOPS][eta 00m:00s]
seq_read: (groupid=0, jobs=1): err= 0: pid=39699: Sat Apr 5 17:53:50 2025
read: IOPS=3091, BW=3091MiB/s (3241MB/s)(181GiB/60005msec)
slat (usec): min=87, max=763, avg=122.50, stdev=14.36
clat (usec): min=1494, max=9504, avg=5047.75, stdev=113.54
lat (usec): min=1590, max=9631, avg=5170.25, stdev=116.81
clat percentiles (usec):
| 1.00th=[ 4621], 5.00th=[ 4621], 10.00th=[ 5080], 20.00th=[ 5080],
| 30.00th=[ 5080], 40.00th=[ 5080], 50.00th=[ 5080], 60.00th=[ 5080],
| 70.00th=[ 5080], 80.00th=[ 5080], 90.00th=[ 5080], 95.00th=[ 5080],
| 99.00th=[ 5145], 99.50th=[ 5145], 99.90th=[ 5211], 99.95th=[ 5276],
| 99.99th=[ 5342]
bw ( MiB/s): min= 2982, max= 3390, per=100.00%, avg=3093.23, stdev=62.70, samples=119
iops : min= 2982, max= 3390, avg=3093.23, stdev=62.70, samples=119
lat (msec) : 2=0.01%, 4=0.01%, 10=99.98%
cpu : usr=0.55%, sys=40.85%, ctx=184275, majf=0, minf=4108
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=185481,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=16
Run status group 0 (all jobs):
READ: bw=3091MiB/s (3241MB/s), 3091MiB/s-3091MiB/s (3241MB/s-3241MB/s), io=181GiB (194GB), run=60005-60005msec
Disk stats (read/write):
md0: ios=1481066/0, sectors=379152896/0, merge=0/0, ticks=7141927/0, in_queue=7141927, util=99.80%, aggrios=370962/1, aggsectors=94966272/0, aggrmerge=0/0, aggrticks=1789031/3, aggrin_queue=1789037, aggrutil=99.73%
nvme0n1: ios=370964/1, sectors=94966784/0, merge=0/0, ticks=1789995/2, in_queue=1789998, util=99.73%
nvme3n1: ios=370960/1, sectors=94965760/0, merge=0/0, ticks=1790479/4, in_queue=1790488, util=99.73%
nvme2n1: ios=370960/1, sectors=94965760/0, merge=0/0, ticks=1785829/5, in_queue=1785840, util=99.73%
nvme1n1: ios=370964/1, sectors=94966784/0, merge=0/0, ticks=1789823/1, in_queue=1789825, util=99.73%
random write and read
root@storage3:/storage/software/benchmarks/fio# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=4k --size=4G --readwrite=randwrite --ramp_time=4
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
fio-3.36
Starting 1 process
test: Laying out IO file (1 file / 4096MiB)
Jobs: 1 (f=1): [w(1)][90.9%][w=105MiB/s][w=26.8k IOPS][eta 00m:04s]
test: (groupid=0, jobs=1): err= 0: pid=39742: Sat Apr 5 17:57:46 2025
write: IOPS=26.6k, BW=104MiB/s (109MB/s)(3685MiB/35443msec); 0 zone resets
slat (usec): min=5, max=509, avg= 8.93, stdev= 2.38
clat (nsec): min=1127, max=1840.1k, avg=27943.29, stdev=4358.93
lat (usec): min=23, max=1850, avg=36.88, stdev= 4.92
clat percentiles (usec):
| 1.00th=[ 25], 5.00th=[ 27], 10.00th=[ 27], 20.00th=[ 28],
| 30.00th=[ 28], 40.00th=[ 28], 50.00th=[ 29], 60.00th=[ 29],
| 70.00th=[ 29], 80.00th=[ 29], 90.00th=[ 30], 95.00th=[ 30],
| 99.00th=[ 32], 99.50th=[ 34], 99.90th=[ 37], 99.95th=[ 39],
| 99.99th=[ 102]
bw ( KiB/s): min=104905, max=111120, per=100.00%, avg=106490.96, stdev=736.46, samples=70
iops : min=26226, max=27780, avg=26622.64, stdev=184.14, samples=70
lat (usec) : 2=0.01%, 20=0.49%, 50=99.49%, 100=0.01%, 250=0.01%
lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01%
lat (msec) : 2=0.01%
cpu : usr=5.80%, sys=26.30%, ctx=943369, majf=0, minf=36
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,943306,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=104MiB/s (109MB/s), 104MiB/s-104MiB/s (109MB/s-109MB/s), io=3685MiB (3864MB), run=35443-35443msec
Disk stats (read/write):
md0: ios=22/1059237, sectors=176/8473896, merge=0/0, ticks=0/24722, in_queue=24722, util=57.09%, aggrios=5/262326, aggsectors=44/2129418, aggrmerge=0/3858, aggrticks=0/6394, aggrin_queue=6404, aggrutil=14.02%
nvme0n1: ios=0/262286, sectors=0/2128936, merge=0/3838, ticks=0/6350, in_queue=6361, util=14.02%
nvme3n1: ios=0/262379, sectors=0/2129792, merge=0/3852, ticks=0/6457, in_queue=6465, util=13.70%
nvme2n1: ios=0/262316, sectors=0/2129256, merge=0/3848, ticks=0/6358, in_queue=6369, util=13.87%
nvme1n1: ios=22/262324, sectors=176/2129688, merge=0/3894, ticks=2/6414, in_queue=6421, util=13.97%
root@storage3:/storage/software/benchmarks/fio# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=4k --size=4G --readwrite=randread --ramp_time=4
test: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
fio-3.36
Starting 1 process
Jobs: 1 (f=1): [r(1)][95.2%][r=51.4MiB/s][r=13.2k IOPS][eta 00m:04s]
test: (groupid=0, jobs=1): err= 0: pid=39874: Sat Apr 5 18:00:14 2025
read: IOPS=13.2k, BW=51.4MiB/s (53.9MB/s)(3899MiB/75842msec)
slat (nsec): min=3461, max=92902, avg=11401.97, stdev=1459.19
clat (usec): min=3, max=1208, avg=62.90, stdev= 3.79
lat (usec): min=37, max=1234, avg=74.30, stdev= 4.44
clat percentiles (usec):
| 1.00th=[ 61], 5.00th=[ 62], 10.00th=[ 63], 20.00th=[ 63],
| 30.00th=[ 63], 40.00th=[ 63], 50.00th=[ 63], 60.00th=[ 63],
| 70.00th=[ 63], 80.00th=[ 64], 90.00th=[ 64], 95.00th=[ 65],
| 99.00th=[ 72], 99.50th=[ 74], 99.90th=[ 120], 99.95th=[ 123],
| 99.99th=[ 137]
bw ( KiB/s): min=52336, max=55470, per=100.00%, avg=52723.17, stdev=267.07, samples=151
iops : min=13084, max=13867, avg=13180.66, stdev=66.74, samples=151
lat (usec) : 4=0.01%, 50=0.20%, 100=99.61%, 250=0.19%, 500=0.01%
lat (msec) : 2=0.01%
cpu : usr=7.02%, sys=22.94%, ctx=998270, majf=0, minf=36
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=998257,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
READ: bw=51.4MiB/s (53.9MB/s), 51.4MiB/s-51.4MiB/s (53.9MB/s-53.9MB/s), io=3899MiB (4089MB), run=75842-75842msec
Disk stats (read/write):
md0: ios=1047487/4, sectors=8379896/32, merge=0/0, ticks=49378/3, in_queue=49381, util=61.91%, aggrios=262144/1, aggsectors=2097152/8, aggrmerge=0/0, aggrticks=15591/2, aggrin_queue=15594, aggrutil=14.89%
nvme0n1: ios=262144/1, sectors=2097152/0, merge=0/0, ticks=15582/2, in_queue=15585, util=14.89%
nvme3n1: ios=262144/1, sectors=2097152/0, merge=0/0, ticks=15601/1, in_queue=15602, util=14.68%
nvme2n1: ios=262144/4, sectors=2097152/32, merge=0/1, ticks=15597/4, in_queue=15603, util=14.68%
nvme1n1: ios=262144/1, sectors=2097152/0, merge=0/0, ticks=15585/1, in_queue=15586, util=14.62%
ioping
root@storage3:/storage/software/benchmarks/fio# ioping /dev/nvme1n1 4 KiB <<< /dev/nvme1n1 (block device 3.64 TiB): request=1 time=11.1 ms (warmup) 4 KiB <<< /dev/nvme1n1 (block device 3.64 TiB): request=2 time=1.27 ms 4 KiB <<< /dev/nvme1n1 (block device 3.64 TiB): request=3 time=1.98 ms 4 KiB <<< /dev/nvme1n1 (block device 3.64 TiB): request=4 time=2.10 ms 4 KiB <<< /dev/nvme1n1 (block device 3.64 TiB): request=5 time=2.05 ms 4 KiB <<< /dev/nvme1n1 (block device 3.64 TiB): request=6 time=2.10 ms 4 KiB <<< /dev/nvme1n1 (block device 3.64 TiB): request=7 time=1.30 ms 4 KiB <<< /dev/nvme1n1 (block device 3.64 TiB): request=8 time=1.39 ms 4 KiB <<< /dev/nvme1n1 (block device 3.64 TiB): request=9 time=1.66 ms ^C --- /dev/nvme1n1 (block device 3.64 TiB) ioping statistics --- 8 requests completed in 13.9 ms, 32 KiB read, 577 iops, 2.26 MiB/s generated 9 requests in 8.14 s, 36 KiB, 1 iops, 4.42 KiB/s min/avg/max/mdev = 1.27 ms / 1.73 ms / 2.10 ms / 345.7 us root@storage3:/storage/software/benchmarks/fio# ioping /dev/md0 4 KiB <<< /dev/md0 (block device 14.6 TiB): request=1 time=11.2 ms (warmup) 4 KiB <<< /dev/md0 (block device 14.6 TiB): request=2 time=11.8 ms 4 KiB <<< /dev/md0 (block device 14.6 TiB): request=3 time=11.8 ms 4 KiB <<< /dev/md0 (block device 14.6 TiB): request=4 time=1.81 ms 4 KiB <<< /dev/md0 (block device 14.6 TiB): request=5 time=1.98 ms 4 KiB <<< /dev/md0 (block device 14.6 TiB): request=6 time=12.0 ms ^C --- /dev/md0 (block device 14.6 TiB) ioping statistics --- 5 requests completed in 39.3 ms, 20 KiB read, 127 iops, 508.7 KiB/s generated 6 requests in 5.85 s, 24 KiB, 1 iops, 4.10 KiB/s min/avg/max/mdev = 1.81 ms / 7.86 ms / 12.0 ms / 4.87 ms
