Page History

	Read	Write	Random write	Random Read
RAIDZ1 (LZ)	894MiB/s-894MiB/s	568MiB/s-568MiB/s	82.5MiB/s-82.5MiB/s	99.2MiB/s-99.2MiB/s

Tech detailsCommands used

Code Block

root@truenas:cd /mnt/pool/shared/user# user
sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=4k --size=4G --readwrite=randwritewrite --ramp_time=4
sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=4k --size=4G --readwrite=write --ramp_time=4
sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=4k --size=4G --readwrite=randwrite --ramp_time=4
sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=4k --size=4G --readwrite=randread --ramp_time=4

Check Power Limits

Code Block

root@truenas:/mnt/pool/shared/user# turbostat 2>&1|grep Watt
RAPL: 43691 sec. Joule Counter Range, at 6 Watts
cpu0: MSR_RAPL_POWER_UNIT: 0x000a0e03 (0.125000 Watts, 0.000061 Joules, 0.000977 sec.)
cpu0: PKG Limit #1: ENabled (15.000 Watts, 8.000000 sec, clamp ENabled)
cpu0: PKG Limit #2: ENabled (15.000 Watts, 0.002441* sec, clamp DISabled)
cpu0: PKG Limit #4: 78.000000 Watts (UNlocked)
cpu0: DRAM Limit: DISabled (0.000 Watts, 0.000977 sec, clamp DISabled)
cpu0: Cores Limit: DISabled (0.000 Watts, 0.000977 sec, clamp DISabled)
cpu0: GFX Limit: DISabled (0.000 Watts, 0.000977 sec, clamp DISabled)

Code Block
root@truenas:/mnt/pool/shared/user# zfs get all pool \|grep comp pool compressratio 1.61x - pool compression lz4 local pool refcompressratio 1.00x -

Code Block
root@truenas:/mnt/pool/shared/user# zfs get all pool/shared \|grep comp pool/shared compressratio 1.41x - pool/shared compression lz4 inherited from pool pool/shared refcompressratio 1.41x -

RAIDZ1 (LZ) commands output

Code Block

root@truenas:/mnt/pool/shared/user# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=4k --size=4G --readwrite=randwrite --ramp_time=4
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
fio-3.33
Starting 1 process
test: Laying out IO file (1 file / 4096MiB)
Jobs: 1 (f=0): [f(1)][100.0%][w=147MiB/s][w=37.6k IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=41307: Thu Apr 10 12:49:43 2025
  write: IOPS=21.1k, BW=82.5MiB/s (86.5MB/s)(3672MiB/44503msec); 0 zone resets
    slat (usec): min=4, max=80062, avg=45.37, stdev=167.42
    clat (nsec): min=614, max=4072.6k, avg=1133.86, stdev=6985.89
     lat (usec): min=5, max=80072, avg=46.50, stdev=167.70
    clat percentiles (nsec):
     |  1.00th=[  740],  5.00th=[  836], 10.00th=[  884], 20.00th=[  924],
     | 30.00th=[  956], 40.00th=[  972], 50.00th=[  996], 60.00th=[ 1020],
     | 70.00th=[ 1048], 80.00th=[ 1080], 90.00th=[ 1192], 95.00th=[ 1384],
     | 99.00th=[ 2008], 99.50th=[ 3312], 99.90th=[24704], 99.95th=[41728],
     | 99.99th=[72192]
   bw (  KiB/s): min=48766, max=137119, per=98.81%, avg=83487.14, stdev=11581.15, samples=87
   iops        : min=12191, max=34279, avg=20872.08, stdev=2895.09, samples=87
  lat (nsec)   : 750=1.20%, 1000=50.42%
  lat (usec)   : 2=47.36%, 4=0.59%, 10=0.14%, 20=0.16%, 50=0.10%
  lat (usec)   : 100=0.02%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
  lat (msec)   : 2=0.01%, 4=0.01%, 10=0.01%
  cpu          : usr=3.69%, sys=85.07%, ctx=107206, majf=0, minf=37
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,940003,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
  WRITE: bw=82.5MiB/s (86.5MB/s), 82.5MiB/s-82.5MiB/s (86.5MB/s-86.5MB/s), io=3672MiB (3850MB), run=44503-44503msec


root@truenas:/mnt/pool/shared/user# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=4k --size=4G --readwrite=randread --ramp_time=4
test: (g=0): rw=randread, bstest: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
fio-3.33
Starting 1 process
test: Laying out IO file (1 file / 4096MiB)
Jobs: 1 (f=01): [fr(1)][10091.0%3%][wr=147MiB160MiB/s][wr=3741.6k0k IOPS][eta 00m:00s04s]
test: (groupid=0, jobs=1): err= 0: pid=4130741962: Thu Apr 10 12:4950:4358 2025
  writeread: IOPS=2125.1k4k, BW=8299.5MiB2MiB/s (86.5MB104MB/s)(3672MiB3708MiB/44503msec); 0 zone resets37391msec)
    slat (usec): min=42, max=80062467, avg=4538.3712, stdev=16711.4250
    clat (nsec): min=614587, max=4072.6k54938, avg=1133738.8669, stdev=6985251.8978
     lat (usec): min=53, max=80072470, avg=4638.5086, stdev=16711.7053
    clat percentiles (nsec):
     |  1.00th=[  740636],  5.00th=[  836668], 10.00th=[  884684], 20.00th=[  924692],
     | 30.00th=[  956700], 40.00th=[  972708], 50.00th=[  996716], 60.00th=[  1020724],
     | 70.00th=[ 1048 740], 80.00th=[  1080756], 90.00th=[ 1192 804], 95.00th=[  1384876],
     | 99.00th=[ 20081012], 99.50th=[ 33121096], 99.90th=[24704 2416], 99.95th=[41728 3056],
     | 99.99th=[7219214144]
   bw (  KiB/s): min=4876691048, max=137119157648, per=98.81%65%, avg=83487100191.1419, stdev=115818691.1521, samples=8774
   iops        : min=1219122762, max=3427939412, avg=2087225047.0876, stdev=28952172.0979, samples=87
  lat (nsec)   : 750=1.20%, 1000=50.42%=74
  lat (usecnsec)   : 2750=4776.36%93%, 41000=0.59%, 10=0.14%, 20=0.16%, 50=0.10%21.93%
  lat (usec)   : 1002=01.02%03%, 2504=0.01%08%, 50010=0.01%, 75020=0.01%, 100050=0.01%
  lat (msecusec)   : 2100=0.01%, 4=0.01%, 10=0.01%
  cpu          : usr=32.69%95%, sys=8596.07%97%, ctx=107206420, majf=0, minf=37
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=949364,0,940003,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
   WRITEREAD: bw=8299.5MiB2MiB/s (86.5MB104MB/s), 8299.5MiB2MiB/s-8299.5MiB2MiB/s (86.5MB104MB/s-86.5MB104MB/s), io=3672MiB3708MiB (3850MB3889MB), run=4450337391-44503msec37391msec


root@truenas:/mnt/pool/shared/user# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=4k --size=4G --readwrite=randreadwrite --ramp_time=4
test: (g=0): rw=randreadwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
fio-3.33
Starting 1 process
Jobs: 1 (f=1): [rW(1)][9153.3%8%][rw=160MiB555MiB/s][rw=41.0k142k IOPS][eta 00m:04s06s]
test: (groupid=0, jobs=1): err= 0: pid=4196243581: Thu Apr 10 12:5053:5842 2025
  readwrite: IOPS=25.4k145k, BW=99.2MiB568MiB/s (104MB595MB/s)(3708MiB/37391msec)1808MiB/3184msec); 0 zone resets
    slat (usec): min=23, max=4677144, avg=38 5.1285, stdev=1113.5087
    clat (nsec): min=587581, max=54938149242, avg=738722.6902, stdev=251731.7807
     lat (usec): min=34, max=4707154, avg=38 6.8657, stdev=1113.5394
    clat percentiles (nsec):
     |  1.00th=[  636],  5.00th=[  668644], 10.00th=[  684652], 20.00th=[  692660],
     | 30.00th=[  700668], 40.00th=[  708676], 50.00th=[  716684], 60.00th=[  724692],
     | 70.00th=[  740700], 80.00th=[  756732], 90.00th=[  804844], 95.00th=[  876892],
     | 99.00th=[ 1012 972], 99.50th=[ 10961144], 99.90th=[ 24161576], 99.95th=[ 30564384],
     | 99.99th=[1414447360]
   bw (  KiB/s): min=91048565370, max=157648595054, per=9899.65%72%, avg=100191579799.1900, stdev=869112657.2107, samples=746
   iops        : min=22762141342, max=39412148763, avg=25047144949.7650, stdev=21723164.7921, samples=746
  lat (nsec)   : 750=7682.93%16%, 1000=2117.93%03%
  lat (usec)   : 2=10.03%73%, 4=0.08%02%, 10=0.01%02%, 20=0.01%, 50=0.01%
  lat (usec)   : 100=0.01%, 250=0.01%
  cpu          : usr=213.95%51%, sys=9684.97%32%, ctx=4201523, majf=0, minf=37
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=9493640,0462792,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
   READWRITE: bw=99.2MiB568MiB/s (104MB595MB/s), 99.2MiB568MiB/s-99.2MiB568MiB/s (104MB595MB/s-104MB595MB/s), io=3708MiB1808MiB (3889MB1896MB), run=373913184-37391msec3184msec

Page tree

Versions Compared

Old Version 1

New Version 2

Key