R730XD---- zfd set primarycache=metadata MAIN_POOL fio --filename=test --direct=1 --rw=randrw --randrepeat=0 --rwmixread=0--iodepth=16 --numjobs=32 --runtime=60 --group_reporting --name=4ktest --size=4G --bs=4k --rwmixread=0--iodepth=16 --numjobs=32 --runtime=60 --group_reporting --name=4ktest --size=4G --bs=4k 4ktest: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B,ioengine=psync, iodepth=1 ... fio-3.28 Starting 32 processes 4ktest: Laying out IO file (1 file / 4096MiB) Jobs: 32 (f=32): [w(32)][100.0%][w=63.4MiB/s][w=16.2k IOPS][eta 00m:00s] 4ktest: (groupid=0, jobs=32): err= 0: pid=18154: Mon Jan 15 15:32:23 2024 write: IOPS=13.3k, BW=52.0MiB/s (54.5MB/s)(3120MiB/60004msec); 0 zone resets clat (usec): min=8, max=4986, avg=2403.07, stdev=658.34 lat (usec): min=8, max=4987, avg=2403.15, stdev=658.35 clat percentiles (usec): | 1.00th=[ 50], 5.00th=[ 1582], 10.00th=[ 1663], 20.00th=[ 1811], | 30.00th=[ 1991], 40.00th=[ 2180], 50.00th=[ 2409], 60.00th=[ 2638], | 70.00th=[ 2868], 80.00th=[ 3097], 90.00th=[ 3261], 95.00th=[ 3294], | 99.00th=[ 3392], 99.50th=[ 3425], 99.90th=[ 3490], 99.95th=[ 3523], | 99.99th=[ 3556] bw ( KiB/s): min=39066, max=187875, per=100.00%, avg=53294.79, stdev=512.20, samples=3808 iops : min= 9760, max=46968, avg=13313.97, stdev=128.06, samples=3808 lat (usec) : 10=0.01%, 20=0.15%, 50=0.86%, 100=0.35%, 250=0.24% lat (usec) : 500=0.36%, 750=0.09%, 1000=0.09% lat (msec) : 2=28.12%, 4=69.74%, 10=0.01% cpu : usr=0.09%, sys=0.90%, ctx=783725, majf=0, minf=0 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8 zfd set primarycache=all MAIN_POOL fio --filename=test --direct=1 --rw=randrw --randrepeat=0 --rwmixread=0--iodepth=16 --numjobs=32 --runtime=60 --group_reporting --name=4ktest --size=4G --bs=4k ioengine=psync, iodepth=1 ... fio-3.28 Starting 32 processes Jobs: 32 (f=32): [w(32)][100.0%][w=63.9MiB/s][w=16.4k IOPS][eta 00m:00s] 4ktest: (groupid=0, jobs=32): err= 0: pid=18297: Mon Jan 15 15:42:23 2024 write: IOPS=13.6k, BW=52.9MiB/s (55.5MB/s)(3177MiB/60004msec); 0 zone resets clat (usec): min=5, max=22144, avg=2359.75, stdev=697.58 lat (usec): min=5, max=22144, avg=2359.85, stdev=697.59 clat percentiles (usec): | 1.00th=[ 28], 5.00th=[ 1549], 10.00th=[ 1631], 20.00th=[ 1795], | 30.00th=[ 1958], 40.00th=[ 2180], 50.00th=[ 2376], 60.00th=[ 2606], | 70.00th=[ 2835], 80.00th=[ 3064], 90.00th=[ 3228], 95.00th=[ 3261], | 99.00th=[ 3326], 99.50th=[ 3359], 99.90th=[ 3425], 99.95th=[ 3425], | 99.99th=[ 3720] bw ( KiB/s): min=39787, max=265597, per=100.00%, avg=54248.56, stdev=721.27, samples=3808 iops : min= 9935, max=66387, avg=13552.69, stdev=180.30, samples=3808 lat (usec) : 10=0.05%, 20=0.31%, 50=1.51%, 100=0.60%, 250=0.34% lat (usec) : 500=0.21%, 750=0.10%, 1000=0.07% lat (msec) : 2=28.81%, 4=68.01%, 10=0.01%, 20=0.01%, 50=0.01% cpu : usr=0.08%, sys=0.90%, ctx=795564, majf=0, minf=0 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8 ________________________________________________________________________________________ fio --filename=test --direct=1 --rw=randrw --randrepeat=0 --rwmixread=100 --iodepth=128 --numjobs=8 --runtime=60 --group_reporting --name=4ktest --ioengine=psync --size=4G --bs=4k 4ktest: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B,ioengine=psync, iodepth=128 ... fio-3.28 Starting 8 processes Jobs: 6 (f=6): [r(3),_(1),r(3),_(1)][92.3%][r=2825MiB/s][r=723k IOPS][eta 00m:01s] 4ktest: (groupid=0, jobs=8): err= 0: pid=18377: Mon Jan 15 15:45:13 2024 read: IOPS=718k, BW=2806MiB/s (2942MB/s)(32.0GiB/11678msec) clat (nsec): min=1826, max=2632.3k, avg=10465.28, stdev=29684.73 lat (nsec): min=1852, max=2632.4k, avg=10491.68, stdev=29684.84 clat percentiles (usec): | 1.00th=[ 5], 5.00th=[ 5], 10.00th=[ 6], 20.00th=[ 6], | 30.00th=[ 6], 40.00th=[ 7], 50.00th=[ 7], 60.00th=[ 8], | 70.00th=[ 8], 80.00th=[ 9], 90.00th=[ 11], 95.00th=[ 16], | 99.00th=[ 130], 99.50th=[ 196], 99.90th=[ 441], 99.95th=[ 562], | 99.99th=[ 807] bw ( MiB/s): min= 2572, max= 3289, per=100.00%, avg=2839.49, stdev=22.75, samples=174 iops : min=658482, max=842085, avg=726907.90, stdev=5823.50, samples=174 lat (usec) : 2=0.01%, 4=0.96%, 10=86.32%, 20=8.81%, 50=2.31% lat (usec) : 100=0.56%, 250=0.58%, 500=0.35%, 750=0.07%, 1000=0.01% lat (msec) : 2=0.01%, 4=0.01% cpu : usr=4.39%, sys=95.57%, ctx=2068, majf=0, minf=0 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8= R750------ zfd set primarycache=metadata NVME-Pool-01 fio --filename=test --direct=1 --rw=randrw --randrepeat=0 --rwmixread=0--iodepth=16 --numjobs=32 --runtime=60 --group_reporting --name=4ktest --size=4G --bs=4k 4ktest: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B,ioengine=psync, iodepth=1 ... fio-3.28 Starting 32 processes 4ktest: Laying out IO file (1 file / 4096MiB) Jobs: 32 (f=32): [w(32)][100.0%][w=48.5MiB/s][w=12.4k IOPS][eta 00m:00s] 4ktest: (groupid=0, jobs=32): err= 0: pid=26893: Mon Jan 15 12:37:00 2024 write: IOPS=10.9k, BW=42.5MiB/s (44.6MB/s)(2552MiB/60005msec); 0 zone resets clat (usec): min=10, max=11109, avg=2931.62, stdev=795.50 lat (usec): min=10, max=11109, avg=2932.27, stdev=795.57 clat percentiles (usec): | 1.00th=[ 92], 5.00th=[ 1926], 10.00th=[ 2008], 20.00th=[ 2212], | 30.00th=[ 2442], 40.00th=[ 2671], 50.00th=[ 2933], 60.00th=[ 3195], | 70.00th=[ 3490], 80.00th=[ 3752], 90.00th=[ 3949], 95.00th=[ 4047], | 99.00th=[ 4178], 99.50th=[ 4228], 99.90th=[ 4293], 99.95th=[ 4359], | 99.99th=[ 4883] bw ( KiB/s): min=30584, max=141641, per=100.00%, avg=43634.17, stdev=406.78, samples=3808 iops : min= 7630, max=35410, avg=10900.03, stdev=101.80, samples=3808 lat (usec) : 20=0.03%, 50=0.29%, 100=0.78%, 250=0.53%, 500=0.15% lat (usec) : 750=0.09%, 1000=0.08% lat (msec) : 2=7.35%, 4=84.06%, 10=6.65%, 20=0.01% cpu : usr=0.36%, sys=2.24%, ctx=646148, majf=0, minf=0 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8 zfd set primarycache=any NVME-Pool-01 fio --filename=test --direct=1 --rw=randrw --randrepeat=0 --rwmixread=0--iodepth=16 --numjobs=32 --runtime=60 --group_reporting --name=4ktest --size=4G --bs=4k 4ktest: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B,ioengine=psync, iodepth=1 ... fio-3.28 Starting 32 processes Jobs: 32 (f=32): [w(32)][100.0%][w=32.5MiB/s][w=8314 IOPS][eta 00m:00s] 4ktest: (groupid=0, jobs=32): err= 0: pid=26963: Mon Jan 15 12:40:08 2024 write: IOPS=11.1k, BW=43.4MiB/s (45.5MB/s)(2602MiB/60005msec); 0 zone resets clat (usec): min=8, max=49721, avg=2875.36, stdev=933.71 lat (usec): min=8, max=49722, avg=2876.01, stdev=933.80 clat percentiles (usec): | 1.00th=[ 50], 5.00th=[ 1860], 10.00th=[ 1975], 20.00th=[ 2180], | 30.00th=[ 2409], 40.00th=[ 2638], 50.00th=[ 2900], 60.00th=[ 3195], | 70.00th=[ 3490], 80.00th=[ 3752], 90.00th=[ 3949], 95.00th=[ 4015], | 99.00th=[ 4146], 99.50th=[ 4178], 99.90th=[ 4293], 99.95th=[ 4293], | 99.99th=[11338] bw ( KiB/s): min=31210, max=251749, per=100.00%, avg=44482.77, stdev=699.81, samples=3808 iops : min= 7789, max=62926, avg=11113.09, stdev=174.98, samples=3808 lat (usec) : 10=0.01%, 20=0.12%, 50=0.90%, 100=1.39%, 250=1.23% lat (usec) : 500=0.24%, 750=0.11%, 1000=0.10% lat (msec) : 2=6.94%, 4=82.94%, 10=6.03%, 20=0.01%, 50=0.01% cpu : usr=0.40%, sys=2.31%, ctx=644611, majf=0, minf=0 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8 _________________________________________________________________________________________________ fio --filename=test --direct=1 --rw=randrw --randrepeat=0 --rwmixread=100 --iodepth=128 --numjobs=8 --runtime=60 --group_reporting --name=4ktest --ioengine=psync --size=4G --bs=4k 4ktest: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B,ioengine=psync, iodepth=128 ... fio-3.28 Starting 8 processes Jobs: 8 (f=8): [r(8)][100.0%][r=2384MiB/s][r=610k IOPS][eta 00m:00s] 4ktest: (groupid=0, jobs=8): err= 0: pid=27080: Mon Jan 15 12:46:33 2024 read: IOPS=500k, BW=1954MiB/s (2049MB/s)(32.0GiB/16770msec) clat (nsec): min=1364, max=4143.2k, avg=15464.48, stdev=19955.32 lat (nsec): min=1395, max=4144.1k, avg=15519.88, stdev=19957.24 clat percentiles (usec): | 1.00th=[ 4], 5.00th=[ 5], 10.00th=[ 6], 20.00th=[ 7], | 30.00th=[ 7], 40.00th=[ 8], 50.00th=[ 8], 60.00th=[ 9], | 70.00th=[ 11], 80.00th=[ 36], 90.00th=[ 41], 95.00th=[ 44], | 99.00th=[ 59], 99.50th=[ 72], 99.90th=[ 137], 99.95th=[ 260], | 99.99th=[ 758] bw ( MiB/s): min= 1282, max= 2463, per=99.23%, avg=1939.01, stdev=22.93, samples=256 iops : min=328439, max=630595, avg=496384.28, stdev=5869.84, samples=256 lat (usec) : 2=0.01%, 4=1.53%, 10=66.60%, 20=9.12%, 50=20.19% lat (usec) : 100=2.38%, 250=0.12%, 500=0.03%, 750=0.01%, 1000=0.01% lat (msec) : 2=0.01%, 4=0.01%, 10=0.01% cpu : usr=4.11%, sys=95.77%, ctx=4439, majf=0, minf=0 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8