root@pve-250:/mnt/pve/pbs-253-nfs# fio --filename=./io-test --size=5GB --direct=1 --rw=randrw --bs=4k --ioengine=libaio --iodepth=256 --runtime=120 --numjobs=4 --time_based --group_reporting --name=iops-test-job --eta-newline=1
iops-test-job: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=256
...
fio-3.33
Starting 4 processes
iops-test-job: Laying out IO file (1 file / 5120MiB)
Jobs: 4 (f=1): [m(4)][2.5%][r=2022KiB/s,w=2038KiB/s][r=505,w=509 IOPS][eta 01m:57s]
Jobs: 4 (f=1): [m(4)][4.2%][r=1184KiB/s,w=1256KiB/s][r=296,w=314 IOPS][eta 01m:55s]
Jobs: 4 (f=1): [m(4)][5.8%][r=1000KiB/s,w=1024KiB/s][r=250,w=256 IOPS][eta 01m:53s]
Jobs: 4 (f=1): [m(4)][7.5%][r=888KiB/s,w=800KiB/s][r=222,w=200 IOPS][eta 01m:51s]
Jobs: 4 (f=1): [m(4)][9.2%][r=1009KiB/s,w=1069KiB/s][r=252,w=267 IOPS][eta 01m:49s]
Jobs: 4 (f=1): [m(4)][10.8%][r=904KiB/s,w=812KiB/s][r=226,w=203 IOPS][eta 01m:47s]
Jobs: 4 (f=1): [m(4)][12.5%][r=1008KiB/s,w=888KiB/s][r=252,w=222 IOPS][eta 01m:45s]
Jobs: 4 (f=1): [m(4)][14.2%][r=1060KiB/s,w=832KiB/s][r=265,w=208 IOPS][eta 01m:43s]
Jobs: 4 (f=1): [m(4)][15.8%][r=1428KiB/s,w=1332KiB/s][r=357,w=333 IOPS][eta 01m:41s]
Jobs: 4 (f=1): [m(4)][17.5%][r=860KiB/s,w=976KiB/s][r=215,w=244 IOPS][eta 01m:39s]
Jobs: 4 (f=1): [m(4)][19.2%][r=1228KiB/s,w=1128KiB/s][r=307,w=282 IOPS][eta 01m:37s]
Jobs: 4 (f=1): [m(4)][20.8%][r=1168KiB/s,w=1164KiB/s][r=292,w=291 IOPS][eta 01m:35s]
Jobs: 4 (f=1): [m(4)][22.5%][r=1145KiB/s,w=1081KiB/s][r=286,w=270 IOPS][eta 01m:33s]
Jobs: 4 (f=1): [m(4)][24.2%][r=1196KiB/s,w=1216KiB/s][r=299,w=304 IOPS][eta 01m:31s]
Jobs: 4 (f=1): [m(4)][25.8%][r=1172KiB/s,w=1184KiB/s][r=293,w=296 IOPS][eta 01m:29s]
Jobs: 4 (f=1): [m(4)][27.5%][r=1013KiB/s,w=1165KiB/s][r=253,w=291 IOPS][eta 01m:27s]
Jobs: 4 (f=1): [m(4)][29.2%][r=1412KiB/s,w=1276KiB/s][r=353,w=319 IOPS][eta 01m:25s]
Jobs: 4 (f=1): [m(4)][30.8%][r=1204KiB/s,w=1308KiB/s][r=301,w=327 IOPS][eta 01m:23s]
Jobs: 4 (f=1): [m(4)][32.5%][r=1113KiB/s,w=1149KiB/s][r=278,w=287 IOPS][eta 01m:21s]
Jobs: 4 (f=1): [m(4)][34.2%][r=1680KiB/s,w=1600KiB/s][r=420,w=400 IOPS][eta 01m:19s]
Jobs: 4 (f=1): [m(4)][35.8%][r=1052KiB/s,w=1068KiB/s][r=263,w=267 IOPS][eta 01m:17s]
Jobs: 4 (f=1): [m(4)][37.5%][r=1144KiB/s,w=1164KiB/s][r=286,w=291 IOPS][eta 01m:15s]
Jobs: 4 (f=1): [m(4)][39.2%][r=1356KiB/s,w=1364KiB/s][r=339,w=341 IOPS][eta 01m:13s]
Jobs: 4 (f=1): [m(4)][40.8%][r=1261KiB/s,w=1385KiB/s][r=315,w=346 IOPS][eta 01m:11s]
Jobs: 4 (f=1): [m(4)][42.5%][r=1517KiB/s,w=1453KiB/s][r=379,w=363 IOPS][eta 01m:09s]
Jobs: 4 (f=1): [m(4)][44.2%][r=1512KiB/s,w=1580KiB/s][r=378,w=395 IOPS][eta 01m:07s]
Jobs: 4 (f=1): [m(4)][45.8%][r=1532KiB/s,w=1612KiB/s][r=383,w=403 IOPS][eta 01m:05s]
Jobs: 4 (f=1): [m(4)][47.5%][r=1480KiB/s,w=1420KiB/s][r=370,w=355 IOPS][eta 01m:03s]
Jobs: 4 (f=1): [m(4)][49.2%][r=1741KiB/s,w=1641KiB/s][r=435,w=410 IOPS][eta 01m:01s]
Jobs: 4 (f=1): [m(4)][50.8%][r=1828KiB/s,w=1788KiB/s][r=457,w=447 IOPS][eta 00m:59s]
Jobs: 4 (f=1): [m(4)][52.5%][r=1496KiB/s,w=1432KiB/s][r=374,w=358 IOPS][eta 00m:57s]
Jobs: 4 (f=1): [m(4)][54.2%][r=1160KiB/s,w=1288KiB/s][r=290,w=322 IOPS][eta 00m:55s]
Jobs: 4 (f=1): [m(4)][55.8%][r=1708KiB/s,w=1896KiB/s][r=427,w=474 IOPS][eta 00m:53s]
Jobs: 4 (f=1): [m(4)][57.5%][r=1645KiB/s,w=1681KiB/s][r=411,w=420 IOPS][eta 00m:51s]
Jobs: 4 (f=1): [m(4)][59.2%][r=1837KiB/s,w=2070KiB/s][r=459,w=517 IOPS][eta 00m:49s]
Jobs: 4 (f=1): [m(4)][60.8%][r=2028KiB/s,w=1900KiB/s][r=507,w=475 IOPS][eta 00m:47s]
Jobs: 4 (f=1): [m(4)][62.5%][r=1640KiB/s,w=1592KiB/s][r=410,w=398 IOPS][eta 00m:45s]
Jobs: 4 (f=1): [m(4)][64.2%][r=2120KiB/s,w=2116KiB/s][r=530,w=529 IOPS][eta 00m:43s]
Jobs: 4 (f=1): [m(4)][65.8%][r=2094KiB/s,w=2098KiB/s][r=523,w=524 IOPS][eta 00m:41s]
Jobs: 4 (f=1): [m(4)][67.5%][r=1861KiB/s,w=1857KiB/s][r=465,w=464 IOPS][eta 00m:39s]
Jobs: 4 (f=1): [m(4)][69.2%][r=1964KiB/s,w=1868KiB/s][r=491,w=467 IOPS][eta 00m:37s]
Jobs: 4 (f=1): [m(4)][70.8%][r=2016KiB/s,w=2148KiB/s][r=504,w=537 IOPS][eta 00m:35s]
Jobs: 4 (f=1): [m(4)][72.5%][r=2254KiB/s,w=2058KiB/s][r=563,w=514 IOPS][eta 00m:33s]
Jobs: 4 (f=1): [m(4)][74.2%][r=2118KiB/s,w=2138KiB/s][r=529,w=534 IOPS][eta 00m:31s]
Jobs: 4 (f=1): [m(4)][75.8%][r=2048KiB/s,w=1776KiB/s][r=512,w=444 IOPS][eta 00m:29s]
Jobs: 4 (f=1): [m(4)][77.5%][r=2076KiB/s,w=2056KiB/s][r=519,w=514 IOPS][eta 00m:27s]
Jobs: 4 (f=1): [m(4)][79.2%][r=1953KiB/s,w=2030KiB/s][r=488,w=507 IOPS][eta 00m:25s]
Jobs: 4 (f=1): [m(4)][80.8%][r=1973KiB/s,w=1793KiB/s][r=493,w=448 IOPS][eta 00m:23s]
Jobs: 4 (f=1): [m(4)][82.5%][r=1656KiB/s,w=1568KiB/s][r=414,w=392 IOPS][eta 00m:21s]
Jobs: 4 (f=1): [m(4)][84.2%][r=2392KiB/s,w=2476KiB/s][r=598,w=619 IOPS][eta 00m:19s]
Jobs: 4 (f=1): [m(4)][85.8%][r=1909KiB/s,w=1649KiB/s][r=477,w=412 IOPS][eta 00m:17s]
Jobs: 4 (f=1): [m(4)][87.5%][r=2000KiB/s,w=1984KiB/s][r=500,w=496 IOPS][eta 00m:15s]
Jobs: 4 (f=1): [m(4)][89.2%][r=2296KiB/s,w=2176KiB/s][r=574,w=544 IOPS][eta 00m:13s]
Jobs: 4 (f=1): [m(4)][90.8%][r=1920KiB/s,w=1912KiB/s][r=480,w=478 IOPS][eta 00m:11s]
Jobs: 4 (f=1): [m(4)][92.5%][r=2620KiB/s,w=2620KiB/s][r=655,w=655 IOPS][eta 00m:09s]
Jobs: 4 (f=1): [m(4)][94.2%][r=2270KiB/s,w=2274KiB/s][r=567,w=568 IOPS][eta 00m:07s]
Jobs: 4 (f=1): [m(4)][95.8%][r=2480KiB/s,w=2416KiB/s][r=620,w=604 IOPS][eta 00m:05s]
Jobs: 4 (f=1): [m(4)][97.5%][r=2220KiB/s,w=2072KiB/s][r=555,w=518 IOPS][eta 00m:03s]
Jobs: 4 (f=1): [m(4)][99.2%][r=2134KiB/s,w=2298KiB/s][r=533,w=574 IOPS][eta 00m:01s]
Jobs: 4 (f=1): [m(4)][100.0%][r=2712KiB/s,w=2760KiB/s][r=678,w=690 IOPS][eta 00m:00s]
iops-test-job: (groupid=0, jobs=4): err= 0: pid=1632667: Wed Jan 3 18:20:40 2024
read: IOPS=409, BW=1637KiB/s (1676kB/s)(192MiB/120272msec)
slat (usec): min=2, max=105, avg= 5.46, stdev= 3.35
clat (usec): min=466, max=768934, avg=242125.87, stdev=91934.43
lat (usec): min=490, max=768938, avg=242131.33, stdev=91934.63
clat percentiles (msec):
| 1.00th=[ 101], 5.00th=[ 127], 10.00th=[ 146], 20.00th=[ 169],
| 30.00th=[ 186], 40.00th=[ 205], 50.00th=[ 222], 60.00th=[ 241],
| 70.00th=[ 271], 80.00th=[ 309], 90.00th=[ 376], 95.00th=[ 435],
| 99.00th=[ 514], 99.50th=[ 550], 99.90th=[ 617], 99.95th=[ 651],
| 99.99th=[ 709]
bw ( KiB/s): min= 624, max= 2856, per=99.99%, avg=1637.15, stdev=503.82, samples=240
iops : min= 156, max= 714, avg=409.28, stdev=125.95, samples=240
write: IOPS=409, BW=1637KiB/s (1677kB/s)(192MiB/120360msec); 0 zone resets
slat (nsec): min=2459, max=99242, avg=5865.26, stdev=3519.69
clat (msec): min=74, max=926, avg=383.00, stdev=128.82
lat (msec): min=74, max=926, avg=383.00, stdev=128.82
clat percentiles (msec):
| 1.00th=[ 197], 5.00th=[ 228], 10.00th=[ 251], 20.00th=[ 279],
| 30.00th=[ 305], 40.00th=[ 326], 50.00th=[ 351], 60.00th=[ 380],
| 70.00th=[ 422], 80.00th=[ 485], 90.00th=[ 575], 95.00th=[ 642],
| 99.00th=[ 768], 99.50th=[ 810], 99.90th=[ 885], 99.95th=[ 911],
| 99.99th=[ 927]
bw ( KiB/s): min= 584, max= 2920, per=99.98%, avg=1637.32, stdev=493.43, samples=240
iops : min= 146, max= 730, avg=409.33, stdev=123.36, samples=240
lat (usec) : 500=0.01%, 1000=0.01%
lat (msec) : 2=0.02%, 4=0.01%, 50=0.01%, 100=0.51%, 250=36.12%
lat (msec) : 500=53.61%, 750=9.12%, 1000=0.60%
cpu : usr=0.06%, sys=0.14%, ctx=46096, majf=0, minf=43
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued rwts: total=49225,49268,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=256
Run status group 0 (all jobs):
READ: bw=1637KiB/s (1676kB/s), 1637KiB/s-1637KiB/s (1676kB/s-1676kB/s), io=192MiB (202MB), run=120272-120272msec
WRITE: bw=1637KiB/s (1677kB/s), 1637KiB/s-1637KiB/s (1677kB/s-1677kB/s), io=192MiB (202MB), run=120360-120360msec