base test [root@centos7-server1 ~]# fio -filename=/dev/vdb -direct=1 -iodepth 128 -thread -rw=randwrite -ioengine=libaio -bs=4k -numjobs=2 -runtime=60 -group_reporting -name=vmtest vmtest: (g=0): rw=randwrite, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=128 ... fio-2.2.10 Starting 2 threads Jobs: 2 (f=2): [w(2)] [100.0% done] [0KB/256KB/0KB /s] [0/64/0 iops] [eta 00m:00s] vmtest: (groupid=0, jobs=2): err= 0: pid=5179: Wed Jan 27 02:47:31 2016 write: io=665868KB, bw=11052KB/s, iops=2763, runt= 60248msec slat (usec): min=3, max=5170.3K, avg=716.73, stdev=26963.78 clat (msec): min=2, max=5314, avg=91.91, stdev=327.22 lat (msec): min=3, max=5314, avg=92.62, stdev=328.54 clat percentiles (msec): | 1.00th=[ 12], 5.00th=[ 16], 10.00th=[ 18], 20.00th=[ 21], | 30.00th=[ 23], 40.00th=[ 25], 50.00th=[ 28], 60.00th=[ 30], | 70.00th=[ 34], 80.00th=[ 41], 90.00th=[ 165], 95.00th=[ 322], | 99.00th=[ 1516], 99.50th=[ 2278], 99.90th=[ 5211], 99.95th=[ 5276], | 99.99th=[ 5342] bw (KB /s): min= 139, max=21123, per=69.25%, avg=7653.90, stdev=5854.55 lat (msec) : 4=0.01%, 10=0.46%, 20=17.28%, 50=65.14%, 100=4.19% lat (msec) : 250=6.42%, 500=3.45%, 750=1.09%, 1000=0.66%, 2000=0.54% lat (msec) : >=2000=0.77% cpu : usr=0.45%, sys=1.01%, ctx=5227, majf=0, minf=9 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1% issued : total=r=0/w=166467/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=128 Run status group 0 (all jobs): WRITE: io=665868KB, aggrb=11052KB/s, minb=11052KB/s, maxb=11052KB/s, mint=60248msec, maxt=60248msec Disk stats (read/write): vdb: ios=186/166394, merge=0/1, ticks=296/8750626, in_queue=8970271, util=100.00% [root@centos7-server1 ~]# [root@centos7-server1 ~]# [root@centos7-server1 ~]# [root@centos7-server1 ~]# fio -filename=/dev/vdb -direct=1 -iodepth 128 -thread -rw=randrad -ioengine=libaio -bs=4k -numjobs=2 -runtime=60 -group_reporting -name=vmtest valid values: read Sequential read : write Sequential write : trim Sequential trim : randread Random read : randwrite Random write : randtrim Random trim : rw Sequential read and write mix : readwrite Sequential read and write mix : randrw Random read and write mix : trimwrite Trim and write mix, trims preceding writes fio: failed parsing rw=randrad [root@centos7-server1 ~]# fio -filename=/dev/vdb -direct=1 -iodepth 128 -thread -rw=randread -ioengine=libaio -bs=4k -numjobs=2 -runtime=60 -group_reporting -name=vmtest vmtest: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=128 ... fio-2.2.10 Starting 2 threads Jobs: 2 (f=2): [r(2)] [100.0% done] [95316KB/0KB/0KB /s] [23.9K/0/0 iops] [eta 00m:00s] vmtest: (groupid=0, jobs=2): err= 0: pid=5193: Wed Jan 27 02:49:00 2016 read : io=5841.4MB, bw=99679KB/s, iops=24919, runt= 60008msec slat (usec): min=2, max=10773, avg=76.26, stdev=406.91 clat (msec): min=1, max=26, avg=10.19, stdev= 1.33 lat (msec): min=1, max=26, avg=10.27, stdev= 1.35 clat percentiles (usec): | 1.00th=[ 7200], 5.00th=[ 8256], 10.00th=[ 8640], 20.00th=[ 9152], | 30.00th=[ 9536], 40.00th=[ 9792], 50.00th=[10176], 60.00th=[10432], | 70.00th=[10688], 80.00th=[11200], 90.00th=[11840], 95.00th=[12480], | 99.00th=[13760], 99.50th=[14400], 99.90th=[16192], 99.95th=[16768], | 99.99th=[18048] bw (KB /s): min=40712, max=55544, per=50.05%, avg=49892.14, stdev=2202.62 lat (msec) : 2=0.01%, 4=0.03%, 10=46.09%, 20=53.88%, 50=0.01% cpu : usr=3.72%, sys=9.07%, ctx=46835, majf=0, minf=266 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1% issued : total=r=1495377/w=0/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=128 Run status group 0 (all jobs): READ: io=5841.4MB, aggrb=99678KB/s, minb=99678KB/s, maxb=99678KB/s, mint=60008msec, maxt=60008msec Disk stats (read/write): vdb: ios=1494567/0, merge=2/0, ticks=8884976/0, in_queue=8886754, util=99.95% [root@centos7-server1 ~]# [root@centos7-server1 ~]# fio -filename=/dev/vdb -direct=1 -iodepth 128 -thread -rw=randread -ioengine=libaio -bs=512k -numjobs=2 -runtime=60 -group_reporting -name=vmtest vmtest: (g=0): rw=randread, bs=512K-512K/512K-512K/512K-512K, ioengine=libaio, iodepth=128 ... fio-2.2.10 Starting 2 threads Jobs: 2 (f=2): [r(2)] [33.2% done] [1114MB/0KB/0KB /s] [2228/0/0 iops] [eta 02m:03s] vmtest: (groupid=0, jobs=2): err= 0: pid=5199: Wed Jan 27 02:50:21 2016 read : io=67962MB, bw=1129.7MB/s, iops=2259, runt= 60162msec slat (usec): min=21, max=48661, avg=878.20, stdev=4845.31 clat (msec): min=11, max=946, avg=112.19, stdev=114.11 lat (msec): min=11, max=971, avg=113.06, stdev=114.19 clat percentiles (msec): | 1.00th=[ 43], 5.00th=[ 53], 10.00th=[ 57], 20.00th=[ 58], | 30.00th=[ 70], 40.00th=[ 82], 50.00th=[ 85], 60.00th=[ 86], | 70.00th=[ 97], 80.00th=[ 114], 90.00th=[ 169], 95.00th=[ 338], | 99.00th=[ 676], 99.50th=[ 758], 99.90th=[ 848], 99.95th=[ 873], | 99.99th=[ 914] bw (KB /s): min=277942, max=617650, per=50.01%, avg=578537.80, stdev=25856.51 lat (msec) : 20=0.01%, 50=4.89%, 100=70.40%, 250=18.21%, 500=3.58% lat (msec) : 750=2.39%, 1000=0.52% cpu : usr=0.41%, sys=4.03%, ctx=4433, majf=0, minf=78 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1% issued : total=r=135924/w=0/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=128 Run status group 0 (all jobs): READ: io=67962MB, aggrb=1129.7MB/s, minb=1129.7MB/s, maxb=1129.7MB/s, mint=60162msec, maxt=60162msec Disk stats (read/write): vdb: ios=135845/0, merge=135924/0, ticks=8768853/0, in_queue=8779505, util=99.98% [root@centos7-server1 ~]# fio -filename=/dev/vdb -direct=1 -iodepth 128 -thread -rw=randwrite -ioengine=libaio -bs=512k -numjobs=2 -runtime=60 -group_reporting -name=vmtest vmtest: (g=0): rw=randwrite, bs=512K-512K/512K-512K/512K-512K, ioengine=libaio, iodepth=128 ... fio-2.2.10 Starting 2 threads Jobs: 2 (f=2): [w(2)] [100.0% done] [0KB/271.5MB/0KB /s] [0/542/0 iops] [eta 00m:00s] vmtest: (groupid=0, jobs=2): err= 0: pid=5205: Wed Jan 27 02:51:45 2016 write: io=15939MB, bw=271285KB/s, iops=529, runt= 60162msec slat (usec): min=36, max=951160, avg=3748.81, stdev=27801.80 clat (msec): min=26, max=2718, avg=478.20, stdev=286.69 lat (msec): min=27, max=2718, avg=481.95, stdev=287.36 clat percentiles (msec): | 1.00th=[ 113], 5.00th=[ 172], 10.00th=[ 219], 20.00th=[ 265], | 30.00th=[ 302], 40.00th=[ 347], 50.00th=[ 396], 60.00th=[ 453], | 70.00th=[ 529], 80.00th=[ 644], 90.00th=[ 898], 95.00th=[ 1090], | 99.00th=[ 1401], 99.50th=[ 1500], 99.90th=[ 1909], 99.95th=[ 2704], | 99.99th=[ 2704] bw (KB /s): min= 2015, max=263168, per=51.81%, avg=140540.45, stdev=55465.39 lat (msec) : 50=0.01%, 100=0.58%, 250=15.63%, 500=50.53%, 750=17.77% lat (msec) : 1000=8.45%, 2000=6.98%, >=2000=0.05% cpu : usr=2.52%, sys=0.98%, ctx=1072, majf=0, minf=9 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.2%, >=64=99.6% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1% issued : total=r=0/w=31877/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0 latency : target=0, window=0, percentile=100.00%, depth=128 Run status group 0 (all jobs): WRITE: io=15939MB, aggrb=271284KB/s, minb=271284KB/s, maxb=271284KB/s, mint=60162msec, maxt=60162msec Disk stats (read/write): vdb: ios=186/31678, merge=0/31811, ticks=886/9131443, in_queue=9157139, util=99.92%