|
[root@ceph102 ~]# ceph osd dump |grep pool
|
|
pool 0 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 1 flags hashpspool stripe_width 0
|
|
pool 1 'volumes' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 1024 pgp_num 1024 last_change 3309 lfor 3309 flags hashpspool tiers 4 read_tier 4 write_tier 4 stripe_width 0
|
|
pool 2 'images' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 128 pgp_num 128 last_change 2416 flags hashpspool stripe_width 0
|
|
pool 3 'vms' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 128 pgp_num 128 last_change 1824 flags hashpspool stripe_width 0
|
|
pool 4 'ssd-pool' replicated size 3 min_size 2 crush_ruleset 1 object_hash rjenkins pg_num 64 pgp_num 64 last_change 3310 flags hashpspool,incomplete_clones tier_of 1 cache_mode writeback hit_set bloom{false_positive_probability: 0.05, target_size: 0, seed: 0} 0s x0 stripe_width 0
|
|
|
|
[root@ceph102 ~]# ceph osd tree
|
|
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
|
|
-5 1.07996 root ssd-root
|
|
30 0.35999 osd.30 up 1.00000 1.00000
|
|
31 0.35999 osd.31 up 1.00000 1.00000
|
|
32 0.35999 osd.32 up 1.00000 1.00000
|
|
-1 8.09967 root default
|
|
-2 2.69989 host ceph104
|
|
1 0.26999 osd.1 up 1.00000 1.00000
|
|
4 0.26999 osd.4 up 1.00000 1.00000
|
|
7 0.26999 osd.7 up 1.00000 1.00000
|
|
10 0.26999 osd.10 up 1.00000 1.00000
|
|
14 0.26999 osd.14 up 1.00000 1.00000
|
|
17 0.26999 osd.17 up 1.00000 1.00000
|
|
20 0.26999 osd.20 up 1.00000 1.00000
|
|
23 0.26999 osd.23 up 1.00000 1.00000
|
|
25 0.26999 osd.25 up 1.00000 1.00000
|
|
27 0.26999 osd.27 up 1.00000 1.00000
|
|
-3 2.69989 host ceph103
|
|
2 0.26999 osd.2 up 1.00000 1.00000
|
|
5 0.26999 osd.5 up 1.00000 1.00000
|
|
9 0.26999 osd.9 up 1.00000 1.00000
|
|
12 0.26999 osd.12 up 1.00000 1.00000
|
|
16 0.26999 osd.16 up 1.00000 1.00000
|
|
19 0.26999 osd.19 up 1.00000 1.00000
|
|
22 0.26999 osd.22 up 1.00000 1.00000
|
|
24 0.26999 osd.24 up 1.00000 1.00000
|
|
28 0.26999 osd.28 up 1.00000 1.00000
|
|
0 0.26999 osd.0 up 1.00000 1.00000
|
|
-4 2.69989 host ceph102
|
|
3 0.26999 osd.3 up 1.00000 1.00000
|
|
6 0.26999 osd.6 up 1.00000 1.00000
|
|
8 0.26999 osd.8 up 1.00000 1.00000
|
|
11 0.26999 osd.11 up 1.00000 1.00000
|
|
13 0.26999 osd.13 up 1.00000 1.00000
|
|
15 0.26999 osd.15 up 1.00000 1.00000
|
|
18 0.26999 osd.18 up 1.00000 1.00000
|
|
21 0.26999 osd.21 up 1.00000 1.00000
|
|
26 0.26999 osd.26 up 1.00000 1.00000
|
|
29 0.26999 osd.29 up 1.00000 1.00000
|
|
[root@ceph102 ~]# ceph osd crush rule dump
|
|
[
|
|
{
|
|
"rule_id": 0,
|
|
"rule_name": "replicated_ruleset",
|
|
"ruleset": 0,
|
|
"type": 1,
|
|
"min_size": 1,
|
|
"max_size": 10,
|
|
"steps": [
|
|
{
|
|
"op": "take",
|
|
"item": -1,
|
|
"item_name": "default"
|
|
},
|
|
{
|
|
"op": "chooseleaf_firstn",
|
|
"num": 0,
|
|
"type": "host"
|
|
},
|
|
{
|
|
"op": "emit"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"rule_id": 1,
|
|
"rule_name": "ssd-rule",
|
|
"ruleset": 1,
|
|
"type": 1,
|
|
"min_size": 1,
|
|
"max_size": 10,
|
|
"steps": [
|
|
{
|
|
"op": "take",
|
|
"item": -5,
|
|
"item_name": "ssd-root"
|
|
},
|
|
{
|
|
"op": "choose_firstn",
|
|
"num": 0,
|
|
"type": "osd"
|
|
},
|
|
{
|
|
"op": "emit"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
|
|
[root@ceph102 ~]# ceph osd df
|
|
ID WEIGHT REWEIGHT SIZE USE AVAIL %USE VAR
|
|
30 0.35999 1.00000 372G 220G 152G 59.15 1.10
|
|
31 0.35999 1.00000 372G 220G 152G 59.15 1.10
|
|
32 0.35999 1.00000 372G 220G 152G 59.15 1.10
|
|
1 0.26999 1.00000 279G 149G 130G 53.37 0.99
|
|
4 0.26999 1.00000 279G 138G 141G 49.50 0.92
|
|
7 0.26999 1.00000 279G 148G 131G 53.00 0.98
|
|
10 0.26999 1.00000 279G 146G 132G 52.58 0.98
|
|
14 0.26999 1.00000 279G 131G 147G 47.25 0.88
|
|
17 0.26999 1.00000 279G 149G 129G 53.62 0.99
|
|
20 0.26999 1.00000 279G 174G 104G 62.65 1.16
|
|
23 0.26999 1.00000 279G 139G 139G 50.12 0.93
|
|
25 0.26999 1.00000 279G 151G 127G 54.19 1.00
|
|
27 0.26999 1.00000 279G 150G 128G 54.06 1.00
|
|
2 0.26999 1.00000 279G 144G 134G 51.92 0.96
|
|
5 0.26999 1.00000 279G 159G 119G 57.25 1.06
|
|
9 0.26999 1.00000 279G 134G 144G 48.29 0.90
|
|
12 0.26999 1.00000 279G 147G 131G 52.81 0.98
|
|
16 0.26999 1.00000 279G 154G 124G 55.24 1.02
|
|
19 0.26999 1.00000 279G 133G 145G 47.83 0.89
|
|
22 0.26999 1.00000 279G 131G 148G 46.91 0.87
|
|
24 0.26999 1.00000 279G 152G 126G 54.72 1.01
|
|
28 0.26999 1.00000 279G 152G 126G 54.76 1.02
|
|
0 0.26999 1.00000 279G 177G 101G 63.50 1.18
|
|
3 0.26999 1.00000 279G 127G 151G 45.76 0.85
|
|
6 0.26999 1.00000 279G 153G 125G 54.99 1.02
|
|
8 0.26999 1.00000 279G 152G 126G 54.66 1.01
|
|
11 0.26999 1.00000 279G 137G 141G 49.28 0.91
|
|
13 0.26999 1.00000 279G 177G 101G 63.68 1.18
|
|
15 0.26999 1.00000 279G 140G 139G 50.14 0.93
|
|
18 0.26999 1.00000 279G 144G 135G 51.61 0.96
|
|
21 0.26999 1.00000 279G 152G 126G 54.63 1.01
|
|
26 0.26999 1.00000 279G 159G 119G 57.14 1.06
|
|
29 0.26999 1.00000 279G 143G 135G 51.44 0.95
|
|
TOTAL 9495G 5120G 4374G 53.93
|
|
MIN/MAX VAR: 0.85/1.18 STDDEV: 4.57
|
|
[root@ceph102 ~]# ceph df
|
|
GLOBAL:
|
|
SIZE AVAIL RAW USED %RAW USED
|
|
9495G 4374G 5120G 53.93
|
|
POOLS:
|
|
NAME ID USED %USED MAX AVAIL OBJECTS
|
|
rbd 0 303G 3.20 1014G 120040
|
|
volumes 1 2010G 21.17 1014G 735455
|
|
images 2 57340M 0.59 1014G 7184
|
|
vms 3 25544M 0.26 1014G 3291
|
|
ssd-pool 4 216G 2.28 152G 69380
|
|
[root@ceph102 ~]#
|
|
|
|
[root@centos7-server1 ~]# fio -filename=/dev/vdc -direct=1 -iodepth 128 -thread -rw=randwrite -ioengine=libaio -bs=4k -numjobs=32 -runtime=60 -group_reporting -name=vmtest
|
|
vmtest: (g=0): rw=randwrite, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=128
|
|
...
|
|
fio-2.2.10
|
|
Starting 32 threads
|
|
Jobs: 32 (f=32): [w(32)] [0.0% done] [0KB/8500KB/0KB /s] [0/2125/0 iops] [eta 05d:22h:45m:46s]
|
|
vmtest: (groupid=0, jobs=32): err= 0: pid=2214: Wed Jan 27 04:53:53 2016
|
|
write: io=466172KB, bw=7739.7KB/s, iops=1934, runt= 60232msec
|
|
slat (usec): min=3, max=1451.8K, avg=16443.02, stdev=85975.27
|
|
clat (msec): min=9, max=7108, avg=2055.04, stdev=770.59
|
|
lat (msec): min=9, max=7108, avg=2071.49, stdev=772.77
|
|
clat percentiles (msec):
|
|
| 1.00th=[ 529], 5.00th=[ 1074], 10.00th=[ 1221], 20.00th=[ 1418],
|
|
| 30.00th=[ 1614], 40.00th=[ 1778], 50.00th=[ 1942], 60.00th=[ 2147],
|
|
| 70.00th=[ 2343], 80.00th=[ 2638], 90.00th=[ 3032], 95.00th=[ 3458],
|
|
| 99.00th=[ 4490], 99.50th=[ 5080], 99.90th=[ 5604], 99.95th=[ 5735],
|
|
| 99.99th=[ 6259]
|
|
bw (KB /s): min= 1, max= 787, per=3.20%, avg=247.92, stdev=133.23
|
|
lat (msec) : 10=0.01%, 20=0.01%, 50=0.02%, 100=0.12%, 250=0.34%
|
|
lat (msec) : 500=0.48%, 750=0.59%, 1000=1.88%, 2000=49.66%, >=2000=46.90%
|
|
cpu : usr=0.05%, sys=0.05%, ctx=6570, majf=0, minf=9
|
|
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.4%, 32=0.9%, >=64=98.3%
|
|
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
|
|
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
|
|
issued : total=r=0/w=116543/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
|
|
latency : target=0, window=0, percentile=100.00%, depth=128
|
|
|
|
Run status group 0 (all jobs):
|
|
WRITE: io=466172KB, aggrb=7739KB/s, minb=7739KB/s, maxb=7739KB/s, mint=60232msec, maxt=60232msec
|
|
|
|
Disk stats (read/write):
|
|
vdc: ios=186/115996, merge=0/0, ticks=439/9224398, in_queue=9229402, util=99.63%
|
|
[root@centos7-server1 ~]#
|
|
|
|
|
|
[root@centos7-server1 ~]# fio -filename=/dev/vdc -direct=1 -iodepth 128 -thread -rw=randread -ioengine=libaio -bs=4k -numjobs=32 -runtime=60 -group_reporting -name=vmtest
|
|
vmtest: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=128
|
|
...
|
|
fio-2.2.10
|
|
Starting 32 threads
|
|
Jobs: 31 (f=31): [r(1),_(1),r(30)] [50.4% done] [71164KB/0KB/0KB /s] [17.8K/0/0 iops] [eta 01m:00s]
|
|
vmtest: (groupid=0, jobs=32): err= 0: pid=2258: Wed Jan 27 04:55:04 2016
|
|
read : io=4121.6MB, bw=70278KB/s, iops=17569, runt= 60054msec
|
|
slat (usec): min=2, max=395943, avg=1814.30, stdev=10100.76
|
|
clat (usec): min=970, max=764228, avg=231074.85, stdev=50232.78
|
|
lat (msec): min=1, max=819, avg=232.89, stdev=50.91
|
|
clat percentiles (msec):
|
|
| 1.00th=[ 137], 5.00th=[ 174], 10.00th=[ 188], 20.00th=[ 200],
|
|
| 30.00th=[ 208], 40.00th=[ 215], 50.00th=[ 223], 60.00th=[ 231],
|
|
| 70.00th=[ 243], 80.00th=[ 260], 90.00th=[ 289], 95.00th=[ 318],
|
|
| 99.00th=[ 392], 99.50th=[ 453], 99.90th=[ 619], 99.95th=[ 644],
|
|
| 99.99th=[ 725]
|
|
bw (KB /s): min= 288, max= 3528, per=3.11%, avg=2188.55, stdev=300.93
|
|
lat (usec) : 1000=0.01%
|
|
lat (msec) : 2=0.01%, 4=0.02%, 10=0.02%, 20=0.03%, 50=0.18%
|
|
lat (msec) : 100=0.22%, 250=75.33%, 500=23.81%, 750=0.39%, 1000=0.01%
|
|
cpu : usr=0.21%, sys=0.40%, ctx=37551, majf=0, minf=4106
|
|
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8%
|
|
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
|
|
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
|
|
issued : total=r=1055115/w=0/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
|
|
latency : target=0, window=0, percentile=100.00%, depth=128
|
|
|
|
Run status group 0 (all jobs):
|
|
READ: io=4121.6MB, aggrb=70277KB/s, minb=70277KB/s, maxb=70277KB/s, mint=60054msec, maxt=60054msec
|
|
|
|
Disk stats (read/write):
|
|
vdc: ios=1052317/0, merge=2/0, ticks=8968586/0, in_queue=8970112, util=99.97%
|
|
[root@centos7-server1 ~]#
|
|
[root@centos7-server1 ~]#
|
|
[root@centos7-server1 ~]#
|
|
[root@centos7-server1 ~]# fio -filename=/dev/vdc -direct=1 -iodepth 128 -thread -rw=randread -ioengine=libaio -bs=512k -numjobs=32 -runtime=60 -group_reporting -name=vmtest
|
|
vmtest: (g=0): rw=randread, bs=512K-512K/512K-512K/512K-512K, ioengine=libaio, iodepth=128
|
|
...
|
|
fio-2.2.10
|
|
Starting 32 threads
|
|
Jobs: 11 (f=11): [_(11),r(2),_(4),r(2),_(1),r(1),_(1),r(2),_(1),r(4),E(1),_(2)] [4.9% done] [2288MB/0KB/0KB /s] [4575/0/0 iops] [eta 19m:59s]
|
|
vmtest: (groupid=0, jobs=32): err= 0: pid=2294: Wed Jan 27 04:56:52 2016
|
|
read : io=64512MB, bw=1066.7MB/s, iops=2133, runt= 60481msec
|
|
slat (usec): min=16, max=981822, avg=14902.46, stdev=82830.07
|
|
clat (msec): min=2, max=3039, avg=1869.38, stdev=309.44
|
|
lat (msec): min=5, max=3441, avg=1884.28, stdev=314.55
|
|
clat percentiles (msec):
|
|
| 1.00th=[ 297], 5.00th=[ 1418], 10.00th=[ 1778], 20.00th=[ 1827],
|
|
| 30.00th=[ 1860], 40.00th=[ 1893], 50.00th=[ 1909], 60.00th=[ 1926],
|
|
| 70.00th=[ 1958], 80.00th=[ 1991], 90.00th=[ 2024], 95.00th=[ 2057],
|
|
| 99.00th=[ 2638], 99.50th=[ 2835], 99.90th=[ 2933], 99.95th=[ 2999],
|
|
| 99.99th=[ 3032]
|
|
bw (KB /s): min= 219, max=380342, per=3.08%, avg=33686.80, stdev=12398.29
|
|
lat (msec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=0.16%, 100=0.16%
|
|
lat (msec) : 250=0.54%, 500=0.91%, 750=0.59%, 1000=0.58%, 2000=82.38%
|
|
lat (msec) : >=2000=14.67%
|
|
cpu : usr=0.02%, sys=0.29%, ctx=4642, majf=0, minf=12805
|
|
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.4%, 32=0.8%, >=64=98.4%
|
|
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
|
|
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
|
|
issued : total=r=129023/w=0/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
|
|
latency : target=0, window=0, percentile=100.00%, depth=128
|
|
|
|
Run status group 0 (all jobs):
|
|
READ: io=64512MB, aggrb=1066.7MB/s, minb=1066.7MB/s, maxb=1066.7MB/s, mint=60481msec, maxt=60481msec
|
|
|
|
Disk stats (read/write):
|
|
vdc: ios=128902/0, merge=129023/0, ticks=8897009/0, in_queue=8906779, util=99.94%
|
|
[root@centos7-server1 ~]#
|
|
[root@centos7-server1 ~]#
|
|
[root@centos7-server1 ~]# fio -filename=/dev/vdc -direct=1 -iodepth 128 -thread -rw=randwrite -ioengine=libaio -bs=512k -numjobs=32 -runtime=60 -group_reporting -name=vmtest
|
|
vmtest: (g=0): rw=randwrite, bs=512K-512K/512K-512K/512K-512K, ioengine=libaio, iodepth=128
|
|
...
|
|
fio-2.2.10
|
|
Starting 32 threads
|
|
Jobs: 30 (f=30): [w(1),_(2),w(29)] [34.1% done] [0KB/264.3MB/0KB /s] [0/528/0 iops] [eta 02m:00s]
|
|
vmtest: (groupid=0, jobs=32): err= 0: pid=2330: Wed Jan 27 04:58:11 2016
|
|
write: io=12389MB, bw=205677KB/s, iops=401, runt= 61681msec
|
|
slat (usec): min=27, max=5172.5K, avg=78284.77, stdev=365112.73
|
|
clat (msec): min=125, max=24993, avg=9165.50, stdev=4566.77
|
|
lat (msec): min=186, max=25323, avg=9243.79, stdev=4582.34
|
|
clat percentiles (msec):
|
|
| 1.00th=[ 420], 5.00th=[ 2573], 10.00th=[ 3916], 20.00th=[ 5342],
|
|
| 30.00th=[ 6128], 40.00th=[ 7177], 50.00th=[ 8717], 60.00th=[10159],
|
|
| 70.00th=[11338], 80.00th=[12780], 90.00th=[15401], 95.00th=[16712],
|
|
| 99.00th=[16712], 99.50th=[16712], 99.90th=[16712], 99.95th=[16712],
|
|
| 99.99th=[16712]
|
|
bw (KB /s): min= 25, max=93391, per=3.86%, avg=7936.56, stdev=10695.74
|
|
lat (msec) : 250=0.20%, 500=1.01%, 750=0.08%, 1000=0.34%, 2000=1.78%
|
|
lat (msec) : >=2000=96.60%
|
|
cpu : usr=0.15%, sys=0.05%, ctx=3316, majf=0, minf=9
|
|
IO depths : 1=0.1%, 2=0.3%, 4=0.5%, 8=1.0%, 16=2.1%, 32=4.1%, >=64=91.9%
|
|
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
|
|
complete : 0=0.0%, 4=99.8%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.2%
|
|
issued : total=r=0/w=24778/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
|
|
latency : target=0, window=0, percentile=100.00%, depth=128
|
|
|
|
Run status group 0 (all jobs):
|
|
WRITE: io=12389MB, aggrb=205676KB/s, minb=205676KB/s, maxb=205676KB/s, mint=61681msec, maxt=61681msec
|
|
|
|
Disk stats (read/write):
|
|
vdc: ios=182/24663, merge=0/24778, ticks=128/9671785, in_queue=9789391, util=99.42%
|
|
[root@centos7-server1 ~]#
|