Project

General

Profile

Actions

Bug #9488

closed

Writing object onto EC pool created with customized ec profile getting hung

Added by Mallikarjun Biradar over 9 years ago. Updated over 9 years ago.

Status:
Rejected
Priority:
Normal
Assignee:
Category:
-
Target version:
-
% Done:

0%

Source:
other
Tags:
Backport:
Regression:
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):

Description

Writing object onto EC pool created with customized EC profile is getting hung.
But, writing object onto EC pool with default EC profile is succeeding.

I have used rados put utility to write object onto EC pool.

ems@rack6-ramp-5:~$ sudo ceph osd erasure-code-profile ls
default
ecprofile
ems@rack6-ramp-5:~$
ems@rack6-ramp-5:~$ sudo ceph osd erasure-code-profile get ecprofile
directory=/usr/lib/ceph/erasure-code
k=22
m=2
plugin=jerasure
technique=reed_sol_van
ems@rack6-ramp-5:~$ sudo ceph osd erasure-code-profile get default
directory=/usr/lib/ceph/erasure-code
k=2
m=1
plugin=jerasure
technique=reed_sol_van
ems@rack6-ramp-5:~$
ems@rack6-ramp-5:~$ sudo ceph osd pool create defprofile_pool 12 12 erasure
pool 'defprofile_pool' created
ems@rack6-ramp-5:~$
ems@rack6-ramp-5:~$ sudo ceph osd lspools
0 rbd,16 coldstor,17 coldstor1,18 coldstor2,19 defprofile_pool,
ems@rack6-ramp-5:~$ sudo ceph osd pool create custprofile_pool 12 12 erasure ecprofile
pool 'custprofile_pool' created
ems@rack6-ramp-5:~$
ems@rack6-ramp-5:~$ sudo ceph osd lspools
0 rbd,16 coldstor,17 coldstor1,18 coldstor2,19 defprofile_pool,20 custprofile_pool,
ems@rack6-ramp-5:~$
ems@rack6-ramp-5:~$ sudo rados put object1 io.fio -p defprofile_pool
ems@rack6-ramp-5:~$
ems@rack6-ramp-5:~$ sudo rados df
pool name category KB objects clones degraded unfound rd rd KB wr wr KB
coldstor - 1 2 0 0 0 0 0 2 2
coldstor1 - 1 2 0 0 0 0 0 2 2
coldstor2 - 0 0 0 0 0 0 0 0 0
custprofile_pool - 0 0 0 0 0 0 0 0 0
defprofile_pool - 1 1 0 0 0 0 0 1 1
rbd - 0 0 0 0 0 0 0 0 0
total used 359368 5
total avail 2307471320
total space 2307830688
ems@rack6-ramp-5:~$
ems@rack6-ramp-5:~$ sudo rados put object1 io.fio -p custprofile_pool

*******getting hung

ems@rack6-ramp-5:~$ sudo strace rados put object1 io.fio -p custprofile_pool
execve("/usr/bin/rados", ["rados", "put", "object1", "io.fio", "-p", "custprofile_pool"], [/* 15 vars /]) = 0
brk(0) = 0x1f67000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2cf8bd8000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=30128, ...}) = 0
mmap(NULL, 30128, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f2cf8bd0000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/librados.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\232\35\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=6976912, ...}) = 0
mmap(NULL, 41040616, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf6294000
mprotect(0x7f2cf68ca000, 2093056, PROT_NONE) = 0
mmap(0x7f2cf6ac9000, 434176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x635000) = 0x7f2cf6ac9000
mmap(0x7f2cf6b33000, 32000744, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f2cf6b33000
mprotect(0x7fffa8b6d000, 4096, PROT_READ|PROT_WRITE|PROT_EXEC|PROT_GROWSDOWN) = 0
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0po\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=141574, ...}) = 0
mmap(NULL, 2217264, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf6076000
mprotect(0x7f2cf608f000, 2093056, PROT_NONE) = 0
mmap(0x7f2cf628e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x18000) = 0x7f2cf628e000
mmap(0x7f2cf6290000, 13616, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f2cf6290000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/x86_64-linux-gnu/libboost_system.so.1.54.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\22\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=14536, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2cf8bcf000
mmap(NULL, 2109728, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf5e72000
mprotect(0x7f2cf5e75000, 2093056, PROT_NONE) = 0
mmap(0x7f2cf6074000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f2cf6074000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/x86_64-linux-gnu/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\265\5\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=979056, ...}) = 0
mmap(NULL, 3159072, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf5b6e000
mprotect(0x7f2cf5c54000, 2093056, PROT_NONE) = 0
mmap(0x7f2cf5e53000, 40960, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xe5000) = 0x7f2cf5e53000
mmap(0x7f2cf5e5d000, 82976, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f2cf5e5d000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20V\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=1071552, ...}) = 0
mmap(NULL, 3166568, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf5868000
mprotect(0x7f2cf596d000, 2093056, PROT_NONE) = 0
mmap(0x7f2cf5b6c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x104000) = 0x7f2cf5b6c000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260
\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=90080, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2cf8bce000
mmap(NULL, 2185952, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf5652000
mprotect(0x7f2cf5668000, 2093056, PROT_NONE) = 0
mmap(0x7f2cf5867000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7f2cf5867000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\37\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1845024, ...}) = 0
mmap(NULL, 3953344, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf528c000
mprotect(0x7f2cf5448000, 2093056, PROT_NONE) = 0
mmap(0x7f2cf5647000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1bb000) = 0x7f2cf5647000
mmap(0x7f2cf564d000, 17088, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f2cf564d000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/x86_64-linux-gnu/libboost_thread.so.1.54.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\257\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=88632, ...}) = 0
mmap(NULL, 2184008, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf5076000
mprotect(0x7f2cf508a000, 2093056, PROT_NONE) = 0
mmap(0x7f2cf5289000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x13000) = 0x7f2cf5289000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/x86_64-linux-gnu/liblttng-ust.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`z\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=303752, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2cf8bcd000
mmap(NULL, 2398776, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf4e2c000
mprotect(0x7f2cf4e70000, 2093056, PROT_NONE) = 0
mmap(0x7f2cf506f000, 28672, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x43000) = 0x7f2cf506f000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\16\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=14664, ...}) = 0
mmap(NULL, 2109744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf4c28000
mprotect(0x7f2cf4c2b000, 2093056, PROT_NONE) = 0
mmap(0x7f2cf4e2a000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f2cf4e2a000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/libcrypto++.so.9", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320Z*\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=5209376, ...}) = 0
mmap(NULL, 7320928, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf452c000
mprotect(0x7f2cf49b7000, 2097152, PROT_NONE) = 0
mmap(0x7f2cf4bb7000, 446464, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x48b000) = 0x7f2cf4bb7000
mmap(0x7f2cf4c24000, 13664, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f2cf4c24000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libuuid.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\26\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=18936, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2cf8bcc000
mmap(NULL, 2113968, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf4327000
mprotect(0x7f2cf432b000, 2093056, PROT_NONE) = 0
mmap(0x7f2cf452a000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f2cf452a000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/librt.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P#\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=31792, ...}) = 0
mmap(NULL, 2129016, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf411f000
mprotect(0x7f2cf4126000, 2093056, PROT_NONE) = 0
mmap(0x7f2cf4325000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f2cf4325000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/x86_64-linux-gnu/liblttng-ust-tracepoint.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\25\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=39168, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2cf8bcb000
mmap(NULL, 2200056, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf3f05000
mprotect(0x7f2cf3f0e000, 2093056, PROT_NONE) = 0
mmap(0x7f2cf410d000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8000) = 0x7f2cf410d000
mmap(0x7f2cf410f000, 61944, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f2cf410f000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/liburcu-bp.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\33\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=26912, ...}) = 0
mmap(NULL, 2122640, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf3cfe000
mprotect(0x7f2cf3d03000, 2097152, PROT_NONE) = 0
mmap(0x7f2cf3f03000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5000) = 0x7f2cf3f03000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/liburcu-cds.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220\21\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=26664, ...}) = 0
mmap(NULL, 2121928, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2cf3af7000
mprotect(0x7f2cf3afd000, 2093056, PROT_NONE) = 0
mmap(0x7f2cf3cfc000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5000) = 0x7f2cf3cfc000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2cf8bca000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2cf8bc9000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2cf8bc7000
arch_prctl(ARCH_SET_FS, 0x7f2cf8bc7840) = 0
mprotect(0x7f2cf5647000, 16384, PROT_READ) = 0
mprotect(0x7f2cf628e000, 4096, PROT_READ) = 0
mprotect(0x7f2cf3cfc000, 4096, PROT_READ) = 0
mprotect(0x7f2cf3f03000, 4096, PROT_READ) = 0
mprotect(0x7f2cf410d000, 4096, PROT_READ) = 0
mprotect(0x7f2cf4325000, 4096, PROT_READ) = 0
mprotect(0x7f2cf452a000, 4096, PROT_READ) = 0
mprotect(0x7f2cf5b6c000, 4096, PROT_READ) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2cf8bc6000
mprotect(0x7f2cf5e53000, 32768, PROT_READ) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2cf8bc4000
mprotect(0x7f2cf4bb7000, 425984, PROT_READ) = 0
mprotect(0x7f2cf4e2a000, 4096, PROT_READ) = 0
mprotect(0x7f2cf506f000, 8192, PROT_READ) = 0
mprotect(0x7f2cf6074000, 4096, PROT_READ) = 0
mprotect(0x7f2cf5289000, 8192, PROT_READ) = 0
mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2cf8bc0000
mprotect(0x7f2cf6ac9000, 339968, PROT_READ) = 0
mprotect(0x64b000, 4096, PROT_READ) = 0
mprotect(0x7f2cf8bda000, 4096, PROT_READ) = 0
munmap(0x7f2cf8bd0000, 30128) = 0
set_tid_address(0x7f2cf8bc7b10) = 8808
set_robust_list(0x7f2cf8bc7b20, 24) = 0
futex(0x7fffa8b6d950, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 1, NULL, 7f2cf8bc7840) = -1 EAGAIN (Resource temporarily unavailable)
rt_sigaction(SIGRTMIN, {0x7f2cf607c9f0, [], SA_RESTORER|SA_SIGINFO, 0x7f2cf6086340}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7f2cf607ca80, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f2cf6086340}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
brk(0) = 0x1f67000
brk(0x1f88000) = 0x1f88000
futex(0x7f2cf5e6f96c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f2cf5e6f978, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f2cf4e2b0d0, FUTEX_WAKE_PRIVATE, 2147483647) = 0
getuid() = 0
geteuid() = 0
rt_sigprocmask(SIG_SETMASK, ~[RTMIN RT_1], [], 8) = 0
mmap(NULL, 8392704, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f2cf32f6000
mprotect(0x7f2cf32f6000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f2cf3af5eb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f2cf3af69d0, tls=0x7f2cf3af6700, child_tidptr=0x7f2cf3af69d0) = 8809
mmap(NULL, 8392704, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f2cf2af5000
mprotect(0x7f2cf2af5000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f2cf32f4eb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f2cf32f59d0, tls=0x7f2cf32f5700, child_tidptr=0x7f2cf32f59d0) = 8810
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
rt_sigprocmask(SIG_BLOCK, [PIPE], [], 8) = 0
mmap(NULL, 8392704, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f2cf22f4000
mprotect(0x7f2cf22f4000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f2cf2af3eb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f2cf2af49d0, tls=0x7f2cf2af4700, child_tidptr=0x7f2cf2af49d0) = 8811
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
open("/etc/ceph/ceph.conf", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=267, ...}) = 0
fstat(3, {st_mode=S_IFREG|0644, st_size=267, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2cf8bd5000
read(3, "[global]\nfsid = 9e0ce1ae-13d7-43"..., 4096) = 267
close(3) = 0
munmap(0x7f2cf8bd5000, 4096) = 0
rt_sigprocmask(SIG_BLOCK, [PIPE], NULL, 8) = 0
rt_sigaction(SIGSEGV, {0x43db80, [], SA_RESTORER|SA_NODEFER|SA_RESETHAND, 0x7f2cf6086340}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGABRT, {0x43db80, [], SA_RESTORER|SA_NODEFER|SA_RESETHAND, 0x7f2cf6086340}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGBUS, {0x43db80, [], SA_RESTORER|SA_NODEFER|SA_RESETHAND, 0x7f2cf6086340}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGILL, {0x43db80, [], SA_RESTORER|SA_NODEFER|SA_RESETHAND, 0x7f2cf6086340}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGFPE, {0x43db80, [], SA_RESTORER|SA_NODEFER|SA_RESETHAND, 0x7f2cf6086340}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGXCPU, {0x43db80, [], SA_RESTORER|SA_NODEFER|SA_RESETHAND, 0x7f2cf6086340}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGXFSZ, {0x43db80, [], SA_RESTORER|SA_NODEFER|SA_RESETHAND, 0x7f2cf6086340}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGSYS, {0x43db80, [], SA_RESTORER|SA_NODEFER|SA_RESETHAND, 0x7f2cf6086340}, {SIG_DFL, [], 0}, 8) = 0
rt_sigprocmask(SIG_BLOCK, [PIPE], [PIPE], 8) = 0
mmap(NULL, 8392704, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f2cf1af3000
mprotect(0x7f2cf1af3000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f2cf22f2eb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f2cf22f39d0, tls=0x7f2cf22f3700, child_tidptr=0x7f2cf22f39d0) = 8812
rt_sigprocmask(SIG_SETMASK, [PIPE], NULL, 8) = 0
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
rt_sigprocmask(SIG_BLOCK, [PIPE], [PIPE], 8) = 0
mmap(NULL, 8392704, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f2cf12f2000
mprotect(0x7f2cf12f2000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f2cf1af1eb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f2cf1af29d0, tls=0x7f2cf1af2700, child_tidptr=0x7f2cf1af29d0) = 8813
rt_sigprocmask(SIG_SETMASK, [PIPE], NULL, 8) = 0
rt_sigprocmask(SIG_BLOCK, [PIPE], [PIPE], 8) = 0
mmap(NULL, 8392704, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f2cf0af1000
mprotect(0x7f2cf0af1000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f2cf12f0eb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f2cf12f19d0, tls=0x7f2cf12f1700, child_tidptr=0x7f2cf12f19d0) = 8814
rt_sigprocmask(SIG_SETMASK, [PIPE], NULL, 8) = 0
rt_sigprocmask(SIG_BLOCK, [PIPE], [PIPE], 8) = 0
mmap(NULL, 8392704, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f2cf02f0000
mprotect(0x7f2cf02f0000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f2cf0aefeb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f2cf0af09d0, tls=0x7f2cf0af0700, child_tidptr=0x7f2cf0af09d0) = 8815
rt_sigprocmask(SIG_SETMASK, [PIPE], NULL, 8) = 0
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
rt_sigprocmask(SIG_BLOCK, [PIPE], [PIPE], 8) = 0
mmap(NULL, 8392704, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f2cefaef000
mprotect(0x7f2cefaef000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f2cf02eeeb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f2cf02ef9d0, tls=0x7f2cf02ef700, child_tidptr=0x7f2cf02ef9d0) = 8816
rt_sigprocmask(SIG_SETMASK, [PIPE], NULL, 8) = 0
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
open("/etc/ceph/ceph.client.admin.keyring", O_RDONLY) = 3
close(3) = 0
open("/etc/ceph/ceph.client.admin.keyring", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=63, ...}) = 0
read(3, "[client.admin]\n\tkey = AQCCBBJUOB"..., 63) = 63
close(3) = 0
futex(0x7f2cf5867850, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
rt_sigprocmask(SIG_BLOCK, [PIPE], [PIPE], 8) = 0
mmap(NULL, 8392704, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f2cef2ee000
mprotect(0x7f2cef2ee000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f2cefaedeb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f2cefaee9d0, tls=0x7f2cefaee700, child_tidptr=0x7f2cefaee9d0) = 8817
rt_sigprocmask(SIG_SETMASK, [PIPE], NULL, 8) = 0
rt_sigprocmask(SIG_BLOCK, [PIPE], [PIPE], 8) = 0
mmap(NULL, 8392704, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f2ceeaed000
mprotect(0x7f2ceeaed000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f2cef2eceb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f2cef2ed9d0, tls=0x7f2cef2ed700, child_tidptr=0x7f2cef2ed9d0) = 8818
rt_sigprocmask(SIG_SETMASK, [PIPE], NULL, 8) = 0
futex(0x1f738d8, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
rt_sigprocmask(SIG_BLOCK, [PIPE], [PIPE], 8) = 0
mmap(NULL, 1052672, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f2cf8abf000
mprotect(0x7f2cf8abf000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f2cf8bbeeb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f2cf8bbf9d0, tls=0x7f2cf8bbf700, child_tidptr=0x7f2cf8bbf9d0) = 8819
rt_sigprocmask(SIG_SETMASK, [PIPE], NULL, 8) = 0
futex(0x1f7c270, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
open("/etc/localtime", O_RDONLY|O_CLOEXEC) = 4
fstat(4, {st_mode=S_IFREG|0644, st_size=265, ...}) = 0
fstat(4, {st_mode=S_IFREG|0644, st_size=265, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2cf8bd5000
read(4, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\0"..., 4096) = 265
lseek(4, -156, SEEK_CUR) = 109
read(4, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\5\0\0\0\5\0\0\0\0"..., 4096) = 156
close(4) = 0
munmap(0x7f2cf8bd5000, 4096) = 0
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f73b64, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 1, {1410888723, 903836000}, ffffffff) = 0
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f738d8, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x1f74e34, FUTEX_CMP_REQUEUE_PRIVATE, 1, 2147483647, 0x1f74de0, 2) = 1
futex(0x1f74de0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f7c344, FUTEX_CMP_REQUEUE_PRIVATE, 1, 2147483647, 0x1f7c270, 20) = 1
futex(0x1f7c270, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f738d8, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f74d74, FUTEX_WAKE_PRIVATE, 1) = 1
rt_sigprocmask(SIG_BLOCK, [PIPE], [PIPE], 8) = 0
mmap(NULL, 8392704, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f2cee1eb000
mprotect(0x7f2cee1eb000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f2cee9eaeb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f2cee9eb9d0, tls=0x7f2cee9eb700, child_tidptr=0x7f2cee9eb9d0) = 8821
rt_sigprocmask(SIG_SETMASK, [PIPE], NULL, 8) = 0
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f7c344, FUTEX_CMP_REQUEUE_PRIVATE, 1, 2147483647, 0x1f7c270, 26) = 1
futex(0x1f7c270, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f738d8, FUTEX_WAKE_PRIVATE, 1) = 1
rt_sigprocmask(SIG_BLOCK, [PIPE], [PIPE], 8) = 0
mmap(NULL, 8392704, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f2ced9ea000
mprotect(0x7f2ced9ea000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f2cee1e9eb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f2cee1ea9d0, tls=0x7f2cee1ea700, child_tidptr=0x7f2cee1ea9d0) = 8822
rt_sigprocmask(SIG_SETMASK, [PIPE], NULL, 8) = 0
futex(0x1f73d08, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f74d68, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
futex(0x1f74d68, FUTEX_WAKE_PRIVATE, 1) = 0
open("io.fio", O_RDONLY) = 4
mmap(NULL, 4198400, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2ced5e9000
read(4, "[RR1_rbd1]\nnew_group\nioengine=li"..., 4194432) = 486
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f6ff54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x1f6ff50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x1f6fed0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x1f7c344, FUTEX_CMP_REQUEUE_PRIVATE, 1, 2147483647, 0x1f7c270, 32) = 1
futex(0x1f7c270, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x7fffa8b6c8fc, FUTEX_WAIT_PRIVATE, 1, NULL


Files

2014-09-16_17-38_ceph_debuginfo.tar.gz (5.96 MB) 2014-09-16_17-38_ceph_debuginfo.tar.gz dmesg, osd & mon logs Mallikarjun Biradar, 09/16/2014 05:14 AM
Actions #1

Updated by Mallikarjun Biradar over 9 years ago

This issue is observed on ceph 0.84

Actions #3

Updated by Loïc Dachary over 9 years ago

  • Status changed from New to Need More Info
  • Assignee changed from Sage Weil to Loïc Dachary

It is the normal behavior when there are not enough hosts to satisfy the crush rules. Do you have 22 hosts available ? Could you share the output of ceph osd tree and ceph osd rule dump ?

Actions #4

Updated by Mallikarjun Biradar over 9 years ago

Hi Loic,

I have 3 OSD hosts and total of 6 OSD's.

ems@rack6-client-5:~$ sudo ceph osd crush rule dump
[ { "rule_id": 0,
"rule_name": "replicated_ruleset",
"ruleset": 0,
"type": 1,
"min_size": 1,
"max_size": 10,
"steps": [ { "op": "take",
"item": -1,
"item_name": "default"}, { "op": "chooseleaf_firstn",
"num": 0,
"type": "host"}, { "op": "emit"}]}, { "rule_id": 1,
"rule_name": "erasure-code",
"ruleset": 1,
"type": 3,
"min_size": 3,
"max_size": 20,
"steps": [ { "op": "set_chooseleaf_tries",
"num": 5}, { "op": "take",
"item": -1,
"item_name": "default"}, { "op": "chooseleaf_indep",
"num": 0,
"type": "host"}, { "op": "emit"}]}, { "rule_id": 2,
"rule_name": "ecprofile",
"ruleset": 2,
"type": 3,
"min_size": 3,
"max_size": 20,
"steps": [ { "op": "set_chooseleaf_tries",
"num": 5}, { "op": "take",
"item": -1,
"item_name": "default"}, { "op": "chooseleaf_indep",
"num": 0,
"type": "host"}, { "op": "emit"}]}, { "rule_id": 3,
"rule_name": " myprofile",
"ruleset": 3,
"type": 3,
"min_size": 3,
"max_size": 20,
"steps": [ { "op": "set_chooseleaf_tries",
"num": 5}, { "op": "take",
"item": -1,
"item_name": "default"}, { "op": "chooseleaf_indep",
"num": 0,
"type": "host"}, { "op": "emit"}]}, { "rule_id": 4,
"rule_name": "coldstor2",
"ruleset": 4,
"type": 3,
"min_size": 3,
"max_size": 20,
"steps": [ { "op": "set_chooseleaf_tries",
"num": 5}, { "op": "take",
"item": -1,
"item_name": "default"}, { "op": "chooseleaf_indep",
"num": 0,
"type": "host"}, { "op": "emit"}]}, { "rule_id": 5,
"rule_name": "custprofile_pool",
"ruleset": 5,
"type": 3,
"min_size": 3,
"max_size": 20,
"steps": [ { "op": "set_chooseleaf_tries",
"num": 5}, { "op": "take",
"item": -1,
"item_name": "default"}, { "op": "chooseleaf_indep",
"num": 0,
"type": "host"}, { "op": "emit"}]}, { "rule_id": 6,
"rule_name": "custPool1",
"ruleset": 6,
"type": 3,
"min_size": 3,
"max_size": 20,
"steps": [ { "op": "set_chooseleaf_tries",
"num": 5}, { "op": "take",
"item": -1,
"item_name": "default"}, { "op": "chooseleaf_indep",
"num": 0,
"type": "host"}, { "op": "emit"}]}]

ems@rack6-client-5:~$ sudo ceph osd tree
  1. id weight type name up/down reweight
    -1 2.16 root default
    -2 0.72 host rack6-storage-4
    0 0.36 osd.0 up 1
    3 0.36 osd.3 up 1
    -3 0.72 host rack6-storage-5
    1 0.36 osd.1 up 1
    4 0.36 osd.4 up 1
    -4 0.72 host rack6-storage-6
    2 0.36 osd.2 up 1
    5 0.36 osd.5 up 1
    ems@rack6-client-5:~$

I have done few experiments on these lines, and these are the observation (My cluster has 3 OSD nodes & 6 OSD's)
1> When created ec pool with profile having k=1 & m = 1, still writing objects gets hung. (Iam not sure whether its a correct profile, please confirm)
2> When created ec pool with profile having k=5 m=3 ruleset-failure-domain=osd, writing objects gets hung.(since we are setting failure domain to osd, it supposed work)

Actions #5

Updated by Loïc Dachary over 9 years ago

  • Status changed from Need More Info to Rejected

k=1 m=1 is not supposed to work, it won't do anything useful. k=5 m=3 totals 8 osds and you only have 6 hence it blocks.

Actions

Also available in: Atom PDF