|
2016-11-23 06:34:37.549716 7f9fbe74f800 0 set uid:gid to 167:167 (ceph:ceph)
|
|
2016-11-23 06:34:37.549726 7f9fbe74f800 0 ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5), process ceph-osd, pid 5572
|
|
2016-11-23 06:34:37.550734 7f9fbe74f800 0 pidfile_write: ignore empty --pid-file
|
|
2016-11-23 06:34:37.560236 7f9fbe74f800 0 filestore(/var/lib/ceph/osd/ceph-14) backend xfs (magic 0x58465342)
|
|
2016-11-23 06:34:37.583954 7f9fbe74f800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
|
|
2016-11-23 06:34:37.583964 7f9fbe74f800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: SEEK_DATA/SEEK_HOLE is disabled via 'filestore seek data hole' config option
|
|
2016-11-23 06:34:37.583979 7f9fbe74f800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: splice is supported
|
|
2016-11-23 06:34:37.597220 7f9fbe74f800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: syncfs(2) syscall fully supported (by glibc and kernel)
|
|
2016-11-23 06:34:37.597263 7f9fbe74f800 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_feature: extsize is disabled by conf
|
|
2016-11-23 06:34:37.683560 7f9fbe74f800 1 leveldb: Recovering log #2220
|
|
2016-11-23 06:34:37.789131 7f9fbe74f800 1 leveldb: Delete type=0 #2220
|
|
|
|
2016-11-23 06:34:37.789164 7f9fbe74f800 1 leveldb: Delete type=2 #2221
|
|
|
|
2016-11-23 06:34:37.789436 7f9fbe74f800 1 leveldb: Delete type=3 #2219
|
|
|
|
2016-11-23 06:34:37.789597 7f9fb0102700 1 leveldb: Compacting 17@0 + 6@1 files
|
|
2016-11-23 06:34:38.048426 7f9fb0102700 1 leveldb: compacted to: files[ 17 6 22 0 0 0 0 ]
|
|
2016-11-23 06:34:38.048443 7f9fb0102700 1 leveldb: Compaction error: Corruption: bad block type
|
|
2016-11-23 06:34:38.061978 7f9fbe74f800 0 filestore(/var/lib/ceph/osd/ceph-14) mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled
|
|
2016-11-23 06:34:43.916929 7f9fbe74f800 1 journal _open /var/lib/ceph/osd/ceph-14/journal fd 18: 91268055040 bytes, block size 4096 bytes, directio = 1, aio = 1
|
|
2016-11-23 06:34:43.918210 7f9fbe74f800 0 filestore(/var/lib/ceph/osd/ceph-14) error (1) Operation not permitted not handled on operation 0x7f9fc90ee0d4 (1633867.0.3, or op 3, counting from 0)
|
|
2016-11-23 06:34:43.918222 7f9fbe74f800 0 filestore(/var/lib/ceph/osd/ceph-14) unexpected error code
|
|
2016-11-23 06:34:43.918223 7f9fbe74f800 0 filestore(/var/lib/ceph/osd/ceph-14) transaction dump:
|
|
{
|
|
"ops": [
|
|
{
|
|
"op_num": 0,
|
|
"op_name": "mkcoll",
|
|
"collection": "8.33d_head"
|
|
},
|
|
{
|
|
"op_num": 1,
|
|
"op_name": "coll_hint",
|
|
"collection": "8.33d_head",
|
|
"type": 1,
|
|
"pg_num": 1024,
|
|
"expected_num_objects": 0
|
|
},
|
|
{
|
|
"op_num": 2,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 3,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_infover": 1
|
|
}
|
|
},
|
|
{
|
|
"op_num": 4,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 5,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_biginfo": 59,
|
|
"_epoch": 4,
|
|
"_info": 859
|
|
}
|
|
},
|
|
{
|
|
"op_num": 6,
|
|
"op_name": "nop"
|
|
}
|
|
]
|
|
}
|
|
|
|
2016-11-23 06:34:43.920687 7f9fbe74f800 -1 os/filestore/FileStore.cc: In function 'void FileStore::_do_transaction(ObjectStore::Transaction&, uint64_t, int, ThreadPool::TPHandle*)' thread 7f9fbe74f800 time 2016-11-23 06:34:43.918275
|
|
os/filestore/FileStore.cc: 2912: FAILED assert(0 == "unexpected error")
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0x7f9fbe1e90ab]
|
|
2: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f9fbdec55f0]
|
|
3: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f9fbdecb2db]
|
|
4: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f9fbdee1b27]
|
|
5: (FileStore::mount()+0x3bf6) [0x7f9fbdeb9326]
|
|
6: (OSD::init()+0x27d) [0x7f9fbdb9434d]
|
|
7: (main()+0x29d1) [0x7f9fbdafd361]
|
|
8: (__libc_start_main()+0xf5) [0x7f9fb9ed4b05]
|
|
9: (()+0x3822a7) [0x7f9fbdb462a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- begin dump of recent events ---
|
|
-67> 2016-11-23 06:34:37.547506 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command perfcounters_dump hook 0x7f9fc8e280b0
|
|
-66> 2016-11-23 06:34:37.547523 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command 1 hook 0x7f9fc8e280b0
|
|
-65> 2016-11-23 06:34:37.547527 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command perf dump hook 0x7f9fc8e280b0
|
|
-64> 2016-11-23 06:34:37.547533 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command perfcounters_schema hook 0x7f9fc8e280b0
|
|
-63> 2016-11-23 06:34:37.547538 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command 2 hook 0x7f9fc8e280b0
|
|
-62> 2016-11-23 06:34:37.547540 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command perf schema hook 0x7f9fc8e280b0
|
|
-61> 2016-11-23 06:34:37.547544 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command perf reset hook 0x7f9fc8e280b0
|
|
-60> 2016-11-23 06:34:37.547549 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command config show hook 0x7f9fc8e280b0
|
|
-59> 2016-11-23 06:34:37.547552 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command config set hook 0x7f9fc8e280b0
|
|
-58> 2016-11-23 06:34:37.547554 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command config get hook 0x7f9fc8e280b0
|
|
-57> 2016-11-23 06:34:37.547557 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command config diff hook 0x7f9fc8e280b0
|
|
-56> 2016-11-23 06:34:37.547559 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command log flush hook 0x7f9fc8e280b0
|
|
-55> 2016-11-23 06:34:37.547561 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command log dump hook 0x7f9fc8e280b0
|
|
-54> 2016-11-23 06:34:37.547564 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command log reopen hook 0x7f9fc8e280b0
|
|
-53> 2016-11-23 06:34:37.549716 7f9fbe74f800 0 set uid:gid to 167:167 (ceph:ceph)
|
|
-52> 2016-11-23 06:34:37.549726 7f9fbe74f800 0 ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5), process ceph-osd, pid 5572
|
|
-51> 2016-11-23 06:34:37.550119 7f9fbe74f800 5 object store type is filestore
|
|
-50> 2016-11-23 06:34:37.550669 7f9fbe74f800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
-49> 2016-11-23 06:34:37.550678 7f9fbe74f800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6804/5572 need_addr=0
|
|
-48> 2016-11-23 06:34:37.550694 7f9fbe74f800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
-47> 2016-11-23 06:34:37.550697 7f9fbe74f800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6804/5572 need_addr=0
|
|
-46> 2016-11-23 06:34:37.550712 7f9fbe74f800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
-45> 2016-11-23 06:34:37.550716 7f9fbe74f800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6805/5572 need_addr=0
|
|
-44> 2016-11-23 06:34:37.550727 7f9fbe74f800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
-43> 2016-11-23 06:34:37.550730 7f9fbe74f800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6805/5572 need_addr=0
|
|
-42> 2016-11-23 06:34:37.550734 7f9fbe74f800 0 pidfile_write: ignore empty --pid-file
|
|
-41> 2016-11-23 06:34:37.552385 7f9fbe74f800 5 asok(0x7f9fc8e34140) init /var/run/ceph/ceph-osd.14.asok
|
|
-40> 2016-11-23 06:34:37.552393 7f9fbe74f800 5 asok(0x7f9fc8e34140) bind_and_listen /var/run/ceph/ceph-osd.14.asok
|
|
-39> 2016-11-23 06:34:37.552411 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command 0 hook 0x7f9fc8e24110
|
|
-38> 2016-11-23 06:34:37.552415 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command version hook 0x7f9fc8e24110
|
|
-37> 2016-11-23 06:34:37.552419 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command git_version hook 0x7f9fc8e24110
|
|
-36> 2016-11-23 06:34:37.552423 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command help hook 0x7f9fc8e281a0
|
|
-35> 2016-11-23 06:34:37.552427 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command get_command_descriptions hook 0x7f9fc8e281b0
|
|
-34> 2016-11-23 06:34:37.552456 7f9fb647b700 5 asok(0x7f9fc8e34140) entry start
|
|
-33> 2016-11-23 06:34:37.552477 7f9fbe74f800 10 monclient(hunting): build_initial_monmap
|
|
-32> 2016-11-23 06:34:37.559627 7f9fbe74f800 5 adding auth protocol: cephx
|
|
-31> 2016-11-23 06:34:37.559633 7f9fbe74f800 5 adding auth protocol: cephx
|
|
-30> 2016-11-23 06:34:37.559755 7f9fbe74f800 5 asok(0x7f9fc8e34140) register_command objecter_requests hook 0x7f9fc8e28270
|
|
-29> 2016-11-23 06:34:37.559798 7f9fbe74f800 1 -- 192.168.23.64:6804/5572 messenger.start
|
|
-28> 2016-11-23 06:34:37.559825 7f9fbe74f800 1 -- :/0 messenger.start
|
|
-27> 2016-11-23 06:34:37.559850 7f9fbe74f800 1 -- 192.168.23.64:6805/5572 messenger.start
|
|
-26> 2016-11-23 06:34:37.559879 7f9fbe74f800 1 -- 172.16.23.64:6805/5572 messenger.start
|
|
-25> 2016-11-23 06:34:37.559899 7f9fbe74f800 1 -- 172.16.23.64:6804/5572 messenger.start
|
|
-24> 2016-11-23 06:34:37.559918 7f9fbe74f800 1 -- :/0 messenger.start
|
|
-23> 2016-11-23 06:34:37.559990 7f9fbe74f800 2 osd.14 0 mounting /var/lib/ceph/osd/ceph-14 /var/lib/ceph/osd/ceph-14/journal
|
|
-22> 2016-11-23 06:34:37.560236 7f9fbe74f800 0 filestore(/var/lib/ceph/osd/ceph-14) backend xfs (magic 0x58465342)
|
|
-21> 2016-11-23 06:34:37.583954 7f9fbe74f800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
|
|
-20> 2016-11-23 06:34:37.583964 7f9fbe74f800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: SEEK_DATA/SEEK_HOLE is disabled via 'filestore seek data hole' config option
|
|
-19> 2016-11-23 06:34:37.583979 7f9fbe74f800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: splice is supported
|
|
-18> 2016-11-23 06:34:37.597220 7f9fbe74f800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: syncfs(2) syscall fully supported (by glibc and kernel)
|
|
-17> 2016-11-23 06:34:37.597263 7f9fbe74f800 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_feature: extsize is disabled by conf
|
|
-16> 2016-11-23 06:34:37.683560 7f9fbe74f800 1 leveldb: Recovering log #2220
|
|
-15> 2016-11-23 06:34:37.789131 7f9fbe74f800 1 leveldb: Delete type=0 #2220
|
|
|
|
-14> 2016-11-23 06:34:37.789164 7f9fbe74f800 1 leveldb: Delete type=2 #2221
|
|
|
|
-13> 2016-11-23 06:34:37.789436 7f9fbe74f800 1 leveldb: Delete type=3 #2219
|
|
|
|
-12> 2016-11-23 06:34:37.789597 7f9fb0102700 1 leveldb: Compacting 17@0 + 6@1 files
|
|
-11> 2016-11-23 06:34:38.048426 7f9fb0102700 1 leveldb: compacted to: files[ 17 6 22 0 0 0 0 ]
|
|
-10> 2016-11-23 06:34:38.048443 7f9fb0102700 1 leveldb: Compaction error: Corruption: bad block type
|
|
-9> 2016-11-23 06:34:38.061978 7f9fbe74f800 0 filestore(/var/lib/ceph/osd/ceph-14) mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled
|
|
-8> 2016-11-23 06:34:43.916866 7f9fbe74f800 2 journal open /var/lib/ceph/osd/ceph-14/journal fsid 67ee4226-c53d-4ded-b5c2-55fe09c3e6dd fs_op_seq 1633866
|
|
-7> 2016-11-23 06:34:43.916929 7f9fbe74f800 1 journal _open /var/lib/ceph/osd/ceph-14/journal fd 18: 91268055040 bytes, block size 4096 bytes, directio = 1, aio = 1
|
|
-6> 2016-11-23 06:34:43.917444 7f9fbe74f800 2 journal read_entry 82652950528 : seq 1633867 1647 bytes
|
|
-5> 2016-11-23 06:34:43.917455 7f9fbe74f800 2 journal read_entry 82652950528 : seq 1633867 1647 bytes
|
|
-4> 2016-11-23 06:34:43.917457 7f9fbe74f800 3 journal journal_replay: applying op seq 1633867
|
|
-3> 2016-11-23 06:34:43.918210 7f9fbe74f800 0 filestore(/var/lib/ceph/osd/ceph-14) error (1) Operation not permitted not handled on operation 0x7f9fc90ee0d4 (1633867.0.3, or op 3, counting from 0)
|
|
-2> 2016-11-23 06:34:43.918222 7f9fbe74f800 0 filestore(/var/lib/ceph/osd/ceph-14) unexpected error code
|
|
-1> 2016-11-23 06:34:43.918223 7f9fbe74f800 0 filestore(/var/lib/ceph/osd/ceph-14) transaction dump:
|
|
{
|
|
"ops": [
|
|
{
|
|
"op_num": 0,
|
|
"op_name": "mkcoll",
|
|
"collection": "8.33d_head"
|
|
},
|
|
{
|
|
"op_num": 1,
|
|
"op_name": "coll_hint",
|
|
"collection": "8.33d_head",
|
|
"type": 1,
|
|
"pg_num": 1024,
|
|
"expected_num_objects": 0
|
|
},
|
|
{
|
|
"op_num": 2,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 3,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_infover": 1
|
|
}
|
|
},
|
|
{
|
|
"op_num": 4,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 5,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_biginfo": 59,
|
|
"_epoch": 4,
|
|
"_info": 859
|
|
}
|
|
},
|
|
{
|
|
"op_num": 6,
|
|
"op_name": "nop"
|
|
}
|
|
]
|
|
}
|
|
|
|
0> 2016-11-23 06:34:43.920687 7f9fbe74f800 -1 os/filestore/FileStore.cc: In function 'void FileStore::_do_transaction(ObjectStore::Transaction&, uint64_t, int, ThreadPool::TPHandle*)' thread 7f9fbe74f800 time 2016-11-23 06:34:43.918275
|
|
os/filestore/FileStore.cc: 2912: FAILED assert(0 == "unexpected error")
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0x7f9fbe1e90ab]
|
|
2: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f9fbdec55f0]
|
|
3: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f9fbdecb2db]
|
|
4: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f9fbdee1b27]
|
|
5: (FileStore::mount()+0x3bf6) [0x7f9fbdeb9326]
|
|
6: (OSD::init()+0x27d) [0x7f9fbdb9434d]
|
|
7: (main()+0x29d1) [0x7f9fbdafd361]
|
|
8: (__libc_start_main()+0xf5) [0x7f9fb9ed4b05]
|
|
9: (()+0x3822a7) [0x7f9fbdb462a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- logging levels ---
|
|
0/ 5 none
|
|
0/ 1 lockdep
|
|
0/ 1 context
|
|
1/ 1 crush
|
|
1/ 5 mds
|
|
1/ 5 mds_balancer
|
|
1/ 5 mds_locker
|
|
1/ 5 mds_log
|
|
1/ 5 mds_log_expire
|
|
1/ 5 mds_migrator
|
|
0/ 1 buffer
|
|
0/ 1 timer
|
|
0/ 1 filer
|
|
0/ 1 striper
|
|
0/ 1 objecter
|
|
0/ 5 rados
|
|
0/ 5 rbd
|
|
0/ 5 rbd_mirror
|
|
0/ 5 rbd_replay
|
|
0/ 5 journaler
|
|
0/ 5 objectcacher
|
|
0/ 5 client
|
|
0/ 5 osd
|
|
0/ 5 optracker
|
|
0/ 5 objclass
|
|
1/ 3 filestore
|
|
1/ 3 journal
|
|
0/ 5 ms
|
|
1/ 5 mon
|
|
0/10 monc
|
|
1/ 5 paxos
|
|
0/ 5 tp
|
|
1/ 5 auth
|
|
1/ 5 crypto
|
|
1/ 1 finisher
|
|
1/ 5 heartbeatmap
|
|
1/ 5 perfcounter
|
|
1/ 5 rgw
|
|
1/10 civetweb
|
|
1/ 5 javaclient
|
|
1/ 5 asok
|
|
1/ 1 throttle
|
|
0/ 0 refs
|
|
1/ 5 xio
|
|
1/ 5 compressor
|
|
1/ 5 newstore
|
|
1/ 5 bluestore
|
|
1/ 5 bluefs
|
|
1/ 3 bdev
|
|
1/ 5 kstore
|
|
4/ 5 rocksdb
|
|
4/ 5 leveldb
|
|
1/ 5 kinetic
|
|
1/ 5 fuse
|
|
-2/-2 (syslog threshold)
|
|
-1/-1 (stderr threshold)
|
|
max_recent 10000
|
|
max_new 1000
|
|
log_file /var/log/ceph/ceph-osd.14.log
|
|
--- end dump of recent events ---
|
|
2016-11-23 06:34:43.923475 7f9fbe74f800 -1 *** Caught signal (Aborted) **
|
|
in thread 7f9fbe74f800 thread_name:ceph-osd
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (()+0x919742) [0x7f9fbe0dd742]
|
|
2: (()+0xf890) [0x7f9fbc458890]
|
|
3: (gsignal()+0x37) [0x7f9fb9ee8187]
|
|
4: (abort()+0x118) [0x7f9fb9ee9538]
|
|
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x265) [0x7f9fbe1e9285]
|
|
6: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f9fbdec55f0]
|
|
7: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f9fbdecb2db]
|
|
8: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f9fbdee1b27]
|
|
9: (FileStore::mount()+0x3bf6) [0x7f9fbdeb9326]
|
|
10: (OSD::init()+0x27d) [0x7f9fbdb9434d]
|
|
11: (main()+0x29d1) [0x7f9fbdafd361]
|
|
12: (__libc_start_main()+0xf5) [0x7f9fb9ed4b05]
|
|
13: (()+0x3822a7) [0x7f9fbdb462a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- begin dump of recent events ---
|
|
0> 2016-11-23 06:34:43.923475 7f9fbe74f800 -1 *** Caught signal (Aborted) **
|
|
in thread 7f9fbe74f800 thread_name:ceph-osd
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (()+0x919742) [0x7f9fbe0dd742]
|
|
2: (()+0xf890) [0x7f9fbc458890]
|
|
3: (gsignal()+0x37) [0x7f9fb9ee8187]
|
|
4: (abort()+0x118) [0x7f9fb9ee9538]
|
|
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x265) [0x7f9fbe1e9285]
|
|
6: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f9fbdec55f0]
|
|
7: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f9fbdecb2db]
|
|
8: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f9fbdee1b27]
|
|
9: (FileStore::mount()+0x3bf6) [0x7f9fbdeb9326]
|
|
10: (OSD::init()+0x27d) [0x7f9fbdb9434d]
|
|
11: (main()+0x29d1) [0x7f9fbdafd361]
|
|
12: (__libc_start_main()+0xf5) [0x7f9fb9ed4b05]
|
|
13: (()+0x3822a7) [0x7f9fbdb462a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- logging levels ---
|
|
0/ 5 none
|
|
0/ 1 lockdep
|
|
0/ 1 context
|
|
1/ 1 crush
|
|
1/ 5 mds
|
|
1/ 5 mds_balancer
|
|
1/ 5 mds_locker
|
|
1/ 5 mds_log
|
|
1/ 5 mds_log_expire
|
|
1/ 5 mds_migrator
|
|
0/ 1 buffer
|
|
0/ 1 timer
|
|
0/ 1 filer
|
|
0/ 1 striper
|
|
0/ 1 objecter
|
|
0/ 5 rados
|
|
0/ 5 rbd
|
|
0/ 5 rbd_mirror
|
|
0/ 5 rbd_replay
|
|
0/ 5 journaler
|
|
0/ 5 objectcacher
|
|
0/ 5 client
|
|
0/ 5 osd
|
|
0/ 5 optracker
|
|
0/ 5 objclass
|
|
1/ 3 filestore
|
|
1/ 3 journal
|
|
0/ 5 ms
|
|
1/ 5 mon
|
|
0/10 monc
|
|
1/ 5 paxos
|
|
0/ 5 tp
|
|
1/ 5 auth
|
|
1/ 5 crypto
|
|
1/ 1 finisher
|
|
1/ 5 heartbeatmap
|
|
1/ 5 perfcounter
|
|
1/ 5 rgw
|
|
1/10 civetweb
|
|
1/ 5 javaclient
|
|
1/ 5 asok
|
|
1/ 1 throttle
|
|
0/ 0 refs
|
|
1/ 5 xio
|
|
1/ 5 compressor
|
|
1/ 5 newstore
|
|
1/ 5 bluestore
|
|
1/ 5 bluefs
|
|
1/ 3 bdev
|
|
1/ 5 kstore
|
|
4/ 5 rocksdb
|
|
4/ 5 leveldb
|
|
1/ 5 kinetic
|
|
1/ 5 fuse
|
|
-2/-2 (syslog threshold)
|
|
-1/-1 (stderr threshold)
|
|
max_recent 10000
|
|
max_new 1000
|
|
log_file /var/log/ceph/ceph-osd.14.log
|
|
--- end dump of recent events ---
|
|
2016-11-23 06:34:44.671178 7f16e45a3800 0 set uid:gid to 167:167 (ceph:ceph)
|
|
2016-11-23 06:34:44.671187 7f16e45a3800 0 ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5), process ceph-osd, pid 6555
|
|
2016-11-23 06:34:44.671765 7f16e45a3800 0 pidfile_write: ignore empty --pid-file
|
|
2016-11-23 06:34:44.681101 7f16e45a3800 0 filestore(/var/lib/ceph/osd/ceph-14) backend xfs (magic 0x58465342)
|
|
2016-11-23 06:34:44.681430 7f16e45a3800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
|
|
2016-11-23 06:34:44.681435 7f16e45a3800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: SEEK_DATA/SEEK_HOLE is disabled via 'filestore seek data hole' config option
|
|
2016-11-23 06:34:44.681448 7f16e45a3800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: splice is supported
|
|
2016-11-23 06:34:44.703724 7f16e45a3800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: syncfs(2) syscall fully supported (by glibc and kernel)
|
|
2016-11-23 06:34:44.703764 7f16e45a3800 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_feature: extsize is disabled by conf
|
|
2016-11-23 06:34:44.704381 7f16e45a3800 1 leveldb: Recovering log #2222
|
|
2016-11-23 06:34:44.779031 7f16e45a3800 1 leveldb: Delete type=3 #2221
|
|
|
|
2016-11-23 06:34:44.779077 7f16e45a3800 1 leveldb: Delete type=2 #2223
|
|
|
|
2016-11-23 06:34:44.779120 7f16e45a3800 1 leveldb: Delete type=0 #2222
|
|
|
|
2016-11-23 06:34:44.779233 7f16d5f56700 1 leveldb: Compacting 17@0 + 6@1 files
|
|
2016-11-23 06:34:44.781280 7f16e45a3800 0 filestore(/var/lib/ceph/osd/ceph-14) mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled
|
|
2016-11-23 06:34:44.782066 7f16d5f56700 1 leveldb: compacted to: files[ 17 6 22 0 0 0 0 ]
|
|
2016-11-23 06:34:44.782078 7f16d5f56700 1 leveldb: Compaction error: Corruption: bad block type
|
|
2016-11-23 06:34:44.787901 7f16e45a3800 1 journal _open /var/lib/ceph/osd/ceph-14/journal fd 18: 91268055040 bytes, block size 4096 bytes, directio = 1, aio = 1
|
|
2016-11-23 06:34:44.789109 7f16e45a3800 0 filestore(/var/lib/ceph/osd/ceph-14) error (1) Operation not permitted not handled on operation 0x7f16ee14c0d4 (1633867.0.3, or op 3, counting from 0)
|
|
2016-11-23 06:34:44.789118 7f16e45a3800 0 filestore(/var/lib/ceph/osd/ceph-14) unexpected error code
|
|
2016-11-23 06:34:44.789119 7f16e45a3800 0 filestore(/var/lib/ceph/osd/ceph-14) transaction dump:
|
|
{
|
|
"ops": [
|
|
{
|
|
"op_num": 0,
|
|
"op_name": "mkcoll",
|
|
"collection": "8.33d_head"
|
|
},
|
|
{
|
|
"op_num": 1,
|
|
"op_name": "coll_hint",
|
|
"collection": "8.33d_head",
|
|
"type": 1,
|
|
"pg_num": 1024,
|
|
"expected_num_objects": 0
|
|
},
|
|
{
|
|
"op_num": 2,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 3,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_infover": 1
|
|
}
|
|
},
|
|
{
|
|
"op_num": 4,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 5,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_biginfo": 59,
|
|
"_epoch": 4,
|
|
"_info": 859
|
|
}
|
|
},
|
|
{
|
|
"op_num": 6,
|
|
"op_name": "nop"
|
|
}
|
|
]
|
|
}
|
|
|
|
2016-11-23 06:34:44.791535 7f16e45a3800 -1 os/filestore/FileStore.cc: In function 'void FileStore::_do_transaction(ObjectStore::Transaction&, uint64_t, int, ThreadPool::TPHandle*)' thread 7f16e45a3800 time 2016-11-23 06:34:44.789164
|
|
os/filestore/FileStore.cc: 2912: FAILED assert(0 == "unexpected error")
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0x7f16e403d0ab]
|
|
2: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f16e3d195f0]
|
|
3: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f16e3d1f2db]
|
|
4: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f16e3d35b27]
|
|
5: (FileStore::mount()+0x3bf6) [0x7f16e3d0d326]
|
|
6: (OSD::init()+0x27d) [0x7f16e39e834d]
|
|
7: (main()+0x29d1) [0x7f16e3951361]
|
|
8: (__libc_start_main()+0xf5) [0x7f16dfd28b05]
|
|
9: (()+0x3822a7) [0x7f16e399a2a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- begin dump of recent events ---
|
|
-67> 2016-11-23 06:34:44.669014 7f16e45a3800 5 asok(0x7f16ede84140) register_command perfcounters_dump hook 0x7f16ede780b0
|
|
-66> 2016-11-23 06:34:44.669033 7f16e45a3800 5 asok(0x7f16ede84140) register_command 1 hook 0x7f16ede780b0
|
|
-65> 2016-11-23 06:34:44.669038 7f16e45a3800 5 asok(0x7f16ede84140) register_command perf dump hook 0x7f16ede780b0
|
|
-64> 2016-11-23 06:34:44.669040 7f16e45a3800 5 asok(0x7f16ede84140) register_command perfcounters_schema hook 0x7f16ede780b0
|
|
-63> 2016-11-23 06:34:44.669042 7f16e45a3800 5 asok(0x7f16ede84140) register_command 2 hook 0x7f16ede780b0
|
|
-62> 2016-11-23 06:34:44.669047 7f16e45a3800 5 asok(0x7f16ede84140) register_command perf schema hook 0x7f16ede780b0
|
|
-61> 2016-11-23 06:34:44.669048 7f16e45a3800 5 asok(0x7f16ede84140) register_command perf reset hook 0x7f16ede780b0
|
|
-60> 2016-11-23 06:34:44.669050 7f16e45a3800 5 asok(0x7f16ede84140) register_command config show hook 0x7f16ede780b0
|
|
-59> 2016-11-23 06:34:44.669055 7f16e45a3800 5 asok(0x7f16ede84140) register_command config set hook 0x7f16ede780b0
|
|
-58> 2016-11-23 06:34:44.669057 7f16e45a3800 5 asok(0x7f16ede84140) register_command config get hook 0x7f16ede780b0
|
|
-57> 2016-11-23 06:34:44.669059 7f16e45a3800 5 asok(0x7f16ede84140) register_command config diff hook 0x7f16ede780b0
|
|
-56> 2016-11-23 06:34:44.669063 7f16e45a3800 5 asok(0x7f16ede84140) register_command log flush hook 0x7f16ede780b0
|
|
-55> 2016-11-23 06:34:44.669065 7f16e45a3800 5 asok(0x7f16ede84140) register_command log dump hook 0x7f16ede780b0
|
|
-54> 2016-11-23 06:34:44.669071 7f16e45a3800 5 asok(0x7f16ede84140) register_command log reopen hook 0x7f16ede780b0
|
|
-53> 2016-11-23 06:34:44.671178 7f16e45a3800 0 set uid:gid to 167:167 (ceph:ceph)
|
|
-52> 2016-11-23 06:34:44.671187 7f16e45a3800 0 ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5), process ceph-osd, pid 6555
|
|
-51> 2016-11-23 06:34:44.671206 7f16e45a3800 5 object store type is filestore
|
|
-50> 2016-11-23 06:34:44.671714 7f16e45a3800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
-49> 2016-11-23 06:34:44.671721 7f16e45a3800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6804/6555 need_addr=0
|
|
-48> 2016-11-23 06:34:44.671735 7f16e45a3800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
-47> 2016-11-23 06:34:44.671738 7f16e45a3800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6804/6555 need_addr=0
|
|
-46> 2016-11-23 06:34:44.671749 7f16e45a3800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
-45> 2016-11-23 06:34:44.671752 7f16e45a3800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6805/6555 need_addr=0
|
|
-44> 2016-11-23 06:34:44.671760 7f16e45a3800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
-43> 2016-11-23 06:34:44.671763 7f16e45a3800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6805/6555 need_addr=0
|
|
-42> 2016-11-23 06:34:44.671765 7f16e45a3800 0 pidfile_write: ignore empty --pid-file
|
|
-41> 2016-11-23 06:34:44.673423 7f16e45a3800 5 asok(0x7f16ede84140) init /var/run/ceph/ceph-osd.14.asok
|
|
-40> 2016-11-23 06:34:44.673431 7f16e45a3800 5 asok(0x7f16ede84140) bind_and_listen /var/run/ceph/ceph-osd.14.asok
|
|
-39> 2016-11-23 06:34:44.673506 7f16e45a3800 5 asok(0x7f16ede84140) register_command 0 hook 0x7f16ede74110
|
|
-38> 2016-11-23 06:34:44.673510 7f16e45a3800 5 asok(0x7f16ede84140) register_command version hook 0x7f16ede74110
|
|
-37> 2016-11-23 06:34:44.673515 7f16e45a3800 5 asok(0x7f16ede84140) register_command git_version hook 0x7f16ede74110
|
|
-36> 2016-11-23 06:34:44.673518 7f16e45a3800 5 asok(0x7f16ede84140) register_command help hook 0x7f16ede781a0
|
|
-35> 2016-11-23 06:34:44.673521 7f16e45a3800 5 asok(0x7f16ede84140) register_command get_command_descriptions hook 0x7f16ede781b0
|
|
-34> 2016-11-23 06:34:44.673554 7f16dc2cf700 5 asok(0x7f16ede84140) entry start
|
|
-33> 2016-11-23 06:34:44.673574 7f16e45a3800 10 monclient(hunting): build_initial_monmap
|
|
-32> 2016-11-23 06:34:44.680717 7f16e45a3800 5 adding auth protocol: cephx
|
|
-31> 2016-11-23 06:34:44.680723 7f16e45a3800 5 adding auth protocol: cephx
|
|
-30> 2016-11-23 06:34:44.680842 7f16e45a3800 5 asok(0x7f16ede84140) register_command objecter_requests hook 0x7f16ede78270
|
|
-29> 2016-11-23 06:34:44.680881 7f16e45a3800 1 -- 192.168.23.64:6804/6555 messenger.start
|
|
-28> 2016-11-23 06:34:44.680905 7f16e45a3800 1 -- :/0 messenger.start
|
|
-27> 2016-11-23 06:34:44.680924 7f16e45a3800 1 -- 192.168.23.64:6805/6555 messenger.start
|
|
-26> 2016-11-23 06:34:44.680942 7f16e45a3800 1 -- 172.16.23.64:6805/6555 messenger.start
|
|
-25> 2016-11-23 06:34:44.680959 7f16e45a3800 1 -- 172.16.23.64:6804/6555 messenger.start
|
|
-24> 2016-11-23 06:34:44.680976 7f16e45a3800 1 -- :/0 messenger.start
|
|
-23> 2016-11-23 06:34:44.681052 7f16e45a3800 2 osd.14 0 mounting /var/lib/ceph/osd/ceph-14 /var/lib/ceph/osd/ceph-14/journal
|
|
-22> 2016-11-23 06:34:44.681101 7f16e45a3800 0 filestore(/var/lib/ceph/osd/ceph-14) backend xfs (magic 0x58465342)
|
|
-21> 2016-11-23 06:34:44.681430 7f16e45a3800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
|
|
-20> 2016-11-23 06:34:44.681435 7f16e45a3800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: SEEK_DATA/SEEK_HOLE is disabled via 'filestore seek data hole' config option
|
|
-19> 2016-11-23 06:34:44.681448 7f16e45a3800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: splice is supported
|
|
-18> 2016-11-23 06:34:44.703724 7f16e45a3800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: syncfs(2) syscall fully supported (by glibc and kernel)
|
|
-17> 2016-11-23 06:34:44.703764 7f16e45a3800 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_feature: extsize is disabled by conf
|
|
-16> 2016-11-23 06:34:44.704381 7f16e45a3800 1 leveldb: Recovering log #2222
|
|
-15> 2016-11-23 06:34:44.779031 7f16e45a3800 1 leveldb: Delete type=3 #2221
|
|
|
|
-14> 2016-11-23 06:34:44.779077 7f16e45a3800 1 leveldb: Delete type=2 #2223
|
|
|
|
-13> 2016-11-23 06:34:44.779120 7f16e45a3800 1 leveldb: Delete type=0 #2222
|
|
|
|
-12> 2016-11-23 06:34:44.779233 7f16d5f56700 1 leveldb: Compacting 17@0 + 6@1 files
|
|
-11> 2016-11-23 06:34:44.781280 7f16e45a3800 0 filestore(/var/lib/ceph/osd/ceph-14) mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled
|
|
-10> 2016-11-23 06:34:44.782066 7f16d5f56700 1 leveldb: compacted to: files[ 17 6 22 0 0 0 0 ]
|
|
-9> 2016-11-23 06:34:44.782078 7f16d5f56700 1 leveldb: Compaction error: Corruption: bad block type
|
|
-8> 2016-11-23 06:34:44.787842 7f16e45a3800 2 journal open /var/lib/ceph/osd/ceph-14/journal fsid 67ee4226-c53d-4ded-b5c2-55fe09c3e6dd fs_op_seq 1633866
|
|
-7> 2016-11-23 06:34:44.787901 7f16e45a3800 1 journal _open /var/lib/ceph/osd/ceph-14/journal fd 18: 91268055040 bytes, block size 4096 bytes, directio = 1, aio = 1
|
|
-6> 2016-11-23 06:34:44.788443 7f16e45a3800 2 journal read_entry 82652950528 : seq 1633867 1647 bytes
|
|
-5> 2016-11-23 06:34:44.788451 7f16e45a3800 2 journal read_entry 82652950528 : seq 1633867 1647 bytes
|
|
-4> 2016-11-23 06:34:44.788455 7f16e45a3800 3 journal journal_replay: applying op seq 1633867
|
|
-3> 2016-11-23 06:34:44.789109 7f16e45a3800 0 filestore(/var/lib/ceph/osd/ceph-14) error (1) Operation not permitted not handled on operation 0x7f16ee14c0d4 (1633867.0.3, or op 3, counting from 0)
|
|
-2> 2016-11-23 06:34:44.789118 7f16e45a3800 0 filestore(/var/lib/ceph/osd/ceph-14) unexpected error code
|
|
-1> 2016-11-23 06:34:44.789119 7f16e45a3800 0 filestore(/var/lib/ceph/osd/ceph-14) transaction dump:
|
|
{
|
|
"ops": [
|
|
{
|
|
"op_num": 0,
|
|
"op_name": "mkcoll",
|
|
"collection": "8.33d_head"
|
|
},
|
|
{
|
|
"op_num": 1,
|
|
"op_name": "coll_hint",
|
|
"collection": "8.33d_head",
|
|
"type": 1,
|
|
"pg_num": 1024,
|
|
"expected_num_objects": 0
|
|
},
|
|
{
|
|
"op_num": 2,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 3,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_infover": 1
|
|
}
|
|
},
|
|
{
|
|
"op_num": 4,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 5,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_biginfo": 59,
|
|
"_epoch": 4,
|
|
"_info": 859
|
|
}
|
|
},
|
|
{
|
|
"op_num": 6,
|
|
"op_name": "nop"
|
|
}
|
|
]
|
|
}
|
|
|
|
0> 2016-11-23 06:34:44.791535 7f16e45a3800 -1 os/filestore/FileStore.cc: In function 'void FileStore::_do_transaction(ObjectStore::Transaction&, uint64_t, int, ThreadPool::TPHandle*)' thread 7f16e45a3800 time 2016-11-23 06:34:44.789164
|
|
os/filestore/FileStore.cc: 2912: FAILED assert(0 == "unexpected error")
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0x7f16e403d0ab]
|
|
2: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f16e3d195f0]
|
|
3: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f16e3d1f2db]
|
|
4: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f16e3d35b27]
|
|
5: (FileStore::mount()+0x3bf6) [0x7f16e3d0d326]
|
|
6: (OSD::init()+0x27d) [0x7f16e39e834d]
|
|
7: (main()+0x29d1) [0x7f16e3951361]
|
|
8: (__libc_start_main()+0xf5) [0x7f16dfd28b05]
|
|
9: (()+0x3822a7) [0x7f16e399a2a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- logging levels ---
|
|
0/ 5 none
|
|
0/ 1 lockdep
|
|
0/ 1 context
|
|
1/ 1 crush
|
|
1/ 5 mds
|
|
1/ 5 mds_balancer
|
|
1/ 5 mds_locker
|
|
1/ 5 mds_log
|
|
1/ 5 mds_log_expire
|
|
1/ 5 mds_migrator
|
|
0/ 1 buffer
|
|
0/ 1 timer
|
|
0/ 1 filer
|
|
0/ 1 striper
|
|
0/ 1 objecter
|
|
0/ 5 rados
|
|
0/ 5 rbd
|
|
0/ 5 rbd_mirror
|
|
0/ 5 rbd_replay
|
|
0/ 5 journaler
|
|
0/ 5 objectcacher
|
|
0/ 5 client
|
|
0/ 5 osd
|
|
0/ 5 optracker
|
|
0/ 5 objclass
|
|
1/ 3 filestore
|
|
1/ 3 journal
|
|
0/ 5 ms
|
|
1/ 5 mon
|
|
0/10 monc
|
|
1/ 5 paxos
|
|
0/ 5 tp
|
|
1/ 5 auth
|
|
1/ 5 crypto
|
|
1/ 1 finisher
|
|
1/ 5 heartbeatmap
|
|
1/ 5 perfcounter
|
|
1/ 5 rgw
|
|
1/10 civetweb
|
|
1/ 5 javaclient
|
|
1/ 5 asok
|
|
1/ 1 throttle
|
|
0/ 0 refs
|
|
1/ 5 xio
|
|
1/ 5 compressor
|
|
1/ 5 newstore
|
|
1/ 5 bluestore
|
|
1/ 5 bluefs
|
|
1/ 3 bdev
|
|
1/ 5 kstore
|
|
4/ 5 rocksdb
|
|
4/ 5 leveldb
|
|
1/ 5 kinetic
|
|
1/ 5 fuse
|
|
-2/-2 (syslog threshold)
|
|
-1/-1 (stderr threshold)
|
|
max_recent 10000
|
|
max_new 1000
|
|
log_file /var/log/ceph/ceph-osd.14.log
|
|
--- end dump of recent events ---
|
|
2016-11-23 06:34:44.794330 7f16e45a3800 -1 *** Caught signal (Aborted) **
|
|
in thread 7f16e45a3800 thread_name:ceph-osd
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (()+0x919742) [0x7f16e3f31742]
|
|
2: (()+0xf890) [0x7f16e22ac890]
|
|
3: (gsignal()+0x37) [0x7f16dfd3c187]
|
|
4: (abort()+0x118) [0x7f16dfd3d538]
|
|
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x265) [0x7f16e403d285]
|
|
6: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f16e3d195f0]
|
|
7: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f16e3d1f2db]
|
|
8: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f16e3d35b27]
|
|
9: (FileStore::mount()+0x3bf6) [0x7f16e3d0d326]
|
|
10: (OSD::init()+0x27d) [0x7f16e39e834d]
|
|
11: (main()+0x29d1) [0x7f16e3951361]
|
|
12: (__libc_start_main()+0xf5) [0x7f16dfd28b05]
|
|
13: (()+0x3822a7) [0x7f16e399a2a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- begin dump of recent events ---
|
|
0> 2016-11-23 06:34:44.794330 7f16e45a3800 -1 *** Caught signal (Aborted) **
|
|
in thread 7f16e45a3800 thread_name:ceph-osd
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (()+0x919742) [0x7f16e3f31742]
|
|
2: (()+0xf890) [0x7f16e22ac890]
|
|
3: (gsignal()+0x37) [0x7f16dfd3c187]
|
|
4: (abort()+0x118) [0x7f16dfd3d538]
|
|
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x265) [0x7f16e403d285]
|
|
6: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f16e3d195f0]
|
|
7: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f16e3d1f2db]
|
|
8: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f16e3d35b27]
|
|
9: (FileStore::mount()+0x3bf6) [0x7f16e3d0d326]
|
|
10: (OSD::init()+0x27d) [0x7f16e39e834d]
|
|
11: (main()+0x29d1) [0x7f16e3951361]
|
|
12: (__libc_start_main()+0xf5) [0x7f16dfd28b05]
|
|
13: (()+0x3822a7) [0x7f16e399a2a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- logging levels ---
|
|
0/ 5 none
|
|
0/ 1 lockdep
|
|
0/ 1 context
|
|
1/ 1 crush
|
|
1/ 5 mds
|
|
1/ 5 mds_balancer
|
|
1/ 5 mds_locker
|
|
1/ 5 mds_log
|
|
1/ 5 mds_log_expire
|
|
1/ 5 mds_migrator
|
|
0/ 1 buffer
|
|
0/ 1 timer
|
|
0/ 1 filer
|
|
0/ 1 striper
|
|
0/ 1 objecter
|
|
0/ 5 rados
|
|
0/ 5 rbd
|
|
0/ 5 rbd_mirror
|
|
0/ 5 rbd_replay
|
|
0/ 5 journaler
|
|
0/ 5 objectcacher
|
|
0/ 5 client
|
|
0/ 5 osd
|
|
0/ 5 optracker
|
|
0/ 5 objclass
|
|
1/ 3 filestore
|
|
1/ 3 journal
|
|
0/ 5 ms
|
|
1/ 5 mon
|
|
0/10 monc
|
|
1/ 5 paxos
|
|
0/ 5 tp
|
|
1/ 5 auth
|
|
1/ 5 crypto
|
|
1/ 1 finisher
|
|
1/ 5 heartbeatmap
|
|
1/ 5 perfcounter
|
|
1/ 5 rgw
|
|
1/10 civetweb
|
|
1/ 5 javaclient
|
|
1/ 5 asok
|
|
1/ 1 throttle
|
|
0/ 0 refs
|
|
1/ 5 xio
|
|
1/ 5 compressor
|
|
1/ 5 newstore
|
|
1/ 5 bluestore
|
|
1/ 5 bluefs
|
|
1/ 3 bdev
|
|
1/ 5 kstore
|
|
4/ 5 rocksdb
|
|
4/ 5 leveldb
|
|
1/ 5 kinetic
|
|
1/ 5 fuse
|
|
-2/-2 (syslog threshold)
|
|
-1/-1 (stderr threshold)
|
|
max_recent 10000
|
|
max_new 1000
|
|
log_file /var/log/ceph/ceph-osd.14.log
|
|
--- end dump of recent events ---
|
|
2016-11-23 06:34:45.451668 7f86fd834800 0 set uid:gid to 167:167 (ceph:ceph)
|
|
2016-11-23 06:34:45.451677 7f86fd834800 0 ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5), process ceph-osd, pid 6780
|
|
2016-11-23 06:34:45.452320 7f86fd834800 0 pidfile_write: ignore empty --pid-file
|
|
2016-11-23 06:34:45.461555 7f86fd834800 0 filestore(/var/lib/ceph/osd/ceph-14) backend xfs (magic 0x58465342)
|
|
2016-11-23 06:34:45.461884 7f86fd834800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
|
|
2016-11-23 06:34:45.461888 7f86fd834800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: SEEK_DATA/SEEK_HOLE is disabled via 'filestore seek data hole' config option
|
|
2016-11-23 06:34:45.461901 7f86fd834800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: splice is supported
|
|
2016-11-23 06:34:45.487159 7f86fd834800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: syncfs(2) syscall fully supported (by glibc and kernel)
|
|
2016-11-23 06:34:45.487203 7f86fd834800 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_feature: extsize is disabled by conf
|
|
2016-11-23 06:34:45.487818 7f86fd834800 1 leveldb: Recovering log #2224
|
|
2016-11-23 06:34:45.545806 7f86fd834800 1 leveldb: Delete type=0 #2224
|
|
|
|
2016-11-23 06:34:45.545834 7f86fd834800 1 leveldb: Delete type=2 #2225
|
|
|
|
2016-11-23 06:34:45.545878 7f86fd834800 1 leveldb: Delete type=3 #2223
|
|
|
|
2016-11-23 06:34:45.545982 7f86ef1e7700 1 leveldb: Compacting 17@0 + 6@1 files
|
|
2016-11-23 06:34:45.548182 7f86fd834800 0 filestore(/var/lib/ceph/osd/ceph-14) mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled
|
|
2016-11-23 06:34:45.548690 7f86ef1e7700 1 leveldb: compacted to: files[ 17 6 22 0 0 0 0 ]
|
|
2016-11-23 06:34:45.548700 7f86ef1e7700 1 leveldb: Compaction error: Corruption: bad block type
|
|
2016-11-23 06:34:45.555098 7f86fd834800 1 journal _open /var/lib/ceph/osd/ceph-14/journal fd 18: 91268055040 bytes, block size 4096 bytes, directio = 1, aio = 1
|
|
2016-11-23 06:34:45.555825 7f86fd834800 0 filestore(/var/lib/ceph/osd/ceph-14) error (1) Operation not permitted not handled on operation 0x7f87089920d4 (1633867.0.3, or op 3, counting from 0)
|
|
2016-11-23 06:34:45.555834 7f86fd834800 0 filestore(/var/lib/ceph/osd/ceph-14) unexpected error code
|
|
2016-11-23 06:34:45.555835 7f86fd834800 0 filestore(/var/lib/ceph/osd/ceph-14) transaction dump:
|
|
{
|
|
"ops": [
|
|
{
|
|
"op_num": 0,
|
|
"op_name": "mkcoll",
|
|
"collection": "8.33d_head"
|
|
},
|
|
{
|
|
"op_num": 1,
|
|
"op_name": "coll_hint",
|
|
"collection": "8.33d_head",
|
|
"type": 1,
|
|
"pg_num": 1024,
|
|
"expected_num_objects": 0
|
|
},
|
|
{
|
|
"op_num": 2,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 3,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_infover": 1
|
|
}
|
|
},
|
|
{
|
|
"op_num": 4,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 5,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_biginfo": 59,
|
|
"_epoch": 4,
|
|
"_info": 859
|
|
}
|
|
},
|
|
{
|
|
"op_num": 6,
|
|
"op_name": "nop"
|
|
}
|
|
]
|
|
}
|
|
|
|
2016-11-23 06:34:45.558263 7f86fd834800 -1 os/filestore/FileStore.cc: In function 'void FileStore::_do_transaction(ObjectStore::Transaction&, uint64_t, int, ThreadPool::TPHandle*)' thread 7f86fd834800 time 2016-11-23 06:34:45.555886
|
|
os/filestore/FileStore.cc: 2912: FAILED assert(0 == "unexpected error")
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0x7f86fd2ce0ab]
|
|
2: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f86fcfaa5f0]
|
|
3: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f86fcfb02db]
|
|
4: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f86fcfc6b27]
|
|
5: (FileStore::mount()+0x3bf6) [0x7f86fcf9e326]
|
|
6: (OSD::init()+0x27d) [0x7f86fcc7934d]
|
|
7: (main()+0x29d1) [0x7f86fcbe2361]
|
|
8: (__libc_start_main()+0xf5) [0x7f86f8fb9b05]
|
|
9: (()+0x3822a7) [0x7f86fcc2b2a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- begin dump of recent events ---
|
|
-67> 2016-11-23 06:34:45.449492 7f86fd834800 5 asok(0x7f87086ca140) register_command perfcounters_dump hook 0x7f87086be0b0
|
|
-66> 2016-11-23 06:34:45.449510 7f86fd834800 5 asok(0x7f87086ca140) register_command 1 hook 0x7f87086be0b0
|
|
-65> 2016-11-23 06:34:45.449515 7f86fd834800 5 asok(0x7f87086ca140) register_command perf dump hook 0x7f87086be0b0
|
|
-64> 2016-11-23 06:34:45.449517 7f86fd834800 5 asok(0x7f87086ca140) register_command perfcounters_schema hook 0x7f87086be0b0
|
|
-63> 2016-11-23 06:34:45.449522 7f86fd834800 5 asok(0x7f87086ca140) register_command 2 hook 0x7f87086be0b0
|
|
-62> 2016-11-23 06:34:45.449523 7f86fd834800 5 asok(0x7f87086ca140) register_command perf schema hook 0x7f87086be0b0
|
|
-61> 2016-11-23 06:34:45.449526 7f86fd834800 5 asok(0x7f87086ca140) register_command perf reset hook 0x7f87086be0b0
|
|
-60> 2016-11-23 06:34:45.449530 7f86fd834800 5 asok(0x7f87086ca140) register_command config show hook 0x7f87086be0b0
|
|
-59> 2016-11-23 06:34:45.449532 7f86fd834800 5 asok(0x7f87086ca140) register_command config set hook 0x7f87086be0b0
|
|
-58> 2016-11-23 06:34:45.449534 7f86fd834800 5 asok(0x7f87086ca140) register_command config get hook 0x7f87086be0b0
|
|
-57> 2016-11-23 06:34:45.449539 7f86fd834800 5 asok(0x7f87086ca140) register_command config diff hook 0x7f87086be0b0
|
|
-56> 2016-11-23 06:34:45.449540 7f86fd834800 5 asok(0x7f87086ca140) register_command log flush hook 0x7f87086be0b0
|
|
-55> 2016-11-23 06:34:45.449542 7f86fd834800 5 asok(0x7f87086ca140) register_command log dump hook 0x7f87086be0b0
|
|
-54> 2016-11-23 06:34:45.449547 7f86fd834800 5 asok(0x7f87086ca140) register_command log reopen hook 0x7f87086be0b0
|
|
-53> 2016-11-23 06:34:45.451668 7f86fd834800 0 set uid:gid to 167:167 (ceph:ceph)
|
|
-52> 2016-11-23 06:34:45.451677 7f86fd834800 0 ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5), process ceph-osd, pid 6780
|
|
-51> 2016-11-23 06:34:45.451697 7f86fd834800 5 object store type is filestore
|
|
-50> 2016-11-23 06:34:45.452263 7f86fd834800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
-49> 2016-11-23 06:34:45.452272 7f86fd834800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6804/6780 need_addr=0
|
|
-48> 2016-11-23 06:34:45.452286 7f86fd834800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
-47> 2016-11-23 06:34:45.452290 7f86fd834800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6804/6780 need_addr=0
|
|
-46> 2016-11-23 06:34:45.452303 7f86fd834800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
-45> 2016-11-23 06:34:45.452306 7f86fd834800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6805/6780 need_addr=0
|
|
-44> 2016-11-23 06:34:45.452315 7f86fd834800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
-43> 2016-11-23 06:34:45.452318 7f86fd834800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6805/6780 need_addr=0
|
|
-42> 2016-11-23 06:34:45.452320 7f86fd834800 0 pidfile_write: ignore empty --pid-file
|
|
-41> 2016-11-23 06:34:45.453949 7f86fd834800 5 asok(0x7f87086ca140) init /var/run/ceph/ceph-osd.14.asok
|
|
-40> 2016-11-23 06:34:45.453957 7f86fd834800 5 asok(0x7f87086ca140) bind_and_listen /var/run/ceph/ceph-osd.14.asok
|
|
-39> 2016-11-23 06:34:45.454028 7f86fd834800 5 asok(0x7f87086ca140) register_command 0 hook 0x7f87086ba110
|
|
-38> 2016-11-23 06:34:45.454033 7f86fd834800 5 asok(0x7f87086ca140) register_command version hook 0x7f87086ba110
|
|
-37> 2016-11-23 06:34:45.454035 7f86fd834800 5 asok(0x7f87086ca140) register_command git_version hook 0x7f87086ba110
|
|
-36> 2016-11-23 06:34:45.454038 7f86fd834800 5 asok(0x7f87086ca140) register_command help hook 0x7f87086be1a0
|
|
-35> 2016-11-23 06:34:45.454042 7f86fd834800 5 asok(0x7f87086ca140) register_command get_command_descriptions hook 0x7f87086be1b0
|
|
-34> 2016-11-23 06:34:45.454073 7f86f5560700 5 asok(0x7f87086ca140) entry start
|
|
-33> 2016-11-23 06:34:45.454091 7f86fd834800 10 monclient(hunting): build_initial_monmap
|
|
-32> 2016-11-23 06:34:45.461172 7f86fd834800 5 adding auth protocol: cephx
|
|
-31> 2016-11-23 06:34:45.461177 7f86fd834800 5 adding auth protocol: cephx
|
|
-30> 2016-11-23 06:34:45.461297 7f86fd834800 5 asok(0x7f87086ca140) register_command objecter_requests hook 0x7f87086be270
|
|
-29> 2016-11-23 06:34:45.461335 7f86fd834800 1 -- 192.168.23.64:6804/6780 messenger.start
|
|
-28> 2016-11-23 06:34:45.461360 7f86fd834800 1 -- :/0 messenger.start
|
|
-27> 2016-11-23 06:34:45.461379 7f86fd834800 1 -- 192.168.23.64:6805/6780 messenger.start
|
|
-26> 2016-11-23 06:34:45.461398 7f86fd834800 1 -- 172.16.23.64:6805/6780 messenger.start
|
|
-25> 2016-11-23 06:34:45.461416 7f86fd834800 1 -- 172.16.23.64:6804/6780 messenger.start
|
|
-24> 2016-11-23 06:34:45.461434 7f86fd834800 1 -- :/0 messenger.start
|
|
-23> 2016-11-23 06:34:45.461506 7f86fd834800 2 osd.14 0 mounting /var/lib/ceph/osd/ceph-14 /var/lib/ceph/osd/ceph-14/journal
|
|
-22> 2016-11-23 06:34:45.461555 7f86fd834800 0 filestore(/var/lib/ceph/osd/ceph-14) backend xfs (magic 0x58465342)
|
|
-21> 2016-11-23 06:34:45.461884 7f86fd834800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
|
|
-20> 2016-11-23 06:34:45.461888 7f86fd834800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: SEEK_DATA/SEEK_HOLE is disabled via 'filestore seek data hole' config option
|
|
-19> 2016-11-23 06:34:45.461901 7f86fd834800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: splice is supported
|
|
-18> 2016-11-23 06:34:45.487159 7f86fd834800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: syncfs(2) syscall fully supported (by glibc and kernel)
|
|
-17> 2016-11-23 06:34:45.487203 7f86fd834800 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_feature: extsize is disabled by conf
|
|
-16> 2016-11-23 06:34:45.487818 7f86fd834800 1 leveldb: Recovering log #2224
|
|
-15> 2016-11-23 06:34:45.545806 7f86fd834800 1 leveldb: Delete type=0 #2224
|
|
|
|
-14> 2016-11-23 06:34:45.545834 7f86fd834800 1 leveldb: Delete type=2 #2225
|
|
|
|
-13> 2016-11-23 06:34:45.545878 7f86fd834800 1 leveldb: Delete type=3 #2223
|
|
|
|
-12> 2016-11-23 06:34:45.545982 7f86ef1e7700 1 leveldb: Compacting 17@0 + 6@1 files
|
|
-11> 2016-11-23 06:34:45.548182 7f86fd834800 0 filestore(/var/lib/ceph/osd/ceph-14) mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled
|
|
-10> 2016-11-23 06:34:45.548690 7f86ef1e7700 1 leveldb: compacted to: files[ 17 6 22 0 0 0 0 ]
|
|
-9> 2016-11-23 06:34:45.548700 7f86ef1e7700 1 leveldb: Compaction error: Corruption: bad block type
|
|
-8> 2016-11-23 06:34:45.555046 7f86fd834800 2 journal open /var/lib/ceph/osd/ceph-14/journal fsid 67ee4226-c53d-4ded-b5c2-55fe09c3e6dd fs_op_seq 1633866
|
|
-7> 2016-11-23 06:34:45.555098 7f86fd834800 1 journal _open /var/lib/ceph/osd/ceph-14/journal fd 18: 91268055040 bytes, block size 4096 bytes, directio = 1, aio = 1
|
|
-6> 2016-11-23 06:34:45.555239 7f86fd834800 2 journal read_entry 82652950528 : seq 1633867 1647 bytes
|
|
-5> 2016-11-23 06:34:45.555248 7f86fd834800 2 journal read_entry 82652950528 : seq 1633867 1647 bytes
|
|
-4> 2016-11-23 06:34:45.555251 7f86fd834800 3 journal journal_replay: applying op seq 1633867
|
|
-3> 2016-11-23 06:34:45.555825 7f86fd834800 0 filestore(/var/lib/ceph/osd/ceph-14) error (1) Operation not permitted not handled on operation 0x7f87089920d4 (1633867.0.3, or op 3, counting from 0)
|
|
-2> 2016-11-23 06:34:45.555834 7f86fd834800 0 filestore(/var/lib/ceph/osd/ceph-14) unexpected error code
|
|
-1> 2016-11-23 06:34:45.555835 7f86fd834800 0 filestore(/var/lib/ceph/osd/ceph-14) transaction dump:
|
|
{
|
|
"ops": [
|
|
{
|
|
"op_num": 0,
|
|
"op_name": "mkcoll",
|
|
"collection": "8.33d_head"
|
|
},
|
|
{
|
|
"op_num": 1,
|
|
"op_name": "coll_hint",
|
|
"collection": "8.33d_head",
|
|
"type": 1,
|
|
"pg_num": 1024,
|
|
"expected_num_objects": 0
|
|
},
|
|
{
|
|
"op_num": 2,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 3,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_infover": 1
|
|
}
|
|
},
|
|
{
|
|
"op_num": 4,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 5,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_biginfo": 59,
|
|
"_epoch": 4,
|
|
"_info": 859
|
|
}
|
|
},
|
|
{
|
|
"op_num": 6,
|
|
"op_name": "nop"
|
|
}
|
|
]
|
|
}
|
|
|
|
0> 2016-11-23 06:34:45.558263 7f86fd834800 -1 os/filestore/FileStore.cc: In function 'void FileStore::_do_transaction(ObjectStore::Transaction&, uint64_t, int, ThreadPool::TPHandle*)' thread 7f86fd834800 time 2016-11-23 06:34:45.555886
|
|
os/filestore/FileStore.cc: 2912: FAILED assert(0 == "unexpected error")
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0x7f86fd2ce0ab]
|
|
2: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f86fcfaa5f0]
|
|
3: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f86fcfb02db]
|
|
4: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f86fcfc6b27]
|
|
5: (FileStore::mount()+0x3bf6) [0x7f86fcf9e326]
|
|
6: (OSD::init()+0x27d) [0x7f86fcc7934d]
|
|
7: (main()+0x29d1) [0x7f86fcbe2361]
|
|
8: (__libc_start_main()+0xf5) [0x7f86f8fb9b05]
|
|
9: (()+0x3822a7) [0x7f86fcc2b2a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- logging levels ---
|
|
0/ 5 none
|
|
0/ 1 lockdep
|
|
0/ 1 context
|
|
1/ 1 crush
|
|
1/ 5 mds
|
|
1/ 5 mds_balancer
|
|
1/ 5 mds_locker
|
|
1/ 5 mds_log
|
|
1/ 5 mds_log_expire
|
|
1/ 5 mds_migrator
|
|
0/ 1 buffer
|
|
0/ 1 timer
|
|
0/ 1 filer
|
|
0/ 1 striper
|
|
0/ 1 objecter
|
|
0/ 5 rados
|
|
0/ 5 rbd
|
|
0/ 5 rbd_mirror
|
|
0/ 5 rbd_replay
|
|
0/ 5 journaler
|
|
0/ 5 objectcacher
|
|
0/ 5 client
|
|
0/ 5 osd
|
|
0/ 5 optracker
|
|
0/ 5 objclass
|
|
1/ 3 filestore
|
|
1/ 3 journal
|
|
0/ 5 ms
|
|
1/ 5 mon
|
|
0/10 monc
|
|
1/ 5 paxos
|
|
0/ 5 tp
|
|
1/ 5 auth
|
|
1/ 5 crypto
|
|
1/ 1 finisher
|
|
1/ 5 heartbeatmap
|
|
1/ 5 perfcounter
|
|
1/ 5 rgw
|
|
1/10 civetweb
|
|
1/ 5 javaclient
|
|
1/ 5 asok
|
|
1/ 1 throttle
|
|
0/ 0 refs
|
|
1/ 5 xio
|
|
1/ 5 compressor
|
|
1/ 5 newstore
|
|
1/ 5 bluestore
|
|
1/ 5 bluefs
|
|
1/ 3 bdev
|
|
1/ 5 kstore
|
|
4/ 5 rocksdb
|
|
4/ 5 leveldb
|
|
1/ 5 kinetic
|
|
1/ 5 fuse
|
|
-2/-2 (syslog threshold)
|
|
-1/-1 (stderr threshold)
|
|
max_recent 10000
|
|
max_new 1000
|
|
log_file /var/log/ceph/ceph-osd.14.log
|
|
--- end dump of recent events ---
|
|
2016-11-23 06:34:45.561059 7f86fd834800 -1 *** Caught signal (Aborted) **
|
|
in thread 7f86fd834800 thread_name:ceph-osd
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (()+0x919742) [0x7f86fd1c2742]
|
|
2: (()+0xf890) [0x7f86fb53d890]
|
|
3: (gsignal()+0x37) [0x7f86f8fcd187]
|
|
4: (abort()+0x118) [0x7f86f8fce538]
|
|
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x265) [0x7f86fd2ce285]
|
|
6: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f86fcfaa5f0]
|
|
7: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f86fcfb02db]
|
|
8: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f86fcfc6b27]
|
|
9: (FileStore::mount()+0x3bf6) [0x7f86fcf9e326]
|
|
10: (OSD::init()+0x27d) [0x7f86fcc7934d]
|
|
11: (main()+0x29d1) [0x7f86fcbe2361]
|
|
12: (__libc_start_main()+0xf5) [0x7f86f8fb9b05]
|
|
13: (()+0x3822a7) [0x7f86fcc2b2a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- begin dump of recent events ---
|
|
0> 2016-11-23 06:34:45.561059 7f86fd834800 -1 *** Caught signal (Aborted) **
|
|
in thread 7f86fd834800 thread_name:ceph-osd
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (()+0x919742) [0x7f86fd1c2742]
|
|
2: (()+0xf890) [0x7f86fb53d890]
|
|
3: (gsignal()+0x37) [0x7f86f8fcd187]
|
|
4: (abort()+0x118) [0x7f86f8fce538]
|
|
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x265) [0x7f86fd2ce285]
|
|
6: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f86fcfaa5f0]
|
|
7: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f86fcfb02db]
|
|
8: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f86fcfc6b27]
|
|
9: (FileStore::mount()+0x3bf6) [0x7f86fcf9e326]
|
|
10: (OSD::init()+0x27d) [0x7f86fcc7934d]
|
|
11: (main()+0x29d1) [0x7f86fcbe2361]
|
|
12: (__libc_start_main()+0xf5) [0x7f86f8fb9b05]
|
|
13: (()+0x3822a7) [0x7f86fcc2b2a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- logging levels ---
|
|
0/ 5 none
|
|
0/ 1 lockdep
|
|
0/ 1 context
|
|
1/ 1 crush
|
|
1/ 5 mds
|
|
1/ 5 mds_balancer
|
|
1/ 5 mds_locker
|
|
1/ 5 mds_log
|
|
1/ 5 mds_log_expire
|
|
1/ 5 mds_migrator
|
|
0/ 1 buffer
|
|
0/ 1 timer
|
|
0/ 1 filer
|
|
0/ 1 striper
|
|
0/ 1 objecter
|
|
0/ 5 rados
|
|
0/ 5 rbd
|
|
0/ 5 rbd_mirror
|
|
0/ 5 rbd_replay
|
|
0/ 5 journaler
|
|
0/ 5 objectcacher
|
|
0/ 5 client
|
|
0/ 5 osd
|
|
0/ 5 optracker
|
|
0/ 5 objclass
|
|
1/ 3 filestore
|
|
1/ 3 journal
|
|
0/ 5 ms
|
|
1/ 5 mon
|
|
0/10 monc
|
|
1/ 5 paxos
|
|
0/ 5 tp
|
|
1/ 5 auth
|
|
1/ 5 crypto
|
|
1/ 1 finisher
|
|
1/ 5 heartbeatmap
|
|
1/ 5 perfcounter
|
|
1/ 5 rgw
|
|
1/10 civetweb
|
|
1/ 5 javaclient
|
|
1/ 5 asok
|
|
1/ 1 throttle
|
|
0/ 0 refs
|
|
1/ 5 xio
|
|
1/ 5 compressor
|
|
1/ 5 newstore
|
|
1/ 5 bluestore
|
|
1/ 5 bluefs
|
|
1/ 3 bdev
|
|
1/ 5 kstore
|
|
4/ 5 rocksdb
|
|
4/ 5 leveldb
|
|
1/ 5 kinetic
|
|
1/ 5 fuse
|
|
-2/-2 (syslog threshold)
|
|
-1/-1 (stderr threshold)
|
|
max_recent 10000
|
|
max_new 1000
|
|
log_file /var/log/ceph/ceph-osd.14.log
|
|
--- end dump of recent events ---
|
|
2016-11-23 06:34:46.443651 7f65d14bb800 0 set uid:gid to 167:167 (ceph:ceph)
|
|
2016-11-23 06:34:46.443663 7f65d14bb800 0 ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5), process ceph-osd, pid 7028
|
|
2016-11-23 06:34:46.444334 7f65d14bb800 0 pidfile_write: ignore empty --pid-file
|
|
2016-11-23 06:34:46.453900 7f65d14bb800 0 filestore(/var/lib/ceph/osd/ceph-14) backend xfs (magic 0x58465342)
|
|
2016-11-23 06:34:46.454229 7f65d14bb800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
|
|
2016-11-23 06:34:46.454235 7f65d14bb800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: SEEK_DATA/SEEK_HOLE is disabled via 'filestore seek data hole' config option
|
|
2016-11-23 06:34:46.454249 7f65d14bb800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: splice is supported
|
|
2016-11-23 06:34:46.478872 7f65d14bb800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: syncfs(2) syscall fully supported (by glibc and kernel)
|
|
2016-11-23 06:34:46.478918 7f65d14bb800 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_feature: extsize is disabled by conf
|
|
2016-11-23 06:34:46.479551 7f65d14bb800 1 leveldb: Recovering log #2226
|
|
2016-11-23 06:34:46.537516 7f65d14bb800 1 leveldb: Delete type=3 #2225
|
|
|
|
2016-11-23 06:34:46.537559 7f65d14bb800 1 leveldb: Delete type=2 #2227
|
|
|
|
2016-11-23 06:34:46.537596 7f65d14bb800 1 leveldb: Delete type=0 #2226
|
|
|
|
2016-11-23 06:34:46.537686 7f65c2e6e700 1 leveldb: Compacting 17@0 + 6@1 files
|
|
2016-11-23 06:34:46.539994 7f65d14bb800 0 filestore(/var/lib/ceph/osd/ceph-14) mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled
|
|
2016-11-23 06:34:46.540756 7f65c2e6e700 1 leveldb: compacted to: files[ 17 6 22 0 0 0 0 ]
|
|
2016-11-23 06:34:46.540766 7f65c2e6e700 1 leveldb: Compaction error: Corruption: bad block type
|
|
2016-11-23 06:34:46.546404 7f65d14bb800 1 journal _open /var/lib/ceph/osd/ceph-14/journal fd 18: 91268055040 bytes, block size 4096 bytes, directio = 1, aio = 1
|
|
2016-11-23 06:34:46.547141 7f65d14bb800 0 filestore(/var/lib/ceph/osd/ceph-14) error (1) Operation not permitted not handled on operation 0x7f65db0d80d4 (1633867.0.3, or op 3, counting from 0)
|
|
2016-11-23 06:34:46.547150 7f65d14bb800 0 filestore(/var/lib/ceph/osd/ceph-14) unexpected error code
|
|
2016-11-23 06:34:46.547151 7f65d14bb800 0 filestore(/var/lib/ceph/osd/ceph-14) transaction dump:
|
|
{
|
|
"ops": [
|
|
{
|
|
"op_num": 0,
|
|
"op_name": "mkcoll",
|
|
"collection": "8.33d_head"
|
|
},
|
|
{
|
|
"op_num": 1,
|
|
"op_name": "coll_hint",
|
|
"collection": "8.33d_head",
|
|
"type": 1,
|
|
"pg_num": 1024,
|
|
"expected_num_objects": 0
|
|
},
|
|
{
|
|
"op_num": 2,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 3,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_infover": 1
|
|
}
|
|
},
|
|
{
|
|
"op_num": 4,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 5,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_biginfo": 59,
|
|
"_epoch": 4,
|
|
"_info": 859
|
|
}
|
|
},
|
|
{
|
|
"op_num": 6,
|
|
"op_name": "nop"
|
|
}
|
|
]
|
|
}
|
|
|
|
2016-11-23 06:34:46.549575 7f65d14bb800 -1 os/filestore/FileStore.cc: In function 'void FileStore::_do_transaction(ObjectStore::Transaction&, uint64_t, int, ThreadPool::TPHandle*)' thread 7f65d14bb800 time 2016-11-23 06:34:46.547195
|
|
os/filestore/FileStore.cc: 2912: FAILED assert(0 == "unexpected error")
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0x7f65d0f550ab]
|
|
2: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f65d0c315f0]
|
|
3: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f65d0c372db]
|
|
4: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f65d0c4db27]
|
|
5: (FileStore::mount()+0x3bf6) [0x7f65d0c25326]
|
|
6: (OSD::init()+0x27d) [0x7f65d090034d]
|
|
7: (main()+0x29d1) [0x7f65d0869361]
|
|
8: (__libc_start_main()+0xf5) [0x7f65ccc40b05]
|
|
9: (()+0x3822a7) [0x7f65d08b22a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- begin dump of recent events ---
|
|
-67> 2016-11-23 06:34:46.441421 7f65d14bb800 5 asok(0x7f65dae10140) register_command perfcounters_dump hook 0x7f65dae040b0
|
|
-66> 2016-11-23 06:34:46.441438 7f65d14bb800 5 asok(0x7f65dae10140) register_command 1 hook 0x7f65dae040b0
|
|
-65> 2016-11-23 06:34:46.441442 7f65d14bb800 5 asok(0x7f65dae10140) register_command perf dump hook 0x7f65dae040b0
|
|
-64> 2016-11-23 06:34:46.441446 7f65d14bb800 5 asok(0x7f65dae10140) register_command perfcounters_schema hook 0x7f65dae040b0
|
|
-63> 2016-11-23 06:34:46.441452 7f65d14bb800 5 asok(0x7f65dae10140) register_command 2 hook 0x7f65dae040b0
|
|
-62> 2016-11-23 06:34:46.441453 7f65d14bb800 5 asok(0x7f65dae10140) register_command perf schema hook 0x7f65dae040b0
|
|
-61> 2016-11-23 06:34:46.441458 7f65d14bb800 5 asok(0x7f65dae10140) register_command perf reset hook 0x7f65dae040b0
|
|
-60> 2016-11-23 06:34:46.441463 7f65d14bb800 5 asok(0x7f65dae10140) register_command config show hook 0x7f65dae040b0
|
|
-59> 2016-11-23 06:34:46.441465 7f65d14bb800 5 asok(0x7f65dae10140) register_command config set hook 0x7f65dae040b0
|
|
-58> 2016-11-23 06:34:46.441468 7f65d14bb800 5 asok(0x7f65dae10140) register_command config get hook 0x7f65dae040b0
|
|
-57> 2016-11-23 06:34:46.441471 7f65d14bb800 5 asok(0x7f65dae10140) register_command config diff hook 0x7f65dae040b0
|
|
-56> 2016-11-23 06:34:46.441474 7f65d14bb800 5 asok(0x7f65dae10140) register_command log flush hook 0x7f65dae040b0
|
|
-55> 2016-11-23 06:34:46.441476 7f65d14bb800 5 asok(0x7f65dae10140) register_command log dump hook 0x7f65dae040b0
|
|
-54> 2016-11-23 06:34:46.441478 7f65d14bb800 5 asok(0x7f65dae10140) register_command log reopen hook 0x7f65dae040b0
|
|
-53> 2016-11-23 06:34:46.443651 7f65d14bb800 0 set uid:gid to 167:167 (ceph:ceph)
|
|
-52> 2016-11-23 06:34:46.443663 7f65d14bb800 0 ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5), process ceph-osd, pid 7028
|
|
-51> 2016-11-23 06:34:46.443695 7f65d14bb800 5 object store type is filestore
|
|
-50> 2016-11-23 06:34:46.444278 7f65d14bb800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
-49> 2016-11-23 06:34:46.444287 7f65d14bb800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6804/7028 need_addr=0
|
|
-48> 2016-11-23 06:34:46.444301 7f65d14bb800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
-47> 2016-11-23 06:34:46.444304 7f65d14bb800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6804/7028 need_addr=0
|
|
-46> 2016-11-23 06:34:46.444316 7f65d14bb800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
-45> 2016-11-23 06:34:46.444319 7f65d14bb800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6805/7028 need_addr=0
|
|
-44> 2016-11-23 06:34:46.444329 7f65d14bb800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
-43> 2016-11-23 06:34:46.444332 7f65d14bb800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6805/7028 need_addr=0
|
|
-42> 2016-11-23 06:34:46.444334 7f65d14bb800 0 pidfile_write: ignore empty --pid-file
|
|
-41> 2016-11-23 06:34:46.446057 7f65d14bb800 5 asok(0x7f65dae10140) init /var/run/ceph/ceph-osd.14.asok
|
|
-40> 2016-11-23 06:34:46.446065 7f65d14bb800 5 asok(0x7f65dae10140) bind_and_listen /var/run/ceph/ceph-osd.14.asok
|
|
-39> 2016-11-23 06:34:46.446138 7f65d14bb800 5 asok(0x7f65dae10140) register_command 0 hook 0x7f65dae00110
|
|
-38> 2016-11-23 06:34:46.446143 7f65d14bb800 5 asok(0x7f65dae10140) register_command version hook 0x7f65dae00110
|
|
-37> 2016-11-23 06:34:46.446146 7f65d14bb800 5 asok(0x7f65dae10140) register_command git_version hook 0x7f65dae00110
|
|
-36> 2016-11-23 06:34:46.446151 7f65d14bb800 5 asok(0x7f65dae10140) register_command help hook 0x7f65dae041a0
|
|
-35> 2016-11-23 06:34:46.446155 7f65d14bb800 5 asok(0x7f65dae10140) register_command get_command_descriptions hook 0x7f65dae041b0
|
|
-34> 2016-11-23 06:34:46.446187 7f65c91e7700 5 asok(0x7f65dae10140) entry start
|
|
-33> 2016-11-23 06:34:46.446205 7f65d14bb800 10 monclient(hunting): build_initial_monmap
|
|
-32> 2016-11-23 06:34:46.453511 7f65d14bb800 5 adding auth protocol: cephx
|
|
-31> 2016-11-23 06:34:46.453516 7f65d14bb800 5 adding auth protocol: cephx
|
|
-30> 2016-11-23 06:34:46.453637 7f65d14bb800 5 asok(0x7f65dae10140) register_command objecter_requests hook 0x7f65dae04270
|
|
-29> 2016-11-23 06:34:46.453678 7f65d14bb800 1 -- 192.168.23.64:6804/7028 messenger.start
|
|
-28> 2016-11-23 06:34:46.453701 7f65d14bb800 1 -- :/0 messenger.start
|
|
-27> 2016-11-23 06:34:46.453719 7f65d14bb800 1 -- 192.168.23.64:6805/7028 messenger.start
|
|
-26> 2016-11-23 06:34:46.453738 7f65d14bb800 1 -- 172.16.23.64:6805/7028 messenger.start
|
|
-25> 2016-11-23 06:34:46.453762 7f65d14bb800 1 -- 172.16.23.64:6804/7028 messenger.start
|
|
-24> 2016-11-23 06:34:46.453781 7f65d14bb800 1 -- :/0 messenger.start
|
|
-23> 2016-11-23 06:34:46.453852 7f65d14bb800 2 osd.14 0 mounting /var/lib/ceph/osd/ceph-14 /var/lib/ceph/osd/ceph-14/journal
|
|
-22> 2016-11-23 06:34:46.453900 7f65d14bb800 0 filestore(/var/lib/ceph/osd/ceph-14) backend xfs (magic 0x58465342)
|
|
-21> 2016-11-23 06:34:46.454229 7f65d14bb800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
|
|
-20> 2016-11-23 06:34:46.454235 7f65d14bb800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: SEEK_DATA/SEEK_HOLE is disabled via 'filestore seek data hole' config option
|
|
-19> 2016-11-23 06:34:46.454249 7f65d14bb800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: splice is supported
|
|
-18> 2016-11-23 06:34:46.478872 7f65d14bb800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: syncfs(2) syscall fully supported (by glibc and kernel)
|
|
-17> 2016-11-23 06:34:46.478918 7f65d14bb800 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_feature: extsize is disabled by conf
|
|
-16> 2016-11-23 06:34:46.479551 7f65d14bb800 1 leveldb: Recovering log #2226
|
|
-15> 2016-11-23 06:34:46.537516 7f65d14bb800 1 leveldb: Delete type=3 #2225
|
|
|
|
-14> 2016-11-23 06:34:46.537559 7f65d14bb800 1 leveldb: Delete type=2 #2227
|
|
|
|
-13> 2016-11-23 06:34:46.537596 7f65d14bb800 1 leveldb: Delete type=0 #2226
|
|
|
|
-12> 2016-11-23 06:34:46.537686 7f65c2e6e700 1 leveldb: Compacting 17@0 + 6@1 files
|
|
-11> 2016-11-23 06:34:46.539994 7f65d14bb800 0 filestore(/var/lib/ceph/osd/ceph-14) mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled
|
|
-10> 2016-11-23 06:34:46.540756 7f65c2e6e700 1 leveldb: compacted to: files[ 17 6 22 0 0 0 0 ]
|
|
-9> 2016-11-23 06:34:46.540766 7f65c2e6e700 1 leveldb: Compaction error: Corruption: bad block type
|
|
-8> 2016-11-23 06:34:46.546349 7f65d14bb800 2 journal open /var/lib/ceph/osd/ceph-14/journal fsid 67ee4226-c53d-4ded-b5c2-55fe09c3e6dd fs_op_seq 1633866
|
|
-7> 2016-11-23 06:34:46.546404 7f65d14bb800 1 journal _open /var/lib/ceph/osd/ceph-14/journal fd 18: 91268055040 bytes, block size 4096 bytes, directio = 1, aio = 1
|
|
-6> 2016-11-23 06:34:46.546546 7f65d14bb800 2 journal read_entry 82652950528 : seq 1633867 1647 bytes
|
|
-5> 2016-11-23 06:34:46.546555 7f65d14bb800 2 journal read_entry 82652950528 : seq 1633867 1647 bytes
|
|
-4> 2016-11-23 06:34:46.546558 7f65d14bb800 3 journal journal_replay: applying op seq 1633867
|
|
-3> 2016-11-23 06:34:46.547141 7f65d14bb800 0 filestore(/var/lib/ceph/osd/ceph-14) error (1) Operation not permitted not handled on operation 0x7f65db0d80d4 (1633867.0.3, or op 3, counting from 0)
|
|
-2> 2016-11-23 06:34:46.547150 7f65d14bb800 0 filestore(/var/lib/ceph/osd/ceph-14) unexpected error code
|
|
-1> 2016-11-23 06:34:46.547151 7f65d14bb800 0 filestore(/var/lib/ceph/osd/ceph-14) transaction dump:
|
|
{
|
|
"ops": [
|
|
{
|
|
"op_num": 0,
|
|
"op_name": "mkcoll",
|
|
"collection": "8.33d_head"
|
|
},
|
|
{
|
|
"op_num": 1,
|
|
"op_name": "coll_hint",
|
|
"collection": "8.33d_head",
|
|
"type": 1,
|
|
"pg_num": 1024,
|
|
"expected_num_objects": 0
|
|
},
|
|
{
|
|
"op_num": 2,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 3,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_infover": 1
|
|
}
|
|
},
|
|
{
|
|
"op_num": 4,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 5,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_biginfo": 59,
|
|
"_epoch": 4,
|
|
"_info": 859
|
|
}
|
|
},
|
|
{
|
|
"op_num": 6,
|
|
"op_name": "nop"
|
|
}
|
|
]
|
|
}
|
|
|
|
0> 2016-11-23 06:34:46.549575 7f65d14bb800 -1 os/filestore/FileStore.cc: In function 'void FileStore::_do_transaction(ObjectStore::Transaction&, uint64_t, int, ThreadPool::TPHandle*)' thread 7f65d14bb800 time 2016-11-23 06:34:46.547195
|
|
os/filestore/FileStore.cc: 2912: FAILED assert(0 == "unexpected error")
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0x7f65d0f550ab]
|
|
2: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f65d0c315f0]
|
|
3: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f65d0c372db]
|
|
4: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f65d0c4db27]
|
|
5: (FileStore::mount()+0x3bf6) [0x7f65d0c25326]
|
|
6: (OSD::init()+0x27d) [0x7f65d090034d]
|
|
7: (main()+0x29d1) [0x7f65d0869361]
|
|
8: (__libc_start_main()+0xf5) [0x7f65ccc40b05]
|
|
9: (()+0x3822a7) [0x7f65d08b22a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- logging levels ---
|
|
0/ 5 none
|
|
0/ 1 lockdep
|
|
0/ 1 context
|
|
1/ 1 crush
|
|
1/ 5 mds
|
|
1/ 5 mds_balancer
|
|
1/ 5 mds_locker
|
|
1/ 5 mds_log
|
|
1/ 5 mds_log_expire
|
|
1/ 5 mds_migrator
|
|
0/ 1 buffer
|
|
0/ 1 timer
|
|
0/ 1 filer
|
|
0/ 1 striper
|
|
0/ 1 objecter
|
|
0/ 5 rados
|
|
0/ 5 rbd
|
|
0/ 5 rbd_mirror
|
|
0/ 5 rbd_replay
|
|
0/ 5 journaler
|
|
0/ 5 objectcacher
|
|
0/ 5 client
|
|
0/ 5 osd
|
|
0/ 5 optracker
|
|
0/ 5 objclass
|
|
1/ 3 filestore
|
|
1/ 3 journal
|
|
0/ 5 ms
|
|
1/ 5 mon
|
|
0/10 monc
|
|
1/ 5 paxos
|
|
0/ 5 tp
|
|
1/ 5 auth
|
|
1/ 5 crypto
|
|
1/ 1 finisher
|
|
1/ 5 heartbeatmap
|
|
1/ 5 perfcounter
|
|
1/ 5 rgw
|
|
1/10 civetweb
|
|
1/ 5 javaclient
|
|
1/ 5 asok
|
|
1/ 1 throttle
|
|
0/ 0 refs
|
|
1/ 5 xio
|
|
1/ 5 compressor
|
|
1/ 5 newstore
|
|
1/ 5 bluestore
|
|
1/ 5 bluefs
|
|
1/ 3 bdev
|
|
1/ 5 kstore
|
|
4/ 5 rocksdb
|
|
4/ 5 leveldb
|
|
1/ 5 kinetic
|
|
1/ 5 fuse
|
|
-2/-2 (syslog threshold)
|
|
-1/-1 (stderr threshold)
|
|
max_recent 10000
|
|
max_new 1000
|
|
log_file /var/log/ceph/ceph-osd.14.log
|
|
--- end dump of recent events ---
|
|
2016-11-23 06:34:46.552378 7f65d14bb800 -1 *** Caught signal (Aborted) **
|
|
in thread 7f65d14bb800 thread_name:ceph-osd
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (()+0x919742) [0x7f65d0e49742]
|
|
2: (()+0xf890) [0x7f65cf1c4890]
|
|
3: (gsignal()+0x37) [0x7f65ccc54187]
|
|
4: (abort()+0x118) [0x7f65ccc55538]
|
|
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x265) [0x7f65d0f55285]
|
|
6: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f65d0c315f0]
|
|
7: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f65d0c372db]
|
|
8: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f65d0c4db27]
|
|
9: (FileStore::mount()+0x3bf6) [0x7f65d0c25326]
|
|
10: (OSD::init()+0x27d) [0x7f65d090034d]
|
|
11: (main()+0x29d1) [0x7f65d0869361]
|
|
12: (__libc_start_main()+0xf5) [0x7f65ccc40b05]
|
|
13: (()+0x3822a7) [0x7f65d08b22a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- begin dump of recent events ---
|
|
0> 2016-11-23 06:34:46.552378 7f65d14bb800 -1 *** Caught signal (Aborted) **
|
|
in thread 7f65d14bb800 thread_name:ceph-osd
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (()+0x919742) [0x7f65d0e49742]
|
|
2: (()+0xf890) [0x7f65cf1c4890]
|
|
3: (gsignal()+0x37) [0x7f65ccc54187]
|
|
4: (abort()+0x118) [0x7f65ccc55538]
|
|
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x265) [0x7f65d0f55285]
|
|
6: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f65d0c315f0]
|
|
7: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f65d0c372db]
|
|
8: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f65d0c4db27]
|
|
9: (FileStore::mount()+0x3bf6) [0x7f65d0c25326]
|
|
10: (OSD::init()+0x27d) [0x7f65d090034d]
|
|
11: (main()+0x29d1) [0x7f65d0869361]
|
|
12: (__libc_start_main()+0xf5) [0x7f65ccc40b05]
|
|
13: (()+0x3822a7) [0x7f65d08b22a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- logging levels ---
|
|
0/ 5 none
|
|
0/ 1 lockdep
|
|
0/ 1 context
|
|
1/ 1 crush
|
|
1/ 5 mds
|
|
1/ 5 mds_balancer
|
|
1/ 5 mds_locker
|
|
1/ 5 mds_log
|
|
1/ 5 mds_log_expire
|
|
1/ 5 mds_migrator
|
|
0/ 1 buffer
|
|
0/ 1 timer
|
|
0/ 1 filer
|
|
0/ 1 striper
|
|
0/ 1 objecter
|
|
0/ 5 rados
|
|
0/ 5 rbd
|
|
0/ 5 rbd_mirror
|
|
0/ 5 rbd_replay
|
|
0/ 5 journaler
|
|
0/ 5 objectcacher
|
|
0/ 5 client
|
|
0/ 5 osd
|
|
0/ 5 optracker
|
|
0/ 5 objclass
|
|
1/ 3 filestore
|
|
1/ 3 journal
|
|
0/ 5 ms
|
|
1/ 5 mon
|
|
0/10 monc
|
|
1/ 5 paxos
|
|
0/ 5 tp
|
|
1/ 5 auth
|
|
1/ 5 crypto
|
|
1/ 1 finisher
|
|
1/ 5 heartbeatmap
|
|
1/ 5 perfcounter
|
|
1/ 5 rgw
|
|
1/10 civetweb
|
|
1/ 5 javaclient
|
|
1/ 5 asok
|
|
1/ 1 throttle
|
|
0/ 0 refs
|
|
1/ 5 xio
|
|
1/ 5 compressor
|
|
1/ 5 newstore
|
|
1/ 5 bluestore
|
|
1/ 5 bluefs
|
|
1/ 3 bdev
|
|
1/ 5 kstore
|
|
4/ 5 rocksdb
|
|
4/ 5 leveldb
|
|
1/ 5 kinetic
|
|
1/ 5 fuse
|
|
-2/-2 (syslog threshold)
|
|
-1/-1 (stderr threshold)
|
|
max_recent 10000
|
|
max_new 1000
|
|
log_file /var/log/ceph/ceph-osd.14.log
|
|
--- end dump of recent events ---
|
|
2016-11-23 06:34:47.416867 7fa1a49db800 0 set uid:gid to 167:167 (ceph:ceph)
|
|
2016-11-23 06:34:47.416876 7fa1a49db800 0 ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5), process ceph-osd, pid 7280
|
|
2016-11-23 06:34:47.417432 7fa1a49db800 0 pidfile_write: ignore empty --pid-file
|
|
2016-11-23 06:34:47.426048 7fa1a49db800 0 filestore(/var/lib/ceph/osd/ceph-14) backend xfs (magic 0x58465342)
|
|
2016-11-23 06:34:47.426352 7fa1a49db800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
|
|
2016-11-23 06:34:47.426357 7fa1a49db800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: SEEK_DATA/SEEK_HOLE is disabled via 'filestore seek data hole' config option
|
|
2016-11-23 06:34:47.426368 7fa1a49db800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: splice is supported
|
|
2016-11-23 06:34:47.453917 7fa1a49db800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: syncfs(2) syscall fully supported (by glibc and kernel)
|
|
2016-11-23 06:34:47.453958 7fa1a49db800 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_feature: extsize is disabled by conf
|
|
2016-11-23 06:34:47.454530 7fa1a49db800 1 leveldb: Recovering log #2228
|
|
2016-11-23 06:34:47.512531 7fa1a49db800 1 leveldb: Delete type=0 #2228
|
|
|
|
2016-11-23 06:34:47.512561 7fa1a49db800 1 leveldb: Delete type=2 #2229
|
|
|
|
2016-11-23 06:34:47.512600 7fa1a49db800 1 leveldb: Delete type=3 #2227
|
|
|
|
2016-11-23 06:34:47.512730 7fa19638e700 1 leveldb: Compacting 17@0 + 6@1 files
|
|
2016-11-23 06:34:47.514827 7fa1a49db800 0 filestore(/var/lib/ceph/osd/ceph-14) mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled
|
|
2016-11-23 06:34:47.515599 7fa19638e700 1 leveldb: compacted to: files[ 17 6 22 0 0 0 0 ]
|
|
2016-11-23 06:34:47.515610 7fa19638e700 1 leveldb: Compaction error: Corruption: bad block type
|
|
2016-11-23 06:34:47.521342 7fa1a49db800 1 journal _open /var/lib/ceph/osd/ceph-14/journal fd 18: 91268055040 bytes, block size 4096 bytes, directio = 1, aio = 1
|
|
2016-11-23 06:34:47.522161 7fa1a49db800 0 filestore(/var/lib/ceph/osd/ceph-14) error (1) Operation not permitted not handled on operation 0x7fa1ae9400d4 (1633867.0.3, or op 3, counting from 0)
|
|
2016-11-23 06:34:47.522170 7fa1a49db800 0 filestore(/var/lib/ceph/osd/ceph-14) unexpected error code
|
|
2016-11-23 06:34:47.522171 7fa1a49db800 0 filestore(/var/lib/ceph/osd/ceph-14) transaction dump:
|
|
{
|
|
"ops": [
|
|
{
|
|
"op_num": 0,
|
|
"op_name": "mkcoll",
|
|
"collection": "8.33d_head"
|
|
},
|
|
{
|
|
"op_num": 1,
|
|
"op_name": "coll_hint",
|
|
"collection": "8.33d_head",
|
|
"type": 1,
|
|
"pg_num": 1024,
|
|
"expected_num_objects": 0
|
|
},
|
|
{
|
|
"op_num": 2,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 3,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_infover": 1
|
|
}
|
|
},
|
|
{
|
|
"op_num": 4,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 5,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_biginfo": 59,
|
|
"_epoch": 4,
|
|
"_info": 859
|
|
}
|
|
},
|
|
{
|
|
"op_num": 6,
|
|
"op_name": "nop"
|
|
}
|
|
]
|
|
}
|
|
|
|
2016-11-23 06:34:47.524591 7fa1a49db800 -1 os/filestore/FileStore.cc: In function 'void FileStore::_do_transaction(ObjectStore::Transaction&, uint64_t, int, ThreadPool::TPHandle*)' thread 7fa1a49db800 time 2016-11-23 06:34:47.522216
|
|
os/filestore/FileStore.cc: 2912: FAILED assert(0 == "unexpected error")
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0x7fa1a44750ab]
|
|
2: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7fa1a41515f0]
|
|
3: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7fa1a41572db]
|
|
4: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7fa1a416db27]
|
|
5: (FileStore::mount()+0x3bf6) [0x7fa1a4145326]
|
|
6: (OSD::init()+0x27d) [0x7fa1a3e2034d]
|
|
7: (main()+0x29d1) [0x7fa1a3d89361]
|
|
8: (__libc_start_main()+0xf5) [0x7fa1a0160b05]
|
|
9: (()+0x3822a7) [0x7fa1a3dd22a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- begin dump of recent events ---
|
|
-67> 2016-11-23 06:34:47.414727 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command perfcounters_dump hook 0x7fa1ae6720b0
|
|
-66> 2016-11-23 06:34:47.414742 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command 1 hook 0x7fa1ae6720b0
|
|
-65> 2016-11-23 06:34:47.414746 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command perf dump hook 0x7fa1ae6720b0
|
|
-64> 2016-11-23 06:34:47.414749 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command perfcounters_schema hook 0x7fa1ae6720b0
|
|
-63> 2016-11-23 06:34:47.414752 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command 2 hook 0x7fa1ae6720b0
|
|
-62> 2016-11-23 06:34:47.414753 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command perf schema hook 0x7fa1ae6720b0
|
|
-61> 2016-11-23 06:34:47.414757 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command perf reset hook 0x7fa1ae6720b0
|
|
-60> 2016-11-23 06:34:47.414761 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command config show hook 0x7fa1ae6720b0
|
|
-59> 2016-11-23 06:34:47.414764 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command config set hook 0x7fa1ae6720b0
|
|
-58> 2016-11-23 06:34:47.414766 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command config get hook 0x7fa1ae6720b0
|
|
-57> 2016-11-23 06:34:47.414768 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command config diff hook 0x7fa1ae6720b0
|
|
-56> 2016-11-23 06:34:47.414771 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command log flush hook 0x7fa1ae6720b0
|
|
-55> 2016-11-23 06:34:47.414773 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command log dump hook 0x7fa1ae6720b0
|
|
-54> 2016-11-23 06:34:47.414776 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command log reopen hook 0x7fa1ae6720b0
|
|
-53> 2016-11-23 06:34:47.416867 7fa1a49db800 0 set uid:gid to 167:167 (ceph:ceph)
|
|
-52> 2016-11-23 06:34:47.416876 7fa1a49db800 0 ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5), process ceph-osd, pid 7280
|
|
-51> 2016-11-23 06:34:47.416894 7fa1a49db800 5 object store type is filestore
|
|
-50> 2016-11-23 06:34:47.417372 7fa1a49db800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
-49> 2016-11-23 06:34:47.417380 7fa1a49db800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6810/7280 need_addr=0
|
|
-48> 2016-11-23 06:34:47.417395 7fa1a49db800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
-47> 2016-11-23 06:34:47.417398 7fa1a49db800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6810/7280 need_addr=0
|
|
-46> 2016-11-23 06:34:47.417413 7fa1a49db800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
-45> 2016-11-23 06:34:47.417416 7fa1a49db800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6811/7280 need_addr=0
|
|
-44> 2016-11-23 06:34:47.417427 7fa1a49db800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
-43> 2016-11-23 06:34:47.417430 7fa1a49db800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6811/7280 need_addr=0
|
|
-42> 2016-11-23 06:34:47.417432 7fa1a49db800 0 pidfile_write: ignore empty --pid-file
|
|
-41> 2016-11-23 06:34:47.418989 7fa1a49db800 5 asok(0x7fa1ae67e140) init /var/run/ceph/ceph-osd.14.asok
|
|
-40> 2016-11-23 06:34:47.418997 7fa1a49db800 5 asok(0x7fa1ae67e140) bind_and_listen /var/run/ceph/ceph-osd.14.asok
|
|
-39> 2016-11-23 06:34:47.419065 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command 0 hook 0x7fa1ae66e110
|
|
-38> 2016-11-23 06:34:47.419069 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command version hook 0x7fa1ae66e110
|
|
-37> 2016-11-23 06:34:47.419072 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command git_version hook 0x7fa1ae66e110
|
|
-36> 2016-11-23 06:34:47.419076 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command help hook 0x7fa1ae6721a0
|
|
-35> 2016-11-23 06:34:47.419081 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command get_command_descriptions hook 0x7fa1ae6721b0
|
|
-34> 2016-11-23 06:34:47.419111 7fa19c707700 5 asok(0x7fa1ae67e140) entry start
|
|
-33> 2016-11-23 06:34:47.419129 7fa1a49db800 10 monclient(hunting): build_initial_monmap
|
|
-32> 2016-11-23 06:34:47.425683 7fa1a49db800 5 adding auth protocol: cephx
|
|
-31> 2016-11-23 06:34:47.425688 7fa1a49db800 5 adding auth protocol: cephx
|
|
-30> 2016-11-23 06:34:47.425800 7fa1a49db800 5 asok(0x7fa1ae67e140) register_command objecter_requests hook 0x7fa1ae672270
|
|
-29> 2016-11-23 06:34:47.425836 7fa1a49db800 1 -- 192.168.23.64:6810/7280 messenger.start
|
|
-28> 2016-11-23 06:34:47.425860 7fa1a49db800 1 -- :/0 messenger.start
|
|
-27> 2016-11-23 06:34:47.425884 7fa1a49db800 1 -- 192.168.23.64:6811/7280 messenger.start
|
|
-26> 2016-11-23 06:34:47.425902 7fa1a49db800 1 -- 172.16.23.64:6811/7280 messenger.start
|
|
-25> 2016-11-23 06:34:47.425919 7fa1a49db800 1 -- 172.16.23.64:6810/7280 messenger.start
|
|
-24> 2016-11-23 06:34:47.425935 7fa1a49db800 1 -- :/0 messenger.start
|
|
-23> 2016-11-23 06:34:47.426003 7fa1a49db800 2 osd.14 0 mounting /var/lib/ceph/osd/ceph-14 /var/lib/ceph/osd/ceph-14/journal
|
|
-22> 2016-11-23 06:34:47.426048 7fa1a49db800 0 filestore(/var/lib/ceph/osd/ceph-14) backend xfs (magic 0x58465342)
|
|
-21> 2016-11-23 06:34:47.426352 7fa1a49db800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
|
|
-20> 2016-11-23 06:34:47.426357 7fa1a49db800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: SEEK_DATA/SEEK_HOLE is disabled via 'filestore seek data hole' config option
|
|
-19> 2016-11-23 06:34:47.426368 7fa1a49db800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: splice is supported
|
|
-18> 2016-11-23 06:34:47.453917 7fa1a49db800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: syncfs(2) syscall fully supported (by glibc and kernel)
|
|
-17> 2016-11-23 06:34:47.453958 7fa1a49db800 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_feature: extsize is disabled by conf
|
|
-16> 2016-11-23 06:34:47.454530 7fa1a49db800 1 leveldb: Recovering log #2228
|
|
-15> 2016-11-23 06:34:47.512531 7fa1a49db800 1 leveldb: Delete type=0 #2228
|
|
|
|
-14> 2016-11-23 06:34:47.512561 7fa1a49db800 1 leveldb: Delete type=2 #2229
|
|
|
|
-13> 2016-11-23 06:34:47.512600 7fa1a49db800 1 leveldb: Delete type=3 #2227
|
|
|
|
-12> 2016-11-23 06:34:47.512730 7fa19638e700 1 leveldb: Compacting 17@0 + 6@1 files
|
|
-11> 2016-11-23 06:34:47.514827 7fa1a49db800 0 filestore(/var/lib/ceph/osd/ceph-14) mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled
|
|
-10> 2016-11-23 06:34:47.515599 7fa19638e700 1 leveldb: compacted to: files[ 17 6 22 0 0 0 0 ]
|
|
-9> 2016-11-23 06:34:47.515610 7fa19638e700 1 leveldb: Compaction error: Corruption: bad block type
|
|
-8> 2016-11-23 06:34:47.521291 7fa1a49db800 2 journal open /var/lib/ceph/osd/ceph-14/journal fsid 67ee4226-c53d-4ded-b5c2-55fe09c3e6dd fs_op_seq 1633866
|
|
-7> 2016-11-23 06:34:47.521342 7fa1a49db800 1 journal _open /var/lib/ceph/osd/ceph-14/journal fd 18: 91268055040 bytes, block size 4096 bytes, directio = 1, aio = 1
|
|
-6> 2016-11-23 06:34:47.521483 7fa1a49db800 2 journal read_entry 82652950528 : seq 1633867 1647 bytes
|
|
-5> 2016-11-23 06:34:47.521492 7fa1a49db800 2 journal read_entry 82652950528 : seq 1633867 1647 bytes
|
|
-4> 2016-11-23 06:34:47.521495 7fa1a49db800 3 journal journal_replay: applying op seq 1633867
|
|
-3> 2016-11-23 06:34:47.522161 7fa1a49db800 0 filestore(/var/lib/ceph/osd/ceph-14) error (1) Operation not permitted not handled on operation 0x7fa1ae9400d4 (1633867.0.3, or op 3, counting from 0)
|
|
-2> 2016-11-23 06:34:47.522170 7fa1a49db800 0 filestore(/var/lib/ceph/osd/ceph-14) unexpected error code
|
|
-1> 2016-11-23 06:34:47.522171 7fa1a49db800 0 filestore(/var/lib/ceph/osd/ceph-14) transaction dump:
|
|
{
|
|
"ops": [
|
|
{
|
|
"op_num": 0,
|
|
"op_name": "mkcoll",
|
|
"collection": "8.33d_head"
|
|
},
|
|
{
|
|
"op_num": 1,
|
|
"op_name": "coll_hint",
|
|
"collection": "8.33d_head",
|
|
"type": 1,
|
|
"pg_num": 1024,
|
|
"expected_num_objects": 0
|
|
},
|
|
{
|
|
"op_num": 2,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 3,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_infover": 1
|
|
}
|
|
},
|
|
{
|
|
"op_num": 4,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 5,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_biginfo": 59,
|
|
"_epoch": 4,
|
|
"_info": 859
|
|
}
|
|
},
|
|
{
|
|
"op_num": 6,
|
|
"op_name": "nop"
|
|
}
|
|
]
|
|
}
|
|
|
|
0> 2016-11-23 06:34:47.524591 7fa1a49db800 -1 os/filestore/FileStore.cc: In function 'void FileStore::_do_transaction(ObjectStore::Transaction&, uint64_t, int, ThreadPool::TPHandle*)' thread 7fa1a49db800 time 2016-11-23 06:34:47.522216
|
|
os/filestore/FileStore.cc: 2912: FAILED assert(0 == "unexpected error")
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0x7fa1a44750ab]
|
|
2: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7fa1a41515f0]
|
|
3: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7fa1a41572db]
|
|
4: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7fa1a416db27]
|
|
5: (FileStore::mount()+0x3bf6) [0x7fa1a4145326]
|
|
6: (OSD::init()+0x27d) [0x7fa1a3e2034d]
|
|
7: (main()+0x29d1) [0x7fa1a3d89361]
|
|
8: (__libc_start_main()+0xf5) [0x7fa1a0160b05]
|
|
9: (()+0x3822a7) [0x7fa1a3dd22a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- logging levels ---
|
|
0/ 5 none
|
|
0/ 1 lockdep
|
|
0/ 1 context
|
|
1/ 1 crush
|
|
1/ 5 mds
|
|
1/ 5 mds_balancer
|
|
1/ 5 mds_locker
|
|
1/ 5 mds_log
|
|
1/ 5 mds_log_expire
|
|
1/ 5 mds_migrator
|
|
0/ 1 buffer
|
|
0/ 1 timer
|
|
0/ 1 filer
|
|
0/ 1 striper
|
|
0/ 1 objecter
|
|
0/ 5 rados
|
|
0/ 5 rbd
|
|
0/ 5 rbd_mirror
|
|
0/ 5 rbd_replay
|
|
0/ 5 journaler
|
|
0/ 5 objectcacher
|
|
0/ 5 client
|
|
0/ 5 osd
|
|
0/ 5 optracker
|
|
0/ 5 objclass
|
|
1/ 3 filestore
|
|
1/ 3 journal
|
|
0/ 5 ms
|
|
1/ 5 mon
|
|
0/10 monc
|
|
1/ 5 paxos
|
|
0/ 5 tp
|
|
1/ 5 auth
|
|
1/ 5 crypto
|
|
1/ 1 finisher
|
|
1/ 5 heartbeatmap
|
|
1/ 5 perfcounter
|
|
1/ 5 rgw
|
|
1/10 civetweb
|
|
1/ 5 javaclient
|
|
1/ 5 asok
|
|
1/ 1 throttle
|
|
0/ 0 refs
|
|
1/ 5 xio
|
|
1/ 5 compressor
|
|
1/ 5 newstore
|
|
1/ 5 bluestore
|
|
1/ 5 bluefs
|
|
1/ 3 bdev
|
|
1/ 5 kstore
|
|
4/ 5 rocksdb
|
|
4/ 5 leveldb
|
|
1/ 5 kinetic
|
|
1/ 5 fuse
|
|
-2/-2 (syslog threshold)
|
|
-1/-1 (stderr threshold)
|
|
max_recent 10000
|
|
max_new 1000
|
|
log_file /var/log/ceph/ceph-osd.14.log
|
|
--- end dump of recent events ---
|
|
2016-11-23 06:34:47.527369 7fa1a49db800 -1 *** Caught signal (Aborted) **
|
|
in thread 7fa1a49db800 thread_name:ceph-osd
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (()+0x919742) [0x7fa1a4369742]
|
|
2: (()+0xf890) [0x7fa1a26e4890]
|
|
3: (gsignal()+0x37) [0x7fa1a0174187]
|
|
4: (abort()+0x118) [0x7fa1a0175538]
|
|
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x265) [0x7fa1a4475285]
|
|
6: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7fa1a41515f0]
|
|
7: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7fa1a41572db]
|
|
8: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7fa1a416db27]
|
|
9: (FileStore::mount()+0x3bf6) [0x7fa1a4145326]
|
|
10: (OSD::init()+0x27d) [0x7fa1a3e2034d]
|
|
11: (main()+0x29d1) [0x7fa1a3d89361]
|
|
12: (__libc_start_main()+0xf5) [0x7fa1a0160b05]
|
|
13: (()+0x3822a7) [0x7fa1a3dd22a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- begin dump of recent events ---
|
|
0> 2016-11-23 06:34:47.527369 7fa1a49db800 -1 *** Caught signal (Aborted) **
|
|
in thread 7fa1a49db800 thread_name:ceph-osd
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (()+0x919742) [0x7fa1a4369742]
|
|
2: (()+0xf890) [0x7fa1a26e4890]
|
|
3: (gsignal()+0x37) [0x7fa1a0174187]
|
|
4: (abort()+0x118) [0x7fa1a0175538]
|
|
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x265) [0x7fa1a4475285]
|
|
6: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7fa1a41515f0]
|
|
7: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7fa1a41572db]
|
|
8: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7fa1a416db27]
|
|
9: (FileStore::mount()+0x3bf6) [0x7fa1a4145326]
|
|
10: (OSD::init()+0x27d) [0x7fa1a3e2034d]
|
|
11: (main()+0x29d1) [0x7fa1a3d89361]
|
|
12: (__libc_start_main()+0xf5) [0x7fa1a0160b05]
|
|
13: (()+0x3822a7) [0x7fa1a3dd22a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- logging levels ---
|
|
0/ 5 none
|
|
0/ 1 lockdep
|
|
0/ 1 context
|
|
1/ 1 crush
|
|
1/ 5 mds
|
|
1/ 5 mds_balancer
|
|
1/ 5 mds_locker
|
|
1/ 5 mds_log
|
|
1/ 5 mds_log_expire
|
|
1/ 5 mds_migrator
|
|
0/ 1 buffer
|
|
0/ 1 timer
|
|
0/ 1 filer
|
|
0/ 1 striper
|
|
0/ 1 objecter
|
|
0/ 5 rados
|
|
0/ 5 rbd
|
|
0/ 5 rbd_mirror
|
|
0/ 5 rbd_replay
|
|
0/ 5 journaler
|
|
0/ 5 objectcacher
|
|
0/ 5 client
|
|
0/ 5 osd
|
|
0/ 5 optracker
|
|
0/ 5 objclass
|
|
1/ 3 filestore
|
|
1/ 3 journal
|
|
0/ 5 ms
|
|
1/ 5 mon
|
|
0/10 monc
|
|
1/ 5 paxos
|
|
0/ 5 tp
|
|
1/ 5 auth
|
|
1/ 5 crypto
|
|
1/ 1 finisher
|
|
1/ 5 heartbeatmap
|
|
1/ 5 perfcounter
|
|
1/ 5 rgw
|
|
1/10 civetweb
|
|
1/ 5 javaclient
|
|
1/ 5 asok
|
|
1/ 1 throttle
|
|
0/ 0 refs
|
|
1/ 5 xio
|
|
1/ 5 compressor
|
|
1/ 5 newstore
|
|
1/ 5 bluestore
|
|
1/ 5 bluefs
|
|
1/ 3 bdev
|
|
1/ 5 kstore
|
|
4/ 5 rocksdb
|
|
4/ 5 leveldb
|
|
1/ 5 kinetic
|
|
1/ 5 fuse
|
|
-2/-2 (syslog threshold)
|
|
-1/-1 (stderr threshold)
|
|
max_recent 10000
|
|
max_new 1000
|
|
log_file /var/log/ceph/ceph-osd.14.log
|
|
--- end dump of recent events ---
|
|
2016-11-24 16:18:01.080187 7f1c570b2800 0 set uid:gid to 167:167 (ceph:ceph)
|
|
2016-11-24 16:18:01.080198 7f1c570b2800 0 ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5), process ceph-osd, pid 5581
|
|
2016-11-24 16:18:01.080836 7f1c570b2800 0 pidfile_write: ignore empty --pid-file
|
|
2016-11-24 16:18:01.090094 7f1c570b2800 0 filestore(/var/lib/ceph/osd/ceph-14) backend xfs (magic 0x58465342)
|
|
2016-11-24 16:18:01.090438 7f1c570b2800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
|
|
2016-11-24 16:18:01.090443 7f1c570b2800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: SEEK_DATA/SEEK_HOLE is disabled via 'filestore seek data hole' config option
|
|
2016-11-24 16:18:01.090457 7f1c570b2800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: splice is supported
|
|
2016-11-24 16:18:01.106209 7f1c570b2800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: syncfs(2) syscall fully supported (by glibc and kernel)
|
|
2016-11-24 16:18:01.106254 7f1c570b2800 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_feature: extsize is disabled by conf
|
|
2016-11-24 16:18:01.106886 7f1c570b2800 1 leveldb: Recovering log #2230
|
|
2016-11-24 16:18:01.173163 7f1c570b2800 1 leveldb: Delete type=3 #2229
|
|
|
|
2016-11-24 16:18:01.173208 7f1c570b2800 1 leveldb: Delete type=2 #2231
|
|
|
|
2016-11-24 16:18:01.173250 7f1c570b2800 1 leveldb: Delete type=0 #2230
|
|
|
|
2016-11-24 16:18:01.173348 7f1c48a65700 1 leveldb: Compacting 17@0 + 6@1 files
|
|
2016-11-24 16:18:01.175337 7f1c570b2800 0 filestore(/var/lib/ceph/osd/ceph-14) mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled
|
|
2016-11-24 16:18:01.181135 7f1c48a65700 1 leveldb: compacted to: files[ 17 6 22 0 0 0 0 ]
|
|
2016-11-24 16:18:01.181146 7f1c48a65700 1 leveldb: Compaction error: Corruption: bad block type
|
|
2016-11-24 16:18:01.182096 7f1c570b2800 1 journal _open /var/lib/ceph/osd/ceph-14/journal fd 18: 91268055040 bytes, block size 4096 bytes, directio = 1, aio = 1
|
|
2016-11-24 16:18:01.183269 7f1c570b2800 0 filestore(/var/lib/ceph/osd/ceph-14) error (1) Operation not permitted not handled on operation 0x7f1c6128a0d4 (1633867.0.3, or op 3, counting from 0)
|
|
2016-11-24 16:18:01.183277 7f1c570b2800 0 filestore(/var/lib/ceph/osd/ceph-14) unexpected error code
|
|
2016-11-24 16:18:01.183278 7f1c570b2800 0 filestore(/var/lib/ceph/osd/ceph-14) transaction dump:
|
|
{
|
|
"ops": [
|
|
{
|
|
"op_num": 0,
|
|
"op_name": "mkcoll",
|
|
"collection": "8.33d_head"
|
|
},
|
|
{
|
|
"op_num": 1,
|
|
"op_name": "coll_hint",
|
|
"collection": "8.33d_head",
|
|
"type": 1,
|
|
"pg_num": 1024,
|
|
"expected_num_objects": 0
|
|
},
|
|
{
|
|
"op_num": 2,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 3,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_infover": 1
|
|
}
|
|
},
|
|
{
|
|
"op_num": 4,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 5,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_biginfo": 59,
|
|
"_epoch": 4,
|
|
"_info": 859
|
|
}
|
|
},
|
|
{
|
|
"op_num": 6,
|
|
"op_name": "nop"
|
|
}
|
|
]
|
|
}
|
|
|
|
2016-11-24 16:18:01.185732 7f1c570b2800 -1 os/filestore/FileStore.cc: In function 'void FileStore::_do_transaction(ObjectStore::Transaction&, uint64_t, int, ThreadPool::TPHandle*)' thread 7f1c570b2800 time 2016-11-24 16:18:01.183326
|
|
os/filestore/FileStore.cc: 2912: FAILED assert(0 == "unexpected error")
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0x7f1c56b4c0ab]
|
|
2: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f1c568285f0]
|
|
3: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f1c5682e2db]
|
|
4: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f1c56844b27]
|
|
5: (FileStore::mount()+0x3bf6) [0x7f1c5681c326]
|
|
6: (OSD::init()+0x27d) [0x7f1c564f734d]
|
|
7: (main()+0x29d1) [0x7f1c56460361]
|
|
8: (__libc_start_main()+0xf5) [0x7f1c52837b05]
|
|
9: (()+0x3822a7) [0x7f1c564a92a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- begin dump of recent events ---
|
|
-67> 2016-11-24 16:18:01.077959 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command perfcounters_dump hook 0x7f1c60fb60b0
|
|
-66> 2016-11-24 16:18:01.077978 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command 1 hook 0x7f1c60fb60b0
|
|
-65> 2016-11-24 16:18:01.077983 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command perf dump hook 0x7f1c60fb60b0
|
|
-64> 2016-11-24 16:18:01.077984 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command perfcounters_schema hook 0x7f1c60fb60b0
|
|
-63> 2016-11-24 16:18:01.077990 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command 2 hook 0x7f1c60fb60b0
|
|
-62> 2016-11-24 16:18:01.077991 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command perf schema hook 0x7f1c60fb60b0
|
|
-61> 2016-11-24 16:18:01.077993 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command perf reset hook 0x7f1c60fb60b0
|
|
-60> 2016-11-24 16:18:01.077998 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command config show hook 0x7f1c60fb60b0
|
|
-59> 2016-11-24 16:18:01.078000 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command config set hook 0x7f1c60fb60b0
|
|
-58> 2016-11-24 16:18:01.078003 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command config get hook 0x7f1c60fb60b0
|
|
-57> 2016-11-24 16:18:01.078007 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command config diff hook 0x7f1c60fb60b0
|
|
-56> 2016-11-24 16:18:01.078009 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command log flush hook 0x7f1c60fb60b0
|
|
-55> 2016-11-24 16:18:01.078011 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command log dump hook 0x7f1c60fb60b0
|
|
-54> 2016-11-24 16:18:01.078016 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command log reopen hook 0x7f1c60fb60b0
|
|
-53> 2016-11-24 16:18:01.080187 7f1c570b2800 0 set uid:gid to 167:167 (ceph:ceph)
|
|
-52> 2016-11-24 16:18:01.080198 7f1c570b2800 0 ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5), process ceph-osd, pid 5581
|
|
-51> 2016-11-24 16:18:01.080230 7f1c570b2800 5 object store type is filestore
|
|
-50> 2016-11-24 16:18:01.080783 7f1c570b2800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
-49> 2016-11-24 16:18:01.080791 7f1c570b2800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6804/5581 need_addr=0
|
|
-48> 2016-11-24 16:18:01.080805 7f1c570b2800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
-47> 2016-11-24 16:18:01.080808 7f1c570b2800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6804/5581 need_addr=0
|
|
-46> 2016-11-24 16:18:01.080819 7f1c570b2800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
-45> 2016-11-24 16:18:01.080822 7f1c570b2800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6805/5581 need_addr=0
|
|
-44> 2016-11-24 16:18:01.080831 7f1c570b2800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
-43> 2016-11-24 16:18:01.080834 7f1c570b2800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6805/5581 need_addr=0
|
|
-42> 2016-11-24 16:18:01.080836 7f1c570b2800 0 pidfile_write: ignore empty --pid-file
|
|
-41> 2016-11-24 16:18:01.082546 7f1c570b2800 5 asok(0x7f1c60fc2140) init /var/run/ceph/ceph-osd.14.asok
|
|
-40> 2016-11-24 16:18:01.082554 7f1c570b2800 5 asok(0x7f1c60fc2140) bind_and_listen /var/run/ceph/ceph-osd.14.asok
|
|
-39> 2016-11-24 16:18:01.082627 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command 0 hook 0x7f1c60fb2110
|
|
-38> 2016-11-24 16:18:01.082631 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command version hook 0x7f1c60fb2110
|
|
-37> 2016-11-24 16:18:01.082633 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command git_version hook 0x7f1c60fb2110
|
|
-36> 2016-11-24 16:18:01.082637 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command help hook 0x7f1c60fb61a0
|
|
-35> 2016-11-24 16:18:01.082640 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command get_command_descriptions hook 0x7f1c60fb61b0
|
|
-34> 2016-11-24 16:18:01.082671 7f1c4edde700 5 asok(0x7f1c60fc2140) entry start
|
|
-33> 2016-11-24 16:18:01.082692 7f1c570b2800 10 monclient(hunting): build_initial_monmap
|
|
-32> 2016-11-24 16:18:01.089710 7f1c570b2800 5 adding auth protocol: cephx
|
|
-31> 2016-11-24 16:18:01.089715 7f1c570b2800 5 adding auth protocol: cephx
|
|
-30> 2016-11-24 16:18:01.089836 7f1c570b2800 5 asok(0x7f1c60fc2140) register_command objecter_requests hook 0x7f1c60fb6270
|
|
-29> 2016-11-24 16:18:01.089877 7f1c570b2800 1 -- 192.168.23.64:6804/5581 messenger.start
|
|
-28> 2016-11-24 16:18:01.089901 7f1c570b2800 1 -- :/0 messenger.start
|
|
-27> 2016-11-24 16:18:01.089919 7f1c570b2800 1 -- 192.168.23.64:6805/5581 messenger.start
|
|
-26> 2016-11-24 16:18:01.089938 7f1c570b2800 1 -- 172.16.23.64:6805/5581 messenger.start
|
|
-25> 2016-11-24 16:18:01.089956 7f1c570b2800 1 -- 172.16.23.64:6804/5581 messenger.start
|
|
-24> 2016-11-24 16:18:01.089973 7f1c570b2800 1 -- :/0 messenger.start
|
|
-23> 2016-11-24 16:18:01.090045 7f1c570b2800 2 osd.14 0 mounting /var/lib/ceph/osd/ceph-14 /var/lib/ceph/osd/ceph-14/journal
|
|
-22> 2016-11-24 16:18:01.090094 7f1c570b2800 0 filestore(/var/lib/ceph/osd/ceph-14) backend xfs (magic 0x58465342)
|
|
-21> 2016-11-24 16:18:01.090438 7f1c570b2800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
|
|
-20> 2016-11-24 16:18:01.090443 7f1c570b2800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: SEEK_DATA/SEEK_HOLE is disabled via 'filestore seek data hole' config option
|
|
-19> 2016-11-24 16:18:01.090457 7f1c570b2800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: splice is supported
|
|
-18> 2016-11-24 16:18:01.106209 7f1c570b2800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: syncfs(2) syscall fully supported (by glibc and kernel)
|
|
-17> 2016-11-24 16:18:01.106254 7f1c570b2800 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_feature: extsize is disabled by conf
|
|
-16> 2016-11-24 16:18:01.106886 7f1c570b2800 1 leveldb: Recovering log #2230
|
|
-15> 2016-11-24 16:18:01.173163 7f1c570b2800 1 leveldb: Delete type=3 #2229
|
|
|
|
-14> 2016-11-24 16:18:01.173208 7f1c570b2800 1 leveldb: Delete type=2 #2231
|
|
|
|
-13> 2016-11-24 16:18:01.173250 7f1c570b2800 1 leveldb: Delete type=0 #2230
|
|
|
|
-12> 2016-11-24 16:18:01.173348 7f1c48a65700 1 leveldb: Compacting 17@0 + 6@1 files
|
|
-11> 2016-11-24 16:18:01.175337 7f1c570b2800 0 filestore(/var/lib/ceph/osd/ceph-14) mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled
|
|
-10> 2016-11-24 16:18:01.181135 7f1c48a65700 1 leveldb: compacted to: files[ 17 6 22 0 0 0 0 ]
|
|
-9> 2016-11-24 16:18:01.181146 7f1c48a65700 1 leveldb: Compaction error: Corruption: bad block type
|
|
-8> 2016-11-24 16:18:01.182039 7f1c570b2800 2 journal open /var/lib/ceph/osd/ceph-14/journal fsid 67ee4226-c53d-4ded-b5c2-55fe09c3e6dd fs_op_seq 1633866
|
|
-7> 2016-11-24 16:18:01.182096 7f1c570b2800 1 journal _open /var/lib/ceph/osd/ceph-14/journal fd 18: 91268055040 bytes, block size 4096 bytes, directio = 1, aio = 1
|
|
-6> 2016-11-24 16:18:01.182582 7f1c570b2800 2 journal read_entry 82652950528 : seq 1633867 1647 bytes
|
|
-5> 2016-11-24 16:18:01.182591 7f1c570b2800 2 journal read_entry 82652950528 : seq 1633867 1647 bytes
|
|
-4> 2016-11-24 16:18:01.182595 7f1c570b2800 3 journal journal_replay: applying op seq 1633867
|
|
-3> 2016-11-24 16:18:01.183269 7f1c570b2800 0 filestore(/var/lib/ceph/osd/ceph-14) error (1) Operation not permitted not handled on operation 0x7f1c6128a0d4 (1633867.0.3, or op 3, counting from 0)
|
|
-2> 2016-11-24 16:18:01.183277 7f1c570b2800 0 filestore(/var/lib/ceph/osd/ceph-14) unexpected error code
|
|
-1> 2016-11-24 16:18:01.183278 7f1c570b2800 0 filestore(/var/lib/ceph/osd/ceph-14) transaction dump:
|
|
{
|
|
"ops": [
|
|
{
|
|
"op_num": 0,
|
|
"op_name": "mkcoll",
|
|
"collection": "8.33d_head"
|
|
},
|
|
{
|
|
"op_num": 1,
|
|
"op_name": "coll_hint",
|
|
"collection": "8.33d_head",
|
|
"type": 1,
|
|
"pg_num": 1024,
|
|
"expected_num_objects": 0
|
|
},
|
|
{
|
|
"op_num": 2,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 3,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_infover": 1
|
|
}
|
|
},
|
|
{
|
|
"op_num": 4,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 5,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_biginfo": 59,
|
|
"_epoch": 4,
|
|
"_info": 859
|
|
}
|
|
},
|
|
{
|
|
"op_num": 6,
|
|
"op_name": "nop"
|
|
}
|
|
]
|
|
}
|
|
|
|
0> 2016-11-24 16:18:01.185732 7f1c570b2800 -1 os/filestore/FileStore.cc: In function 'void FileStore::_do_transaction(ObjectStore::Transaction&, uint64_t, int, ThreadPool::TPHandle*)' thread 7f1c570b2800 time 2016-11-24 16:18:01.183326
|
|
os/filestore/FileStore.cc: 2912: FAILED assert(0 == "unexpected error")
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0x7f1c56b4c0ab]
|
|
2: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f1c568285f0]
|
|
3: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f1c5682e2db]
|
|
4: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f1c56844b27]
|
|
5: (FileStore::mount()+0x3bf6) [0x7f1c5681c326]
|
|
6: (OSD::init()+0x27d) [0x7f1c564f734d]
|
|
7: (main()+0x29d1) [0x7f1c56460361]
|
|
8: (__libc_start_main()+0xf5) [0x7f1c52837b05]
|
|
9: (()+0x3822a7) [0x7f1c564a92a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- logging levels ---
|
|
0/ 5 none
|
|
0/ 1 lockdep
|
|
0/ 1 context
|
|
1/ 1 crush
|
|
1/ 5 mds
|
|
1/ 5 mds_balancer
|
|
1/ 5 mds_locker
|
|
1/ 5 mds_log
|
|
1/ 5 mds_log_expire
|
|
1/ 5 mds_migrator
|
|
0/ 1 buffer
|
|
0/ 1 timer
|
|
0/ 1 filer
|
|
0/ 1 striper
|
|
0/ 1 objecter
|
|
0/ 5 rados
|
|
0/ 5 rbd
|
|
0/ 5 rbd_mirror
|
|
0/ 5 rbd_replay
|
|
0/ 5 journaler
|
|
0/ 5 objectcacher
|
|
0/ 5 client
|
|
0/ 5 osd
|
|
0/ 5 optracker
|
|
0/ 5 objclass
|
|
1/ 3 filestore
|
|
1/ 3 journal
|
|
0/ 5 ms
|
|
1/ 5 mon
|
|
0/10 monc
|
|
1/ 5 paxos
|
|
0/ 5 tp
|
|
1/ 5 auth
|
|
1/ 5 crypto
|
|
1/ 1 finisher
|
|
1/ 5 heartbeatmap
|
|
1/ 5 perfcounter
|
|
1/ 5 rgw
|
|
1/10 civetweb
|
|
1/ 5 javaclient
|
|
1/ 5 asok
|
|
1/ 1 throttle
|
|
0/ 0 refs
|
|
1/ 5 xio
|
|
1/ 5 compressor
|
|
1/ 5 newstore
|
|
1/ 5 bluestore
|
|
1/ 5 bluefs
|
|
1/ 3 bdev
|
|
1/ 5 kstore
|
|
4/ 5 rocksdb
|
|
4/ 5 leveldb
|
|
1/ 5 kinetic
|
|
1/ 5 fuse
|
|
-2/-2 (syslog threshold)
|
|
-1/-1 (stderr threshold)
|
|
max_recent 10000
|
|
max_new 1000
|
|
log_file /var/log/ceph/ceph-osd.14.log
|
|
--- end dump of recent events ---
|
|
2016-11-24 16:18:01.188581 7f1c570b2800 -1 *** Caught signal (Aborted) **
|
|
in thread 7f1c570b2800 thread_name:ceph-osd
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (()+0x919742) [0x7f1c56a40742]
|
|
2: (()+0xf890) [0x7f1c54dbb890]
|
|
3: (gsignal()+0x37) [0x7f1c5284b187]
|
|
4: (abort()+0x118) [0x7f1c5284c538]
|
|
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x265) [0x7f1c56b4c285]
|
|
6: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f1c568285f0]
|
|
7: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f1c5682e2db]
|
|
8: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f1c56844b27]
|
|
9: (FileStore::mount()+0x3bf6) [0x7f1c5681c326]
|
|
10: (OSD::init()+0x27d) [0x7f1c564f734d]
|
|
11: (main()+0x29d1) [0x7f1c56460361]
|
|
12: (__libc_start_main()+0xf5) [0x7f1c52837b05]
|
|
13: (()+0x3822a7) [0x7f1c564a92a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- begin dump of recent events ---
|
|
0> 2016-11-24 16:18:01.188581 7f1c570b2800 -1 *** Caught signal (Aborted) **
|
|
in thread 7f1c570b2800 thread_name:ceph-osd
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (()+0x919742) [0x7f1c56a40742]
|
|
2: (()+0xf890) [0x7f1c54dbb890]
|
|
3: (gsignal()+0x37) [0x7f1c5284b187]
|
|
4: (abort()+0x118) [0x7f1c5284c538]
|
|
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x265) [0x7f1c56b4c285]
|
|
6: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7f1c568285f0]
|
|
7: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7f1c5682e2db]
|
|
8: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7f1c56844b27]
|
|
9: (FileStore::mount()+0x3bf6) [0x7f1c5681c326]
|
|
10: (OSD::init()+0x27d) [0x7f1c564f734d]
|
|
11: (main()+0x29d1) [0x7f1c56460361]
|
|
12: (__libc_start_main()+0xf5) [0x7f1c52837b05]
|
|
13: (()+0x3822a7) [0x7f1c564a92a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- logging levels ---
|
|
0/ 5 none
|
|
0/ 1 lockdep
|
|
0/ 1 context
|
|
1/ 1 crush
|
|
1/ 5 mds
|
|
1/ 5 mds_balancer
|
|
1/ 5 mds_locker
|
|
1/ 5 mds_log
|
|
1/ 5 mds_log_expire
|
|
1/ 5 mds_migrator
|
|
0/ 1 buffer
|
|
0/ 1 timer
|
|
0/ 1 filer
|
|
0/ 1 striper
|
|
0/ 1 objecter
|
|
0/ 5 rados
|
|
0/ 5 rbd
|
|
0/ 5 rbd_mirror
|
|
0/ 5 rbd_replay
|
|
0/ 5 journaler
|
|
0/ 5 objectcacher
|
|
0/ 5 client
|
|
0/ 5 osd
|
|
0/ 5 optracker
|
|
0/ 5 objclass
|
|
1/ 3 filestore
|
|
1/ 3 journal
|
|
0/ 5 ms
|
|
1/ 5 mon
|
|
0/10 monc
|
|
1/ 5 paxos
|
|
0/ 5 tp
|
|
1/ 5 auth
|
|
1/ 5 crypto
|
|
1/ 1 finisher
|
|
1/ 5 heartbeatmap
|
|
1/ 5 perfcounter
|
|
1/ 5 rgw
|
|
1/10 civetweb
|
|
1/ 5 javaclient
|
|
1/ 5 asok
|
|
1/ 1 throttle
|
|
0/ 0 refs
|
|
1/ 5 xio
|
|
1/ 5 compressor
|
|
1/ 5 newstore
|
|
1/ 5 bluestore
|
|
1/ 5 bluefs
|
|
1/ 3 bdev
|
|
1/ 5 kstore
|
|
4/ 5 rocksdb
|
|
4/ 5 leveldb
|
|
1/ 5 kinetic
|
|
1/ 5 fuse
|
|
-2/-2 (syslog threshold)
|
|
-1/-1 (stderr threshold)
|
|
max_recent 10000
|
|
max_new 1000
|
|
log_file /var/log/ceph/ceph-osd.14.log
|
|
--- end dump of recent events ---
|
|
2016-11-24 17:24:01.320983 7fcab17d4800 0 set uid:gid to 167:167 (ceph:ceph)
|
|
2016-11-24 17:24:01.320994 7fcab17d4800 0 ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5), process ceph-osd, pid 22827
|
|
2016-11-24 17:24:01.321024 7fcab17d4800 5 object store type is filestore
|
|
2016-11-24 17:24:01.321593 7fcab17d4800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
2016-11-24 17:24:01.321601 7fcab17d4800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6804/22827 need_addr=0
|
|
2016-11-24 17:24:01.321618 7fcab17d4800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
2016-11-24 17:24:01.321621 7fcab17d4800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6804/22827 need_addr=0
|
|
2016-11-24 17:24:01.321633 7fcab17d4800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
2016-11-24 17:24:01.321636 7fcab17d4800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6805/22827 need_addr=0
|
|
2016-11-24 17:24:01.321646 7fcab17d4800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
2016-11-24 17:24:01.321648 7fcab17d4800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6805/22827 need_addr=0
|
|
2016-11-24 17:24:01.321651 7fcab17d4800 0 pidfile_write: ignore empty --pid-file
|
|
2016-11-24 17:24:01.323425 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) dump_stop
|
|
2016-11-24 17:24:01.328354 7fcab17d4800 10 ErasureCodePluginSelectJerasure: load: jerasure_sse4
|
|
2016-11-24 17:24:01.330950 7fcab17d4800 10 load: jerasure load: lrc load: isa
|
|
2016-11-24 17:24:01.331121 7fcab17d4800 5 filestore(/var/lib/ceph/osd/ceph-14) test_mount basedir /var/lib/ceph/osd/ceph-14 journal /var/lib/ceph/osd/ceph-14/journal
|
|
2016-11-24 17:24:01.331142 7fcab17d4800 1 -- 192.168.23.64:6804/22827 messenger.start
|
|
2016-11-24 17:24:01.331166 7fcab17d4800 1 -- :/0 messenger.start
|
|
2016-11-24 17:24:01.331184 7fcab17d4800 1 -- 192.168.23.64:6805/22827 messenger.start
|
|
2016-11-24 17:24:01.331207 7fcab17d4800 1 -- 172.16.23.64:6805/22827 messenger.start
|
|
2016-11-24 17:24:01.331225 7fcab17d4800 1 -- 172.16.23.64:6804/22827 messenger.start
|
|
2016-11-24 17:24:01.331243 7fcab17d4800 1 -- :/0 messenger.start
|
|
2016-11-24 17:24:01.331318 7fcab17d4800 2 osd.14 0 mounting /var/lib/ceph/osd/ceph-14 /var/lib/ceph/osd/ceph-14/journal
|
|
2016-11-24 17:24:01.331325 7fcab17d4800 5 filestore(/var/lib/ceph/osd/ceph-14) basedir /var/lib/ceph/osd/ceph-14 journal /var/lib/ceph/osd/ceph-14/journal
|
|
2016-11-24 17:24:01.331337 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) mount fsid is 67ee4226-c53d-4ded-b5c2-55fe09c3e6dd
|
|
2016-11-24 17:24:01.331356 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) version_stamp_is_valid was 4 vs target 4
|
|
2016-11-24 17:24:01.331375 7fcab17d4800 0 filestore(/var/lib/ceph/osd/ceph-14) backend xfs (magic 0x58465342)
|
|
2016-11-24 17:24:01.331720 7fcab17d4800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
|
|
2016-11-24 17:24:01.331724 7fcab17d4800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: SEEK_DATA/SEEK_HOLE is disabled via 'filestore seek data hole' config option
|
|
2016-11-24 17:24:01.331738 7fcab17d4800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: splice is supported
|
|
2016-11-24 17:24:01.345723 7fcab17d4800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: syncfs(2) syscall fully supported (by glibc and kernel)
|
|
2016-11-24 17:24:01.345774 7fcab17d4800 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_feature: extsize is disabled by conf
|
|
2016-11-24 17:24:01.346157 7fcab17d4800 5 filestore(/var/lib/ceph/osd/ceph-14) mount op_seq is 1633866
|
|
2016-11-24 17:24:01.346474 7fcab17d4800 1 leveldb: Recovering log #2232
|
|
2016-11-24 17:24:01.412665 7fcab17d4800 1 leveldb: Delete type=0 #2232
|
|
|
|
2016-11-24 17:24:01.412695 7fcab17d4800 1 leveldb: Delete type=2 #2233
|
|
|
|
2016-11-24 17:24:01.412739 7fcab17d4800 1 leveldb: Delete type=3 #2231
|
|
|
|
2016-11-24 17:24:01.412860 7fcaa3187700 1 leveldb: Compacting 17@0 + 6@1 files
|
|
2016-11-24 17:24:01.415087 7fcab17d4800 20 filestore (init)dbobjectmap: seq is 244823
|
|
2016-11-24 17:24:01.415095 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) open_journal at /var/lib/ceph/osd/ceph-14/journal
|
|
2016-11-24 17:24:01.415113 7fcab17d4800 0 filestore(/var/lib/ceph/osd/ceph-14) mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled
|
|
2016-11-24 17:24:01.415116 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) list_collections
|
|
2016-11-24 17:24:01.421641 7fcaa3187700 1 leveldb: compacted to: files[ 17 6 22 0 0 0 0 ]
|
|
2016-11-24 17:24:01.421655 7fcaa3187700 1 leveldb: Compaction error: Corruption: bad block type
|
|
2016-11-24 17:24:01.422119 7fcaa1734700 20 filestore(/var/lib/ceph/osd/ceph-14) sync_entry waiting for max_interval 5.000000
|
|
2016-11-24 17:24:01.422172 7fcab17d4800 1 journal _open /var/lib/ceph/osd/ceph-14/journal fd 18: 91268055040 bytes, block size 4096 bytes, directio = 1, aio = 1
|
|
2016-11-24 17:24:01.422385 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) _do_transaction on 0x7fcabb848fc0
|
|
2016-11-24 17:24:01.422399 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) _check_replay_guard object has 1633867.0.1 > current pos 1633867.0.0, now or in future, SKIPPING REPLAY
|
|
2016-11-24 17:24:01.422410 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) _check_replay_guard object has 1633867.0.1 == current pos 1633867.0.1, in_progress=false, SKIPPING REPLAY
|
|
2016-11-24 17:24:01.422418 7fcab17d4800 20 filestore(/var/lib/ceph/osd/ceph-14) _check_global_replay_guard no xattr
|
|
2016-11-24 17:24:01.422441 7fcab17d4800 20 filestore(/var/lib/ceph/osd/ceph-14) _check_replay_guard no xattr
|
|
2016-11-24 17:24:01.422446 7fcab17d4800 15 filestore(/var/lib/ceph/osd/ceph-14) touch 8.33d_head/#8:bcc00000::::head#
|
|
2016-11-24 17:24:01.422459 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) touch 8.33d_head/#8:bcc00000::::head# = 0
|
|
2016-11-24 17:24:01.422466 7fcab17d4800 15 filestore(/var/lib/ceph/osd/ceph-14) _omap_setkeys 8.33d_head/#8:bcc00000::::head#
|
|
2016-11-24 17:24:01.423030 7fcab17d4800 10 filestore oid: #8:bcc00000::::head# not skipping op, *spos 1633867.0.3
|
|
2016-11-24 17:24:01.423034 7fcab17d4800 10 filestore > header.spos 0.0.0
|
|
2016-11-24 17:24:01.423048 7fcab17d4800 20 filestore(/var/lib/ceph/osd/ceph-14) _omap_setkeys 8.33d_head/#8:bcc00000::::head# = -1
|
|
2016-11-24 17:24:01.423052 7fcab17d4800 0 filestore(/var/lib/ceph/osd/ceph-14) error (1) Operation not permitted not handled on operation 0x7fcabbaec0d4 (1633867.0.3, or op 3, counting from 0)
|
|
2016-11-24 17:24:01.423059 7fcab17d4800 0 filestore(/var/lib/ceph/osd/ceph-14) unexpected error code
|
|
2016-11-24 17:24:01.423060 7fcab17d4800 0 filestore(/var/lib/ceph/osd/ceph-14) transaction dump:
|
|
{
|
|
"ops": [
|
|
{
|
|
"op_num": 0,
|
|
"op_name": "mkcoll",
|
|
"collection": "8.33d_head"
|
|
},
|
|
{
|
|
"op_num": 1,
|
|
"op_name": "coll_hint",
|
|
"collection": "8.33d_head",
|
|
"type": 1,
|
|
"pg_num": 1024,
|
|
"expected_num_objects": 0
|
|
},
|
|
{
|
|
"op_num": 2,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 3,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_infover": 1
|
|
}
|
|
},
|
|
{
|
|
"op_num": 4,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 5,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_biginfo": 59,
|
|
"_epoch": 4,
|
|
"_info": 859
|
|
}
|
|
},
|
|
{
|
|
"op_num": 6,
|
|
"op_name": "nop"
|
|
}
|
|
]
|
|
}
|
|
|
|
2016-11-24 17:24:01.425504 7fcab17d4800 -1 os/filestore/FileStore.cc: In function 'void FileStore::_do_transaction(ObjectStore::Transaction&, uint64_t, int, ThreadPool::TPHandle*)' thread 7fcab17d4800 time 2016-11-24 17:24:01.423101
|
|
os/filestore/FileStore.cc: 2912: FAILED assert(0 == "unexpected error")
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0x7fcab126e0ab]
|
|
2: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7fcab0f4a5f0]
|
|
3: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7fcab0f502db]
|
|
4: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7fcab0f66b27]
|
|
5: (FileStore::mount()+0x3bf6) [0x7fcab0f3e326]
|
|
6: (OSD::init()+0x27d) [0x7fcab0c1934d]
|
|
7: (main()+0x29d1) [0x7fcab0b82361]
|
|
8: (__libc_start_main()+0xf5) [0x7fcaacf59b05]
|
|
9: (()+0x3822a7) [0x7fcab0bcb2a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- begin dump of recent events ---
|
|
-90> 2016-11-24 17:24:01.318306 7fcab17d4800 5 asok(0x7fcabb82e140) register_command perfcounters_dump hook 0x7fcabb8220b0
|
|
-89> 2016-11-24 17:24:01.318326 7fcab17d4800 5 asok(0x7fcabb82e140) register_command 1 hook 0x7fcabb8220b0
|
|
-88> 2016-11-24 17:24:01.318331 7fcab17d4800 5 asok(0x7fcabb82e140) register_command perf dump hook 0x7fcabb8220b0
|
|
-87> 2016-11-24 17:24:01.318338 7fcab17d4800 5 asok(0x7fcabb82e140) register_command perfcounters_schema hook 0x7fcabb8220b0
|
|
-86> 2016-11-24 17:24:01.318339 7fcab17d4800 5 asok(0x7fcabb82e140) register_command 2 hook 0x7fcabb8220b0
|
|
-85> 2016-11-24 17:24:01.318346 7fcab17d4800 5 asok(0x7fcabb82e140) register_command perf schema hook 0x7fcabb8220b0
|
|
-84> 2016-11-24 17:24:01.318348 7fcab17d4800 5 asok(0x7fcabb82e140) register_command perf reset hook 0x7fcabb8220b0
|
|
-83> 2016-11-24 17:24:01.318352 7fcab17d4800 5 asok(0x7fcabb82e140) register_command config show hook 0x7fcabb8220b0
|
|
-82> 2016-11-24 17:24:01.318357 7fcab17d4800 5 asok(0x7fcabb82e140) register_command config set hook 0x7fcabb8220b0
|
|
-81> 2016-11-24 17:24:01.318359 7fcab17d4800 5 asok(0x7fcabb82e140) register_command config get hook 0x7fcabb8220b0
|
|
-80> 2016-11-24 17:24:01.318361 7fcab17d4800 5 asok(0x7fcabb82e140) register_command config diff hook 0x7fcabb8220b0
|
|
-79> 2016-11-24 17:24:01.318365 7fcab17d4800 5 asok(0x7fcabb82e140) register_command log flush hook 0x7fcabb8220b0
|
|
-78> 2016-11-24 17:24:01.318367 7fcab17d4800 5 asok(0x7fcabb82e140) register_command log dump hook 0x7fcabb8220b0
|
|
-77> 2016-11-24 17:24:01.318370 7fcab17d4800 5 asok(0x7fcabb82e140) register_command log reopen hook 0x7fcabb8220b0
|
|
-76> 2016-11-24 17:24:01.320983 7fcab17d4800 0 set uid:gid to 167:167 (ceph:ceph)
|
|
-75> 2016-11-24 17:24:01.320994 7fcab17d4800 0 ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5), process ceph-osd, pid 22827
|
|
-74> 2016-11-24 17:24:01.321024 7fcab17d4800 5 object store type is filestore
|
|
-73> 2016-11-24 17:24:01.321593 7fcab17d4800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
-72> 2016-11-24 17:24:01.321601 7fcab17d4800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6804/22827 need_addr=0
|
|
-71> 2016-11-24 17:24:01.321618 7fcab17d4800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
-70> 2016-11-24 17:24:01.321621 7fcab17d4800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6804/22827 need_addr=0
|
|
-69> 2016-11-24 17:24:01.321633 7fcab17d4800 1 -- 172.16.23.64:0/0 learned my addr 172.16.23.64:0/0
|
|
-68> 2016-11-24 17:24:01.321636 7fcab17d4800 1 accepter.accepter.bind my_inst.addr is 172.16.23.64:6805/22827 need_addr=0
|
|
-67> 2016-11-24 17:24:01.321646 7fcab17d4800 1 -- 192.168.23.64:0/0 learned my addr 192.168.23.64:0/0
|
|
-66> 2016-11-24 17:24:01.321648 7fcab17d4800 1 accepter.accepter.bind my_inst.addr is 192.168.23.64:6805/22827 need_addr=0
|
|
-65> 2016-11-24 17:24:01.321651 7fcab17d4800 0 pidfile_write: ignore empty --pid-file
|
|
-64> 2016-11-24 17:24:01.323425 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) dump_stop
|
|
-63> 2016-11-24 17:24:01.323438 7fcab17d4800 5 asok(0x7fcabb82e140) init /var/run/ceph/ceph-osd.14.asok
|
|
-62> 2016-11-24 17:24:01.323445 7fcab17d4800 5 asok(0x7fcabb82e140) bind_and_listen /var/run/ceph/ceph-osd.14.asok
|
|
-61> 2016-11-24 17:24:01.323518 7fcab17d4800 5 asok(0x7fcabb82e140) register_command 0 hook 0x7fcabb81e110
|
|
-60> 2016-11-24 17:24:01.323523 7fcab17d4800 5 asok(0x7fcabb82e140) register_command version hook 0x7fcabb81e110
|
|
-59> 2016-11-24 17:24:01.323527 7fcab17d4800 5 asok(0x7fcabb82e140) register_command git_version hook 0x7fcabb81e110
|
|
-58> 2016-11-24 17:24:01.323530 7fcab17d4800 5 asok(0x7fcabb82e140) register_command help hook 0x7fcabb8221a0
|
|
-57> 2016-11-24 17:24:01.323533 7fcab17d4800 5 asok(0x7fcabb82e140) register_command get_command_descriptions hook 0x7fcabb8221b0
|
|
-56> 2016-11-24 17:24:01.323571 7fcaa9500700 5 asok(0x7fcabb82e140) entry start
|
|
-55> 2016-11-24 17:24:01.323582 7fcab17d4800 10 monclient(hunting): build_initial_monmap
|
|
-54> 2016-11-24 17:24:01.328354 7fcab17d4800 10 ErasureCodePluginSelectJerasure: load: jerasure_sse4
|
|
-53> 2016-11-24 17:24:01.330950 7fcab17d4800 10 load: jerasure load: lrc load: isa
|
|
-52> 2016-11-24 17:24:01.330966 7fcab17d4800 5 adding auth protocol: cephx
|
|
-51> 2016-11-24 17:24:01.330969 7fcab17d4800 5 adding auth protocol: cephx
|
|
-50> 2016-11-24 17:24:01.331098 7fcab17d4800 5 asok(0x7fcabb82e140) register_command objecter_requests hook 0x7fcabb822270
|
|
-49> 2016-11-24 17:24:01.331121 7fcab17d4800 5 filestore(/var/lib/ceph/osd/ceph-14) test_mount basedir /var/lib/ceph/osd/ceph-14 journal /var/lib/ceph/osd/ceph-14/journal
|
|
-48> 2016-11-24 17:24:01.331142 7fcab17d4800 1 -- 192.168.23.64:6804/22827 messenger.start
|
|
-47> 2016-11-24 17:24:01.331166 7fcab17d4800 1 -- :/0 messenger.start
|
|
-46> 2016-11-24 17:24:01.331184 7fcab17d4800 1 -- 192.168.23.64:6805/22827 messenger.start
|
|
-45> 2016-11-24 17:24:01.331207 7fcab17d4800 1 -- 172.16.23.64:6805/22827 messenger.start
|
|
-44> 2016-11-24 17:24:01.331225 7fcab17d4800 1 -- 172.16.23.64:6804/22827 messenger.start
|
|
-43> 2016-11-24 17:24:01.331243 7fcab17d4800 1 -- :/0 messenger.start
|
|
-42> 2016-11-24 17:24:01.331318 7fcab17d4800 2 osd.14 0 mounting /var/lib/ceph/osd/ceph-14 /var/lib/ceph/osd/ceph-14/journal
|
|
-41> 2016-11-24 17:24:01.331325 7fcab17d4800 5 filestore(/var/lib/ceph/osd/ceph-14) basedir /var/lib/ceph/osd/ceph-14 journal /var/lib/ceph/osd/ceph-14/journal
|
|
-40> 2016-11-24 17:24:01.331337 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) mount fsid is 67ee4226-c53d-4ded-b5c2-55fe09c3e6dd
|
|
-39> 2016-11-24 17:24:01.331356 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) version_stamp_is_valid was 4 vs target 4
|
|
-38> 2016-11-24 17:24:01.331375 7fcab17d4800 0 filestore(/var/lib/ceph/osd/ceph-14) backend xfs (magic 0x58465342)
|
|
-37> 2016-11-24 17:24:01.331720 7fcab17d4800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: FIEMAP ioctl is disabled via 'filestore fiemap' config option
|
|
-36> 2016-11-24 17:24:01.331724 7fcab17d4800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: SEEK_DATA/SEEK_HOLE is disabled via 'filestore seek data hole' config option
|
|
-35> 2016-11-24 17:24:01.331738 7fcab17d4800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: splice is supported
|
|
-34> 2016-11-24 17:24:01.345723 7fcab17d4800 0 genericfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_features: syncfs(2) syscall fully supported (by glibc and kernel)
|
|
-33> 2016-11-24 17:24:01.345774 7fcab17d4800 0 xfsfilestorebackend(/var/lib/ceph/osd/ceph-14) detect_feature: extsize is disabled by conf
|
|
-32> 2016-11-24 17:24:01.346157 7fcab17d4800 5 filestore(/var/lib/ceph/osd/ceph-14) mount op_seq is 1633866
|
|
-31> 2016-11-24 17:24:01.346474 7fcab17d4800 1 leveldb: Recovering log #2232
|
|
-30> 2016-11-24 17:24:01.412665 7fcab17d4800 1 leveldb: Delete type=0 #2232
|
|
|
|
-29> 2016-11-24 17:24:01.412695 7fcab17d4800 1 leveldb: Delete type=2 #2233
|
|
|
|
-28> 2016-11-24 17:24:01.412739 7fcab17d4800 1 leveldb: Delete type=3 #2231
|
|
|
|
-27> 2016-11-24 17:24:01.412860 7fcaa3187700 1 leveldb: Compacting 17@0 + 6@1 files
|
|
-26> 2016-11-24 17:24:01.415087 7fcab17d4800 20 filestore (init)dbobjectmap: seq is 244823
|
|
-25> 2016-11-24 17:24:01.415095 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) open_journal at /var/lib/ceph/osd/ceph-14/journal
|
|
-24> 2016-11-24 17:24:01.415113 7fcab17d4800 0 filestore(/var/lib/ceph/osd/ceph-14) mount: enabling WRITEAHEAD journal mode: checkpoint is not enabled
|
|
-23> 2016-11-24 17:24:01.415116 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) list_collections
|
|
-22> 2016-11-24 17:24:01.421641 7fcaa3187700 1 leveldb: compacted to: files[ 17 6 22 0 0 0 0 ]
|
|
-21> 2016-11-24 17:24:01.421655 7fcaa3187700 1 leveldb: Compaction error: Corruption: bad block type
|
|
-20> 2016-11-24 17:24:01.422115 7fcab17d4800 2 journal open /var/lib/ceph/osd/ceph-14/journal fsid 67ee4226-c53d-4ded-b5c2-55fe09c3e6dd fs_op_seq 1633866
|
|
-19> 2016-11-24 17:24:01.422119 7fcaa1734700 20 filestore(/var/lib/ceph/osd/ceph-14) sync_entry waiting for max_interval 5.000000
|
|
-18> 2016-11-24 17:24:01.422172 7fcab17d4800 1 journal _open /var/lib/ceph/osd/ceph-14/journal fd 18: 91268055040 bytes, block size 4096 bytes, directio = 1, aio = 1
|
|
-17> 2016-11-24 17:24:01.422357 7fcab17d4800 2 journal read_entry 82652950528 : seq 1633867 1647 bytes
|
|
-16> 2016-11-24 17:24:01.422367 7fcab17d4800 2 journal read_entry 82652950528 : seq 1633867 1647 bytes
|
|
-15> 2016-11-24 17:24:01.422369 7fcab17d4800 3 journal journal_replay: applying op seq 1633867
|
|
-14> 2016-11-24 17:24:01.422385 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) _do_transaction on 0x7fcabb848fc0
|
|
-13> 2016-11-24 17:24:01.422399 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) _check_replay_guard object has 1633867.0.1 > current pos 1633867.0.0, now or in future, SKIPPING REPLAY
|
|
-12> 2016-11-24 17:24:01.422410 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) _check_replay_guard object has 1633867.0.1 == current pos 1633867.0.1, in_progress=false, SKIPPING REPLAY
|
|
-11> 2016-11-24 17:24:01.422418 7fcab17d4800 20 filestore(/var/lib/ceph/osd/ceph-14) _check_global_replay_guard no xattr
|
|
-10> 2016-11-24 17:24:01.422441 7fcab17d4800 20 filestore(/var/lib/ceph/osd/ceph-14) _check_replay_guard no xattr
|
|
-9> 2016-11-24 17:24:01.422446 7fcab17d4800 15 filestore(/var/lib/ceph/osd/ceph-14) touch 8.33d_head/#8:bcc00000::::head#
|
|
-8> 2016-11-24 17:24:01.422459 7fcab17d4800 10 filestore(/var/lib/ceph/osd/ceph-14) touch 8.33d_head/#8:bcc00000::::head# = 0
|
|
-7> 2016-11-24 17:24:01.422466 7fcab17d4800 15 filestore(/var/lib/ceph/osd/ceph-14) _omap_setkeys 8.33d_head/#8:bcc00000::::head#
|
|
-6> 2016-11-24 17:24:01.423030 7fcab17d4800 10 filestore oid: #8:bcc00000::::head# not skipping op, *spos 1633867.0.3
|
|
-5> 2016-11-24 17:24:01.423034 7fcab17d4800 10 filestore > header.spos 0.0.0
|
|
-4> 2016-11-24 17:24:01.423048 7fcab17d4800 20 filestore(/var/lib/ceph/osd/ceph-14) _omap_setkeys 8.33d_head/#8:bcc00000::::head# = -1
|
|
-3> 2016-11-24 17:24:01.423052 7fcab17d4800 0 filestore(/var/lib/ceph/osd/ceph-14) error (1) Operation not permitted not handled on operation 0x7fcabbaec0d4 (1633867.0.3, or op 3, counting from 0)
|
|
-2> 2016-11-24 17:24:01.423059 7fcab17d4800 0 filestore(/var/lib/ceph/osd/ceph-14) unexpected error code
|
|
-1> 2016-11-24 17:24:01.423060 7fcab17d4800 0 filestore(/var/lib/ceph/osd/ceph-14) transaction dump:
|
|
{
|
|
"ops": [
|
|
{
|
|
"op_num": 0,
|
|
"op_name": "mkcoll",
|
|
"collection": "8.33d_head"
|
|
},
|
|
{
|
|
"op_num": 1,
|
|
"op_name": "coll_hint",
|
|
"collection": "8.33d_head",
|
|
"type": 1,
|
|
"pg_num": 1024,
|
|
"expected_num_objects": 0
|
|
},
|
|
{
|
|
"op_num": 2,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 3,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_infover": 1
|
|
}
|
|
},
|
|
{
|
|
"op_num": 4,
|
|
"op_name": "touch",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#"
|
|
},
|
|
{
|
|
"op_num": 5,
|
|
"op_name": "omap_setkeys",
|
|
"collection": "8.33d_head",
|
|
"oid": "#8:bcc00000::::head#",
|
|
"attr_lens": {
|
|
"_biginfo": 59,
|
|
"_epoch": 4,
|
|
"_info": 859
|
|
}
|
|
},
|
|
{
|
|
"op_num": 6,
|
|
"op_name": "nop"
|
|
}
|
|
]
|
|
}
|
|
|
|
0> 2016-11-24 17:24:01.425504 7fcab17d4800 -1 os/filestore/FileStore.cc: In function 'void FileStore::_do_transaction(ObjectStore::Transaction&, uint64_t, int, ThreadPool::TPHandle*)' thread 7fcab17d4800 time 2016-11-24 17:24:01.423101
|
|
os/filestore/FileStore.cc: 2912: FAILED assert(0 == "unexpected error")
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0x7fcab126e0ab]
|
|
2: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7fcab0f4a5f0]
|
|
3: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7fcab0f502db]
|
|
4: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7fcab0f66b27]
|
|
5: (FileStore::mount()+0x3bf6) [0x7fcab0f3e326]
|
|
6: (OSD::init()+0x27d) [0x7fcab0c1934d]
|
|
7: (main()+0x29d1) [0x7fcab0b82361]
|
|
8: (__libc_start_main()+0xf5) [0x7fcaacf59b05]
|
|
9: (()+0x3822a7) [0x7fcab0bcb2a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- logging levels ---
|
|
0/ 5 none
|
|
0/ 1 lockdep
|
|
0/ 1 context
|
|
1/ 1 crush
|
|
1/ 5 mds
|
|
1/ 5 mds_balancer
|
|
1/ 5 mds_locker
|
|
1/ 5 mds_log
|
|
1/ 5 mds_log_expire
|
|
1/ 5 mds_migrator
|
|
0/ 1 buffer
|
|
0/ 1 timer
|
|
0/ 1 filer
|
|
0/ 1 striper
|
|
0/ 1 objecter
|
|
0/ 5 rados
|
|
0/ 5 rbd
|
|
0/ 5 rbd_mirror
|
|
0/ 5 rbd_replay
|
|
0/ 5 journaler
|
|
0/ 5 objectcacher
|
|
0/ 5 client
|
|
20/20 osd
|
|
0/ 5 optracker
|
|
0/ 5 objclass
|
|
20/20 filestore
|
|
1/ 3 journal
|
|
1/ 1 ms
|
|
1/ 5 mon
|
|
0/10 monc
|
|
1/ 5 paxos
|
|
0/ 5 tp
|
|
1/ 5 auth
|
|
1/ 5 crypto
|
|
1/ 1 finisher
|
|
1/ 5 heartbeatmap
|
|
1/ 5 perfcounter
|
|
1/ 5 rgw
|
|
1/10 civetweb
|
|
1/ 5 javaclient
|
|
1/ 5 asok
|
|
1/ 1 throttle
|
|
0/ 0 refs
|
|
1/ 5 xio
|
|
1/ 5 compressor
|
|
1/ 5 newstore
|
|
1/ 5 bluestore
|
|
1/ 5 bluefs
|
|
1/ 3 bdev
|
|
1/ 5 kstore
|
|
4/ 5 rocksdb
|
|
4/ 5 leveldb
|
|
1/ 5 kinetic
|
|
1/ 5 fuse
|
|
-2/-2 (syslog threshold)
|
|
-1/-1 (stderr threshold)
|
|
max_recent 10000
|
|
max_new 1000
|
|
log_file /var/log/ceph/ceph-osd.14.log
|
|
--- end dump of recent events ---
|
|
2016-11-24 17:24:01.428411 7fcab17d4800 -1 *** Caught signal (Aborted) **
|
|
in thread 7fcab17d4800 thread_name:ceph-osd
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (()+0x919742) [0x7fcab1162742]
|
|
2: (()+0xf890) [0x7fcaaf4dd890]
|
|
3: (gsignal()+0x37) [0x7fcaacf6d187]
|
|
4: (abort()+0x118) [0x7fcaacf6e538]
|
|
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x265) [0x7fcab126e285]
|
|
6: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7fcab0f4a5f0]
|
|
7: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7fcab0f502db]
|
|
8: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7fcab0f66b27]
|
|
9: (FileStore::mount()+0x3bf6) [0x7fcab0f3e326]
|
|
10: (OSD::init()+0x27d) [0x7fcab0c1934d]
|
|
11: (main()+0x29d1) [0x7fcab0b82361]
|
|
12: (__libc_start_main()+0xf5) [0x7fcaacf59b05]
|
|
13: (()+0x3822a7) [0x7fcab0bcb2a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- begin dump of recent events ---
|
|
0> 2016-11-24 17:24:01.428411 7fcab17d4800 -1 *** Caught signal (Aborted) **
|
|
in thread 7fcab17d4800 thread_name:ceph-osd
|
|
|
|
ceph version 10.2.2-90-g308eb8b (308eb8be25a544f1ad2f482de8e59f8bb15dd4f5)
|
|
1: (()+0x919742) [0x7fcab1162742]
|
|
2: (()+0xf890) [0x7fcaaf4dd890]
|
|
3: (gsignal()+0x37) [0x7fcaacf6d187]
|
|
4: (abort()+0x118) [0x7fcaacf6e538]
|
|
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x265) [0x7fcab126e285]
|
|
6: (FileStore::_do_transaction(ObjectStore::Transaction&, unsigned long, int, ThreadPool::TPHandle*)+0xee0) [0x7fcab0f4a5f0]
|
|
7: (FileStore::_do_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, unsigned long, ThreadPool::TPHandle*)+0x3b) [0x7fcab0f502db]
|
|
8: (JournalingObjectStore::journal_replay(unsigned long)+0x8c7) [0x7fcab0f66b27]
|
|
9: (FileStore::mount()+0x3bf6) [0x7fcab0f3e326]
|
|
10: (OSD::init()+0x27d) [0x7fcab0c1934d]
|
|
11: (main()+0x29d1) [0x7fcab0b82361]
|
|
12: (__libc_start_main()+0xf5) [0x7fcaacf59b05]
|
|
13: (()+0x3822a7) [0x7fcab0bcb2a7]
|
|
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
|
|
|
|
--- logging levels ---
|
|
0/ 5 none
|
|
0/ 1 lockdep
|
|
0/ 1 context
|
|
1/ 1 crush
|
|
1/ 5 mds
|
|
1/ 5 mds_balancer
|
|
1/ 5 mds_locker
|
|
1/ 5 mds_log
|
|
1/ 5 mds_log_expire
|
|
1/ 5 mds_migrator
|
|
0/ 1 buffer
|
|
0/ 1 timer
|
|
0/ 1 filer
|
|
0/ 1 striper
|
|
0/ 1 objecter
|
|
0/ 5 rados
|
|
0/ 5 rbd
|
|
0/ 5 rbd_mirror
|
|
0/ 5 rbd_replay
|
|
0/ 5 journaler
|
|
0/ 5 objectcacher
|
|
0/ 5 client
|
|
20/20 osd
|
|
0/ 5 optracker
|
|
0/ 5 objclass
|
|
20/20 filestore
|
|
1/ 3 journal
|
|
1/ 1 ms
|
|
1/ 5 mon
|
|
0/10 monc
|
|
1/ 5 paxos
|
|
0/ 5 tp
|
|
1/ 5 auth
|
|
1/ 5 crypto
|
|
1/ 1 finisher
|
|
1/ 5 heartbeatmap
|
|
1/ 5 perfcounter
|
|
1/ 5 rgw
|
|
1/10 civetweb
|
|
1/ 5 javaclient
|
|
1/ 5 asok
|
|
1/ 1 throttle
|
|
0/ 0 refs
|
|
1/ 5 xio
|
|
1/ 5 compressor
|
|
1/ 5 newstore
|
|
1/ 5 bluestore
|
|
1/ 5 bluefs
|
|
1/ 3 bdev
|
|
1/ 5 kstore
|
|
4/ 5 rocksdb
|
|
4/ 5 leveldb
|
|
1/ 5 kinetic
|
|
1/ 5 fuse
|
|
-2/-2 (syslog threshold)
|
|
-1/-1 (stderr threshold)
|
|
max_recent 10000
|
|
max_new 1000
|
|
log_file /var/log/ceph/ceph-osd.14.log
|
|
--- end dump of recent events ---
|