Project

General

Profile

vstartrunner_output_with_master.log

complete vstart_runner output - Ramana Raja, 07/14/2016 04:58 PM

Download (181 KB)

 
1
[root@bzn build]# LD_LIBRARY_PATH=/home/rraja/git/ceph/build/lib PYTHONPATH=/home/rraja/git/teuthology/:/home/rraja/git/ceph-qa-suite/:/home/rraja/git/ceph/src/pybind:/home/rraja/git/ceph/build/lib/cython_modules/lib.linux-x86_64-2.7/ python /home/rraja/git/ceph-qa-suite/tasks/cephfs/vstart_runner.py --interactive tasks.cephfs.test_volume_client.TestVolumeClient
2
2016-07-14 21:49:05,160.160 INFO:__main__:run args=['ps', '-u0']
3
2016-07-14 21:49:05,160.160 INFO:__main__:Running ['ps', '-u0']
4
2016-07-14 21:49:05,175.175 WARNING:__main__:Killing stray process  3133 ?        00:00:03 ceph-mds
5
2016-07-14 21:49:05,178.178 INFO:__main__:run args=['./bin/ceph', 'auth', 'get-or-create', 'client.0', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
6
2016-07-14 21:49:05,178.178 INFO:__main__:Running ['./bin/ceph', 'auth', 'get-or-create', 'client.0', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
7
2016-07-14 21:49:05,392.392 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
8
2016-07-14 21:49:05,392.392 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
9
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.0': No such file or directory
10
2016-07-14 21:49:05,400.400 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.0
11
2016-07-14 21:49:05,401.401 INFO:__main__:run args=['./bin/ceph', 'auth', 'get-or-create', 'client.1', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
12
2016-07-14 21:49:05,401.401 INFO:__main__:Running ['./bin/ceph', 'auth', 'get-or-create', 'client.1', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
13
2016-07-14 21:49:05,601.601 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
14
2016-07-14 21:49:05,601.601 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
15
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.1': No such file or directory
16
2016-07-14 21:49:05,608.608 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.1
17
2016-07-14 21:49:05,609.609 INFO:__main__:run args=['./bin/ceph', 'auth', 'get-or-create', 'client.2', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
18
2016-07-14 21:49:05,609.609 INFO:__main__:Running ['./bin/ceph', 'auth', 'get-or-create', 'client.2', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
19
2016-07-14 21:49:05,825.825 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
20
2016-07-14 21:49:05,825.825 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
21
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.2': No such file or directory
22
2016-07-14 21:49:05,833.833 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.2
23
2016-07-14 21:49:05,833.833 INFO:__main__:run args=['./bin/ceph', 'auth', 'get-or-create', 'client.3', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
24
2016-07-14 21:49:05,834.834 INFO:__main__:Running ['./bin/ceph', 'auth', 'get-or-create', 'client.3', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
25
2016-07-14 21:49:06,038.038 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
26
2016-07-14 21:49:06,038.038 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
27
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.3': No such file or directory
28
2016-07-14 21:49:06,046.046 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.3
29
2016-07-14 21:49:06,046.046 INFO:__main__:Discovered MDS IDs: ['a']
30
2016-07-14 21:49:06,048.048 INFO:__main__:run args=['./bin/ceph', 'tell', 'osd.*', 'injectargs', '--osd-mon-report-interval-max', '5']
31
2016-07-14 21:49:06,048.048 INFO:__main__:Running ['./bin/ceph', 'tell', 'osd.*', 'injectargs', '--osd-mon-report-interval-max', '5']
32
2016-07-14 21:49:06,188.188 INFO:__main__:Searching for existing instance osd_mon_report_interval_max/osd
33
2016-07-14 21:49:06,190.190 INFO:__main__:Searching for existing instance mds log max segments/mds
34
2016-07-14 21:49:06,190.190 INFO:__main__:Found string to replace at 1722
35
2016-07-14 21:49:06,190.190 INFO:__main__:Searching for existing instance osd_mon_report_interval_max/osd
36
2016-07-14 21:49:06,191.191 INFO:__main__:Searching for existing instance mds log max segments/mds
37
2016-07-14 21:49:06,192.192 INFO:__main__:Searching for existing instance mds root ino uid/global
38
2016-07-14 21:49:06,192.192 INFO:__main__:Searching for existing instance osd_mon_report_interval_max/osd
39
2016-07-14 21:49:06,193.193 INFO:__main__:Searching for existing instance mds log max segments/mds
40
2016-07-14 21:49:06,193.193 INFO:__main__:Searching for existing instance mds root ino uid/global
41
2016-07-14 21:49:06,193.193 INFO:__main__:Searching for existing instance mds root ino gid/global
42
2016-07-14 21:49:06,193.193 INFO:__main__:Searching for existing instance osd_mon_report_interval_max/osd
43
2016-07-14 21:49:06,194.194 INFO:__main__:Executing modules: ['tasks.cephfs.test_volume_client.TestVolumeClient']
44
2016-07-14 21:49:06,198.198 INFO:__main__:Loaded: [<unittest.suite.TestSuite tests=[<tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_15303>, <tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_data_isolated>, <tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_default_prefix>, <tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_evict_client>, <tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_idempotency>, <tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_lifecycle>, <tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_purge>, <tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_readonly_authorization>]>]
45
2016-07-14 21:49:06,198.198 INFO:__main__:Disabling 0 tests because of is_for_teuthology or needs_trimming
46
2016-07-14 21:49:06,199.199 INFO:__main__:Starting test: test_15303 (tasks.cephfs.test_volume_client.TestVolumeClient)
47
2016-07-14 21:49:06,199.199 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
48
2016-07-14 21:49:06,199.199 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
49
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.0': No such file or directory
50
2016-07-14 21:49:06,207.207 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.0
51
2016-07-14 21:49:06,207.207 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
52
2016-07-14 21:49:06,207.207 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
53
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.1': No such file or directory
54
2016-07-14 21:49:06,216.216 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.1
55
2016-07-14 21:49:06,217.217 INFO:__main__:run args=['ps', '-u0']
56
2016-07-14 21:49:06,217.217 INFO:__main__:Running ['ps', '-u0']
57
2016-07-14 21:49:06,239.239 INFO:__main__:No match for mds a: PID TTY          TIME CMD
58
    1 ?        00:00:02 systemd
59
    2 ?        00:00:00 kthreadd
60
    3 ?        00:00:00 ksoftirqd/0
61
    5 ?        00:00:00 kworker/0:0H
62
    7 ?        00:00:36 rcu_sched
63
    8 ?        00:00:00 rcu_bh
64
    9 ?        00:00:15 rcuos/0
65
   10 ?        00:00:00 rcuob/0
66
   11 ?        00:00:00 migration/0
67
   12 ?        00:00:00 watchdog/0
68
   13 ?        00:00:00 watchdog/1
69
   14 ?        00:00:00 migration/1
70
   15 ?        00:00:00 ksoftirqd/1
71
   17 ?        00:00:00 kworker/1:0H
72
   18 ?        00:00:06 rcuos/1
73
   19 ?        00:00:00 rcuob/1
74
   20 ?        00:00:00 watchdog/2
75
   21 ?        00:00:00 migration/2
76
   22 ?        00:00:00 ksoftirqd/2
77
   24 ?        00:00:00 kworker/2:0H
78
   25 ?        00:00:14 rcuos/2
79
   26 ?        00:00:00 rcuob/2
80
   27 ?        00:00:00 watchdog/3
81
   28 ?        00:00:00 migration/3
82
   29 ?        00:00:00 ksoftirqd/3
83
   31 ?        00:00:00 kworker/3:0H
84
   32 ?        00:00:05 rcuos/3
85
   33 ?        00:00:00 rcuob/3
86
   34 ?        00:00:00 kdevtmpfs
87
   35 ?        00:00:00 netns
88
   36 ?        00:00:00 writeback
89
   37 ?        00:00:00 ksmd
90
   38 ?        00:00:00 khugepaged
91
   39 ?        00:00:00 crypto
92
   40 ?        00:00:00 kintegrityd
93
   41 ?        00:00:00 bioset
94
   42 ?        00:00:00 kblockd
95
   43 ?        00:00:00 ata_sff
96
   44 ?        00:00:00 md
97
   45 ?        00:00:00 devfreq_wq
98
   51 ?        00:00:13 kswapd0
99
   52 ?        00:00:00 vmstat
100
   93 ?        00:00:00 kthrotld
101
   94 ?        00:00:00 acpi_thermal_pm
102
   95 ?        00:00:00 scsi_eh_0
103
   96 ?        00:00:00 scsi_tmf_0
104
   97 ?        00:00:00 scsi_eh_1
105
   98 ?        00:00:00 scsi_tmf_1
106
   99 ?        00:00:00 scsi_eh_2
107
  100 ?        00:00:00 scsi_tmf_2
108
  104 ?        00:00:00 kpsmoused
109
  106 ?        00:00:00 dm_bufio_cache
110
  107 ?        00:00:00 ipv6_addrconf
111
  138 ?        00:00:00 deferwq
112
  140 ?        00:00:00 bioset
113
  178 ?        00:00:00 kauditd
114
  389 ?        00:00:01 kworker/0:1H
115
  399 ?        00:00:00 kworker/1:1H
116
  400 ?        00:00:00 kworker/3:1H
117
  420 ?        00:00:00 rtsx_pci_sdmmc_
118
  425 ?        00:00:00 kworker/2:1H
119
  613 ?        00:00:00 kdmflush
120
  621 ?        00:00:00 bioset
121
  622 ?        00:00:00 kcryptd_io
122
  623 ?        00:00:00 kcryptd
123
  624 ?        00:00:08 dmcrypt_write
124
  625 ?        00:00:00 bioset
125
  688 ?        00:00:00 kdmflush
126
  691 ?        00:00:00 bioset
127
  700 ?        00:00:00 kdmflush
128
  701 ?        00:00:00 bioset
129
  724 ?        00:00:00 jbd2/dm-1-8
130
  725 ?        00:00:00 ext4-rsv-conver
131
  823 ?        00:00:02 systemd-journal
132
  861 ?        00:00:00 rpciod
133
  880 ?        00:00:00 systemd-udevd
134
  910 ?        00:00:00 ktpacpid
135
  918 ?        00:00:00 irq/47-mei_me
136
  949 ?        00:00:00 kmemstick
137
  968 ?        00:00:00 cfg80211
138
  984 ?        00:00:07 irq/50-iwlwifi
139
  999 ?        00:00:00 kworker/u17:0
140
 1001 ?        00:00:00 hci0
141
 1002 ?        00:00:00 hci0
142
 1007 ?        00:00:00 kworker/u17:2
143
 1015 ?        00:00:00 kdmflush
144
 1016 ?        00:00:00 bioset
145
 1018 ?        00:00:00 kvm-irqfd-clean
146
 1071 ?        00:00:00 jbd2/sda1-8
147
 1072 ?        00:00:00 ext4-rsv-conver
148
 1077 ?        00:00:06 jbd2/dm-3-8
149
 1078 ?        00:00:00 ext4-rsv-conver
150
 1105 ?        00:00:00 auditd
151
 1119 ?        00:00:00 audispd
152
 1123 ?        00:00:00 sedispatch
153
 1133 ?        00:00:00 bluetoothd
154
 1137 ?        00:00:00 mcelog
155
 1139 ?        00:00:39 rngd
156
 1146 ?        00:00:00 gssproxy
157
 1154 ?        00:00:00 ModemManager
158
 1158 ?        00:00:04 udisksd
159
 1159 ?        00:00:00 firewalld
160
 1169 ?        00:00:00 systemd-logind
161
 1173 ?        00:00:00 accounts-daemon
162
 1177 ?        00:00:00 alsactl
163
 1249 ?        00:00:00 abrtd
164
 1262 ?        00:00:00 iprt-VBoxWQueue
165
 1268 ?        00:00:00 iprt-VBoxTscThr
166
 1281 ?        00:00:00 abrt-dump-journ
167
 1286 ?        00:00:00 abrt-dump-journ
168
 1324 ?        00:00:12 NetworkManager
169
 1376 ?        00:00:00 libvirtd
170
 1393 ?        00:00:00 crond
171
 1396 ?        00:00:00 atd
172
 1398 ?        00:00:00 gdm
173
 1463 ?        00:00:01 wpa_supplicant
174
 1509 ?        00:00:00 gdm-session-wor
175
 1582 ?        00:00:00 upowerd
176
 1680 ?        00:00:10 packagekitd
177
 1839 ?        00:00:00 dhclient
178
 1903 ?        00:00:00 gdm-session-wor
179
 1938 tty2     00:00:00 xf86-video-inte
180
 2067 ?        00:00:00 krfcommd
181
 2206 ?        00:00:00 cupsd
182
 2235 ?        00:00:00 fwupd
183
 2448 ?        00:00:00 dhclient
184
 2633 ?        00:00:00 kworker/0:1
185
 3387 ?        00:00:00 kworker/3:1
186
 3927 ?        00:00:04 kworker/u16:0
187
 5168 ?        00:00:00 kworker/1:3
188
13278 ?        00:00:00 kworker/u16:5
189
13551 ?        00:00:00 kworker/u16:1
190
13663 ?        00:00:00 kworker/1:1
191
13668 ?        00:00:00 kworker/3:0
192
13681 ?        00:00:00 kworker/2:2
193
13694 ?        00:00:00 kworker/0:3
194
13724 ?        00:00:00 kworker/u16:4
195
13765 ?        00:00:00 kworker/1:2
196
13781 ?        00:00:00 kworker/2:0
197
13794 ?        00:00:00 kworker/3:3
198
13799 ?        00:00:00 kworker/0:2
199
14835 pts/1    00:00:00 sudo
200
14836 ?        00:00:00 fprintd
201
14848 pts/1    00:00:00 su
202
14853 pts/1    00:00:00 bash
203
14902 ?        00:00:00 kworker/2:3
204
14903 pts/1    00:00:00 python
205
15074 pts/1    00:00:00 ps
206
16120 ?        00:00:00 kworker/2:1
207
18809 ?        00:00:00 ceph-msgr
208
18811 ?        00:00:00 rbd
209
28322 ?        00:00:00 systemd
210
28332 ?        00:00:00 (sd-pam)
211
28799 ?        00:00:10 kworker/u16:2
212
29866 ?        00:00:00 kworker/3:2
213
30529 ?        00:00:00 dio/dm-3
214
31742 ?        00:00:00 kworker/0:0
215
32711 ?        00:00:00 kworker/1:0
216
2016-07-14 21:49:06,240.240 ERROR:__main__:tried to stop a non-running daemon
217
2016-07-14 21:49:06,240.240 INFO:__main__:run args=['./bin/ceph', 'fs', 'ls', '--format=json-pretty']
218
2016-07-14 21:49:06,240.240 INFO:__main__:Running ['./bin/ceph', 'fs', 'ls', '--format=json-pretty']
219
2016-07-14 21:49:06,428.428 INFO:__main__:run args=['./bin/ceph', 'fs', 'set', u'cephfs_a', 'cluster_down', 'true']
220
2016-07-14 21:49:06,428.428 INFO:__main__:Running ['./bin/ceph', 'fs', 'set', u'cephfs_a', 'cluster_down', 'true']
221
2016-07-14 21:49:07,071.071 INFO:__main__:run args=['./bin/ceph', 'fs', 'get', u'cephfs_a', '--format=json-pretty']
222
2016-07-14 21:49:07,071.071 INFO:__main__:Running ['./bin/ceph', 'fs', 'get', u'cephfs_a', '--format=json-pretty']
223
2016-07-14 21:49:07,257.257 INFO:__main__:run args=['./bin/ceph', 'mds', 'fail', '4113']
224
2016-07-14 21:49:07,257.257 INFO:__main__:Running ['./bin/ceph', 'mds', 'fail', '4113']
225
2016-07-14 21:49:08,055.055 INFO:__main__:run args=['./bin/ceph', 'fs', 'rm', u'cephfs_a', '--yes-i-really-mean-it']
226
2016-07-14 21:49:08,055.055 INFO:__main__:Running ['./bin/ceph', 'fs', 'rm', u'cephfs_a', '--yes-i-really-mean-it']
227
2016-07-14 21:49:09,055.055 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_metadata_a', u'cephfs_metadata_a', '--yes-i-really-really-mean-it']
228
2016-07-14 21:49:09,055.055 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_metadata_a', u'cephfs_metadata_a', '--yes-i-really-really-mean-it']
229
2016-07-14 21:49:10,155.155 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_data_a', u'cephfs_data_a', '--yes-i-really-really-mean-it']
230
2016-07-14 21:49:10,155.155 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_data_a', u'cephfs_data_a', '--yes-i-really-really-mean-it']
231
2016-07-14 21:49:11,269.269 INFO:__main__:run args=['./bin/ceph', 'daemon', 'osd.0', 'config', 'get', 'osd_mon_report_interval_max']
232
2016-07-14 21:49:11,270.270 INFO:__main__:Running ['./bin/ceph', 'daemon', 'osd.0', 'config', 'get', 'osd_mon_report_interval_max']
233
2016-07-14 21:49:11,385.385 INFO:tasks.cephfs.filesystem:_json_asok output: {
234
    "osd_mon_report_interval_max": "5"
235
}
236

    
237
2016-07-14 21:49:11,385.385 INFO:__main__:run args=['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
238
2016-07-14 21:49:11,385.385 INFO:__main__:Running ['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
239
2016-07-14 21:49:11,567.567 INFO:__main__:run args=['./bin/ceph', 'osd', 'blacklist', 'clear']
240
2016-07-14 21:49:11,567.567 INFO:__main__:Running ['./bin/ceph', 'osd', 'blacklist', 'clear']
241
2016-07-14 21:49:12,259.259 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.0', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
242
2016-07-14 21:49:12,259.259 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.0', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
243
2016-07-14 21:49:12,493.493 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.1', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
244
2016-07-14 21:49:12,494.494 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.1', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
245
2016-07-14 21:49:12,754.754 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.2', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
246
2016-07-14 21:49:12,754.754 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.2', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
247
2016-07-14 21:49:12,972.972 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.3', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
248
2016-07-14 21:49:12,972.972 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.3', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
249
2016-07-14 21:49:13,187.187 INFO:tasks.cephfs.cephfs_test_case:['0', '1', '2', '3']
250
2016-07-14 21:49:13,187.187 INFO:__main__:run args=['./bin/ceph', 'auth', 'list', '--format=json-pretty']
251
2016-07-14 21:49:13,188.188 INFO:__main__:Running ['./bin/ceph', 'auth', 'list', '--format=json-pretty']
252
2016-07-14 21:49:13,380.380 INFO:tasks.cephfs.filesystem:Creating filesystem 'cephfs'
253
2016-07-14 21:49:13,381.381 INFO:__main__:run args=['./bin/ceph', 'daemon', 'mon.a', 'config', 'get', 'mon_pg_warn_min_per_osd']
254
2016-07-14 21:49:13,381.381 INFO:__main__:Running ['./bin/ceph', 'daemon', 'mon.a', 'config', 'get', 'mon_pg_warn_min_per_osd']
255
2016-07-14 21:49:13,489.489 INFO:tasks.cephfs.filesystem:_json_asok output: {
256
    "mon_pg_warn_min_per_osd": "3"
257
}
258

    
259
2016-07-14 21:49:13,489.489 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_metadata', '9']
260
2016-07-14 21:49:13,490.490 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_metadata', '9']
261
2016-07-14 21:49:14,342.342 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_data', '9']
262
2016-07-14 21:49:14,342.342 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_data', '9']
263
2016-07-14 21:49:15,351.351 INFO:__main__:run args=['./bin/ceph', 'fs', 'new', 'cephfs', 'cephfs_metadata', 'cephfs_data']
264
2016-07-14 21:49:15,351.351 INFO:__main__:Running ['./bin/ceph', 'fs', 'new', 'cephfs', 'cephfs_metadata', 'cephfs_data']
265
2016-07-14 21:49:16,416.416 INFO:__main__:run args=['ps', '-u0']
266
2016-07-14 21:49:16,416.416 INFO:__main__:Running ['ps', '-u0']
267
2016-07-14 21:49:16,440.440 INFO:__main__:No match for mds a: PID TTY          TIME CMD
268
    1 ?        00:00:02 systemd
269
    2 ?        00:00:00 kthreadd
270
    3 ?        00:00:00 ksoftirqd/0
271
    5 ?        00:00:00 kworker/0:0H
272
    7 ?        00:00:36 rcu_sched
273
    8 ?        00:00:00 rcu_bh
274
    9 ?        00:00:15 rcuos/0
275
   10 ?        00:00:00 rcuob/0
276
   11 ?        00:00:00 migration/0
277
   12 ?        00:00:00 watchdog/0
278
   13 ?        00:00:00 watchdog/1
279
   14 ?        00:00:00 migration/1
280
   15 ?        00:00:00 ksoftirqd/1
281
   17 ?        00:00:00 kworker/1:0H
282
   18 ?        00:00:06 rcuos/1
283
   19 ?        00:00:00 rcuob/1
284
   20 ?        00:00:00 watchdog/2
285
   21 ?        00:00:00 migration/2
286
   22 ?        00:00:00 ksoftirqd/2
287
   24 ?        00:00:00 kworker/2:0H
288
   25 ?        00:00:14 rcuos/2
289
   26 ?        00:00:00 rcuob/2
290
   27 ?        00:00:00 watchdog/3
291
   28 ?        00:00:00 migration/3
292
   29 ?        00:00:00 ksoftirqd/3
293
   31 ?        00:00:00 kworker/3:0H
294
   32 ?        00:00:05 rcuos/3
295
   33 ?        00:00:00 rcuob/3
296
   34 ?        00:00:00 kdevtmpfs
297
   35 ?        00:00:00 netns
298
   36 ?        00:00:00 writeback
299
   37 ?        00:00:00 ksmd
300
   38 ?        00:00:00 khugepaged
301
   39 ?        00:00:00 crypto
302
   40 ?        00:00:00 kintegrityd
303
   41 ?        00:00:00 bioset
304
   42 ?        00:00:00 kblockd
305
   43 ?        00:00:00 ata_sff
306
   44 ?        00:00:00 md
307
   45 ?        00:00:00 devfreq_wq
308
   51 ?        00:00:13 kswapd0
309
   52 ?        00:00:00 vmstat
310
   93 ?        00:00:00 kthrotld
311
   94 ?        00:00:00 acpi_thermal_pm
312
   95 ?        00:00:00 scsi_eh_0
313
   96 ?        00:00:00 scsi_tmf_0
314
   97 ?        00:00:00 scsi_eh_1
315
   98 ?        00:00:00 scsi_tmf_1
316
   99 ?        00:00:00 scsi_eh_2
317
  100 ?        00:00:00 scsi_tmf_2
318
  104 ?        00:00:00 kpsmoused
319
  106 ?        00:00:00 dm_bufio_cache
320
  107 ?        00:00:00 ipv6_addrconf
321
  138 ?        00:00:00 deferwq
322
  140 ?        00:00:00 bioset
323
  178 ?        00:00:00 kauditd
324
  389 ?        00:00:01 kworker/0:1H
325
  399 ?        00:00:00 kworker/1:1H
326
  400 ?        00:00:00 kworker/3:1H
327
  420 ?        00:00:00 rtsx_pci_sdmmc_
328
  425 ?        00:00:00 kworker/2:1H
329
  613 ?        00:00:00 kdmflush
330
  621 ?        00:00:00 bioset
331
  622 ?        00:00:00 kcryptd_io
332
  623 ?        00:00:00 kcryptd
333
  624 ?        00:00:08 dmcrypt_write
334
  625 ?        00:00:00 bioset
335
  688 ?        00:00:00 kdmflush
336
  691 ?        00:00:00 bioset
337
  700 ?        00:00:00 kdmflush
338
  701 ?        00:00:00 bioset
339
  724 ?        00:00:00 jbd2/dm-1-8
340
  725 ?        00:00:00 ext4-rsv-conver
341
  823 ?        00:00:02 systemd-journal
342
  861 ?        00:00:00 rpciod
343
  880 ?        00:00:00 systemd-udevd
344
  910 ?        00:00:00 ktpacpid
345
  918 ?        00:00:00 irq/47-mei_me
346
  949 ?        00:00:00 kmemstick
347
  968 ?        00:00:00 cfg80211
348
  984 ?        00:00:07 irq/50-iwlwifi
349
  999 ?        00:00:00 kworker/u17:0
350
 1001 ?        00:00:00 hci0
351
 1002 ?        00:00:00 hci0
352
 1007 ?        00:00:00 kworker/u17:2
353
 1015 ?        00:00:00 kdmflush
354
 1016 ?        00:00:00 bioset
355
 1018 ?        00:00:00 kvm-irqfd-clean
356
 1071 ?        00:00:00 jbd2/sda1-8
357
 1072 ?        00:00:00 ext4-rsv-conver
358
 1077 ?        00:00:06 jbd2/dm-3-8
359
 1078 ?        00:00:00 ext4-rsv-conver
360
 1105 ?        00:00:00 auditd
361
 1119 ?        00:00:00 audispd
362
 1123 ?        00:00:00 sedispatch
363
 1133 ?        00:00:00 bluetoothd
364
 1137 ?        00:00:00 mcelog
365
 1139 ?        00:00:39 rngd
366
 1146 ?        00:00:00 gssproxy
367
 1154 ?        00:00:00 ModemManager
368
 1158 ?        00:00:04 udisksd
369
 1159 ?        00:00:00 firewalld
370
 1169 ?        00:00:00 systemd-logind
371
 1173 ?        00:00:00 accounts-daemon
372
 1177 ?        00:00:00 alsactl
373
 1249 ?        00:00:00 abrtd
374
 1262 ?        00:00:00 iprt-VBoxWQueue
375
 1268 ?        00:00:00 iprt-VBoxTscThr
376
 1281 ?        00:00:00 abrt-dump-journ
377
 1286 ?        00:00:00 abrt-dump-journ
378
 1324 ?        00:00:12 NetworkManager
379
 1376 ?        00:00:00 libvirtd
380
 1393 ?        00:00:00 crond
381
 1396 ?        00:00:00 atd
382
 1398 ?        00:00:00 gdm
383
 1463 ?        00:00:01 wpa_supplicant
384
 1509 ?        00:00:00 gdm-session-wor
385
 1582 ?        00:00:00 upowerd
386
 1680 ?        00:00:10 packagekitd
387
 1839 ?        00:00:00 dhclient
388
 1903 ?        00:00:00 gdm-session-wor
389
 1938 tty2     00:00:00 xf86-video-inte
390
 2067 ?        00:00:00 krfcommd
391
 2206 ?        00:00:00 cupsd
392
 2235 ?        00:00:00 fwupd
393
 2448 ?        00:00:00 dhclient
394
 2633 ?        00:00:00 kworker/0:1
395
 3387 ?        00:00:00 kworker/3:1
396
 3927 ?        00:00:04 kworker/u16:0
397
 5168 ?        00:00:00 kworker/1:3
398
13278 ?        00:00:00 kworker/u16:5
399
13551 ?        00:00:00 kworker/u16:1
400
13663 ?        00:00:00 kworker/1:1
401
13668 ?        00:00:00 kworker/3:0
402
13681 ?        00:00:00 kworker/2:2
403
13694 ?        00:00:00 kworker/0:3
404
13724 ?        00:00:00 kworker/u16:4
405
13765 ?        00:00:00 kworker/1:2
406
13781 ?        00:00:00 kworker/2:0
407
13794 ?        00:00:00 kworker/3:3
408
13799 ?        00:00:00 kworker/0:2
409
14835 pts/1    00:00:00 sudo
410
14836 ?        00:00:00 fprintd
411
14848 pts/1    00:00:00 su
412
14853 pts/1    00:00:00 bash
413
14902 ?        00:00:00 kworker/2:3
414
14903 pts/1    00:00:00 python
415
15599 pts/1    00:00:00 ps
416
16120 ?        00:00:00 kworker/2:1
417
18809 ?        00:00:00 ceph-msgr
418
18811 ?        00:00:00 rbd
419
28322 ?        00:00:00 systemd
420
28332 ?        00:00:00 (sd-pam)
421
28799 ?        00:00:10 kworker/u16:2
422
29866 ?        00:00:00 kworker/3:2
423
30529 ?        00:00:00 dio/dm-3
424
31742 ?        00:00:00 kworker/0:0
425
32711 ?        00:00:00 kworker/1:0
426
2016-07-14 21:49:16,441.441 INFO:__main__:run args=['./bin/./ceph-mds', '-i', 'a']
427
2016-07-14 21:49:16,441.441 INFO:__main__:Running ['./bin/./ceph-mds', '-i', 'a']
428
2016-07-14 21:49:16,471.471 INFO:__main__:run args=['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
429
2016-07-14 21:49:16,472.472 INFO:__main__:Running ['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
430
2016-07-14 21:49:16,667.667 INFO:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {u'session_autoclose': 300, u'up': {u'mds_0': 4123}, u'last_failure_osd_epoch': 0, u'in': [0], u'last_failure': 0, u'max_file_size': 1099511627776, u'explicitly_allowed_features': 0, u'damaged': [], u'tableserver': 0, u'metadata_pool': 3, u'failed': [], u'epoch': 10, u'flags': 0, u'max_mds': 1, u'compat': {u'compat': {}, u'ro_compat': {}, u'incompat': {u'feature_8': u'file layout v2', u'feature_2': u'client writeable ranges', u'feature_3': u'default file layouts on dirs', u'feature_1': u'base v0.20', u'feature_6': u'dirfrag is stored in omap', u'feature_4': u'dir inode in separate object', u'feature_5': u'mds uses versioned encoding'}}, u'data_pools': [4], u'info': {u'gid_4123': {u'standby_for_rank': -1, u'export_targets': [], u'name': u'a', u'incarnation': 10, u'standby_replay': False, u'state_seq': 1, u'standby_for_fscid': -1, u'state': u'up:creating', u'gid': 4123, u'features': 576460752032890879, u'rank': 0, u'standby_for_name': u'', u'addr': u'10.70.1.141:6812/14766'}}, u'fs_name': u'cephfs', u'created': u'2016-07-14 21:49:15.555766', u'enabled': True, u'modified': u'2016-07-14 21:49:15.555766', u'session_timeout': 60, u'stopped': [], u'ever_allowed_features': 0, u'root': 0}
431
2016-07-14 21:49:16,668.668 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_4123:up:creating
432
2016-07-14 21:49:17,668.668 INFO:__main__:run args=['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
433
2016-07-14 21:49:17,668.668 INFO:__main__:Running ['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
434
2016-07-14 21:49:17,866.866 INFO:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {u'session_autoclose': 300, u'up': {u'mds_0': 4123}, u'last_failure_osd_epoch': 0, u'in': [0], u'last_failure': 0, u'max_file_size': 1099511627776, u'explicitly_allowed_features': 0, u'damaged': [], u'tableserver': 0, u'metadata_pool': 3, u'failed': [], u'epoch': 11, u'flags': 0, u'max_mds': 1, u'compat': {u'compat': {}, u'ro_compat': {}, u'incompat': {u'feature_8': u'file layout v2', u'feature_2': u'client writeable ranges', u'feature_3': u'default file layouts on dirs', u'feature_1': u'base v0.20', u'feature_6': u'dirfrag is stored in omap', u'feature_4': u'dir inode in separate object', u'feature_5': u'mds uses versioned encoding'}}, u'data_pools': [4], u'info': {u'gid_4123': {u'standby_for_rank': -1, u'export_targets': [], u'name': u'a', u'incarnation': 10, u'standby_replay': False, u'state_seq': 4, u'standby_for_fscid': -1, u'state': u'up:active', u'gid': 4123, u'features': 576460752032890879, u'rank': 0, u'standby_for_name': u'', u'addr': u'10.70.1.141:6812/14766'}}, u'fs_name': u'cephfs', u'created': u'2016-07-14 21:49:15.555766', u'enabled': True, u'modified': u'2016-07-14 21:49:15.555766', u'session_timeout': 60, u'stopped': [], u'ever_allowed_features': 0, u'root': 0}
435
2016-07-14 21:49:17,866.866 INFO:tasks.cephfs.filesystem:are_daemons_healthy: 1/1
436
2016-07-14 21:49:17,867.867 INFO:__main__:run args=['./bin/ceph', 'daemon', 'mds.a', 'status']
437
2016-07-14 21:49:17,867.867 INFO:__main__:Running ['./bin/ceph', 'daemon', 'mds.a', 'status']
438
2016-07-14 21:49:17,968.968 INFO:tasks.cephfs.filesystem:_json_asok output: {
439
    "cluster_fsid": "ae66e242-d103-408a-b4cf-f160f95d365c",
440
    "whoami": 0,
441
    "want_state": "up:active",
442
    "state": "up:active",
443
    "mdsmap_epoch": 11,
444
    "osdmap_epoch": 17,
445
    "osdmap_epoch_barrier": 17
446
}
447

    
448
2016-07-14 21:49:17,969.969 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
449
2016-07-14 21:49:17,969.969 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
450
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.0': No such file or directory
451
2016-07-14 21:49:17,983.983 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.0
452
2016-07-14 21:49:17,984.984 INFO:__main__:run args=['mkdir', '--', '/tmp/tmp0EnRmD/mnt.0']
453
2016-07-14 21:49:17,984.984 INFO:__main__:Running ['mkdir', '--', '/tmp/tmp0EnRmD/mnt.0']
454
2016-07-14 21:49:17,991.991 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
455
2016-07-14 21:49:17,992.992 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
456
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
457
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
458
2016-07-14 21:49:18,007.007 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
459
2016-07-14 21:49:18,008.008 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
460
2016-07-14 21:49:18,019.019 INFO:__main__:Pre-mount connections: [43, 47]
461
2016-07-14 21:49:18,019.019 INFO:__main__:run args=['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0']
462
2016-07-14 21:49:18,020.020 INFO:__main__:Running ['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0']
463
2016-07-14 21:49:18,026.026 INFO:__main__:Mounting client.0 with pid 15669
464
2016-07-14 21:49:18,026.026 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
465
2016-07-14 21:49:18,027.027 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
466
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
467
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
468
2016-07-14 21:49:18,043.043 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
469
2016-07-14 21:49:18,044.044 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
470
2016-07-14 21:49:19,059.059 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
471
2016-07-14 21:49:19,060.060 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
472
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
473
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
474
2016-07-14 21:49:19,069.069 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
475
2016-07-14 21:49:19,069.069 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
476
2016-07-14 21:49:19,080.080 INFO:__main__:Post-mount connections: [43, 47, 48]
477
2016-07-14 21:49:19,080.080 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
478
2016-07-14 21:49:19,081.081 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
479
2016-07-14 21:49:19,089.089 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.0
480
2016-07-14 21:49:19,089.089 INFO:__main__:run args=['sudo', 'chmod', '1777', '/tmp/tmp0EnRmD/mnt.0']
481
2016-07-14 21:49:19,089.089 INFO:__main__:Running ['chmod', '1777', '/tmp/tmp0EnRmD/mnt.0']
482
2016-07-14 21:49:19,101.101 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
483
2016-07-14 21:49:19,101.101 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
484
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.1': No such file or directory
485
2016-07-14 21:49:19,112.112 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.1
486
2016-07-14 21:49:19,112.112 INFO:__main__:run args=['mkdir', '--', '/tmp/tmp0EnRmD/mnt.1']
487
2016-07-14 21:49:19,113.113 INFO:__main__:Running ['mkdir', '--', '/tmp/tmp0EnRmD/mnt.1']
488
2016-07-14 21:49:19,121.121 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
489
2016-07-14 21:49:19,121.121 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
490
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
491
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
492
2016-07-14 21:49:19,133.133 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
493
2016-07-14 21:49:19,134.134 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
494
2016-07-14 21:49:19,143.143 INFO:__main__:Pre-mount connections: [43, 47, 48]
495
2016-07-14 21:49:19,143.143 INFO:__main__:run args=['./bin/ceph-fuse', '-f', '--name', 'client.1', '/tmp/tmp0EnRmD/mnt.1']
496
2016-07-14 21:49:19,144.144 INFO:__main__:Running ['./bin/ceph-fuse', '-f', '--name', 'client.1', '/tmp/tmp0EnRmD/mnt.1']
497
2016-07-14 21:49:19,152.152 INFO:__main__:Mounting client.1 with pid 15711
498
2016-07-14 21:49:19,152.152 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
499
2016-07-14 21:49:19,152.152 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
500
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
501
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
502
2016-07-14 21:49:19,165.165 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
503
2016-07-14 21:49:19,166.166 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
504
2016-07-14 21:49:20,176.176 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
505
2016-07-14 21:49:20,176.176 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
506
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
507
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
508
2016-07-14 21:49:20,185.185 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
509
2016-07-14 21:49:20,185.185 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
510
2016-07-14 21:49:20,193.193 INFO:__main__:Post-mount connections: [43, 47, 48, 49]
511
2016-07-14 21:49:20,194.194 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
512
2016-07-14 21:49:20,194.194 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
513
2016-07-14 21:49:20,202.202 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.1
514
2016-07-14 21:49:20,202.202 INFO:__main__:run args=['sudo', 'chmod', '1777', '/tmp/tmp0EnRmD/mnt.1']
515
2016-07-14 21:49:20,202.202 INFO:__main__:Running ['chmod', '1777', '/tmp/tmp0EnRmD/mnt.1']
516
2016-07-14 21:49:20,218.218 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
517
2016-07-14 21:49:20,218.218 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
518
2016-07-14 21:49:20,229.229 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.0
519
2016-07-14 21:49:20,229.229 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on local...
520
2016-07-14 21:49:20,229.229 INFO:__main__:run args=['sudo', 'fusermount', '-u', '/tmp/tmp0EnRmD/mnt.0']
521
2016-07-14 21:49:20,230.230 INFO:__main__:Running ['fusermount', '-u', '/tmp/tmp0EnRmD/mnt.0']
522
2016-07-14 21:49:20,260.260 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
523
2016-07-14 21:49:20,261.261 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
524
2016-07-14 21:49:20,270.270 INFO:teuthology.orchestra.run:waiting for 900
525
2016-07-14 21:49:26,277.277 INFO:__main__:run args=['rmdir', '--', '/tmp/tmp0EnRmD/mnt.0']
526
2016-07-14 21:49:26,277.277 INFO:__main__:Running ['rmdir', '--', '/tmp/tmp0EnRmD/mnt.0']
527
2016-07-14 21:49:26,286.286 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
528
2016-07-14 21:49:26,287.287 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
529
2016-07-14 21:49:26,294.294 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.1
530
2016-07-14 21:49:26,294.294 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on local...
531
2016-07-14 21:49:26,294.294 INFO:__main__:run args=['sudo', 'fusermount', '-u', '/tmp/tmp0EnRmD/mnt.1']
532
2016-07-14 21:49:26,294.294 INFO:__main__:Running ['fusermount', '-u', '/tmp/tmp0EnRmD/mnt.1']
533
2016-07-14 21:49:26,323.323 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
534
2016-07-14 21:49:26,323.323 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
535
2016-07-14 21:49:26,333.333 INFO:teuthology.orchestra.run:waiting for 900
536
2016-07-14 21:49:32,338.338 INFO:__main__:run args=['rmdir', '--', '/tmp/tmp0EnRmD/mnt.1']
537
2016-07-14 21:49:32,339.339 INFO:__main__:Running ['rmdir', '--', '/tmp/tmp0EnRmD/mnt.1']
538
2016-07-14 21:49:32,346.346 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
539
2016-07-14 21:49:32,347.347 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
540
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.2': No such file or directory
541
2016-07-14 21:49:32,358.358 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.2
542
2016-07-14 21:49:32,359.359 INFO:__main__:run args=['rmdir', '--', '/tmp/tmp0EnRmD/mnt.2']
543
2016-07-14 21:49:32,359.359 INFO:__main__:Running ['rmdir', '--', '/tmp/tmp0EnRmD/mnt.2']
544
rmdir: failed to remove '/tmp/tmp0EnRmD/mnt.2': No such file or directory
545
2016-07-14 21:49:32,371.371 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
546
2016-07-14 21:49:32,372.372 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
547
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.3': No such file or directory
548
2016-07-14 21:49:32,382.382 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.3
549
2016-07-14 21:49:32,383.383 INFO:__main__:run args=['rmdir', '--', '/tmp/tmp0EnRmD/mnt.3']
550
2016-07-14 21:49:32,384.384 INFO:__main__:Running ['rmdir', '--', '/tmp/tmp0EnRmD/mnt.3']
551
rmdir: failed to remove '/tmp/tmp0EnRmD/mnt.3': No such file or directory
552
2016-07-14 21:49:32,392.392 INFO:__main__:run args=['mkdir', '--', '/tmp/tmp0EnRmD/mnt.0']
553
2016-07-14 21:49:32,392.392 INFO:__main__:Running ['mkdir', '--', '/tmp/tmp0EnRmD/mnt.0']
554
2016-07-14 21:49:32,405.405 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
555
2016-07-14 21:49:32,405.405 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
556
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
557
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
558
2016-07-14 21:49:32,419.419 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
559
2016-07-14 21:49:32,420.420 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
560
2016-07-14 21:49:32,427.427 INFO:__main__:Pre-mount connections: [43, 47]
561
2016-07-14 21:49:32,427.427 INFO:__main__:run args=['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0']
562
2016-07-14 21:49:32,427.427 INFO:__main__:Running ['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0']
563
2016-07-14 21:49:32,440.440 INFO:__main__:Mounting client.0 with pid 15797
564
2016-07-14 21:49:32,441.441 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
565
2016-07-14 21:49:32,441.441 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
566
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
567
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
568
2016-07-14 21:49:32,456.456 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
569
2016-07-14 21:49:32,456.456 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
570
2016-07-14 21:49:33,469.469 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
571
2016-07-14 21:49:33,469.469 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
572
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
573
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
574
2016-07-14 21:49:33,477.477 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
575
2016-07-14 21:49:33,477.477 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
576
2016-07-14 21:49:33,485.485 INFO:__main__:Post-mount connections: [43, 47, 48]
577
2016-07-14 21:49:33,486.486 INFO:__main__:run args=['mkdir', 'parent1']
578
2016-07-14 21:49:33,486.486 INFO:__main__:Running ['mkdir', 'parent1']
579
2016-07-14 21:49:33,520.520 INFO:__main__:run args=['mkdir', 'parent2']
580
2016-07-14 21:49:33,520.520 INFO:__main__:Running ['mkdir', 'parent2']
581
2016-07-14 21:49:33,531.531 INFO:__main__:run args=['mkdir', 'parent1/mydir']
582
2016-07-14 21:49:33,531.531 INFO:__main__:Running ['mkdir', 'parent1/mydir']
583
2016-07-14 21:49:33,557.557 INFO:__main__:run args=['mkdir', '--', '/tmp/tmp0EnRmD/mnt.1']
584
2016-07-14 21:49:33,557.557 INFO:__main__:Running ['mkdir', '--', '/tmp/tmp0EnRmD/mnt.1']
585
2016-07-14 21:49:33,564.564 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
586
2016-07-14 21:49:33,564.564 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
587
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
588
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
589
2016-07-14 21:49:33,576.576 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
590
2016-07-14 21:49:33,576.576 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
591
2016-07-14 21:49:33,588.588 INFO:__main__:Pre-mount connections: [43, 47, 48]
592
2016-07-14 21:49:33,589.589 INFO:__main__:run args=['./bin/ceph-fuse', '-f', '--name', 'client.1', '/tmp/tmp0EnRmD/mnt.1']
593
2016-07-14 21:49:33,589.589 INFO:__main__:Running ['./bin/ceph-fuse', '-f', '--name', 'client.1', '/tmp/tmp0EnRmD/mnt.1']
594
2016-07-14 21:49:33,596.596 INFO:__main__:Mounting client.1 with pid 15840
595
2016-07-14 21:49:33,596.596 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
596
2016-07-14 21:49:33,596.596 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
597
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
598
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
599
2016-07-14 21:49:33,612.612 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
600
2016-07-14 21:49:33,612.612 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
601
2016-07-14 21:49:34,626.626 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
602
2016-07-14 21:49:34,626.626 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
603
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
604
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
605
2016-07-14 21:49:34,635.635 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
606
2016-07-14 21:49:34,635.635 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
607
2016-07-14 21:49:34,643.643 INFO:__main__:Post-mount connections: [43, 47, 48, 49]
608
2016-07-14 21:49:34,644.644 INFO:__main__:run args=['touch', 'parent1/mydir/afile']
609
2016-07-14 21:49:34,644.644 INFO:__main__:Running ['touch', 'parent1/mydir/afile']
610
2016-07-14 21:49:34,693.693 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
611
2016-07-14 21:49:34,694.694 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
612
2016-07-14 21:49:34,706.706 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.1
613
2016-07-14 21:49:34,706.706 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on local...
614
2016-07-14 21:49:34,706.706 INFO:__main__:run args=['sudo', 'fusermount', '-u', '/tmp/tmp0EnRmD/mnt.1']
615
2016-07-14 21:49:34,706.706 INFO:__main__:Running ['fusermount', '-u', '/tmp/tmp0EnRmD/mnt.1']
616
2016-07-14 21:49:34,728.728 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
617
2016-07-14 21:49:34,729.729 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
618
2016-07-14 21:49:34,742.742 INFO:teuthology.orchestra.run:waiting for 900
619
2016-07-14 21:49:40,745.745 INFO:__main__:run args=['rmdir', '--', '/tmp/tmp0EnRmD/mnt.1']
620
2016-07-14 21:49:40,745.745 INFO:__main__:Running ['rmdir', '--', '/tmp/tmp0EnRmD/mnt.1']
621
2016-07-14 21:49:40,752.752 INFO:__main__:run args=['ls', 'parent1/mydir']
622
2016-07-14 21:49:40,752.752 INFO:__main__:Running ['ls', 'parent1/mydir']
623
2016-07-14 21:49:40,761.761 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
624
2016-07-14 21:49:40,762.762 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
625
2016-07-14 21:49:40,769.769 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.0
626
2016-07-14 21:49:40,769.769 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on local...
627
2016-07-14 21:49:40,769.769 INFO:__main__:run args=['sudo', 'fusermount', '-u', '/tmp/tmp0EnRmD/mnt.0']
628
2016-07-14 21:49:40,769.769 INFO:__main__:Running ['fusermount', '-u', '/tmp/tmp0EnRmD/mnt.0']
629
2016-07-14 21:49:40,801.801 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
630
2016-07-14 21:49:40,801.801 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
631
2016-07-14 21:49:40,816.816 INFO:__main__:kill 
632
2016-07-14 21:49:40,816.816 INFO:__main__:kill: killing pid 15797 (['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0'])
633
ceph-fuse[15797]: starting ceph client
634
2016-07-14 21:49:32.455607 7fa938e2cf40 -1 WARNING: the following dangerous and experimental features are enabled: *
635
2016-07-14 21:49:32.455746 7fa938e2cf40 -1 WARNING: the following dangerous and experimental features are enabled: *
636
2016-07-14 21:49:32.457752 7fa938e2cf40 -1 WARNING: the following dangerous and experimental features are enabled: *
637
2016-07-14 21:49:32.458410 7fa938e2cf40 -1 init, newargv = 0x9b09080 newargc=11
638
ceph-fuse[15797]: starting fuse
639
ceph-fuse[15797]: fuse finished with error 0 and tester_r 0
640
2016-07-14 21:49:40,818.818 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.0']
641
2016-07-14 21:49:40,818.818 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.0']
642
2016-07-14 21:49:40,829.829 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
643
2016-07-14 21:49:40,829.829 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
644
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.1': No such file or directory
645
2016-07-14 21:49:40,837.837 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.1
646
2016-07-14 21:49:40,837.837 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.1']
647
2016-07-14 21:49:40,838.838 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.1']
648
2016-07-14 21:49:40,850.850 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
649
2016-07-14 21:49:40,850.850 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
650
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.2': No such file or directory
651
2016-07-14 21:49:40,861.861 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.2
652
2016-07-14 21:49:40,861.861 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.2']
653
2016-07-14 21:49:40,862.862 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.2']
654
2016-07-14 21:49:40,869.869 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
655
2016-07-14 21:49:40,869.869 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
656
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.3': No such file or directory
657
2016-07-14 21:49:40,881.881 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.3
658
2016-07-14 21:49:40,881.881 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.3']
659
2016-07-14 21:49:40,881.881 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.3']
660
2016-07-14 21:49:40,889.889 INFO:__main__:test_15303 (tasks.cephfs.test_volume_client.TestVolumeClient) ... ok
661
2016-07-14 21:49:40,890.890 INFO:__main__:Stopped test: test_15303 (tasks.cephfs.test_volume_client.TestVolumeClient) in 34.690646s
662
2016-07-14 21:49:40,891.891 INFO:__main__:Starting test: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)
663
2016-07-14 21:49:40,891.891 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
664
2016-07-14 21:49:40,892.892 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
665
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.0': No such file or directory
666
2016-07-14 21:49:40,901.901 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.0
667
2016-07-14 21:49:40,901.901 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
668
2016-07-14 21:49:40,901.901 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
669
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.1': No such file or directory
670
2016-07-14 21:49:40,916.916 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.1
671
2016-07-14 21:49:40,916.916 INFO:__main__:run args=['ps', '-u0']
672
2016-07-14 21:49:40,917.917 INFO:__main__:Running ['ps', '-u0']
673
2016-07-14 21:49:40,947.947 INFO:__main__:No match for mds a: PID TTY          TIME CMD
674
    1 ?        00:00:02 systemd
675
    2 ?        00:00:00 kthreadd
676
    3 ?        00:00:00 ksoftirqd/0
677
    5 ?        00:00:00 kworker/0:0H
678
    7 ?        00:00:36 rcu_sched
679
    8 ?        00:00:00 rcu_bh
680
    9 ?        00:00:15 rcuos/0
681
   10 ?        00:00:00 rcuob/0
682
   11 ?        00:00:00 migration/0
683
   12 ?        00:00:00 watchdog/0
684
   13 ?        00:00:00 watchdog/1
685
   14 ?        00:00:00 migration/1
686
   15 ?        00:00:00 ksoftirqd/1
687
   17 ?        00:00:00 kworker/1:0H
688
   18 ?        00:00:06 rcuos/1
689
   19 ?        00:00:00 rcuob/1
690
   20 ?        00:00:00 watchdog/2
691
   21 ?        00:00:00 migration/2
692
   22 ?        00:00:00 ksoftirqd/2
693
   24 ?        00:00:00 kworker/2:0H
694
   25 ?        00:00:14 rcuos/2
695
   26 ?        00:00:00 rcuob/2
696
   27 ?        00:00:00 watchdog/3
697
   28 ?        00:00:00 migration/3
698
   29 ?        00:00:00 ksoftirqd/3
699
   31 ?        00:00:00 kworker/3:0H
700
   32 ?        00:00:05 rcuos/3
701
   33 ?        00:00:00 rcuob/3
702
   34 ?        00:00:00 kdevtmpfs
703
   35 ?        00:00:00 netns
704
   36 ?        00:00:00 writeback
705
   37 ?        00:00:00 ksmd
706
   38 ?        00:00:00 khugepaged
707
   39 ?        00:00:00 crypto
708
   40 ?        00:00:00 kintegrityd
709
   41 ?        00:00:00 bioset
710
   42 ?        00:00:00 kblockd
711
   43 ?        00:00:00 ata_sff
712
   44 ?        00:00:00 md
713
   45 ?        00:00:00 devfreq_wq
714
   51 ?        00:00:13 kswapd0
715
   52 ?        00:00:00 vmstat
716
   93 ?        00:00:00 kthrotld
717
   94 ?        00:00:00 acpi_thermal_pm
718
   95 ?        00:00:00 scsi_eh_0
719
   96 ?        00:00:00 scsi_tmf_0
720
   97 ?        00:00:00 scsi_eh_1
721
   98 ?        00:00:00 scsi_tmf_1
722
   99 ?        00:00:00 scsi_eh_2
723
  100 ?        00:00:00 scsi_tmf_2
724
  104 ?        00:00:00 kpsmoused
725
  106 ?        00:00:00 dm_bufio_cache
726
  107 ?        00:00:00 ipv6_addrconf
727
  138 ?        00:00:00 deferwq
728
  140 ?        00:00:00 bioset
729
  178 ?        00:00:00 kauditd
730
  389 ?        00:00:01 kworker/0:1H
731
  399 ?        00:00:00 kworker/1:1H
732
  400 ?        00:00:00 kworker/3:1H
733
  420 ?        00:00:00 rtsx_pci_sdmmc_
734
  425 ?        00:00:00 kworker/2:1H
735
  613 ?        00:00:00 kdmflush
736
  621 ?        00:00:00 bioset
737
  622 ?        00:00:00 kcryptd_io
738
  623 ?        00:00:00 kcryptd
739
  624 ?        00:00:08 dmcrypt_write
740
  625 ?        00:00:00 bioset
741
  688 ?        00:00:00 kdmflush
742
  691 ?        00:00:00 bioset
743
  700 ?        00:00:00 kdmflush
744
  701 ?        00:00:00 bioset
745
  724 ?        00:00:00 jbd2/dm-1-8
746
  725 ?        00:00:00 ext4-rsv-conver
747
  823 ?        00:00:02 systemd-journal
748
  861 ?        00:00:00 rpciod
749
  880 ?        00:00:00 systemd-udevd
750
  910 ?        00:00:00 ktpacpid
751
  918 ?        00:00:00 irq/47-mei_me
752
  949 ?        00:00:00 kmemstick
753
  968 ?        00:00:00 cfg80211
754
  984 ?        00:00:07 irq/50-iwlwifi
755
  999 ?        00:00:00 kworker/u17:0
756
 1001 ?        00:00:00 hci0
757
 1002 ?        00:00:00 hci0
758
 1007 ?        00:00:00 kworker/u17:2
759
 1015 ?        00:00:00 kdmflush
760
 1016 ?        00:00:00 bioset
761
 1018 ?        00:00:00 kvm-irqfd-clean
762
 1071 ?        00:00:00 jbd2/sda1-8
763
 1072 ?        00:00:00 ext4-rsv-conver
764
 1077 ?        00:00:06 jbd2/dm-3-8
765
 1078 ?        00:00:00 ext4-rsv-conver
766
 1105 ?        00:00:00 auditd
767
 1119 ?        00:00:00 audispd
768
 1123 ?        00:00:00 sedispatch
769
 1133 ?        00:00:00 bluetoothd
770
 1137 ?        00:00:00 mcelog
771
 1139 ?        00:00:39 rngd
772
 1146 ?        00:00:00 gssproxy
773
 1154 ?        00:00:00 ModemManager
774
 1158 ?        00:00:04 udisksd
775
 1159 ?        00:00:00 firewalld
776
 1169 ?        00:00:00 systemd-logind
777
 1173 ?        00:00:00 accounts-daemon
778
 1177 ?        00:00:00 alsactl
779
 1249 ?        00:00:00 abrtd
780
 1262 ?        00:00:00 iprt-VBoxWQueue
781
 1268 ?        00:00:00 iprt-VBoxTscThr
782
 1281 ?        00:00:00 abrt-dump-journ
783
 1286 ?        00:00:00 abrt-dump-journ
784
 1324 ?        00:00:12 NetworkManager
785
 1376 ?        00:00:00 libvirtd
786
 1393 ?        00:00:00 crond
787
 1396 ?        00:00:00 atd
788
 1398 ?        00:00:00 gdm
789
 1463 ?        00:00:01 wpa_supplicant
790
 1509 ?        00:00:00 gdm-session-wor
791
 1582 ?        00:00:00 upowerd
792
 1680 ?        00:00:10 packagekitd
793
 1839 ?        00:00:00 dhclient
794
 1903 ?        00:00:00 gdm-session-wor
795
 1938 tty2     00:00:00 xf86-video-inte
796
 2067 ?        00:00:00 krfcommd
797
 2206 ?        00:00:00 cupsd
798
 2235 ?        00:00:00 fwupd
799
 2448 ?        00:00:00 dhclient
800
 2633 ?        00:00:00 kworker/0:1
801
 3387 ?        00:00:00 kworker/3:1
802
 3927 ?        00:00:04 kworker/u16:0
803
 5168 ?        00:00:00 kworker/1:3
804
13278 ?        00:00:00 kworker/u16:5
805
13551 ?        00:00:00 kworker/u16:1
806
13663 ?        00:00:00 kworker/1:1
807
13668 ?        00:00:00 kworker/3:0
808
13681 ?        00:00:00 kworker/2:2
809
13694 ?        00:00:00 kworker/0:3
810
13724 ?        00:00:00 kworker/u16:4
811
13765 ?        00:00:00 kworker/1:2
812
13781 ?        00:00:00 kworker/2:0
813
13794 ?        00:00:00 kworker/3:3
814
13799 ?        00:00:00 kworker/0:2
815
14835 pts/1    00:00:00 sudo
816
14848 pts/1    00:00:00 su
817
14853 pts/1    00:00:00 bash
818
14902 ?        00:00:00 kworker/2:3
819
14903 pts/1    00:00:00 python
820
15764 ?        00:00:00 fprintd
821
15897 pts/1    00:00:00 ps
822
16120 ?        00:00:00 kworker/2:1
823
18809 ?        00:00:00 ceph-msgr
824
18811 ?        00:00:00 rbd
825
28322 ?        00:00:00 systemd
826
28332 ?        00:00:00 (sd-pam)
827
28799 ?        00:00:10 kworker/u16:2
828
29866 ?        00:00:00 kworker/3:2
829
30529 ?        00:00:00 dio/dm-3
830
31742 ?        00:00:00 kworker/0:0
831
32711 ?        00:00:00 kworker/1:0
832
2016-07-14 21:49:40,948.948 ERROR:__main__:tried to stop a non-running daemon
833
2016-07-14 21:49:40,949.949 INFO:__main__:run args=['./bin/ceph', 'fs', 'ls', '--format=json-pretty']
834
2016-07-14 21:49:40,950.950 INFO:__main__:Running ['./bin/ceph', 'fs', 'ls', '--format=json-pretty']
835
2016-07-14 21:49:41,133.133 INFO:__main__:run args=['./bin/ceph', 'fs', 'set', u'cephfs', 'cluster_down', 'true']
836
2016-07-14 21:49:41,133.133 INFO:__main__:Running ['./bin/ceph', 'fs', 'set', u'cephfs', 'cluster_down', 'true']
837
2016-07-14 21:49:41,578.578 INFO:__main__:run args=['./bin/ceph', 'fs', 'get', u'cephfs', '--format=json-pretty']
838
2016-07-14 21:49:41,578.578 INFO:__main__:Running ['./bin/ceph', 'fs', 'get', u'cephfs', '--format=json-pretty']
839
2016-07-14 21:49:41,770.770 INFO:__main__:run args=['./bin/ceph', 'mds', 'fail', '4123']
840
2016-07-14 21:49:41,770.770 INFO:__main__:Running ['./bin/ceph', 'mds', 'fail', '4123']
841
2016-07-14 21:49:42,617.617 INFO:__main__:run args=['./bin/ceph', 'fs', 'rm', u'cephfs', '--yes-i-really-mean-it']
842
2016-07-14 21:49:42,617.617 INFO:__main__:Running ['./bin/ceph', 'fs', 'rm', u'cephfs', '--yes-i-really-mean-it']
843
2016-07-14 21:49:43,655.655 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_metadata', u'cephfs_metadata', '--yes-i-really-really-mean-it']
844
2016-07-14 21:49:43,655.655 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_metadata', u'cephfs_metadata', '--yes-i-really-really-mean-it']
845
2016-07-14 21:49:44,708.708 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_data', u'cephfs_data', '--yes-i-really-really-mean-it']
846
2016-07-14 21:49:44,708.708 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_data', u'cephfs_data', '--yes-i-really-really-mean-it']
847
2016-07-14 21:49:45,790.790 INFO:__main__:run args=['./bin/ceph', 'daemon', 'osd.0', 'config', 'get', 'osd_mon_report_interval_max']
848
2016-07-14 21:49:45,790.790 INFO:__main__:Running ['./bin/ceph', 'daemon', 'osd.0', 'config', 'get', 'osd_mon_report_interval_max']
849
2016-07-14 21:49:45,890.890 INFO:tasks.cephfs.filesystem:_json_asok output: {
850
    "osd_mon_report_interval_max": "5"
851
}
852

    
853
2016-07-14 21:49:45,891.891 INFO:__main__:run args=['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
854
2016-07-14 21:49:45,891.891 INFO:__main__:Running ['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
855
2016-07-14 21:49:46,093.093 INFO:__main__:run args=['./bin/ceph', 'osd', 'blacklist', 'clear']
856
2016-07-14 21:49:46,093.093 INFO:__main__:Running ['./bin/ceph', 'osd', 'blacklist', 'clear']
857
2016-07-14 21:49:46,835.835 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.0', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
858
2016-07-14 21:49:46,835.835 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.0', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
859
2016-07-14 21:49:47,034.034 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.1', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
860
2016-07-14 21:49:47,034.034 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.1', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
861
2016-07-14 21:49:47,255.255 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.2', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
862
2016-07-14 21:49:47,255.255 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.2', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
863
2016-07-14 21:49:47,445.445 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.3', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
864
2016-07-14 21:49:47,446.446 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.3', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
865
2016-07-14 21:49:47,665.665 INFO:tasks.cephfs.cephfs_test_case:['0', '1', '2', '3']
866
2016-07-14 21:49:47,665.665 INFO:__main__:run args=['./bin/ceph', 'auth', 'list', '--format=json-pretty']
867
2016-07-14 21:49:47,666.666 INFO:__main__:Running ['./bin/ceph', 'auth', 'list', '--format=json-pretty']
868
2016-07-14 21:49:47,855.855 INFO:tasks.cephfs.filesystem:Creating filesystem 'cephfs'
869
2016-07-14 21:49:47,855.855 INFO:__main__:run args=['./bin/ceph', 'daemon', 'mon.a', 'config', 'get', 'mon_pg_warn_min_per_osd']
870
2016-07-14 21:49:47,855.855 INFO:__main__:Running ['./bin/ceph', 'daemon', 'mon.a', 'config', 'get', 'mon_pg_warn_min_per_osd']
871
2016-07-14 21:49:47,957.957 INFO:tasks.cephfs.filesystem:_json_asok output: {
872
    "mon_pg_warn_min_per_osd": "3"
873
}
874

    
875
2016-07-14 21:49:47,958.958 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_metadata', '9']
876
2016-07-14 21:49:47,958.958 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_metadata', '9']
877
2016-07-14 21:49:48,860.860 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_data', '9']
878
2016-07-14 21:49:48,860.860 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_data', '9']
879
2016-07-14 21:49:49,917.917 INFO:__main__:run args=['./bin/ceph', 'fs', 'new', 'cephfs', 'cephfs_metadata', 'cephfs_data']
880
2016-07-14 21:49:49,918.918 INFO:__main__:Running ['./bin/ceph', 'fs', 'new', 'cephfs', 'cephfs_metadata', 'cephfs_data']
881
2016-07-14 21:49:50,479.479 INFO:__main__:run args=['ps', '-u0']
882
2016-07-14 21:49:50,479.479 INFO:__main__:Running ['ps', '-u0']
883
2016-07-14 21:49:50,496.496 INFO:__main__:No match for mds a: PID TTY          TIME CMD
884
    1 ?        00:00:02 systemd
885
    2 ?        00:00:00 kthreadd
886
    3 ?        00:00:00 ksoftirqd/0
887
    5 ?        00:00:00 kworker/0:0H
888
    7 ?        00:00:36 rcu_sched
889
    8 ?        00:00:00 rcu_bh
890
    9 ?        00:00:15 rcuos/0
891
   10 ?        00:00:00 rcuob/0
892
   11 ?        00:00:00 migration/0
893
   12 ?        00:00:00 watchdog/0
894
   13 ?        00:00:00 watchdog/1
895
   14 ?        00:00:00 migration/1
896
   15 ?        00:00:00 ksoftirqd/1
897
   17 ?        00:00:00 kworker/1:0H
898
   18 ?        00:00:06 rcuos/1
899
   19 ?        00:00:00 rcuob/1
900
   20 ?        00:00:00 watchdog/2
901
   21 ?        00:00:00 migration/2
902
   22 ?        00:00:00 ksoftirqd/2
903
   24 ?        00:00:00 kworker/2:0H
904
   25 ?        00:00:14 rcuos/2
905
   26 ?        00:00:00 rcuob/2
906
   27 ?        00:00:00 watchdog/3
907
   28 ?        00:00:00 migration/3
908
   29 ?        00:00:00 ksoftirqd/3
909
   31 ?        00:00:00 kworker/3:0H
910
   32 ?        00:00:05 rcuos/3
911
   33 ?        00:00:00 rcuob/3
912
   34 ?        00:00:00 kdevtmpfs
913
   35 ?        00:00:00 netns
914
   36 ?        00:00:00 writeback
915
   37 ?        00:00:00 ksmd
916
   38 ?        00:00:00 khugepaged
917
   39 ?        00:00:00 crypto
918
   40 ?        00:00:00 kintegrityd
919
   41 ?        00:00:00 bioset
920
   42 ?        00:00:00 kblockd
921
   43 ?        00:00:00 ata_sff
922
   44 ?        00:00:00 md
923
   45 ?        00:00:00 devfreq_wq
924
   51 ?        00:00:13 kswapd0
925
   52 ?        00:00:00 vmstat
926
   93 ?        00:00:00 kthrotld
927
   94 ?        00:00:00 acpi_thermal_pm
928
   95 ?        00:00:00 scsi_eh_0
929
   96 ?        00:00:00 scsi_tmf_0
930
   97 ?        00:00:00 scsi_eh_1
931
   98 ?        00:00:00 scsi_tmf_1
932
   99 ?        00:00:00 scsi_eh_2
933
  100 ?        00:00:00 scsi_tmf_2
934
  104 ?        00:00:00 kpsmoused
935
  106 ?        00:00:00 dm_bufio_cache
936
  107 ?        00:00:00 ipv6_addrconf
937
  138 ?        00:00:00 deferwq
938
  140 ?        00:00:00 bioset
939
  178 ?        00:00:00 kauditd
940
  389 ?        00:00:01 kworker/0:1H
941
  399 ?        00:00:00 kworker/1:1H
942
  400 ?        00:00:00 kworker/3:1H
943
  420 ?        00:00:00 rtsx_pci_sdmmc_
944
  425 ?        00:00:00 kworker/2:1H
945
  613 ?        00:00:00 kdmflush
946
  621 ?        00:00:00 bioset
947
  622 ?        00:00:00 kcryptd_io
948
  623 ?        00:00:00 kcryptd
949
  624 ?        00:00:08 dmcrypt_write
950
  625 ?        00:00:00 bioset
951
  688 ?        00:00:00 kdmflush
952
  691 ?        00:00:00 bioset
953
  700 ?        00:00:00 kdmflush
954
  701 ?        00:00:00 bioset
955
  724 ?        00:00:00 jbd2/dm-1-8
956
  725 ?        00:00:00 ext4-rsv-conver
957
  823 ?        00:00:02 systemd-journal
958
  861 ?        00:00:00 rpciod
959
  880 ?        00:00:00 systemd-udevd
960
  910 ?        00:00:00 ktpacpid
961
  918 ?        00:00:00 irq/47-mei_me
962
  949 ?        00:00:00 kmemstick
963
  968 ?        00:00:00 cfg80211
964
  984 ?        00:00:07 irq/50-iwlwifi
965
  999 ?        00:00:00 kworker/u17:0
966
 1001 ?        00:00:00 hci0
967
 1002 ?        00:00:00 hci0
968
 1007 ?        00:00:00 kworker/u17:2
969
 1015 ?        00:00:00 kdmflush
970
 1016 ?        00:00:00 bioset
971
 1018 ?        00:00:00 kvm-irqfd-clean
972
 1071 ?        00:00:00 jbd2/sda1-8
973
 1072 ?        00:00:00 ext4-rsv-conver
974
 1077 ?        00:00:06 jbd2/dm-3-8
975
 1078 ?        00:00:00 ext4-rsv-conver
976
 1105 ?        00:00:00 auditd
977
 1119 ?        00:00:00 audispd
978
 1123 ?        00:00:00 sedispatch
979
 1133 ?        00:00:00 bluetoothd
980
 1137 ?        00:00:00 mcelog
981
 1139 ?        00:00:39 rngd
982
 1146 ?        00:00:00 gssproxy
983
 1154 ?        00:00:00 ModemManager
984
 1158 ?        00:00:04 udisksd
985
 1159 ?        00:00:00 firewalld
986
 1169 ?        00:00:00 systemd-logind
987
 1173 ?        00:00:00 accounts-daemon
988
 1177 ?        00:00:00 alsactl
989
 1249 ?        00:00:00 abrtd
990
 1262 ?        00:00:00 iprt-VBoxWQueue
991
 1268 ?        00:00:00 iprt-VBoxTscThr
992
 1281 ?        00:00:00 abrt-dump-journ
993
 1286 ?        00:00:00 abrt-dump-journ
994
 1324 ?        00:00:12 NetworkManager
995
 1376 ?        00:00:00 libvirtd
996
 1393 ?        00:00:00 crond
997
 1396 ?        00:00:00 atd
998
 1398 ?        00:00:00 gdm
999
 1463 ?        00:00:01 wpa_supplicant
1000
 1509 ?        00:00:00 gdm-session-wor
1001
 1582 ?        00:00:00 upowerd
1002
 1680 ?        00:00:10 packagekitd
1003
 1839 ?        00:00:00 dhclient
1004
 1903 ?        00:00:00 gdm-session-wor
1005
 1938 tty2     00:00:00 xf86-video-inte
1006
 2067 ?        00:00:00 krfcommd
1007
 2206 ?        00:00:00 cupsd
1008
 2235 ?        00:00:00 fwupd
1009
 2448 ?        00:00:00 dhclient
1010
 2633 ?        00:00:00 kworker/0:1
1011
 3387 ?        00:00:00 kworker/3:1
1012
 3927 ?        00:00:04 kworker/u16:0
1013
 5168 ?        00:00:00 kworker/1:3
1014
13278 ?        00:00:00 kworker/u16:5
1015
13551 ?        00:00:00 kworker/u16:1
1016
13663 ?        00:00:00 kworker/1:1
1017
13668 ?        00:00:00 kworker/3:0
1018
13681 ?        00:00:00 kworker/2:2
1019
13694 ?        00:00:00 kworker/0:3
1020
13724 ?        00:00:00 kworker/u16:4
1021
13765 ?        00:00:00 kworker/1:2
1022
13781 ?        00:00:00 kworker/2:0
1023
13794 ?        00:00:00 kworker/3:3
1024
13799 ?        00:00:00 kworker/0:2
1025
14835 pts/1    00:00:00 sudo
1026
14848 pts/1    00:00:00 su
1027
14853 pts/1    00:00:00 bash
1028
14902 ?        00:00:00 kworker/2:3
1029
14903 pts/1    00:00:00 python
1030
15764 ?        00:00:00 fprintd
1031
16120 ?        00:00:00 kworker/2:1
1032
16419 pts/1    00:00:00 ps
1033
18809 ?        00:00:00 ceph-msgr
1034
18811 ?        00:00:00 rbd
1035
28322 ?        00:00:00 systemd
1036
28332 ?        00:00:00 (sd-pam)
1037
28799 ?        00:00:10 kworker/u16:2
1038
29866 ?        00:00:00 kworker/3:2
1039
30529 ?        00:00:00 dio/dm-3
1040
31742 ?        00:00:00 kworker/0:0
1041
32711 ?        00:00:00 kworker/1:0
1042
2016-07-14 21:49:50,497.497 INFO:__main__:run args=['./bin/./ceph-mds', '-i', 'a']
1043
2016-07-14 21:49:50,497.497 INFO:__main__:Running ['./bin/./ceph-mds', '-i', 'a']
1044
2016-07-14 21:49:50,532.532 INFO:__main__:run args=['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
1045
2016-07-14 21:49:50,533.533 INFO:__main__:Running ['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
1046
2016-07-14 21:49:50,736.736 INFO:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {u'session_autoclose': 300, u'up': {u'mds_0': 4147}, u'last_failure_osd_epoch': 0, u'in': [0], u'last_failure': 0, u'max_file_size': 1099511627776, u'explicitly_allowed_features': 0, u'damaged': [], u'tableserver': 0, u'metadata_pool': 5, u'failed': [], u'epoch': 15, u'flags': 0, u'max_mds': 1, u'compat': {u'compat': {}, u'ro_compat': {}, u'incompat': {u'feature_8': u'file layout v2', u'feature_2': u'client writeable ranges', u'feature_3': u'default file layouts on dirs', u'feature_1': u'base v0.20', u'feature_6': u'dirfrag is stored in omap', u'feature_4': u'dir inode in separate object', u'feature_5': u'mds uses versioned encoding'}}, u'data_pools': [6], u'info': {u'gid_4147': {u'standby_for_rank': -1, u'export_targets': [], u'name': u'a', u'incarnation': 15, u'standby_replay': False, u'state_seq': 1, u'standby_for_fscid': -1, u'state': u'up:creating', u'gid': 4147, u'features': 576460752032890879, u'rank': 0, u'standby_for_name': u'', u'addr': u'10.70.1.141:6812/15180'}}, u'fs_name': u'cephfs', u'created': u'2016-07-14 21:49:50.110782', u'enabled': True, u'modified': u'2016-07-14 21:49:50.110782', u'session_timeout': 60, u'stopped': [], u'ever_allowed_features': 0, u'root': 0}
1047
2016-07-14 21:49:50,736.736 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_4147:up:creating
1048
2016-07-14 21:49:51,736.736 INFO:__main__:run args=['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
1049
2016-07-14 21:49:51,737.737 INFO:__main__:Running ['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
1050
2016-07-14 21:49:51,921.921 INFO:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {u'session_autoclose': 300, u'up': {u'mds_0': 4147}, u'last_failure_osd_epoch': 0, u'in': [0], u'last_failure': 0, u'max_file_size': 1099511627776, u'explicitly_allowed_features': 0, u'damaged': [], u'tableserver': 0, u'metadata_pool': 5, u'failed': [], u'epoch': 15, u'flags': 0, u'max_mds': 1, u'compat': {u'compat': {}, u'ro_compat': {}, u'incompat': {u'feature_8': u'file layout v2', u'feature_2': u'client writeable ranges', u'feature_3': u'default file layouts on dirs', u'feature_1': u'base v0.20', u'feature_6': u'dirfrag is stored in omap', u'feature_4': u'dir inode in separate object', u'feature_5': u'mds uses versioned encoding'}}, u'data_pools': [6], u'info': {u'gid_4147': {u'standby_for_rank': -1, u'export_targets': [], u'name': u'a', u'incarnation': 15, u'standby_replay': False, u'state_seq': 1, u'standby_for_fscid': -1, u'state': u'up:creating', u'gid': 4147, u'features': 576460752032890879, u'rank': 0, u'standby_for_name': u'', u'addr': u'10.70.1.141:6812/15180'}}, u'fs_name': u'cephfs', u'created': u'2016-07-14 21:49:50.110782', u'enabled': True, u'modified': u'2016-07-14 21:49:50.110782', u'session_timeout': 60, u'stopped': [], u'ever_allowed_features': 0, u'root': 0}
1051
2016-07-14 21:49:51,921.921 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_4147:up:creating
1052
2016-07-14 21:49:52,922.922 INFO:__main__:run args=['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
1053
2016-07-14 21:49:52,922.922 INFO:__main__:Running ['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
1054
2016-07-14 21:49:53,106.106 INFO:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {u'session_autoclose': 300, u'up': {u'mds_0': 4147}, u'last_failure_osd_epoch': 0, u'in': [0], u'last_failure': 0, u'max_file_size': 1099511627776, u'explicitly_allowed_features': 0, u'damaged': [], u'tableserver': 0, u'metadata_pool': 5, u'failed': [], u'epoch': 16, u'flags': 0, u'max_mds': 1, u'compat': {u'compat': {}, u'ro_compat': {}, u'incompat': {u'feature_8': u'file layout v2', u'feature_2': u'client writeable ranges', u'feature_3': u'default file layouts on dirs', u'feature_1': u'base v0.20', u'feature_6': u'dirfrag is stored in omap', u'feature_4': u'dir inode in separate object', u'feature_5': u'mds uses versioned encoding'}}, u'data_pools': [6], u'info': {u'gid_4147': {u'standby_for_rank': -1, u'export_targets': [], u'name': u'a', u'incarnation': 15, u'standby_replay': False, u'state_seq': 4, u'standby_for_fscid': -1, u'state': u'up:active', u'gid': 4147, u'features': 576460752032890879, u'rank': 0, u'standby_for_name': u'', u'addr': u'10.70.1.141:6812/15180'}}, u'fs_name': u'cephfs', u'created': u'2016-07-14 21:49:50.110782', u'enabled': True, u'modified': u'2016-07-14 21:49:50.110782', u'session_timeout': 60, u'stopped': [], u'ever_allowed_features': 0, u'root': 0}
1055
2016-07-14 21:49:53,106.106 INFO:tasks.cephfs.filesystem:are_daemons_healthy: 1/1
1056
2016-07-14 21:49:53,106.106 INFO:__main__:run args=['./bin/ceph', 'daemon', 'mds.a', 'status']
1057
2016-07-14 21:49:53,107.107 INFO:__main__:Running ['./bin/ceph', 'daemon', 'mds.a', 'status']
1058
2016-07-14 21:49:53,223.223 INFO:tasks.cephfs.filesystem:_json_asok output: {
1059
    "cluster_fsid": "ae66e242-d103-408a-b4cf-f160f95d365c",
1060
    "whoami": 0,
1061
    "want_state": "up:active",
1062
    "state": "up:active",
1063
    "mdsmap_epoch": 16,
1064
    "osdmap_epoch": 24,
1065
    "osdmap_epoch_barrier": 24
1066
}
1067

    
1068
2016-07-14 21:49:53,224.224 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1069
2016-07-14 21:49:53,224.224 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1070
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.0': No such file or directory
1071
2016-07-14 21:49:53,237.237 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.0
1072
2016-07-14 21:49:53,237.237 INFO:__main__:run args=['mkdir', '--', '/tmp/tmp0EnRmD/mnt.0']
1073
2016-07-14 21:49:53,238.238 INFO:__main__:Running ['mkdir', '--', '/tmp/tmp0EnRmD/mnt.0']
1074
2016-07-14 21:49:53,247.247 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1075
2016-07-14 21:49:53,247.247 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1076
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1077
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1078
2016-07-14 21:49:53,263.263 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1079
2016-07-14 21:49:53,264.264 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1080
2016-07-14 21:49:53,275.275 INFO:__main__:Pre-mount connections: [43, 47]
1081
2016-07-14 21:49:53,275.275 INFO:__main__:run args=['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0']
1082
2016-07-14 21:49:53,275.275 INFO:__main__:Running ['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0']
1083
2016-07-14 21:49:53,282.282 INFO:__main__:Mounting client.0 with pid 16513
1084
2016-07-14 21:49:53,283.283 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1085
2016-07-14 21:49:53,283.283 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1086
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1087
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1088
2016-07-14 21:49:53,297.297 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1089
2016-07-14 21:49:53,297.297 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1090
2016-07-14 21:49:54,312.312 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1091
2016-07-14 21:49:54,313.313 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1092
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1093
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1094
2016-07-14 21:49:54,321.321 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1095
2016-07-14 21:49:54,321.321 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1096
2016-07-14 21:49:54,332.332 INFO:__main__:Post-mount connections: [43, 47, 48]
1097
2016-07-14 21:49:54,333.333 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1098
2016-07-14 21:49:54,334.334 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1099
2016-07-14 21:49:54,343.343 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.0
1100
2016-07-14 21:49:54,343.343 INFO:__main__:run args=['sudo', 'chmod', '1777', '/tmp/tmp0EnRmD/mnt.0']
1101
2016-07-14 21:49:54,344.344 INFO:__main__:Running ['chmod', '1777', '/tmp/tmp0EnRmD/mnt.0']
1102
2016-07-14 21:49:54,357.357 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1103
2016-07-14 21:49:54,357.357 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1104
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.1': No such file or directory
1105
2016-07-14 21:49:54,364.364 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.1
1106
2016-07-14 21:49:54,364.364 INFO:__main__:run args=['mkdir', '--', '/tmp/tmp0EnRmD/mnt.1']
1107
2016-07-14 21:49:54,365.365 INFO:__main__:Running ['mkdir', '--', '/tmp/tmp0EnRmD/mnt.1']
1108
2016-07-14 21:49:54,372.372 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1109
2016-07-14 21:49:54,373.373 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1110
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1111
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1112
2016-07-14 21:49:54,388.388 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1113
2016-07-14 21:49:54,388.388 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1114
2016-07-14 21:49:54,396.396 INFO:__main__:Pre-mount connections: [43, 47, 48]
1115
2016-07-14 21:49:54,396.396 INFO:__main__:run args=['./bin/ceph-fuse', '-f', '--name', 'client.1', '/tmp/tmp0EnRmD/mnt.1']
1116
2016-07-14 21:49:54,396.396 INFO:__main__:Running ['./bin/ceph-fuse', '-f', '--name', 'client.1', '/tmp/tmp0EnRmD/mnt.1']
1117
2016-07-14 21:49:54,408.408 INFO:__main__:Mounting client.1 with pid 16555
1118
2016-07-14 21:49:54,410.410 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1119
2016-07-14 21:49:54,411.411 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1120
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1121
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1122
2016-07-14 21:49:54,426.426 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1123
2016-07-14 21:49:54,427.427 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1124
2016-07-14 21:49:55,440.440 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1125
2016-07-14 21:49:55,440.440 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1126
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1127
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1128
2016-07-14 21:49:55,450.450 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1129
2016-07-14 21:49:55,452.452 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1130
2016-07-14 21:49:55,461.461 INFO:__main__:Post-mount connections: [43, 47, 48, 49]
1131
2016-07-14 21:49:55,461.461 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1132
2016-07-14 21:49:55,461.461 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1133
2016-07-14 21:49:55,476.476 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.1
1134
2016-07-14 21:49:55,476.476 INFO:__main__:run args=['sudo', 'chmod', '1777', '/tmp/tmp0EnRmD/mnt.1']
1135
2016-07-14 21:49:55,476.476 INFO:__main__:Running ['chmod', '1777', '/tmp/tmp0EnRmD/mnt.1']
1136
2016-07-14 21:49:55,487.487 INFO:__main__:Searching for existing instance mon pg warn max per osd/global
1137
2016-07-14 21:49:55,488.488 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1138
2016-07-14 21:49:55,489.489 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1139
2016-07-14 21:49:55,497.497 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.1
1140
2016-07-14 21:49:55,497.497 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on local...
1141
2016-07-14 21:49:55,498.498 INFO:__main__:run args=['sudo', 'fusermount', '-u', '/tmp/tmp0EnRmD/mnt.1']
1142
2016-07-14 21:49:55,498.498 INFO:__main__:Running ['fusermount', '-u', '/tmp/tmp0EnRmD/mnt.1']
1143
2016-07-14 21:49:55,524.524 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1144
2016-07-14 21:49:55,524.524 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1145
2016-07-14 21:49:55,534.534 INFO:teuthology.orchestra.run:waiting for 900
1146
2016-07-14 21:50:01,538.538 INFO:__main__:run args=['rmdir', '--', '/tmp/tmp0EnRmD/mnt.1']
1147
2016-07-14 21:50:01,539.539 INFO:__main__:Running ['rmdir', '--', '/tmp/tmp0EnRmD/mnt.1']
1148
2016-07-14 21:50:01,546.546 INFO:__main__:run args=['./bin/ceph', 'auth', 'get-or-create', 'client.manila', 'mds', 'allow *', 'osd', 'allow rw', 'mon', 'allow *']
1149
2016-07-14 21:50:01,546.546 INFO:__main__:Running ['./bin/ceph', 'auth', 'get-or-create', 'client.manila', 'mds', 'allow *', 'osd', 'allow rw', 'mon', 'allow *']
1150
2016-07-14 21:50:01,739.739 INFO:__main__:run args=['sudo', 'python', '-c', 'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))', '/home/rraja/git/ceph/build/client.manila.keyring']
1151
2016-07-14 21:50:01,739.739 INFO:__main__:Running ['python', '-c', 'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))', '/home/rraja/git/ceph/build/client.manila.keyring']
1152
2016-07-14 21:50:01,763.763 INFO:__main__:Searching for existing instance mon pg warn max per osd/global
1153
2016-07-14 21:50:01,763.763 INFO:__main__:Searching for existing instance keyring/client.manila
1154
2016-07-14 21:50:01,764.764 INFO:__main__:run args=['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
1155
2016-07-14 21:50:01,764.764 INFO:__main__:Running ['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
1156
2016-07-14 21:50:01,940.940 INFO:__main__:run args=['./bin/ceph', 'daemon', 'mon.a', 'config', 'get', 'mon_pg_warn_max_per_osd']
1157
2016-07-14 21:50:01,940.940 INFO:__main__:Running ['./bin/ceph', 'daemon', 'mon.a', 'config', 'get', 'mon_pg_warn_max_per_osd']
1158
2016-07-14 21:50:02,048.048 INFO:tasks.cephfs.filesystem:_json_asok output: {
1159
    "mon_pg_warn_max_per_osd": "300"
1160
}
1161

    
1162
2016-07-14 21:50:02,048.048 INFO:tasks.cephfs.test_volume_client:max_per_osd 300
1163
2016-07-14 21:50:02,048.048 INFO:tasks.cephfs.test_volume_client:osd_count 3
1164
2016-07-14 21:50:02,048.048 INFO:tasks.cephfs.test_volume_client:max_overall 900
1165
2016-07-14 21:50:02,049.049 INFO:tasks.cephfs.test_volume_client:existing_pg_count 26
1166
2016-07-14 21:50:02,049.049 INFO:tasks.cephfs.test_volume_client:expected_pg_num 87
1167
2016-07-14 21:50:02,049.049 INFO:__main__:run args=['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
1168
2016-07-14 21:50:02,049.049 INFO:__main__:Running ['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
1169
2016-07-14 21:50:02,234.234 INFO:__main__:run args=['python', '-c', '\nfrom ceph_volume_client import CephFSVolumeClient, VolumePath\nimport logging\nlog = logging.getLogger("ceph_volume_client")\nlog.addHandler(logging.StreamHandler())\nlog.setLevel(logging.DEBUG)\nvc = CephFSVolumeClient("manila", "./ceph.conf", "ceph", None, None)\nvc.connect()\n\nvp = VolumePath("grpid", "volid")\nvc.create_volume(vp, 10, data_isolated=True)\n\nvc.disconnect()\n        ']
1170
2016-07-14 21:50:02,235.235 INFO:__main__:Running ['python', '-c', '\nfrom ceph_volume_client import CephFSVolumeClient, VolumePath\nimport logging\nlog = logging.getLogger("ceph_volume_client")\nlog.addHandler(logging.StreamHandler())\nlog.setLevel(logging.DEBUG)\nvc = CephFSVolumeClient("manila", "./ceph.conf", "ceph", None, None)\nvc.connect()\n\nvp = VolumePath("grpid", "volid")\nvc.create_volume(vp, 10, data_isolated=True)\n\nvc.disconnect()\n        ']
1171
src/tcmalloc.cc:283] Attempt to free invalid pointer 0x561bc63d3940 
1172
2016-07-14 21:50:02,503.503 INFO:__main__:test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient) ... ERROR
1173
2016-07-14 21:50:02,504.504 ERROR:__main__:Traceback (most recent call last):
1174
  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/test_volume_client.py", line 370, in test_data_isolated
1175
    guest_entity=guest_entity
1176
  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/test_volume_client.py", line 34, in _volume_client_python
1177
    """.format(payload=script, conf_path=client.config_path, vol_prefix=vol_prefix, ns_prefix=ns_prefix))
1178
  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/mount.py", line 134, in run_python
1179
    p.wait()
1180
  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/vstart_runner.py", line 113, in wait
1181
    raise CommandFailedError(self.args, self.exitstatus)
1182
CommandFailedError: Command failed with status -6: ['python', '-c', '\nfrom ceph_volume_client import CephFSVolumeClient, VolumePath\nimport logging\nlog = logging.getLogger("ceph_volume_client")\nlog.addHandler(logging.StreamHandler())\nlog.setLevel(logging.DEBUG)\nvc = CephFSVolumeClient("manila", "./ceph.conf", "ceph", None, None)\nvc.connect()\n\nvp = VolumePath("grpid", "volid")\nvc.create_volume(vp, 10, data_isolated=True)\n\nvc.disconnect()\n        ']
1183

    
1184
2016-07-14 21:50:02,504.504 ERROR:__main__:Error in test 'test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)', going interactive
1185
Ceph test interactive mode, use ctx to interact with the cluster, press control-D to exit...
1186
>>> 
1187
2016-07-14 21:56:08,890.890 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1188
2016-07-14 21:56:08,890.890 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1189
2016-07-14 21:56:08,901.901 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.0
1190
2016-07-14 21:56:08,901.901 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on local...
1191
2016-07-14 21:56:08,901.901 INFO:__main__:run args=['sudo', 'fusermount', '-u', '/tmp/tmp0EnRmD/mnt.0']
1192
2016-07-14 21:56:08,901.901 INFO:__main__:Running ['fusermount', '-u', '/tmp/tmp0EnRmD/mnt.0']
1193
2016-07-14 21:56:08,920.920 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1194
2016-07-14 21:56:08,920.920 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1195
2016-07-14 21:56:08,932.932 INFO:__main__:kill 
1196
2016-07-14 21:56:08,932.932 INFO:__main__:kill: killing pid 16513 (['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0'])
1197
ceph-fuse[16513]: starting ceph client
1198
2016-07-14 21:49:53.297548 7facfb952f40 -1 WARNING: the following dangerous and experimental features are enabled: *
1199
2016-07-14 21:49:53.297636 7facfb952f40 -1 WARNING: the following dangerous and experimental features are enabled: *
1200
2016-07-14 21:49:53.303164 7facfb952f40 -1 WARNING: the following dangerous and experimental features are enabled: *
1201
2016-07-14 21:49:53.303889 7facfb952f40 -1 init, newargv = 0xa149080 newargc=11
1202
ceph-fuse[16513]: starting fuse
1203
ceph-fuse[16513]: fuse finished with error 0 and tester_r 0
1204
2016-07-14 21:56:08,936.936 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.0']
1205
2016-07-14 21:56:08,936.936 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.0']
1206
2016-07-14 21:56:08,945.945 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1207
2016-07-14 21:56:08,945.945 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1208
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.1': No such file or directory
1209
2016-07-14 21:56:08,956.956 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.1
1210
2016-07-14 21:56:08,956.956 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.1']
1211
2016-07-14 21:56:08,956.956 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.1']
1212
2016-07-14 21:56:08,968.968 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
1213
2016-07-14 21:56:08,969.969 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
1214
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.2': No such file or directory
1215
2016-07-14 21:56:08,976.976 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.2
1216
2016-07-14 21:56:08,977.977 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.2']
1217
2016-07-14 21:56:08,977.977 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.2']
1218
2016-07-14 21:56:08,985.985 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
1219
2016-07-14 21:56:08,985.985 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
1220
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.3': No such file or directory
1221
2016-07-14 21:56:08,995.995 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.3
1222
2016-07-14 21:56:08,995.995 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.3']
1223
2016-07-14 21:56:08,995.995 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.3']
1224
2016-07-14 21:56:09,006.006 INFO:__main__:Searching for existing instance mon pg warn max per osd/global
1225
2016-07-14 21:56:09,007.007 INFO:__main__:Stopped test: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient) in 388.115495s
1226
2016-07-14 21:56:09,007.007 INFO:__main__:
1227
2016-07-14 21:56:09,007.007 INFO:__main__:======================================================================
1228
2016-07-14 21:56:09,007.007 INFO:__main__:ERROR: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)
1229
2016-07-14 21:56:09,007.007 INFO:__main__:----------------------------------------------------------------------
1230
2016-07-14 21:56:09,007.007 INFO:__main__:Traceback (most recent call last):
1231
2016-07-14 21:56:09,007.007 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/test_volume_client.py", line 370, in test_data_isolated
1232
2016-07-14 21:56:09,007.007 INFO:__main__:    guest_entity=guest_entity
1233
2016-07-14 21:56:09,007.007 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/test_volume_client.py", line 34, in _volume_client_python
1234
2016-07-14 21:56:09,008.008 INFO:__main__:    """.format(payload=script, conf_path=client.config_path, vol_prefix=vol_prefix, ns_prefix=ns_prefix))
1235
2016-07-14 21:56:09,008.008 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/mount.py", line 134, in run_python
1236
2016-07-14 21:56:09,008.008 INFO:__main__:    p.wait()
1237
2016-07-14 21:56:09,008.008 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/vstart_runner.py", line 113, in wait
1238
2016-07-14 21:56:09,008.008 INFO:__main__:    raise CommandFailedError(self.args, self.exitstatus)
1239
2016-07-14 21:56:09,008.008 INFO:__main__:CommandFailedError: Command failed with status -6: ['python', '-c', '\nfrom ceph_volume_client import CephFSVolumeClient, VolumePath\nimport logging\nlog = logging.getLogger("ceph_volume_client")\nlog.addHandler(logging.StreamHandler())\nlog.setLevel(logging.DEBUG)\nvc = CephFSVolumeClient("manila", "./ceph.conf", "ceph", None, None)\nvc.connect()\n\nvp = VolumePath("grpid", "volid")\nvc.create_volume(vp, 10, data_isolated=True)\n\nvc.disconnect()\n        ']
1240
2016-07-14 21:56:09,008.008 INFO:__main__:
1241
2016-07-14 21:56:09,008.008 INFO:__main__:----------------------------------------------------------------------
1242
2016-07-14 21:56:09,008.008 INFO:__main__:Ran 2 tests in 422.808s
1243
2016-07-14 21:56:09,008.008 INFO:__main__:
1244
2016-07-14 21:56:09,008.008 INFO:__main__:FAILED (errors=1)
1245
2016-07-14 21:56:09,008.008 INFO:__main__:
1246
2016-07-14 21:56:09,008.008 INFO:__main__:======================================================================
1247
2016-07-14 21:56:09,008.008 INFO:__main__:ERROR: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)
1248
2016-07-14 21:56:09,009.009 INFO:__main__:----------------------------------------------------------------------
1249
2016-07-14 21:56:09,009.009 INFO:__main__:Traceback (most recent call last):
1250
2016-07-14 21:56:09,009.009 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/test_volume_client.py", line 370, in test_data_isolated
1251
2016-07-14 21:56:09,009.009 INFO:__main__:    guest_entity=guest_entity
1252
2016-07-14 21:56:09,009.009 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/test_volume_client.py", line 34, in _volume_client_python
1253
2016-07-14 21:56:09,009.009 INFO:__main__:    """.format(payload=script, conf_path=client.config_path, vol_prefix=vol_prefix, ns_prefix=ns_prefix))
1254
2016-07-14 21:56:09,009.009 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/mount.py", line 134, in run_python
1255
2016-07-14 21:56:09,009.009 INFO:__main__:    p.wait()
1256
2016-07-14 21:56:09,009.009 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/vstart_runner.py", line 113, in wait
1257
2016-07-14 21:56:09,009.009 INFO:__main__:    raise CommandFailedError(self.args, self.exitstatus)
1258
2016-07-14 21:56:09,009.009 INFO:__main__:CommandFailedError: Command failed with status -6: ['python', '-c', '\nfrom ceph_volume_client import CephFSVolumeClient, VolumePath\nimport logging\nlog = logging.getLogger("ceph_volume_client")\nlog.addHandler(logging.StreamHandler())\nlog.setLevel(logging.DEBUG)\nvc = CephFSVolumeClient("manila", "./ceph.conf", "ceph", None, None)\nvc.connect()\n\nvp = VolumePath("grpid", "volid")\nvc.create_volume(vp, 10, data_isolated=True)\n\nvc.disconnect()\n        ']
1259
2016-07-14 21:56:09,009.009 INFO:__main__:
1260
[root@bzn build]# LD_LIBRARY_PATH=/home/rraja/git/ceph/build/lib PYTHONPATH=/home/rraja/git/teuthology/:/home/rraja/git/ceph-qa-suite/:/home/rraja/git/ceph/src/pybind:/home/rraja/git/ceph/build/lib/cython_modules/lib.linux-x86_64-2.7/ python /home/rraja/git/ceph-qa-suite/tasks/cephfs/vstart_runner.py --interactive tasks.cephfs.test_volume_client.TestVolumeClient
1261
2016-07-14 21:49:05,160.160 INFO:__main__:run args=['ps', '-u0']
1262
2016-07-14 21:49:05,160.160 INFO:__main__:Running ['ps', '-u0']
1263
2016-07-14 21:49:05,175.175 WARNING:__main__:Killing stray process  3133 ?        00:00:03 ceph-mds
1264
2016-07-14 21:49:05,178.178 INFO:__main__:run args=['./bin/ceph', 'auth', 'get-or-create', 'client.0', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
1265
2016-07-14 21:49:05,178.178 INFO:__main__:Running ['./bin/ceph', 'auth', 'get-or-create', 'client.0', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
1266
2016-07-14 21:49:05,392.392 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1267
2016-07-14 21:49:05,392.392 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1268
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.0': No such file or directory
1269
2016-07-14 21:49:05,400.400 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.0
1270
2016-07-14 21:49:05,401.401 INFO:__main__:run args=['./bin/ceph', 'auth', 'get-or-create', 'client.1', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
1271
2016-07-14 21:49:05,401.401 INFO:__main__:Running ['./bin/ceph', 'auth', 'get-or-create', 'client.1', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
1272
2016-07-14 21:49:05,601.601 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1273
2016-07-14 21:49:05,601.601 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1274
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.1': No such file or directory
1275
2016-07-14 21:49:05,608.608 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.1
1276
2016-07-14 21:49:05,609.609 INFO:__main__:run args=['./bin/ceph', 'auth', 'get-or-create', 'client.2', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
1277
2016-07-14 21:49:05,609.609 INFO:__main__:Running ['./bin/ceph', 'auth', 'get-or-create', 'client.2', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
1278
2016-07-14 21:49:05,825.825 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
1279
2016-07-14 21:49:05,825.825 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
1280
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.2': No such file or directory
1281
2016-07-14 21:49:05,833.833 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.2
1282
2016-07-14 21:49:05,833.833 INFO:__main__:run args=['./bin/ceph', 'auth', 'get-or-create', 'client.3', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
1283
2016-07-14 21:49:05,834.834 INFO:__main__:Running ['./bin/ceph', 'auth', 'get-or-create', 'client.3', 'osd', 'allow rw', 'mds', 'allow', 'mon', 'allow r']
1284
2016-07-14 21:49:06,038.038 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
1285
2016-07-14 21:49:06,038.038 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
1286
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.3': No such file or directory
1287
2016-07-14 21:49:06,046.046 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.3
1288
2016-07-14 21:49:06,046.046 INFO:__main__:Discovered MDS IDs: ['a']
1289
2016-07-14 21:49:06,048.048 INFO:__main__:run args=['./bin/ceph', 'tell', 'osd.*', 'injectargs', '--osd-mon-report-interval-max', '5']
1290
2016-07-14 21:49:06,048.048 INFO:__main__:Running ['./bin/ceph', 'tell', 'osd.*', 'injectargs', '--osd-mon-report-interval-max', '5']
1291
2016-07-14 21:49:06,188.188 INFO:__main__:Searching for existing instance osd_mon_report_interval_max/osd
1292
2016-07-14 21:49:06,190.190 INFO:__main__:Searching for existing instance mds log max segments/mds
1293
2016-07-14 21:49:06,190.190 INFO:__main__:Found string to replace at 1722
1294
2016-07-14 21:49:06,190.190 INFO:__main__:Searching for existing instance osd_mon_report_interval_max/osd
1295
2016-07-14 21:49:06,191.191 INFO:__main__:Searching for existing instance mds log max segments/mds
1296
2016-07-14 21:49:06,192.192 INFO:__main__:Searching for existing instance mds root ino uid/global
1297
2016-07-14 21:49:06,192.192 INFO:__main__:Searching for existing instance osd_mon_report_interval_max/osd
1298
2016-07-14 21:49:06,193.193 INFO:__main__:Searching for existing instance mds log max segments/mds
1299
2016-07-14 21:49:06,193.193 INFO:__main__:Searching for existing instance mds root ino uid/global
1300
2016-07-14 21:49:06,193.193 INFO:__main__:Searching for existing instance mds root ino gid/global
1301
2016-07-14 21:49:06,193.193 INFO:__main__:Searching for existing instance osd_mon_report_interval_max/osd
1302
2016-07-14 21:49:06,194.194 INFO:__main__:Executing modules: ['tasks.cephfs.test_volume_client.TestVolumeClient']
1303
2016-07-14 21:49:06,198.198 INFO:__main__:Loaded: [<unittest.suite.TestSuite tests=[<tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_15303>, <tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_data_isolated>, <tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_default_prefix>, <tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_evict_client>, <tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_idempotency>, <tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_lifecycle>, <tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_purge>, <tasks.cephfs.test_volume_client.TestVolumeClient testMethod=test_readonly_authorization>]>]
1304
2016-07-14 21:49:06,198.198 INFO:__main__:Disabling 0 tests because of is_for_teuthology or needs_trimming
1305
2016-07-14 21:49:06,199.199 INFO:__main__:Starting test: test_15303 (tasks.cephfs.test_volume_client.TestVolumeClient)
1306
2016-07-14 21:49:06,199.199 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1307
2016-07-14 21:49:06,199.199 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1308
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.0': No such file or directory
1309
2016-07-14 21:49:06,207.207 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.0
1310
2016-07-14 21:49:06,207.207 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1311
2016-07-14 21:49:06,207.207 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1312
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.1': No such file or directory
1313
2016-07-14 21:49:06,216.216 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.1
1314
2016-07-14 21:49:06,217.217 INFO:__main__:run args=['ps', '-u0']
1315
2016-07-14 21:49:06,217.217 INFO:__main__:Running ['ps', '-u0']
1316
2016-07-14 21:49:06,239.239 INFO:__main__:No match for mds a: PID TTY          TIME CMD
1317
    1 ?        00:00:02 systemd
1318
    2 ?        00:00:00 kthreadd
1319
    3 ?        00:00:00 ksoftirqd/0
1320
    5 ?        00:00:00 kworker/0:0H
1321
    7 ?        00:00:36 rcu_sched
1322
    8 ?        00:00:00 rcu_bh
1323
    9 ?        00:00:15 rcuos/0
1324
   10 ?        00:00:00 rcuob/0
1325
   11 ?        00:00:00 migration/0
1326
   12 ?        00:00:00 watchdog/0
1327
   13 ?        00:00:00 watchdog/1
1328
   14 ?        00:00:00 migration/1
1329
   15 ?        00:00:00 ksoftirqd/1
1330
   17 ?        00:00:00 kworker/1:0H
1331
   18 ?        00:00:06 rcuos/1
1332
   19 ?        00:00:00 rcuob/1
1333
   20 ?        00:00:00 watchdog/2
1334
   21 ?        00:00:00 migration/2
1335
   22 ?        00:00:00 ksoftirqd/2
1336
   24 ?        00:00:00 kworker/2:0H
1337
   25 ?        00:00:14 rcuos/2
1338
   26 ?        00:00:00 rcuob/2
1339
   27 ?        00:00:00 watchdog/3
1340
   28 ?        00:00:00 migration/3
1341
   29 ?        00:00:00 ksoftirqd/3
1342
   31 ?        00:00:00 kworker/3:0H
1343
   32 ?        00:00:05 rcuos/3
1344
   33 ?        00:00:00 rcuob/3
1345
   34 ?        00:00:00 kdevtmpfs
1346
   35 ?        00:00:00 netns
1347
   36 ?        00:00:00 writeback
1348
   37 ?        00:00:00 ksmd
1349
   38 ?        00:00:00 khugepaged
1350
   39 ?        00:00:00 crypto
1351
   40 ?        00:00:00 kintegrityd
1352
   41 ?        00:00:00 bioset
1353
   42 ?        00:00:00 kblockd
1354
   43 ?        00:00:00 ata_sff
1355
   44 ?        00:00:00 md
1356
   45 ?        00:00:00 devfreq_wq
1357
   51 ?        00:00:13 kswapd0
1358
   52 ?        00:00:00 vmstat
1359
   93 ?        00:00:00 kthrotld
1360
   94 ?        00:00:00 acpi_thermal_pm
1361
   95 ?        00:00:00 scsi_eh_0
1362
   96 ?        00:00:00 scsi_tmf_0
1363
   97 ?        00:00:00 scsi_eh_1
1364
   98 ?        00:00:00 scsi_tmf_1
1365
   99 ?        00:00:00 scsi_eh_2
1366
  100 ?        00:00:00 scsi_tmf_2
1367
  104 ?        00:00:00 kpsmoused
1368
  106 ?        00:00:00 dm_bufio_cache
1369
  107 ?        00:00:00 ipv6_addrconf
1370
  138 ?        00:00:00 deferwq
1371
  140 ?        00:00:00 bioset
1372
  178 ?        00:00:00 kauditd
1373
  389 ?        00:00:01 kworker/0:1H
1374
  399 ?        00:00:00 kworker/1:1H
1375
  400 ?        00:00:00 kworker/3:1H
1376
  420 ?        00:00:00 rtsx_pci_sdmmc_
1377
  425 ?        00:00:00 kworker/2:1H
1378
  613 ?        00:00:00 kdmflush
1379
  621 ?        00:00:00 bioset
1380
  622 ?        00:00:00 kcryptd_io
1381
  623 ?        00:00:00 kcryptd
1382
  624 ?        00:00:08 dmcrypt_write
1383
  625 ?        00:00:00 bioset
1384
  688 ?        00:00:00 kdmflush
1385
  691 ?        00:00:00 bioset
1386
  700 ?        00:00:00 kdmflush
1387
  701 ?        00:00:00 bioset
1388
  724 ?        00:00:00 jbd2/dm-1-8
1389
  725 ?        00:00:00 ext4-rsv-conver
1390
  823 ?        00:00:02 systemd-journal
1391
  861 ?        00:00:00 rpciod
1392
  880 ?        00:00:00 systemd-udevd
1393
  910 ?        00:00:00 ktpacpid
1394
  918 ?        00:00:00 irq/47-mei_me
1395
  949 ?        00:00:00 kmemstick
1396
  968 ?        00:00:00 cfg80211
1397
  984 ?        00:00:07 irq/50-iwlwifi
1398
  999 ?        00:00:00 kworker/u17:0
1399
 1001 ?        00:00:00 hci0
1400
 1002 ?        00:00:00 hci0
1401
 1007 ?        00:00:00 kworker/u17:2
1402
 1015 ?        00:00:00 kdmflush
1403
 1016 ?        00:00:00 bioset
1404
 1018 ?        00:00:00 kvm-irqfd-clean
1405
 1071 ?        00:00:00 jbd2/sda1-8
1406
 1072 ?        00:00:00 ext4-rsv-conver
1407
 1077 ?        00:00:06 jbd2/dm-3-8
1408
 1078 ?        00:00:00 ext4-rsv-conver
1409
 1105 ?        00:00:00 auditd
1410
 1119 ?        00:00:00 audispd
1411
 1123 ?        00:00:00 sedispatch
1412
 1133 ?        00:00:00 bluetoothd
1413
 1137 ?        00:00:00 mcelog
1414
 1139 ?        00:00:39 rngd
1415
 1146 ?        00:00:00 gssproxy
1416
 1154 ?        00:00:00 ModemManager
1417
 1158 ?        00:00:04 udisksd
1418
 1159 ?        00:00:00 firewalld
1419
 1169 ?        00:00:00 systemd-logind
1420
 1173 ?        00:00:00 accounts-daemon
1421
 1177 ?        00:00:00 alsactl
1422
 1249 ?        00:00:00 abrtd
1423
 1262 ?        00:00:00 iprt-VBoxWQueue
1424
 1268 ?        00:00:00 iprt-VBoxTscThr
1425
 1281 ?        00:00:00 abrt-dump-journ
1426
 1286 ?        00:00:00 abrt-dump-journ
1427
 1324 ?        00:00:12 NetworkManager
1428
 1376 ?        00:00:00 libvirtd
1429
 1393 ?        00:00:00 crond
1430
 1396 ?        00:00:00 atd
1431
 1398 ?        00:00:00 gdm
1432
 1463 ?        00:00:01 wpa_supplicant
1433
 1509 ?        00:00:00 gdm-session-wor
1434
 1582 ?        00:00:00 upowerd
1435
 1680 ?        00:00:10 packagekitd
1436
 1839 ?        00:00:00 dhclient
1437
 1903 ?        00:00:00 gdm-session-wor
1438
 1938 tty2     00:00:00 xf86-video-inte
1439
 2067 ?        00:00:00 krfcommd
1440
 2206 ?        00:00:00 cupsd
1441
 2235 ?        00:00:00 fwupd
1442
 2448 ?        00:00:00 dhclient
1443
 2633 ?        00:00:00 kworker/0:1
1444
 3387 ?        00:00:00 kworker/3:1
1445
 3927 ?        00:00:04 kworker/u16:0
1446
 5168 ?        00:00:00 kworker/1:3
1447
13278 ?        00:00:00 kworker/u16:5
1448
13551 ?        00:00:00 kworker/u16:1
1449
13663 ?        00:00:00 kworker/1:1
1450
13668 ?        00:00:00 kworker/3:0
1451
13681 ?        00:00:00 kworker/2:2
1452
13694 ?        00:00:00 kworker/0:3
1453
13724 ?        00:00:00 kworker/u16:4
1454
13765 ?        00:00:00 kworker/1:2
1455
13781 ?        00:00:00 kworker/2:0
1456
13794 ?        00:00:00 kworker/3:3
1457
13799 ?        00:00:00 kworker/0:2
1458
14835 pts/1    00:00:00 sudo
1459
14836 ?        00:00:00 fprintd
1460
14848 pts/1    00:00:00 su
1461
14853 pts/1    00:00:00 bash
1462
14902 ?        00:00:00 kworker/2:3
1463
14903 pts/1    00:00:00 python
1464
15074 pts/1    00:00:00 ps
1465
16120 ?        00:00:00 kworker/2:1
1466
18809 ?        00:00:00 ceph-msgr
1467
18811 ?        00:00:00 rbd
1468
28322 ?        00:00:00 systemd
1469
28332 ?        00:00:00 (sd-pam)
1470
28799 ?        00:00:10 kworker/u16:2
1471
29866 ?        00:00:00 kworker/3:2
1472
30529 ?        00:00:00 dio/dm-3
1473
31742 ?        00:00:00 kworker/0:0
1474
32711 ?        00:00:00 kworker/1:0
1475
2016-07-14 21:49:06,240.240 ERROR:__main__:tried to stop a non-running daemon
1476
2016-07-14 21:49:06,240.240 INFO:__main__:run args=['./bin/ceph', 'fs', 'ls', '--format=json-pretty']
1477
2016-07-14 21:49:06,240.240 INFO:__main__:Running ['./bin/ceph', 'fs', 'ls', '--format=json-pretty']
1478
2016-07-14 21:49:06,428.428 INFO:__main__:run args=['./bin/ceph', 'fs', 'set', u'cephfs_a', 'cluster_down', 'true']
1479
2016-07-14 21:49:06,428.428 INFO:__main__:Running ['./bin/ceph', 'fs', 'set', u'cephfs_a', 'cluster_down', 'true']
1480
2016-07-14 21:49:07,071.071 INFO:__main__:run args=['./bin/ceph', 'fs', 'get', u'cephfs_a', '--format=json-pretty']
1481
2016-07-14 21:49:07,071.071 INFO:__main__:Running ['./bin/ceph', 'fs', 'get', u'cephfs_a', '--format=json-pretty']
1482
2016-07-14 21:49:07,257.257 INFO:__main__:run args=['./bin/ceph', 'mds', 'fail', '4113']
1483
2016-07-14 21:49:07,257.257 INFO:__main__:Running ['./bin/ceph', 'mds', 'fail', '4113']
1484
2016-07-14 21:49:08,055.055 INFO:__main__:run args=['./bin/ceph', 'fs', 'rm', u'cephfs_a', '--yes-i-really-mean-it']
1485
2016-07-14 21:49:08,055.055 INFO:__main__:Running ['./bin/ceph', 'fs', 'rm', u'cephfs_a', '--yes-i-really-mean-it']
1486
2016-07-14 21:49:09,055.055 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_metadata_a', u'cephfs_metadata_a', '--yes-i-really-really-mean-it']
1487
2016-07-14 21:49:09,055.055 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_metadata_a', u'cephfs_metadata_a', '--yes-i-really-really-mean-it']
1488
2016-07-14 21:49:10,155.155 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_data_a', u'cephfs_data_a', '--yes-i-really-really-mean-it']
1489
2016-07-14 21:49:10,155.155 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_data_a', u'cephfs_data_a', '--yes-i-really-really-mean-it']
1490
2016-07-14 21:49:11,269.269 INFO:__main__:run args=['./bin/ceph', 'daemon', 'osd.0', 'config', 'get', 'osd_mon_report_interval_max']
1491
2016-07-14 21:49:11,270.270 INFO:__main__:Running ['./bin/ceph', 'daemon', 'osd.0', 'config', 'get', 'osd_mon_report_interval_max']
1492
2016-07-14 21:49:11,385.385 INFO:tasks.cephfs.filesystem:_json_asok output: {
1493
    "osd_mon_report_interval_max": "5"
1494
}
1495

    
1496
2016-07-14 21:49:11,385.385 INFO:__main__:run args=['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
1497
2016-07-14 21:49:11,385.385 INFO:__main__:Running ['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
1498
2016-07-14 21:49:11,567.567 INFO:__main__:run args=['./bin/ceph', 'osd', 'blacklist', 'clear']
1499
2016-07-14 21:49:11,567.567 INFO:__main__:Running ['./bin/ceph', 'osd', 'blacklist', 'clear']
1500
2016-07-14 21:49:12,259.259 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.0', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
1501
2016-07-14 21:49:12,259.259 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.0', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
1502
2016-07-14 21:49:12,493.493 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.1', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
1503
2016-07-14 21:49:12,494.494 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.1', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
1504
2016-07-14 21:49:12,754.754 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.2', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
1505
2016-07-14 21:49:12,754.754 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.2', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
1506
2016-07-14 21:49:12,972.972 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.3', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
1507
2016-07-14 21:49:12,972.972 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.3', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
1508
2016-07-14 21:49:13,187.187 INFO:tasks.cephfs.cephfs_test_case:['0', '1', '2', '3']
1509
2016-07-14 21:49:13,187.187 INFO:__main__:run args=['./bin/ceph', 'auth', 'list', '--format=json-pretty']
1510
2016-07-14 21:49:13,188.188 INFO:__main__:Running ['./bin/ceph', 'auth', 'list', '--format=json-pretty']
1511
2016-07-14 21:49:13,380.380 INFO:tasks.cephfs.filesystem:Creating filesystem 'cephfs'
1512
2016-07-14 21:49:13,381.381 INFO:__main__:run args=['./bin/ceph', 'daemon', 'mon.a', 'config', 'get', 'mon_pg_warn_min_per_osd']
1513
2016-07-14 21:49:13,381.381 INFO:__main__:Running ['./bin/ceph', 'daemon', 'mon.a', 'config', 'get', 'mon_pg_warn_min_per_osd']
1514
2016-07-14 21:49:13,489.489 INFO:tasks.cephfs.filesystem:_json_asok output: {
1515
    "mon_pg_warn_min_per_osd": "3"
1516
}
1517

    
1518
2016-07-14 21:49:13,489.489 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_metadata', '9']
1519
2016-07-14 21:49:13,490.490 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_metadata', '9']
1520
2016-07-14 21:49:14,342.342 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_data', '9']
1521
2016-07-14 21:49:14,342.342 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_data', '9']
1522
2016-07-14 21:49:15,351.351 INFO:__main__:run args=['./bin/ceph', 'fs', 'new', 'cephfs', 'cephfs_metadata', 'cephfs_data']
1523
2016-07-14 21:49:15,351.351 INFO:__main__:Running ['./bin/ceph', 'fs', 'new', 'cephfs', 'cephfs_metadata', 'cephfs_data']
1524
2016-07-14 21:49:16,416.416 INFO:__main__:run args=['ps', '-u0']
1525
2016-07-14 21:49:16,416.416 INFO:__main__:Running ['ps', '-u0']
1526
2016-07-14 21:49:16,440.440 INFO:__main__:No match for mds a: PID TTY          TIME CMD
1527
    1 ?        00:00:02 systemd
1528
    2 ?        00:00:00 kthreadd
1529
    3 ?        00:00:00 ksoftirqd/0
1530
    5 ?        00:00:00 kworker/0:0H
1531
    7 ?        00:00:36 rcu_sched
1532
    8 ?        00:00:00 rcu_bh
1533
    9 ?        00:00:15 rcuos/0
1534
   10 ?        00:00:00 rcuob/0
1535
   11 ?        00:00:00 migration/0
1536
   12 ?        00:00:00 watchdog/0
1537
   13 ?        00:00:00 watchdog/1
1538
   14 ?        00:00:00 migration/1
1539
   15 ?        00:00:00 ksoftirqd/1
1540
   17 ?        00:00:00 kworker/1:0H
1541
   18 ?        00:00:06 rcuos/1
1542
   19 ?        00:00:00 rcuob/1
1543
   20 ?        00:00:00 watchdog/2
1544
   21 ?        00:00:00 migration/2
1545
   22 ?        00:00:00 ksoftirqd/2
1546
   24 ?        00:00:00 kworker/2:0H
1547
   25 ?        00:00:14 rcuos/2
1548
   26 ?        00:00:00 rcuob/2
1549
   27 ?        00:00:00 watchdog/3
1550
   28 ?        00:00:00 migration/3
1551
   29 ?        00:00:00 ksoftirqd/3
1552
   31 ?        00:00:00 kworker/3:0H
1553
   32 ?        00:00:05 rcuos/3
1554
   33 ?        00:00:00 rcuob/3
1555
   34 ?        00:00:00 kdevtmpfs
1556
   35 ?        00:00:00 netns
1557
   36 ?        00:00:00 writeback
1558
   37 ?        00:00:00 ksmd
1559
   38 ?        00:00:00 khugepaged
1560
   39 ?        00:00:00 crypto
1561
   40 ?        00:00:00 kintegrityd
1562
   41 ?        00:00:00 bioset
1563
   42 ?        00:00:00 kblockd
1564
   43 ?        00:00:00 ata_sff
1565
   44 ?        00:00:00 md
1566
   45 ?        00:00:00 devfreq_wq
1567
   51 ?        00:00:13 kswapd0
1568
   52 ?        00:00:00 vmstat
1569
   93 ?        00:00:00 kthrotld
1570
   94 ?        00:00:00 acpi_thermal_pm
1571
   95 ?        00:00:00 scsi_eh_0
1572
   96 ?        00:00:00 scsi_tmf_0
1573
   97 ?        00:00:00 scsi_eh_1
1574
   98 ?        00:00:00 scsi_tmf_1
1575
   99 ?        00:00:00 scsi_eh_2
1576
  100 ?        00:00:00 scsi_tmf_2
1577
  104 ?        00:00:00 kpsmoused
1578
  106 ?        00:00:00 dm_bufio_cache
1579
  107 ?        00:00:00 ipv6_addrconf
1580
  138 ?        00:00:00 deferwq
1581
  140 ?        00:00:00 bioset
1582
  178 ?        00:00:00 kauditd
1583
  389 ?        00:00:01 kworker/0:1H
1584
  399 ?        00:00:00 kworker/1:1H
1585
  400 ?        00:00:00 kworker/3:1H
1586
  420 ?        00:00:00 rtsx_pci_sdmmc_
1587
  425 ?        00:00:00 kworker/2:1H
1588
  613 ?        00:00:00 kdmflush
1589
  621 ?        00:00:00 bioset
1590
  622 ?        00:00:00 kcryptd_io
1591
  623 ?        00:00:00 kcryptd
1592
  624 ?        00:00:08 dmcrypt_write
1593
  625 ?        00:00:00 bioset
1594
  688 ?        00:00:00 kdmflush
1595
  691 ?        00:00:00 bioset
1596
  700 ?        00:00:00 kdmflush
1597
  701 ?        00:00:00 bioset
1598
  724 ?        00:00:00 jbd2/dm-1-8
1599
  725 ?        00:00:00 ext4-rsv-conver
1600
  823 ?        00:00:02 systemd-journal
1601
  861 ?        00:00:00 rpciod
1602
  880 ?        00:00:00 systemd-udevd
1603
  910 ?        00:00:00 ktpacpid
1604
  918 ?        00:00:00 irq/47-mei_me
1605
  949 ?        00:00:00 kmemstick
1606
  968 ?        00:00:00 cfg80211
1607
  984 ?        00:00:07 irq/50-iwlwifi
1608
  999 ?        00:00:00 kworker/u17:0
1609
 1001 ?        00:00:00 hci0
1610
 1002 ?        00:00:00 hci0
1611
 1007 ?        00:00:00 kworker/u17:2
1612
 1015 ?        00:00:00 kdmflush
1613
 1016 ?        00:00:00 bioset
1614
 1018 ?        00:00:00 kvm-irqfd-clean
1615
 1071 ?        00:00:00 jbd2/sda1-8
1616
 1072 ?        00:00:00 ext4-rsv-conver
1617
 1077 ?        00:00:06 jbd2/dm-3-8
1618
 1078 ?        00:00:00 ext4-rsv-conver
1619
 1105 ?        00:00:00 auditd
1620
 1119 ?        00:00:00 audispd
1621
 1123 ?        00:00:00 sedispatch
1622
 1133 ?        00:00:00 bluetoothd
1623
 1137 ?        00:00:00 mcelog
1624
 1139 ?        00:00:39 rngd
1625
 1146 ?        00:00:00 gssproxy
1626
 1154 ?        00:00:00 ModemManager
1627
 1158 ?        00:00:04 udisksd
1628
 1159 ?        00:00:00 firewalld
1629
 1169 ?        00:00:00 systemd-logind
1630
 1173 ?        00:00:00 accounts-daemon
1631
 1177 ?        00:00:00 alsactl
1632
 1249 ?        00:00:00 abrtd
1633
 1262 ?        00:00:00 iprt-VBoxWQueue
1634
 1268 ?        00:00:00 iprt-VBoxTscThr
1635
 1281 ?        00:00:00 abrt-dump-journ
1636
 1286 ?        00:00:00 abrt-dump-journ
1637
 1324 ?        00:00:12 NetworkManager
1638
 1376 ?        00:00:00 libvirtd
1639
 1393 ?        00:00:00 crond
1640
 1396 ?        00:00:00 atd
1641
 1398 ?        00:00:00 gdm
1642
 1463 ?        00:00:01 wpa_supplicant
1643
 1509 ?        00:00:00 gdm-session-wor
1644
 1582 ?        00:00:00 upowerd
1645
 1680 ?        00:00:10 packagekitd
1646
 1839 ?        00:00:00 dhclient
1647
 1903 ?        00:00:00 gdm-session-wor
1648
 1938 tty2     00:00:00 xf86-video-inte
1649
 2067 ?        00:00:00 krfcommd
1650
 2206 ?        00:00:00 cupsd
1651
 2235 ?        00:00:00 fwupd
1652
 2448 ?        00:00:00 dhclient
1653
 2633 ?        00:00:00 kworker/0:1
1654
 3387 ?        00:00:00 kworker/3:1
1655
 3927 ?        00:00:04 kworker/u16:0
1656
 5168 ?        00:00:00 kworker/1:3
1657
13278 ?        00:00:00 kworker/u16:5
1658
13551 ?        00:00:00 kworker/u16:1
1659
13663 ?        00:00:00 kworker/1:1
1660
13668 ?        00:00:00 kworker/3:0
1661
13681 ?        00:00:00 kworker/2:2
1662
13694 ?        00:00:00 kworker/0:3
1663
13724 ?        00:00:00 kworker/u16:4
1664
13765 ?        00:00:00 kworker/1:2
1665
13781 ?        00:00:00 kworker/2:0
1666
13794 ?        00:00:00 kworker/3:3
1667
13799 ?        00:00:00 kworker/0:2
1668
14835 pts/1    00:00:00 sudo
1669
14836 ?        00:00:00 fprintd
1670
14848 pts/1    00:00:00 su
1671
14853 pts/1    00:00:00 bash
1672
14902 ?        00:00:00 kworker/2:3
1673
14903 pts/1    00:00:00 python
1674
15599 pts/1    00:00:00 ps
1675
16120 ?        00:00:00 kworker/2:1
1676
18809 ?        00:00:00 ceph-msgr
1677
18811 ?        00:00:00 rbd
1678
28322 ?        00:00:00 systemd
1679
28332 ?        00:00:00 (sd-pam)
1680
28799 ?        00:00:10 kworker/u16:2
1681
29866 ?        00:00:00 kworker/3:2
1682
30529 ?        00:00:00 dio/dm-3
1683
31742 ?        00:00:00 kworker/0:0
1684
32711 ?        00:00:00 kworker/1:0
1685
2016-07-14 21:49:16,441.441 INFO:__main__:run args=['./bin/./ceph-mds', '-i', 'a']
1686
2016-07-14 21:49:16,441.441 INFO:__main__:Running ['./bin/./ceph-mds', '-i', 'a']
1687
2016-07-14 21:49:16,471.471 INFO:__main__:run args=['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
1688
2016-07-14 21:49:16,472.472 INFO:__main__:Running ['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
1689
2016-07-14 21:49:16,667.667 INFO:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {u'session_autoclose': 300, u'up': {u'mds_0': 4123}, u'last_failure_osd_epoch': 0, u'in': [0], u'last_failure': 0, u'max_file_size': 1099511627776, u'explicitly_allowed_features': 0, u'damaged': [], u'tableserver': 0, u'metadata_pool': 3, u'failed': [], u'epoch': 10, u'flags': 0, u'max_mds': 1, u'compat': {u'compat': {}, u'ro_compat': {}, u'incompat': {u'feature_8': u'file layout v2', u'feature_2': u'client writeable ranges', u'feature_3': u'default file layouts on dirs', u'feature_1': u'base v0.20', u'feature_6': u'dirfrag is stored in omap', u'feature_4': u'dir inode in separate object', u'feature_5': u'mds uses versioned encoding'}}, u'data_pools': [4], u'info': {u'gid_4123': {u'standby_for_rank': -1, u'export_targets': [], u'name': u'a', u'incarnation': 10, u'standby_replay': False, u'state_seq': 1, u'standby_for_fscid': -1, u'state': u'up:creating', u'gid': 4123, u'features': 576460752032890879, u'rank': 0, u'standby_for_name': u'', u'addr': u'10.70.1.141:6812/14766'}}, u'fs_name': u'cephfs', u'created': u'2016-07-14 21:49:15.555766', u'enabled': True, u'modified': u'2016-07-14 21:49:15.555766', u'session_timeout': 60, u'stopped': [], u'ever_allowed_features': 0, u'root': 0}
1690
2016-07-14 21:49:16,668.668 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_4123:up:creating
1691
2016-07-14 21:49:17,668.668 INFO:__main__:run args=['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
1692
2016-07-14 21:49:17,668.668 INFO:__main__:Running ['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
1693
2016-07-14 21:49:17,866.866 INFO:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {u'session_autoclose': 300, u'up': {u'mds_0': 4123}, u'last_failure_osd_epoch': 0, u'in': [0], u'last_failure': 0, u'max_file_size': 1099511627776, u'explicitly_allowed_features': 0, u'damaged': [], u'tableserver': 0, u'metadata_pool': 3, u'failed': [], u'epoch': 11, u'flags': 0, u'max_mds': 1, u'compat': {u'compat': {}, u'ro_compat': {}, u'incompat': {u'feature_8': u'file layout v2', u'feature_2': u'client writeable ranges', u'feature_3': u'default file layouts on dirs', u'feature_1': u'base v0.20', u'feature_6': u'dirfrag is stored in omap', u'feature_4': u'dir inode in separate object', u'feature_5': u'mds uses versioned encoding'}}, u'data_pools': [4], u'info': {u'gid_4123': {u'standby_for_rank': -1, u'export_targets': [], u'name': u'a', u'incarnation': 10, u'standby_replay': False, u'state_seq': 4, u'standby_for_fscid': -1, u'state': u'up:active', u'gid': 4123, u'features': 576460752032890879, u'rank': 0, u'standby_for_name': u'', u'addr': u'10.70.1.141:6812/14766'}}, u'fs_name': u'cephfs', u'created': u'2016-07-14 21:49:15.555766', u'enabled': True, u'modified': u'2016-07-14 21:49:15.555766', u'session_timeout': 60, u'stopped': [], u'ever_allowed_features': 0, u'root': 0}
1694
2016-07-14 21:49:17,866.866 INFO:tasks.cephfs.filesystem:are_daemons_healthy: 1/1
1695
2016-07-14 21:49:17,867.867 INFO:__main__:run args=['./bin/ceph', 'daemon', 'mds.a', 'status']
1696
2016-07-14 21:49:17,867.867 INFO:__main__:Running ['./bin/ceph', 'daemon', 'mds.a', 'status']
1697
2016-07-14 21:49:17,968.968 INFO:tasks.cephfs.filesystem:_json_asok output: {
1698
    "cluster_fsid": "ae66e242-d103-408a-b4cf-f160f95d365c",
1699
    "whoami": 0,
1700
    "want_state": "up:active",
1701
    "state": "up:active",
1702
    "mdsmap_epoch": 11,
1703
    "osdmap_epoch": 17,
1704
    "osdmap_epoch_barrier": 17
1705
}
1706

    
1707
2016-07-14 21:49:17,969.969 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1708
2016-07-14 21:49:17,969.969 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1709
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.0': No such file or directory
1710
2016-07-14 21:49:17,983.983 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.0
1711
2016-07-14 21:49:17,984.984 INFO:__main__:run args=['mkdir', '--', '/tmp/tmp0EnRmD/mnt.0']
1712
2016-07-14 21:49:17,984.984 INFO:__main__:Running ['mkdir', '--', '/tmp/tmp0EnRmD/mnt.0']
1713
2016-07-14 21:49:17,991.991 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1714
2016-07-14 21:49:17,992.992 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1715
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1716
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1717
2016-07-14 21:49:18,007.007 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1718
2016-07-14 21:49:18,008.008 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1719
2016-07-14 21:49:18,019.019 INFO:__main__:Pre-mount connections: [43, 47]
1720
2016-07-14 21:49:18,019.019 INFO:__main__:run args=['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0']
1721
2016-07-14 21:49:18,020.020 INFO:__main__:Running ['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0']
1722
2016-07-14 21:49:18,026.026 INFO:__main__:Mounting client.0 with pid 15669
1723
2016-07-14 21:49:18,026.026 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1724
2016-07-14 21:49:18,027.027 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1725
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1726
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1727
2016-07-14 21:49:18,043.043 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1728
2016-07-14 21:49:18,044.044 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1729
2016-07-14 21:49:19,059.059 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1730
2016-07-14 21:49:19,060.060 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1731
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1732
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1733
2016-07-14 21:49:19,069.069 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1734
2016-07-14 21:49:19,069.069 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1735
2016-07-14 21:49:19,080.080 INFO:__main__:Post-mount connections: [43, 47, 48]
1736
2016-07-14 21:49:19,080.080 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1737
2016-07-14 21:49:19,081.081 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1738
2016-07-14 21:49:19,089.089 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.0
1739
2016-07-14 21:49:19,089.089 INFO:__main__:run args=['sudo', 'chmod', '1777', '/tmp/tmp0EnRmD/mnt.0']
1740
2016-07-14 21:49:19,089.089 INFO:__main__:Running ['chmod', '1777', '/tmp/tmp0EnRmD/mnt.0']
1741
2016-07-14 21:49:19,101.101 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1742
2016-07-14 21:49:19,101.101 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1743
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.1': No such file or directory
1744
2016-07-14 21:49:19,112.112 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.1
1745
2016-07-14 21:49:19,112.112 INFO:__main__:run args=['mkdir', '--', '/tmp/tmp0EnRmD/mnt.1']
1746
2016-07-14 21:49:19,113.113 INFO:__main__:Running ['mkdir', '--', '/tmp/tmp0EnRmD/mnt.1']
1747
2016-07-14 21:49:19,121.121 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1748
2016-07-14 21:49:19,121.121 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1749
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1750
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1751
2016-07-14 21:49:19,133.133 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1752
2016-07-14 21:49:19,134.134 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1753
2016-07-14 21:49:19,143.143 INFO:__main__:Pre-mount connections: [43, 47, 48]
1754
2016-07-14 21:49:19,143.143 INFO:__main__:run args=['./bin/ceph-fuse', '-f', '--name', 'client.1', '/tmp/tmp0EnRmD/mnt.1']
1755
2016-07-14 21:49:19,144.144 INFO:__main__:Running ['./bin/ceph-fuse', '-f', '--name', 'client.1', '/tmp/tmp0EnRmD/mnt.1']
1756
2016-07-14 21:49:19,152.152 INFO:__main__:Mounting client.1 with pid 15711
1757
2016-07-14 21:49:19,152.152 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1758
2016-07-14 21:49:19,152.152 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1759
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1760
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1761
2016-07-14 21:49:19,165.165 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1762
2016-07-14 21:49:19,166.166 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1763
2016-07-14 21:49:20,176.176 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1764
2016-07-14 21:49:20,176.176 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1765
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1766
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1767
2016-07-14 21:49:20,185.185 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1768
2016-07-14 21:49:20,185.185 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1769
2016-07-14 21:49:20,193.193 INFO:__main__:Post-mount connections: [43, 47, 48, 49]
1770
2016-07-14 21:49:20,194.194 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1771
2016-07-14 21:49:20,194.194 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1772
2016-07-14 21:49:20,202.202 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.1
1773
2016-07-14 21:49:20,202.202 INFO:__main__:run args=['sudo', 'chmod', '1777', '/tmp/tmp0EnRmD/mnt.1']
1774
2016-07-14 21:49:20,202.202 INFO:__main__:Running ['chmod', '1777', '/tmp/tmp0EnRmD/mnt.1']
1775
2016-07-14 21:49:20,218.218 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1776
2016-07-14 21:49:20,218.218 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1777
2016-07-14 21:49:20,229.229 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.0
1778
2016-07-14 21:49:20,229.229 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on local...
1779
2016-07-14 21:49:20,229.229 INFO:__main__:run args=['sudo', 'fusermount', '-u', '/tmp/tmp0EnRmD/mnt.0']
1780
2016-07-14 21:49:20,230.230 INFO:__main__:Running ['fusermount', '-u', '/tmp/tmp0EnRmD/mnt.0']
1781
2016-07-14 21:49:20,260.260 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1782
2016-07-14 21:49:20,261.261 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1783
2016-07-14 21:49:20,270.270 INFO:teuthology.orchestra.run:waiting for 900
1784
2016-07-14 21:49:26,277.277 INFO:__main__:run args=['rmdir', '--', '/tmp/tmp0EnRmD/mnt.0']
1785
2016-07-14 21:49:26,277.277 INFO:__main__:Running ['rmdir', '--', '/tmp/tmp0EnRmD/mnt.0']
1786
2016-07-14 21:49:26,286.286 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1787
2016-07-14 21:49:26,287.287 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1788
2016-07-14 21:49:26,294.294 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.1
1789
2016-07-14 21:49:26,294.294 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on local...
1790
2016-07-14 21:49:26,294.294 INFO:__main__:run args=['sudo', 'fusermount', '-u', '/tmp/tmp0EnRmD/mnt.1']
1791
2016-07-14 21:49:26,294.294 INFO:__main__:Running ['fusermount', '-u', '/tmp/tmp0EnRmD/mnt.1']
1792
2016-07-14 21:49:26,323.323 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1793
2016-07-14 21:49:26,323.323 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1794
2016-07-14 21:49:26,333.333 INFO:teuthology.orchestra.run:waiting for 900
1795
2016-07-14 21:49:32,338.338 INFO:__main__:run args=['rmdir', '--', '/tmp/tmp0EnRmD/mnt.1']
1796
2016-07-14 21:49:32,339.339 INFO:__main__:Running ['rmdir', '--', '/tmp/tmp0EnRmD/mnt.1']
1797
2016-07-14 21:49:32,346.346 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
1798
2016-07-14 21:49:32,347.347 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
1799
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.2': No such file or directory
1800
2016-07-14 21:49:32,358.358 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.2
1801
2016-07-14 21:49:32,359.359 INFO:__main__:run args=['rmdir', '--', '/tmp/tmp0EnRmD/mnt.2']
1802
2016-07-14 21:49:32,359.359 INFO:__main__:Running ['rmdir', '--', '/tmp/tmp0EnRmD/mnt.2']
1803
rmdir: failed to remove '/tmp/tmp0EnRmD/mnt.2': No such file or directory
1804
2016-07-14 21:49:32,371.371 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
1805
2016-07-14 21:49:32,372.372 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
1806
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.3': No such file or directory
1807
2016-07-14 21:49:32,382.382 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.3
1808
2016-07-14 21:49:32,383.383 INFO:__main__:run args=['rmdir', '--', '/tmp/tmp0EnRmD/mnt.3']
1809
2016-07-14 21:49:32,384.384 INFO:__main__:Running ['rmdir', '--', '/tmp/tmp0EnRmD/mnt.3']
1810
rmdir: failed to remove '/tmp/tmp0EnRmD/mnt.3': No such file or directory
1811
2016-07-14 21:49:32,392.392 INFO:__main__:run args=['mkdir', '--', '/tmp/tmp0EnRmD/mnt.0']
1812
2016-07-14 21:49:32,392.392 INFO:__main__:Running ['mkdir', '--', '/tmp/tmp0EnRmD/mnt.0']
1813
2016-07-14 21:49:32,405.405 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1814
2016-07-14 21:49:32,405.405 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1815
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1816
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1817
2016-07-14 21:49:32,419.419 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1818
2016-07-14 21:49:32,420.420 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1819
2016-07-14 21:49:32,427.427 INFO:__main__:Pre-mount connections: [43, 47]
1820
2016-07-14 21:49:32,427.427 INFO:__main__:run args=['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0']
1821
2016-07-14 21:49:32,427.427 INFO:__main__:Running ['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0']
1822
2016-07-14 21:49:32,440.440 INFO:__main__:Mounting client.0 with pid 15797
1823
2016-07-14 21:49:32,441.441 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1824
2016-07-14 21:49:32,441.441 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1825
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1826
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1827
2016-07-14 21:49:32,456.456 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1828
2016-07-14 21:49:32,456.456 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1829
2016-07-14 21:49:33,469.469 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1830
2016-07-14 21:49:33,469.469 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1831
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1832
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1833
2016-07-14 21:49:33,477.477 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1834
2016-07-14 21:49:33,477.477 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1835
2016-07-14 21:49:33,485.485 INFO:__main__:Post-mount connections: [43, 47, 48]
1836
2016-07-14 21:49:33,486.486 INFO:__main__:run args=['mkdir', 'parent1']
1837
2016-07-14 21:49:33,486.486 INFO:__main__:Running ['mkdir', 'parent1']
1838
2016-07-14 21:49:33,520.520 INFO:__main__:run args=['mkdir', 'parent2']
1839
2016-07-14 21:49:33,520.520 INFO:__main__:Running ['mkdir', 'parent2']
1840
2016-07-14 21:49:33,531.531 INFO:__main__:run args=['mkdir', 'parent1/mydir']
1841
2016-07-14 21:49:33,531.531 INFO:__main__:Running ['mkdir', 'parent1/mydir']
1842
2016-07-14 21:49:33,557.557 INFO:__main__:run args=['mkdir', '--', '/tmp/tmp0EnRmD/mnt.1']
1843
2016-07-14 21:49:33,557.557 INFO:__main__:Running ['mkdir', '--', '/tmp/tmp0EnRmD/mnt.1']
1844
2016-07-14 21:49:33,564.564 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1845
2016-07-14 21:49:33,564.564 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1846
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1847
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1848
2016-07-14 21:49:33,576.576 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1849
2016-07-14 21:49:33,576.576 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1850
2016-07-14 21:49:33,588.588 INFO:__main__:Pre-mount connections: [43, 47, 48]
1851
2016-07-14 21:49:33,589.589 INFO:__main__:run args=['./bin/ceph-fuse', '-f', '--name', 'client.1', '/tmp/tmp0EnRmD/mnt.1']
1852
2016-07-14 21:49:33,589.589 INFO:__main__:Running ['./bin/ceph-fuse', '-f', '--name', 'client.1', '/tmp/tmp0EnRmD/mnt.1']
1853
2016-07-14 21:49:33,596.596 INFO:__main__:Mounting client.1 with pid 15840
1854
2016-07-14 21:49:33,596.596 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1855
2016-07-14 21:49:33,596.596 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1856
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1857
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1858
2016-07-14 21:49:33,612.612 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1859
2016-07-14 21:49:33,612.612 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1860
2016-07-14 21:49:34,626.626 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1861
2016-07-14 21:49:34,626.626 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
1862
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
1863
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
1864
2016-07-14 21:49:34,635.635 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
1865
2016-07-14 21:49:34,635.635 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
1866
2016-07-14 21:49:34,643.643 INFO:__main__:Post-mount connections: [43, 47, 48, 49]
1867
2016-07-14 21:49:34,644.644 INFO:__main__:run args=['touch', 'parent1/mydir/afile']
1868
2016-07-14 21:49:34,644.644 INFO:__main__:Running ['touch', 'parent1/mydir/afile']
1869
2016-07-14 21:49:34,693.693 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1870
2016-07-14 21:49:34,694.694 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1871
2016-07-14 21:49:34,706.706 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.1
1872
2016-07-14 21:49:34,706.706 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on local...
1873
2016-07-14 21:49:34,706.706 INFO:__main__:run args=['sudo', 'fusermount', '-u', '/tmp/tmp0EnRmD/mnt.1']
1874
2016-07-14 21:49:34,706.706 INFO:__main__:Running ['fusermount', '-u', '/tmp/tmp0EnRmD/mnt.1']
1875
2016-07-14 21:49:34,728.728 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1876
2016-07-14 21:49:34,729.729 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1877
2016-07-14 21:49:34,742.742 INFO:teuthology.orchestra.run:waiting for 900
1878
2016-07-14 21:49:40,745.745 INFO:__main__:run args=['rmdir', '--', '/tmp/tmp0EnRmD/mnt.1']
1879
2016-07-14 21:49:40,745.745 INFO:__main__:Running ['rmdir', '--', '/tmp/tmp0EnRmD/mnt.1']
1880
2016-07-14 21:49:40,752.752 INFO:__main__:run args=['ls', 'parent1/mydir']
1881
2016-07-14 21:49:40,752.752 INFO:__main__:Running ['ls', 'parent1/mydir']
1882
2016-07-14 21:49:40,761.761 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1883
2016-07-14 21:49:40,762.762 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1884
2016-07-14 21:49:40,769.769 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.0
1885
2016-07-14 21:49:40,769.769 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on local...
1886
2016-07-14 21:49:40,769.769 INFO:__main__:run args=['sudo', 'fusermount', '-u', '/tmp/tmp0EnRmD/mnt.0']
1887
2016-07-14 21:49:40,769.769 INFO:__main__:Running ['fusermount', '-u', '/tmp/tmp0EnRmD/mnt.0']
1888
2016-07-14 21:49:40,801.801 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1889
2016-07-14 21:49:40,801.801 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1890
2016-07-14 21:49:40,816.816 INFO:__main__:kill 
1891
2016-07-14 21:49:40,816.816 INFO:__main__:kill: killing pid 15797 (['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0'])
1892
ceph-fuse[15797]: starting ceph client
1893
2016-07-14 21:49:32.455607 7fa938e2cf40 -1 WARNING: the following dangerous and experimental features are enabled: *
1894
2016-07-14 21:49:32.455746 7fa938e2cf40 -1 WARNING: the following dangerous and experimental features are enabled: *
1895
2016-07-14 21:49:32.457752 7fa938e2cf40 -1 WARNING: the following dangerous and experimental features are enabled: *
1896
2016-07-14 21:49:32.458410 7fa938e2cf40 -1 init, newargv = 0x9b09080 newargc=11
1897
ceph-fuse[15797]: starting fuse
1898
ceph-fuse[15797]: fuse finished with error 0 and tester_r 0
1899
2016-07-14 21:49:40,818.818 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.0']
1900
2016-07-14 21:49:40,818.818 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.0']
1901
2016-07-14 21:49:40,829.829 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1902
2016-07-14 21:49:40,829.829 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1903
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.1': No such file or directory
1904
2016-07-14 21:49:40,837.837 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.1
1905
2016-07-14 21:49:40,837.837 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.1']
1906
2016-07-14 21:49:40,838.838 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.1']
1907
2016-07-14 21:49:40,850.850 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
1908
2016-07-14 21:49:40,850.850 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
1909
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.2': No such file or directory
1910
2016-07-14 21:49:40,861.861 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.2
1911
2016-07-14 21:49:40,861.861 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.2']
1912
2016-07-14 21:49:40,862.862 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.2']
1913
2016-07-14 21:49:40,869.869 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
1914
2016-07-14 21:49:40,869.869 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
1915
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.3': No such file or directory
1916
2016-07-14 21:49:40,881.881 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.3
1917
2016-07-14 21:49:40,881.881 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.3']
1918
2016-07-14 21:49:40,881.881 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.3']
1919
2016-07-14 21:49:40,889.889 INFO:__main__:test_15303 (tasks.cephfs.test_volume_client.TestVolumeClient) ... ok
1920
2016-07-14 21:49:40,890.890 INFO:__main__:Stopped test: test_15303 (tasks.cephfs.test_volume_client.TestVolumeClient) in 34.690646s
1921
2016-07-14 21:49:40,891.891 INFO:__main__:Starting test: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)
1922
2016-07-14 21:49:40,891.891 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1923
2016-07-14 21:49:40,892.892 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
1924
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.0': No such file or directory
1925
2016-07-14 21:49:40,901.901 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.0
1926
2016-07-14 21:49:40,901.901 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1927
2016-07-14 21:49:40,901.901 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
1928
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.1': No such file or directory
1929
2016-07-14 21:49:40,916.916 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.1
1930
2016-07-14 21:49:40,916.916 INFO:__main__:run args=['ps', '-u0']
1931
2016-07-14 21:49:40,917.917 INFO:__main__:Running ['ps', '-u0']
1932
2016-07-14 21:49:40,947.947 INFO:__main__:No match for mds a: PID TTY          TIME CMD
1933
    1 ?        00:00:02 systemd
1934
    2 ?        00:00:00 kthreadd
1935
    3 ?        00:00:00 ksoftirqd/0
1936
    5 ?        00:00:00 kworker/0:0H
1937
    7 ?        00:00:36 rcu_sched
1938
    8 ?        00:00:00 rcu_bh
1939
    9 ?        00:00:15 rcuos/0
1940
   10 ?        00:00:00 rcuob/0
1941
   11 ?        00:00:00 migration/0
1942
   12 ?        00:00:00 watchdog/0
1943
   13 ?        00:00:00 watchdog/1
1944
   14 ?        00:00:00 migration/1
1945
   15 ?        00:00:00 ksoftirqd/1
1946
   17 ?        00:00:00 kworker/1:0H
1947
   18 ?        00:00:06 rcuos/1
1948
   19 ?        00:00:00 rcuob/1
1949
   20 ?        00:00:00 watchdog/2
1950
   21 ?        00:00:00 migration/2
1951
   22 ?        00:00:00 ksoftirqd/2
1952
   24 ?        00:00:00 kworker/2:0H
1953
   25 ?        00:00:14 rcuos/2
1954
   26 ?        00:00:00 rcuob/2
1955
   27 ?        00:00:00 watchdog/3
1956
   28 ?        00:00:00 migration/3
1957
   29 ?        00:00:00 ksoftirqd/3
1958
   31 ?        00:00:00 kworker/3:0H
1959
   32 ?        00:00:05 rcuos/3
1960
   33 ?        00:00:00 rcuob/3
1961
   34 ?        00:00:00 kdevtmpfs
1962
   35 ?        00:00:00 netns
1963
   36 ?        00:00:00 writeback
1964
   37 ?        00:00:00 ksmd
1965
   38 ?        00:00:00 khugepaged
1966
   39 ?        00:00:00 crypto
1967
   40 ?        00:00:00 kintegrityd
1968
   41 ?        00:00:00 bioset
1969
   42 ?        00:00:00 kblockd
1970
   43 ?        00:00:00 ata_sff
1971
   44 ?        00:00:00 md
1972
   45 ?        00:00:00 devfreq_wq
1973
   51 ?        00:00:13 kswapd0
1974
   52 ?        00:00:00 vmstat
1975
   93 ?        00:00:00 kthrotld
1976
   94 ?        00:00:00 acpi_thermal_pm
1977
   95 ?        00:00:00 scsi_eh_0
1978
   96 ?        00:00:00 scsi_tmf_0
1979
   97 ?        00:00:00 scsi_eh_1
1980
   98 ?        00:00:00 scsi_tmf_1
1981
   99 ?        00:00:00 scsi_eh_2
1982
  100 ?        00:00:00 scsi_tmf_2
1983
  104 ?        00:00:00 kpsmoused
1984
  106 ?        00:00:00 dm_bufio_cache
1985
  107 ?        00:00:00 ipv6_addrconf
1986
  138 ?        00:00:00 deferwq
1987
  140 ?        00:00:00 bioset
1988
  178 ?        00:00:00 kauditd
1989
  389 ?        00:00:01 kworker/0:1H
1990
  399 ?        00:00:00 kworker/1:1H
1991
  400 ?        00:00:00 kworker/3:1H
1992
  420 ?        00:00:00 rtsx_pci_sdmmc_
1993
  425 ?        00:00:00 kworker/2:1H
1994
  613 ?        00:00:00 kdmflush
1995
  621 ?        00:00:00 bioset
1996
  622 ?        00:00:00 kcryptd_io
1997
  623 ?        00:00:00 kcryptd
1998
  624 ?        00:00:08 dmcrypt_write
1999
  625 ?        00:00:00 bioset
2000
  688 ?        00:00:00 kdmflush
2001
  691 ?        00:00:00 bioset
2002
  700 ?        00:00:00 kdmflush
2003
  701 ?        00:00:00 bioset
2004
  724 ?        00:00:00 jbd2/dm-1-8
2005
  725 ?        00:00:00 ext4-rsv-conver
2006
  823 ?        00:00:02 systemd-journal
2007
  861 ?        00:00:00 rpciod
2008
  880 ?        00:00:00 systemd-udevd
2009
  910 ?        00:00:00 ktpacpid
2010
  918 ?        00:00:00 irq/47-mei_me
2011
  949 ?        00:00:00 kmemstick
2012
  968 ?        00:00:00 cfg80211
2013
  984 ?        00:00:07 irq/50-iwlwifi
2014
  999 ?        00:00:00 kworker/u17:0
2015
 1001 ?        00:00:00 hci0
2016
 1002 ?        00:00:00 hci0
2017
 1007 ?        00:00:00 kworker/u17:2
2018
 1015 ?        00:00:00 kdmflush
2019
 1016 ?        00:00:00 bioset
2020
 1018 ?        00:00:00 kvm-irqfd-clean
2021
 1071 ?        00:00:00 jbd2/sda1-8
2022
 1072 ?        00:00:00 ext4-rsv-conver
2023
 1077 ?        00:00:06 jbd2/dm-3-8
2024
 1078 ?        00:00:00 ext4-rsv-conver
2025
 1105 ?        00:00:00 auditd
2026
 1119 ?        00:00:00 audispd
2027
 1123 ?        00:00:00 sedispatch
2028
 1133 ?        00:00:00 bluetoothd
2029
 1137 ?        00:00:00 mcelog
2030
 1139 ?        00:00:39 rngd
2031
 1146 ?        00:00:00 gssproxy
2032
 1154 ?        00:00:00 ModemManager
2033
 1158 ?        00:00:04 udisksd
2034
 1159 ?        00:00:00 firewalld
2035
 1169 ?        00:00:00 systemd-logind
2036
 1173 ?        00:00:00 accounts-daemon
2037
 1177 ?        00:00:00 alsactl
2038
 1249 ?        00:00:00 abrtd
2039
 1262 ?        00:00:00 iprt-VBoxWQueue
2040
 1268 ?        00:00:00 iprt-VBoxTscThr
2041
 1281 ?        00:00:00 abrt-dump-journ
2042
 1286 ?        00:00:00 abrt-dump-journ
2043
 1324 ?        00:00:12 NetworkManager
2044
 1376 ?        00:00:00 libvirtd
2045
 1393 ?        00:00:00 crond
2046
 1396 ?        00:00:00 atd
2047
 1398 ?        00:00:00 gdm
2048
 1463 ?        00:00:01 wpa_supplicant
2049
 1509 ?        00:00:00 gdm-session-wor
2050
 1582 ?        00:00:00 upowerd
2051
 1680 ?        00:00:10 packagekitd
2052
 1839 ?        00:00:00 dhclient
2053
 1903 ?        00:00:00 gdm-session-wor
2054
 1938 tty2     00:00:00 xf86-video-inte
2055
 2067 ?        00:00:00 krfcommd
2056
 2206 ?        00:00:00 cupsd
2057
 2235 ?        00:00:00 fwupd
2058
 2448 ?        00:00:00 dhclient
2059
 2633 ?        00:00:00 kworker/0:1
2060
 3387 ?        00:00:00 kworker/3:1
2061
 3927 ?        00:00:04 kworker/u16:0
2062
 5168 ?        00:00:00 kworker/1:3
2063
13278 ?        00:00:00 kworker/u16:5
2064
13551 ?        00:00:00 kworker/u16:1
2065
13663 ?        00:00:00 kworker/1:1
2066
13668 ?        00:00:00 kworker/3:0
2067
13681 ?        00:00:00 kworker/2:2
2068
13694 ?        00:00:00 kworker/0:3
2069
13724 ?        00:00:00 kworker/u16:4
2070
13765 ?        00:00:00 kworker/1:2
2071
13781 ?        00:00:00 kworker/2:0
2072
13794 ?        00:00:00 kworker/3:3
2073
13799 ?        00:00:00 kworker/0:2
2074
14835 pts/1    00:00:00 sudo
2075
14848 pts/1    00:00:00 su
2076
14853 pts/1    00:00:00 bash
2077
14902 ?        00:00:00 kworker/2:3
2078
14903 pts/1    00:00:00 python
2079
15764 ?        00:00:00 fprintd
2080
15897 pts/1    00:00:00 ps
2081
16120 ?        00:00:00 kworker/2:1
2082
18809 ?        00:00:00 ceph-msgr
2083
18811 ?        00:00:00 rbd
2084
28322 ?        00:00:00 systemd
2085
28332 ?        00:00:00 (sd-pam)
2086
28799 ?        00:00:10 kworker/u16:2
2087
29866 ?        00:00:00 kworker/3:2
2088
30529 ?        00:00:00 dio/dm-3
2089
31742 ?        00:00:00 kworker/0:0
2090
32711 ?        00:00:00 kworker/1:0
2091
2016-07-14 21:49:40,948.948 ERROR:__main__:tried to stop a non-running daemon
2092
2016-07-14 21:49:40,949.949 INFO:__main__:run args=['./bin/ceph', 'fs', 'ls', '--format=json-pretty']
2093
2016-07-14 21:49:40,950.950 INFO:__main__:Running ['./bin/ceph', 'fs', 'ls', '--format=json-pretty']
2094
2016-07-14 21:49:41,133.133 INFO:__main__:run args=['./bin/ceph', 'fs', 'set', u'cephfs', 'cluster_down', 'true']
2095
2016-07-14 21:49:41,133.133 INFO:__main__:Running ['./bin/ceph', 'fs', 'set', u'cephfs', 'cluster_down', 'true']
2096
2016-07-14 21:49:41,578.578 INFO:__main__:run args=['./bin/ceph', 'fs', 'get', u'cephfs', '--format=json-pretty']
2097
2016-07-14 21:49:41,578.578 INFO:__main__:Running ['./bin/ceph', 'fs', 'get', u'cephfs', '--format=json-pretty']
2098
2016-07-14 21:49:41,770.770 INFO:__main__:run args=['./bin/ceph', 'mds', 'fail', '4123']
2099
2016-07-14 21:49:41,770.770 INFO:__main__:Running ['./bin/ceph', 'mds', 'fail', '4123']
2100
2016-07-14 21:49:42,617.617 INFO:__main__:run args=['./bin/ceph', 'fs', 'rm', u'cephfs', '--yes-i-really-mean-it']
2101
2016-07-14 21:49:42,617.617 INFO:__main__:Running ['./bin/ceph', 'fs', 'rm', u'cephfs', '--yes-i-really-mean-it']
2102
2016-07-14 21:49:43,655.655 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_metadata', u'cephfs_metadata', '--yes-i-really-really-mean-it']
2103
2016-07-14 21:49:43,655.655 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_metadata', u'cephfs_metadata', '--yes-i-really-really-mean-it']
2104
2016-07-14 21:49:44,708.708 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_data', u'cephfs_data', '--yes-i-really-really-mean-it']
2105
2016-07-14 21:49:44,708.708 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'delete', u'cephfs_data', u'cephfs_data', '--yes-i-really-really-mean-it']
2106
2016-07-14 21:49:45,790.790 INFO:__main__:run args=['./bin/ceph', 'daemon', 'osd.0', 'config', 'get', 'osd_mon_report_interval_max']
2107
2016-07-14 21:49:45,790.790 INFO:__main__:Running ['./bin/ceph', 'daemon', 'osd.0', 'config', 'get', 'osd_mon_report_interval_max']
2108
2016-07-14 21:49:45,890.890 INFO:tasks.cephfs.filesystem:_json_asok output: {
2109
    "osd_mon_report_interval_max": "5"
2110
}
2111

    
2112
2016-07-14 21:49:45,891.891 INFO:__main__:run args=['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
2113
2016-07-14 21:49:45,891.891 INFO:__main__:Running ['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
2114
2016-07-14 21:49:46,093.093 INFO:__main__:run args=['./bin/ceph', 'osd', 'blacklist', 'clear']
2115
2016-07-14 21:49:46,093.093 INFO:__main__:Running ['./bin/ceph', 'osd', 'blacklist', 'clear']
2116
2016-07-14 21:49:46,835.835 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.0', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
2117
2016-07-14 21:49:46,835.835 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.0', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
2118
2016-07-14 21:49:47,034.034 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.1', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
2119
2016-07-14 21:49:47,034.034 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.1', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
2120
2016-07-14 21:49:47,255.255 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.2', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
2121
2016-07-14 21:49:47,255.255 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.2', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
2122
2016-07-14 21:49:47,445.445 INFO:__main__:run args=['./bin/ceph', 'auth', 'caps', 'client.3', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
2123
2016-07-14 21:49:47,446.446 INFO:__main__:Running ['./bin/ceph', 'auth', 'caps', 'client.3', 'mds', 'allow', 'mon', 'allow r', 'osd', 'allow rw pool=cephfs_data']
2124
2016-07-14 21:49:47,665.665 INFO:tasks.cephfs.cephfs_test_case:['0', '1', '2', '3']
2125
2016-07-14 21:49:47,665.665 INFO:__main__:run args=['./bin/ceph', 'auth', 'list', '--format=json-pretty']
2126
2016-07-14 21:49:47,666.666 INFO:__main__:Running ['./bin/ceph', 'auth', 'list', '--format=json-pretty']
2127
2016-07-14 21:49:47,855.855 INFO:tasks.cephfs.filesystem:Creating filesystem 'cephfs'
2128
2016-07-14 21:49:47,855.855 INFO:__main__:run args=['./bin/ceph', 'daemon', 'mon.a', 'config', 'get', 'mon_pg_warn_min_per_osd']
2129
2016-07-14 21:49:47,855.855 INFO:__main__:Running ['./bin/ceph', 'daemon', 'mon.a', 'config', 'get', 'mon_pg_warn_min_per_osd']
2130
2016-07-14 21:49:47,957.957 INFO:tasks.cephfs.filesystem:_json_asok output: {
2131
    "mon_pg_warn_min_per_osd": "3"
2132
}
2133

    
2134
2016-07-14 21:49:47,958.958 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_metadata', '9']
2135
2016-07-14 21:49:47,958.958 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_metadata', '9']
2136
2016-07-14 21:49:48,860.860 INFO:__main__:run args=['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_data', '9']
2137
2016-07-14 21:49:48,860.860 INFO:__main__:Running ['./bin/ceph', 'osd', 'pool', 'create', 'cephfs_data', '9']
2138
2016-07-14 21:49:49,917.917 INFO:__main__:run args=['./bin/ceph', 'fs', 'new', 'cephfs', 'cephfs_metadata', 'cephfs_data']
2139
2016-07-14 21:49:49,918.918 INFO:__main__:Running ['./bin/ceph', 'fs', 'new', 'cephfs', 'cephfs_metadata', 'cephfs_data']
2140
2016-07-14 21:49:50,479.479 INFO:__main__:run args=['ps', '-u0']
2141
2016-07-14 21:49:50,479.479 INFO:__main__:Running ['ps', '-u0']
2142
2016-07-14 21:49:50,496.496 INFO:__main__:No match for mds a: PID TTY          TIME CMD
2143
    1 ?        00:00:02 systemd
2144
    2 ?        00:00:00 kthreadd
2145
    3 ?        00:00:00 ksoftirqd/0
2146
    5 ?        00:00:00 kworker/0:0H
2147
    7 ?        00:00:36 rcu_sched
2148
    8 ?        00:00:00 rcu_bh
2149
    9 ?        00:00:15 rcuos/0
2150
   10 ?        00:00:00 rcuob/0
2151
   11 ?        00:00:00 migration/0
2152
   12 ?        00:00:00 watchdog/0
2153
   13 ?        00:00:00 watchdog/1
2154
   14 ?        00:00:00 migration/1
2155
   15 ?        00:00:00 ksoftirqd/1
2156
   17 ?        00:00:00 kworker/1:0H
2157
   18 ?        00:00:06 rcuos/1
2158
   19 ?        00:00:00 rcuob/1
2159
   20 ?        00:00:00 watchdog/2
2160
   21 ?        00:00:00 migration/2
2161
   22 ?        00:00:00 ksoftirqd/2
2162
   24 ?        00:00:00 kworker/2:0H
2163
   25 ?        00:00:14 rcuos/2
2164
   26 ?        00:00:00 rcuob/2
2165
   27 ?        00:00:00 watchdog/3
2166
   28 ?        00:00:00 migration/3
2167
   29 ?        00:00:00 ksoftirqd/3
2168
   31 ?        00:00:00 kworker/3:0H
2169
   32 ?        00:00:05 rcuos/3
2170
   33 ?        00:00:00 rcuob/3
2171
   34 ?        00:00:00 kdevtmpfs
2172
   35 ?        00:00:00 netns
2173
   36 ?        00:00:00 writeback
2174
   37 ?        00:00:00 ksmd
2175
   38 ?        00:00:00 khugepaged
2176
   39 ?        00:00:00 crypto
2177
   40 ?        00:00:00 kintegrityd
2178
   41 ?        00:00:00 bioset
2179
   42 ?        00:00:00 kblockd
2180
   43 ?        00:00:00 ata_sff
2181
   44 ?        00:00:00 md
2182
   45 ?        00:00:00 devfreq_wq
2183
   51 ?        00:00:13 kswapd0
2184
   52 ?        00:00:00 vmstat
2185
   93 ?        00:00:00 kthrotld
2186
   94 ?        00:00:00 acpi_thermal_pm
2187
   95 ?        00:00:00 scsi_eh_0
2188
   96 ?        00:00:00 scsi_tmf_0
2189
   97 ?        00:00:00 scsi_eh_1
2190
   98 ?        00:00:00 scsi_tmf_1
2191
   99 ?        00:00:00 scsi_eh_2
2192
  100 ?        00:00:00 scsi_tmf_2
2193
  104 ?        00:00:00 kpsmoused
2194
  106 ?        00:00:00 dm_bufio_cache
2195
  107 ?        00:00:00 ipv6_addrconf
2196
  138 ?        00:00:00 deferwq
2197
  140 ?        00:00:00 bioset
2198
  178 ?        00:00:00 kauditd
2199
  389 ?        00:00:01 kworker/0:1H
2200
  399 ?        00:00:00 kworker/1:1H
2201
  400 ?        00:00:00 kworker/3:1H
2202
  420 ?        00:00:00 rtsx_pci_sdmmc_
2203
  425 ?        00:00:00 kworker/2:1H
2204
  613 ?        00:00:00 kdmflush
2205
  621 ?        00:00:00 bioset
2206
  622 ?        00:00:00 kcryptd_io
2207
  623 ?        00:00:00 kcryptd
2208
  624 ?        00:00:08 dmcrypt_write
2209
  625 ?        00:00:00 bioset
2210
  688 ?        00:00:00 kdmflush
2211
  691 ?        00:00:00 bioset
2212
  700 ?        00:00:00 kdmflush
2213
  701 ?        00:00:00 bioset
2214
  724 ?        00:00:00 jbd2/dm-1-8
2215
  725 ?        00:00:00 ext4-rsv-conver
2216
  823 ?        00:00:02 systemd-journal
2217
  861 ?        00:00:00 rpciod
2218
  880 ?        00:00:00 systemd-udevd
2219
  910 ?        00:00:00 ktpacpid
2220
  918 ?        00:00:00 irq/47-mei_me
2221
  949 ?        00:00:00 kmemstick
2222
  968 ?        00:00:00 cfg80211
2223
  984 ?        00:00:07 irq/50-iwlwifi
2224
  999 ?        00:00:00 kworker/u17:0
2225
 1001 ?        00:00:00 hci0
2226
 1002 ?        00:00:00 hci0
2227
 1007 ?        00:00:00 kworker/u17:2
2228
 1015 ?        00:00:00 kdmflush
2229
 1016 ?        00:00:00 bioset
2230
 1018 ?        00:00:00 kvm-irqfd-clean
2231
 1071 ?        00:00:00 jbd2/sda1-8
2232
 1072 ?        00:00:00 ext4-rsv-conver
2233
 1077 ?        00:00:06 jbd2/dm-3-8
2234
 1078 ?        00:00:00 ext4-rsv-conver
2235
 1105 ?        00:00:00 auditd
2236
 1119 ?        00:00:00 audispd
2237
 1123 ?        00:00:00 sedispatch
2238
 1133 ?        00:00:00 bluetoothd
2239
 1137 ?        00:00:00 mcelog
2240
 1139 ?        00:00:39 rngd
2241
 1146 ?        00:00:00 gssproxy
2242
 1154 ?        00:00:00 ModemManager
2243
 1158 ?        00:00:04 udisksd
2244
 1159 ?        00:00:00 firewalld
2245
 1169 ?        00:00:00 systemd-logind
2246
 1173 ?        00:00:00 accounts-daemon
2247
 1177 ?        00:00:00 alsactl
2248
 1249 ?        00:00:00 abrtd
2249
 1262 ?        00:00:00 iprt-VBoxWQueue
2250
 1268 ?        00:00:00 iprt-VBoxTscThr
2251
 1281 ?        00:00:00 abrt-dump-journ
2252
 1286 ?        00:00:00 abrt-dump-journ
2253
 1324 ?        00:00:12 NetworkManager
2254
 1376 ?        00:00:00 libvirtd
2255
 1393 ?        00:00:00 crond
2256
 1396 ?        00:00:00 atd
2257
 1398 ?        00:00:00 gdm
2258
 1463 ?        00:00:01 wpa_supplicant
2259
 1509 ?        00:00:00 gdm-session-wor
2260
 1582 ?        00:00:00 upowerd
2261
 1680 ?        00:00:10 packagekitd
2262
 1839 ?        00:00:00 dhclient
2263
 1903 ?        00:00:00 gdm-session-wor
2264
 1938 tty2     00:00:00 xf86-video-inte
2265
 2067 ?        00:00:00 krfcommd
2266
 2206 ?        00:00:00 cupsd
2267
 2235 ?        00:00:00 fwupd
2268
 2448 ?        00:00:00 dhclient
2269
 2633 ?        00:00:00 kworker/0:1
2270
 3387 ?        00:00:00 kworker/3:1
2271
 3927 ?        00:00:04 kworker/u16:0
2272
 5168 ?        00:00:00 kworker/1:3
2273
13278 ?        00:00:00 kworker/u16:5
2274
13551 ?        00:00:00 kworker/u16:1
2275
13663 ?        00:00:00 kworker/1:1
2276
13668 ?        00:00:00 kworker/3:0
2277
13681 ?        00:00:00 kworker/2:2
2278
13694 ?        00:00:00 kworker/0:3
2279
13724 ?        00:00:00 kworker/u16:4
2280
13765 ?        00:00:00 kworker/1:2
2281
13781 ?        00:00:00 kworker/2:0
2282
13794 ?        00:00:00 kworker/3:3
2283
13799 ?        00:00:00 kworker/0:2
2284
14835 pts/1    00:00:00 sudo
2285
14848 pts/1    00:00:00 su
2286
14853 pts/1    00:00:00 bash
2287
14902 ?        00:00:00 kworker/2:3
2288
14903 pts/1    00:00:00 python
2289
15764 ?        00:00:00 fprintd
2290
16120 ?        00:00:00 kworker/2:1
2291
16419 pts/1    00:00:00 ps
2292
18809 ?        00:00:00 ceph-msgr
2293
18811 ?        00:00:00 rbd
2294
28322 ?        00:00:00 systemd
2295
28332 ?        00:00:00 (sd-pam)
2296
28799 ?        00:00:10 kworker/u16:2
2297
29866 ?        00:00:00 kworker/3:2
2298
30529 ?        00:00:00 dio/dm-3
2299
31742 ?        00:00:00 kworker/0:0
2300
32711 ?        00:00:00 kworker/1:0
2301
2016-07-14 21:49:50,497.497 INFO:__main__:run args=['./bin/./ceph-mds', '-i', 'a']
2302
2016-07-14 21:49:50,497.497 INFO:__main__:Running ['./bin/./ceph-mds', '-i', 'a']
2303
2016-07-14 21:49:50,532.532 INFO:__main__:run args=['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
2304
2016-07-14 21:49:50,533.533 INFO:__main__:Running ['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
2305
2016-07-14 21:49:50,736.736 INFO:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {u'session_autoclose': 300, u'up': {u'mds_0': 4147}, u'last_failure_osd_epoch': 0, u'in': [0], u'last_failure': 0, u'max_file_size': 1099511627776, u'explicitly_allowed_features': 0, u'damaged': [], u'tableserver': 0, u'metadata_pool': 5, u'failed': [], u'epoch': 15, u'flags': 0, u'max_mds': 1, u'compat': {u'compat': {}, u'ro_compat': {}, u'incompat': {u'feature_8': u'file layout v2', u'feature_2': u'client writeable ranges', u'feature_3': u'default file layouts on dirs', u'feature_1': u'base v0.20', u'feature_6': u'dirfrag is stored in omap', u'feature_4': u'dir inode in separate object', u'feature_5': u'mds uses versioned encoding'}}, u'data_pools': [6], u'info': {u'gid_4147': {u'standby_for_rank': -1, u'export_targets': [], u'name': u'a', u'incarnation': 15, u'standby_replay': False, u'state_seq': 1, u'standby_for_fscid': -1, u'state': u'up:creating', u'gid': 4147, u'features': 576460752032890879, u'rank': 0, u'standby_for_name': u'', u'addr': u'10.70.1.141:6812/15180'}}, u'fs_name': u'cephfs', u'created': u'2016-07-14 21:49:50.110782', u'enabled': True, u'modified': u'2016-07-14 21:49:50.110782', u'session_timeout': 60, u'stopped': [], u'ever_allowed_features': 0, u'root': 0}
2306
2016-07-14 21:49:50,736.736 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_4147:up:creating
2307
2016-07-14 21:49:51,736.736 INFO:__main__:run args=['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
2308
2016-07-14 21:49:51,737.737 INFO:__main__:Running ['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
2309
2016-07-14 21:49:51,921.921 INFO:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {u'session_autoclose': 300, u'up': {u'mds_0': 4147}, u'last_failure_osd_epoch': 0, u'in': [0], u'last_failure': 0, u'max_file_size': 1099511627776, u'explicitly_allowed_features': 0, u'damaged': [], u'tableserver': 0, u'metadata_pool': 5, u'failed': [], u'epoch': 15, u'flags': 0, u'max_mds': 1, u'compat': {u'compat': {}, u'ro_compat': {}, u'incompat': {u'feature_8': u'file layout v2', u'feature_2': u'client writeable ranges', u'feature_3': u'default file layouts on dirs', u'feature_1': u'base v0.20', u'feature_6': u'dirfrag is stored in omap', u'feature_4': u'dir inode in separate object', u'feature_5': u'mds uses versioned encoding'}}, u'data_pools': [6], u'info': {u'gid_4147': {u'standby_for_rank': -1, u'export_targets': [], u'name': u'a', u'incarnation': 15, u'standby_replay': False, u'state_seq': 1, u'standby_for_fscid': -1, u'state': u'up:creating', u'gid': 4147, u'features': 576460752032890879, u'rank': 0, u'standby_for_name': u'', u'addr': u'10.70.1.141:6812/15180'}}, u'fs_name': u'cephfs', u'created': u'2016-07-14 21:49:50.110782', u'enabled': True, u'modified': u'2016-07-14 21:49:50.110782', u'session_timeout': 60, u'stopped': [], u'ever_allowed_features': 0, u'root': 0}
2310
2016-07-14 21:49:51,921.921 WARNING:tasks.cephfs.filesystem:Unhealthy mds state gid_4147:up:creating
2311
2016-07-14 21:49:52,922.922 INFO:__main__:run args=['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
2312
2016-07-14 21:49:52,922.922 INFO:__main__:Running ['./bin/ceph', 'fs', 'get', 'cephfs', '--format=json-pretty']
2313
2016-07-14 21:49:53,106.106 INFO:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {u'session_autoclose': 300, u'up': {u'mds_0': 4147}, u'last_failure_osd_epoch': 0, u'in': [0], u'last_failure': 0, u'max_file_size': 1099511627776, u'explicitly_allowed_features': 0, u'damaged': [], u'tableserver': 0, u'metadata_pool': 5, u'failed': [], u'epoch': 16, u'flags': 0, u'max_mds': 1, u'compat': {u'compat': {}, u'ro_compat': {}, u'incompat': {u'feature_8': u'file layout v2', u'feature_2': u'client writeable ranges', u'feature_3': u'default file layouts on dirs', u'feature_1': u'base v0.20', u'feature_6': u'dirfrag is stored in omap', u'feature_4': u'dir inode in separate object', u'feature_5': u'mds uses versioned encoding'}}, u'data_pools': [6], u'info': {u'gid_4147': {u'standby_for_rank': -1, u'export_targets': [], u'name': u'a', u'incarnation': 15, u'standby_replay': False, u'state_seq': 4, u'standby_for_fscid': -1, u'state': u'up:active', u'gid': 4147, u'features': 576460752032890879, u'rank': 0, u'standby_for_name': u'', u'addr': u'10.70.1.141:6812/15180'}}, u'fs_name': u'cephfs', u'created': u'2016-07-14 21:49:50.110782', u'enabled': True, u'modified': u'2016-07-14 21:49:50.110782', u'session_timeout': 60, u'stopped': [], u'ever_allowed_features': 0, u'root': 0}
2314
2016-07-14 21:49:53,106.106 INFO:tasks.cephfs.filesystem:are_daemons_healthy: 1/1
2315
2016-07-14 21:49:53,106.106 INFO:__main__:run args=['./bin/ceph', 'daemon', 'mds.a', 'status']
2316
2016-07-14 21:49:53,107.107 INFO:__main__:Running ['./bin/ceph', 'daemon', 'mds.a', 'status']
2317
2016-07-14 21:49:53,223.223 INFO:tasks.cephfs.filesystem:_json_asok output: {
2318
    "cluster_fsid": "ae66e242-d103-408a-b4cf-f160f95d365c",
2319
    "whoami": 0,
2320
    "want_state": "up:active",
2321
    "state": "up:active",
2322
    "mdsmap_epoch": 16,
2323
    "osdmap_epoch": 24,
2324
    "osdmap_epoch_barrier": 24
2325
}
2326

    
2327
2016-07-14 21:49:53,224.224 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
2328
2016-07-14 21:49:53,224.224 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
2329
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.0': No such file or directory
2330
2016-07-14 21:49:53,237.237 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.0
2331
2016-07-14 21:49:53,237.237 INFO:__main__:run args=['mkdir', '--', '/tmp/tmp0EnRmD/mnt.0']
2332
2016-07-14 21:49:53,238.238 INFO:__main__:Running ['mkdir', '--', '/tmp/tmp0EnRmD/mnt.0']
2333
2016-07-14 21:49:53,247.247 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
2334
2016-07-14 21:49:53,247.247 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
2335
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
2336
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
2337
2016-07-14 21:49:53,263.263 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
2338
2016-07-14 21:49:53,264.264 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
2339
2016-07-14 21:49:53,275.275 INFO:__main__:Pre-mount connections: [43, 47]
2340
2016-07-14 21:49:53,275.275 INFO:__main__:run args=['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0']
2341
2016-07-14 21:49:53,275.275 INFO:__main__:Running ['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0']
2342
2016-07-14 21:49:53,282.282 INFO:__main__:Mounting client.0 with pid 16513
2343
2016-07-14 21:49:53,283.283 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
2344
2016-07-14 21:49:53,283.283 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
2345
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
2346
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
2347
2016-07-14 21:49:53,297.297 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
2348
2016-07-14 21:49:53,297.297 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
2349
2016-07-14 21:49:54,312.312 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
2350
2016-07-14 21:49:54,313.313 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
2351
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
2352
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
2353
2016-07-14 21:49:54,321.321 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
2354
2016-07-14 21:49:54,321.321 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
2355
2016-07-14 21:49:54,332.332 INFO:__main__:Post-mount connections: [43, 47, 48]
2356
2016-07-14 21:49:54,333.333 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
2357
2016-07-14 21:49:54,334.334 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
2358
2016-07-14 21:49:54,343.343 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.0
2359
2016-07-14 21:49:54,343.343 INFO:__main__:run args=['sudo', 'chmod', '1777', '/tmp/tmp0EnRmD/mnt.0']
2360
2016-07-14 21:49:54,344.344 INFO:__main__:Running ['chmod', '1777', '/tmp/tmp0EnRmD/mnt.0']
2361
2016-07-14 21:49:54,357.357 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
2362
2016-07-14 21:49:54,357.357 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
2363
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.1': No such file or directory
2364
2016-07-14 21:49:54,364.364 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.1
2365
2016-07-14 21:49:54,364.364 INFO:__main__:run args=['mkdir', '--', '/tmp/tmp0EnRmD/mnt.1']
2366
2016-07-14 21:49:54,365.365 INFO:__main__:Running ['mkdir', '--', '/tmp/tmp0EnRmD/mnt.1']
2367
2016-07-14 21:49:54,372.372 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
2368
2016-07-14 21:49:54,373.373 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
2369
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
2370
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
2371
2016-07-14 21:49:54,388.388 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
2372
2016-07-14 21:49:54,388.388 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
2373
2016-07-14 21:49:54,396.396 INFO:__main__:Pre-mount connections: [43, 47, 48]
2374
2016-07-14 21:49:54,396.396 INFO:__main__:run args=['./bin/ceph-fuse', '-f', '--name', 'client.1', '/tmp/tmp0EnRmD/mnt.1']
2375
2016-07-14 21:49:54,396.396 INFO:__main__:Running ['./bin/ceph-fuse', '-f', '--name', 'client.1', '/tmp/tmp0EnRmD/mnt.1']
2376
2016-07-14 21:49:54,408.408 INFO:__main__:Mounting client.1 with pid 16555
2377
2016-07-14 21:49:54,410.410 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
2378
2016-07-14 21:49:54,411.411 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
2379
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
2380
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
2381
2016-07-14 21:49:54,426.426 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
2382
2016-07-14 21:49:54,427.427 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
2383
2016-07-14 21:49:55,440.440 INFO:__main__:run args=['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
2384
2016-07-14 21:49:55,440.440 INFO:__main__:Running ['mount', '-t', 'fusectl', '/sys/fs/fuse/connections', '/sys/fs/fuse/connections']
2385
mount: /sys/fs/fuse/connections is already mounted or /sys/fs/fuse/connections busy
2386
       /sys/fs/fuse/connections is already mounted on /sys/fs/fuse/connections
2387
2016-07-14 21:49:55,450.450 INFO:__main__:run args=['ls', '/sys/fs/fuse/connections']
2388
2016-07-14 21:49:55,452.452 INFO:__main__:Running ['ls', '/sys/fs/fuse/connections']
2389
2016-07-14 21:49:55,461.461 INFO:__main__:Post-mount connections: [43, 47, 48, 49]
2390
2016-07-14 21:49:55,461.461 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
2391
2016-07-14 21:49:55,461.461 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
2392
2016-07-14 21:49:55,476.476 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.1
2393
2016-07-14 21:49:55,476.476 INFO:__main__:run args=['sudo', 'chmod', '1777', '/tmp/tmp0EnRmD/mnt.1']
2394
2016-07-14 21:49:55,476.476 INFO:__main__:Running ['chmod', '1777', '/tmp/tmp0EnRmD/mnt.1']
2395
2016-07-14 21:49:55,487.487 INFO:__main__:Searching for existing instance mon pg warn max per osd/global
2396
2016-07-14 21:49:55,488.488 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
2397
2016-07-14 21:49:55,489.489 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
2398
2016-07-14 21:49:55,497.497 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.1
2399
2016-07-14 21:49:55,497.497 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on local...
2400
2016-07-14 21:49:55,498.498 INFO:__main__:run args=['sudo', 'fusermount', '-u', '/tmp/tmp0EnRmD/mnt.1']
2401
2016-07-14 21:49:55,498.498 INFO:__main__:Running ['fusermount', '-u', '/tmp/tmp0EnRmD/mnt.1']
2402
2016-07-14 21:49:55,524.524 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
2403
2016-07-14 21:49:55,524.524 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
2404
2016-07-14 21:49:55,534.534 INFO:teuthology.orchestra.run:waiting for 900
2405
2016-07-14 21:50:01,538.538 INFO:__main__:run args=['rmdir', '--', '/tmp/tmp0EnRmD/mnt.1']
2406
2016-07-14 21:50:01,539.539 INFO:__main__:Running ['rmdir', '--', '/tmp/tmp0EnRmD/mnt.1']
2407
2016-07-14 21:50:01,546.546 INFO:__main__:run args=['./bin/ceph', 'auth', 'get-or-create', 'client.manila', 'mds', 'allow *', 'osd', 'allow rw', 'mon', 'allow *']
2408
2016-07-14 21:50:01,546.546 INFO:__main__:Running ['./bin/ceph', 'auth', 'get-or-create', 'client.manila', 'mds', 'allow *', 'osd', 'allow rw', 'mon', 'allow *']
2409
2016-07-14 21:50:01,739.739 INFO:__main__:run args=['sudo', 'python', '-c', 'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))', '/home/rraja/git/ceph/build/client.manila.keyring']
2410
2016-07-14 21:50:01,739.739 INFO:__main__:Running ['python', '-c', 'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))', '/home/rraja/git/ceph/build/client.manila.keyring']
2411
2016-07-14 21:50:01,763.763 INFO:__main__:Searching for existing instance mon pg warn max per osd/global
2412
2016-07-14 21:50:01,763.763 INFO:__main__:Searching for existing instance keyring/client.manila
2413
2016-07-14 21:50:01,764.764 INFO:__main__:run args=['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
2414
2016-07-14 21:50:01,764.764 INFO:__main__:Running ['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
2415
2016-07-14 21:50:01,940.940 INFO:__main__:run args=['./bin/ceph', 'daemon', 'mon.a', 'config', 'get', 'mon_pg_warn_max_per_osd']
2416
2016-07-14 21:50:01,940.940 INFO:__main__:Running ['./bin/ceph', 'daemon', 'mon.a', 'config', 'get', 'mon_pg_warn_max_per_osd']
2417
2016-07-14 21:50:02,048.048 INFO:tasks.cephfs.filesystem:_json_asok output: {
2418
    "mon_pg_warn_max_per_osd": "300"
2419
}
2420

    
2421
2016-07-14 21:50:02,048.048 INFO:tasks.cephfs.test_volume_client:max_per_osd 300
2422
2016-07-14 21:50:02,048.048 INFO:tasks.cephfs.test_volume_client:osd_count 3
2423
2016-07-14 21:50:02,048.048 INFO:tasks.cephfs.test_volume_client:max_overall 900
2424
2016-07-14 21:50:02,049.049 INFO:tasks.cephfs.test_volume_client:existing_pg_count 26
2425
2016-07-14 21:50:02,049.049 INFO:tasks.cephfs.test_volume_client:expected_pg_num 87
2426
2016-07-14 21:50:02,049.049 INFO:__main__:run args=['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
2427
2016-07-14 21:50:02,049.049 INFO:__main__:Running ['./bin/ceph', 'osd', 'dump', '--format=json-pretty']
2428
2016-07-14 21:50:02,234.234 INFO:__main__:run args=['python', '-c', '\nfrom ceph_volume_client import CephFSVolumeClient, VolumePath\nimport logging\nlog = logging.getLogger("ceph_volume_client")\nlog.addHandler(logging.StreamHandler())\nlog.setLevel(logging.DEBUG)\nvc = CephFSVolumeClient("manila", "./ceph.conf", "ceph", None, None)\nvc.connect()\n\nvp = VolumePath("grpid", "volid")\nvc.create_volume(vp, 10, data_isolated=True)\n\nvc.disconnect()\n        ']
2429
2016-07-14 21:50:02,235.235 INFO:__main__:Running ['python', '-c', '\nfrom ceph_volume_client import CephFSVolumeClient, VolumePath\nimport logging\nlog = logging.getLogger("ceph_volume_client")\nlog.addHandler(logging.StreamHandler())\nlog.setLevel(logging.DEBUG)\nvc = CephFSVolumeClient("manila", "./ceph.conf", "ceph", None, None)\nvc.connect()\n\nvp = VolumePath("grpid", "volid")\nvc.create_volume(vp, 10, data_isolated=True)\n\nvc.disconnect()\n        ']
2430
src/tcmalloc.cc:283] Attempt to free invalid pointer 0x561bc63d3940 
2431
2016-07-14 21:50:02,503.503 INFO:__main__:test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient) ... ERROR
2432
2016-07-14 21:50:02,504.504 ERROR:__main__:Traceback (most recent call last):
2433
  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/test_volume_client.py", line 370, in test_data_isolated
2434
    guest_entity=guest_entity
2435
  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/test_volume_client.py", line 34, in _volume_client_python
2436
    """.format(payload=script, conf_path=client.config_path, vol_prefix=vol_prefix, ns_prefix=ns_prefix))
2437
  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/mount.py", line 134, in run_python
2438
    p.wait()
2439
  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/vstart_runner.py", line 113, in wait
2440
    raise CommandFailedError(self.args, self.exitstatus)
2441
CommandFailedError: Command failed with status -6: ['python', '-c', '\nfrom ceph_volume_client import CephFSVolumeClient, VolumePath\nimport logging\nlog = logging.getLogger("ceph_volume_client")\nlog.addHandler(logging.StreamHandler())\nlog.setLevel(logging.DEBUG)\nvc = CephFSVolumeClient("manila", "./ceph.conf", "ceph", None, None)\nvc.connect()\n\nvp = VolumePath("grpid", "volid")\nvc.create_volume(vp, 10, data_isolated=True)\n\nvc.disconnect()\n        ']
2442

    
2443
2016-07-14 21:50:02,504.504 ERROR:__main__:Error in test 'test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)', going interactive
2444
Ceph test interactive mode, use ctx to interact with the cluster, press control-D to exit...
2445
>>> 
2446
2016-07-14 21:56:08,890.890 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
2447
2016-07-14 21:56:08,890.890 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
2448
2016-07-14 21:56:08,901.901 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /tmp/tmp0EnRmD/mnt.0
2449
2016-07-14 21:56:08,901.901 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on local...
2450
2016-07-14 21:56:08,901.901 INFO:__main__:run args=['sudo', 'fusermount', '-u', '/tmp/tmp0EnRmD/mnt.0']
2451
2016-07-14 21:56:08,901.901 INFO:__main__:Running ['fusermount', '-u', '/tmp/tmp0EnRmD/mnt.0']
2452
2016-07-14 21:56:08,920.920 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
2453
2016-07-14 21:56:08,920.920 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.0']
2454
2016-07-14 21:56:08,932.932 INFO:__main__:kill 
2455
2016-07-14 21:56:08,932.932 INFO:__main__:kill: killing pid 16513 (['./bin/ceph-fuse', '-f', '--name', 'client.0', '/tmp/tmp0EnRmD/mnt.0'])
2456
ceph-fuse[16513]: starting ceph client
2457
2016-07-14 21:49:53.297548 7facfb952f40 -1 WARNING: the following dangerous and experimental features are enabled: *
2458
2016-07-14 21:49:53.297636 7facfb952f40 -1 WARNING: the following dangerous and experimental features are enabled: *
2459
2016-07-14 21:49:53.303164 7facfb952f40 -1 WARNING: the following dangerous and experimental features are enabled: *
2460
2016-07-14 21:49:53.303889 7facfb952f40 -1 init, newargv = 0xa149080 newargc=11
2461
ceph-fuse[16513]: starting fuse
2462
ceph-fuse[16513]: fuse finished with error 0 and tester_r 0
2463
2016-07-14 21:56:08,936.936 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.0']
2464
2016-07-14 21:56:08,936.936 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.0']
2465
2016-07-14 21:56:08,945.945 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
2466
2016-07-14 21:56:08,945.945 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.1']
2467
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.1': No such file or directory
2468
2016-07-14 21:56:08,956.956 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.1
2469
2016-07-14 21:56:08,956.956 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.1']
2470
2016-07-14 21:56:08,956.956 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.1']
2471
2016-07-14 21:56:08,968.968 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
2472
2016-07-14 21:56:08,969.969 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.2']
2473
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.2': No such file or directory
2474
2016-07-14 21:56:08,976.976 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.2
2475
2016-07-14 21:56:08,977.977 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.2']
2476
2016-07-14 21:56:08,977.977 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.2']
2477
2016-07-14 21:56:08,985.985 INFO:__main__:run args=['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
2478
2016-07-14 21:56:08,985.985 INFO:__main__:Running ['stat', '--file-system', '--printf=%T\n', '--', '/tmp/tmp0EnRmD/mnt.3']
2479
stat: cannot read file system information for '/tmp/tmp0EnRmD/mnt.3': No such file or directory
2480
2016-07-14 21:56:08,995.995 INFO:tasks.cephfs.fuse_mount:mount point does not exist: /tmp/tmp0EnRmD/mnt.3
2481
2016-07-14 21:56:08,995.995 INFO:__main__:run args=['rm', '-rf', '/tmp/tmp0EnRmD/mnt.3']
2482
2016-07-14 21:56:08,995.995 INFO:__main__:Running ['rm', '-rf', '/tmp/tmp0EnRmD/mnt.3']
2483
2016-07-14 21:56:09,006.006 INFO:__main__:Searching for existing instance mon pg warn max per osd/global
2484
2016-07-14 21:56:09,007.007 INFO:__main__:Stopped test: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient) in 388.115495s
2485
2016-07-14 21:56:09,007.007 INFO:__main__:
2486
2016-07-14 21:56:09,007.007 INFO:__main__:======================================================================
2487
2016-07-14 21:56:09,007.007 INFO:__main__:ERROR: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)
2488
2016-07-14 21:56:09,007.007 INFO:__main__:----------------------------------------------------------------------
2489
2016-07-14 21:56:09,007.007 INFO:__main__:Traceback (most recent call last):
2490
2016-07-14 21:56:09,007.007 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/test_volume_client.py", line 370, in test_data_isolated
2491
2016-07-14 21:56:09,007.007 INFO:__main__:    guest_entity=guest_entity
2492
2016-07-14 21:56:09,007.007 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/test_volume_client.py", line 34, in _volume_client_python
2493
2016-07-14 21:56:09,008.008 INFO:__main__:    """.format(payload=script, conf_path=client.config_path, vol_prefix=vol_prefix, ns_prefix=ns_prefix))
2494
2016-07-14 21:56:09,008.008 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/mount.py", line 134, in run_python
2495
2016-07-14 21:56:09,008.008 INFO:__main__:    p.wait()
2496
2016-07-14 21:56:09,008.008 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/vstart_runner.py", line 113, in wait
2497
2016-07-14 21:56:09,008.008 INFO:__main__:    raise CommandFailedError(self.args, self.exitstatus)
2498
2016-07-14 21:56:09,008.008 INFO:__main__:CommandFailedError: Command failed with status -6: ['python', '-c', '\nfrom ceph_volume_client import CephFSVolumeClient, VolumePath\nimport logging\nlog = logging.getLogger("ceph_volume_client")\nlog.addHandler(logging.StreamHandler())\nlog.setLevel(logging.DEBUG)\nvc = CephFSVolumeClient("manila", "./ceph.conf", "ceph", None, None)\nvc.connect()\n\nvp = VolumePath("grpid", "volid")\nvc.create_volume(vp, 10, data_isolated=True)\n\nvc.disconnect()\n        ']
2499
2016-07-14 21:56:09,008.008 INFO:__main__:
2500
2016-07-14 21:56:09,008.008 INFO:__main__:----------------------------------------------------------------------
2501
2016-07-14 21:56:09,008.008 INFO:__main__:Ran 2 tests in 422.808s
2502
2016-07-14 21:56:09,008.008 INFO:__main__:
2503
2016-07-14 21:56:09,008.008 INFO:__main__:FAILED (errors=1)
2504
2016-07-14 21:56:09,008.008 INFO:__main__:
2505
2016-07-14 21:56:09,008.008 INFO:__main__:======================================================================
2506
2016-07-14 21:56:09,008.008 INFO:__main__:ERROR: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)
2507
2016-07-14 21:56:09,009.009 INFO:__main__:----------------------------------------------------------------------
2508
2016-07-14 21:56:09,009.009 INFO:__main__:Traceback (most recent call last):
2509
2016-07-14 21:56:09,009.009 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/test_volume_client.py", line 370, in test_data_isolated
2510
2016-07-14 21:56:09,009.009 INFO:__main__:    guest_entity=guest_entity
2511
2016-07-14 21:56:09,009.009 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/test_volume_client.py", line 34, in _volume_client_python
2512
2016-07-14 21:56:09,009.009 INFO:__main__:    """.format(payload=script, conf_path=client.config_path, vol_prefix=vol_prefix, ns_prefix=ns_prefix))
2513
2016-07-14 21:56:09,009.009 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/mount.py", line 134, in run_python
2514
2016-07-14 21:56:09,009.009 INFO:__main__:    p.wait()
2515
2016-07-14 21:56:09,009.009 INFO:__main__:  File "/home/rraja/git/ceph-qa-suite/tasks/cephfs/vstart_runner.py", line 113, in wait
2516
2016-07-14 21:56:09,009.009 INFO:__main__:    raise CommandFailedError(self.args, self.exitstatus)
2517
2016-07-14 21:56:09,009.009 INFO:__main__:CommandFailedError: Command failed with status -6: ['python', '-c', '\nfrom ceph_volume_client import CephFSVolumeClient, VolumePath\nimport logging\nlog = logging.getLogger("ceph_volume_client")\nlog.addHandler(logging.StreamHandler())\nlog.setLevel(logging.DEBUG)\nvc = CephFSVolumeClient("manila", "./ceph.conf", "ceph", None, None)\nvc.connect()\n\nvp = VolumePath("grpid", "volid")\nvc.create_volume(vp, 10, data_isolated=True)\n\nvc.disconnect()\n        ']
2518
2016-07-14 21:56:09,009.009 INFO:__main__:
2519