Actions
Bug #46274
openSome unit tests leak lvs and pvs calls, insufficient patching.
Status:
New
Priority:
Normal
Assignee:
-
Target version:
-
% Done:
0%
Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
=================================== FAILURES ===================================
___________________ TestFunctionalCall.test_unicode_encoding ___________________
self = <ceph_volume.tests.test_process.TestFunctionalCall object at 0x7fad40f7b9e8>
def test_unicode_encoding(self):
> process.call(['echo', u'\xd0'])
ceph_volume/tests/test_process.py:83:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
command = ['/bin/echo', '�'], kw = {}, executable = '/bin/echo'
terminal_verbose = False, logfile_verbose = True, verbose_on_failure = True
show_command = False, command_msg = 'Running command: /bin/echo �', stdin = None
def call(command, **kw):
"""
Similar to ``subprocess.Popen`` with the following changes:
* returns stdout, stderr, and exit code (vs. just the exit code)
* logs the full contents of stderr and stdout (separately) to the file log
By default, no terminal output is given, not even the command that is going
to run.
Useful when system calls are needed to act on output, and that same output
shouldn't get displayed on the terminal.
Optionally, the command can be displayed on the terminal and the log file,
and log file output can be turned off. This is useful to prevent sensitive
output going to stderr/stdout and being captured on a log file.
:param terminal_verbose: Log command output to terminal, defaults to False, and
it is forcefully set to True if a return code is non-zero
:param logfile_verbose: Log stderr/stdout output to log file. Defaults to True
:param verbose_on_failure: On a non-zero exit status, it will forcefully set logging ON for
the terminal. Defaults to True
"""
executable = which(command.pop(0))
command.insert(0, executable)
terminal_verbose = kw.pop('terminal_verbose', False)
logfile_verbose = kw.pop('logfile_verbose', True)
verbose_on_failure = kw.pop('verbose_on_failure', True)
show_command = kw.pop('show_command', False)
command_msg = "Running command: %s" % ' '.join(command)
stdin = kw.pop('stdin', None)
logger.info(command_msg)
if show_command:
terminal.write(command_msg)
process = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
close_fds=True,
**kw
)
if stdin:
stdout_stream, stderr_stream = process.communicate(as_bytes(stdin))
else:
stdout_stream = process.stdout.read()
stderr_stream = process.stderr.read()
returncode = process.wait()
if not isinstance(stdout_stream, str):
> stdout_stream = stdout_stream.decode('utf-8')
E UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd0 in position 0: invalid continuation byte
ceph_volume/process.py:211: UnicodeDecodeError
------------------------------ Captured log call -------------------------------
INFO ceph_volume.process:process.py:191 Running command: /bin/echo �
________ TestFullReport.test_physical_2nd_device_gets_reported[journal] ________
self = <ceph_volume.tests.devices.lvm.test_listing.TestFullReport object at 0x7fad40bcfdd8>
type_ = 'journal'
monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7fad40bd3a58>
@pytest.mark.parametrize('type_', ['journal', 'db', 'wal'])
def test_physical_2nd_device_gets_reported(self, type_, monkeypatch):
tags = ('ceph.osd_id=0,ceph.{t}_uuid=x,ceph.type=data,'
'ceph.{t}_device=/dev/sda1').format(t=type_)
osd = api.Volume(lv_name='volume1', lv_uuid='y', lv_tags=tags,
vg_name='VolGroup', lv_path='/dev/VolGroup/lv')
monkeypatch.setattr(lvm.listing.api, 'get_lvs', lambda **kwargs:
[osd])
> result = lvm.listing.List([]).full_report()
ceph_volume/tests/devices/lvm/test_listing.py:165:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
ceph_volume/devices/lvm/listing.py:141: in full_report
return self.create_report(api.get_lvs())
ceph_volume/devices/lvm/listing.py:113: in create_report
pvs = api.get_pvs(filters={'lv_uuid': lv.lv_uuid})
ceph_volume/api/lvm.py:1466: in get_pvs
stdout, stderr, returncode = process.call(args, verbose_on_failure=False)
ceph_volume/process.py:201: in call
**kw
/usr/lib/python3.5/subprocess.py:947: in __init__
restore_signals, start_new_session)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <subprocess.Popen object at 0x7fad40bd3ba8>
args = ['pvs', '--no-heading', '--readonly', '--separator=";"', '-S', 'lv_uuid=y', ...]
executable = b'pvs', preexec_fn = None, close_fds = True, pass_fds = ()
cwd = None, env = None, startupinfo = None, creationflags = 0, shell = False
p2cread = 12, p2cwrite = 13, c2pread = 14, c2pwrite = 15, errread = 16
errwrite = 17, restore_signals = True, start_new_session = False
def _execute_child(self, args, executable, preexec_fn, close_fds,
pass_fds, cwd, env,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
restore_signals, start_new_session):
"""Execute program (POSIX version)"""
if isinstance(args, (str, bytes)):
args = [args]
else:
args = list(args)
if shell:
args = ["/bin/sh", "-c"] + args
if executable:
args[0] = executable
if executable is None:
executable = args[0]
orig_executable = executable
# For transferring possible exec failure from child to parent.
# Data format: "exception name:hex errno:description"
# Pickle is not used; it is complex and involves memory allocation.
errpipe_read, errpipe_write = os.pipe()
# errpipe_write must not be in the standard io 0, 1, or 2 fd range.
low_fds_to_close = []
while errpipe_write < 3:
low_fds_to_close.append(errpipe_write)
errpipe_write = os.dup(errpipe_write)
for low_fd in low_fds_to_close:
os.close(low_fd)
try:
try:
# We must avoid complex work that could involve
# malloc or free in the child process to avoid
# potential deadlocks, thus we do all this here.
# and pass it to fork_exec()
if env is not None:
env_list = [os.fsencode(k) + b'=' + os.fsencode(v)
for k, v in env.items()]
else:
env_list = None # Use execv instead of execve.
executable = os.fsencode(executable)
if os.path.dirname(executable):
executable_list = (executable,)
else:
# This matches the behavior of os._execvpe().
executable_list = tuple(
os.path.join(os.fsencode(dir), executable)
for dir in os.get_exec_path(env))
fds_to_keep = set(pass_fds)
fds_to_keep.add(errpipe_write)
self.pid = _posixsubprocess.fork_exec(
args, executable_list,
close_fds, sorted(fds_to_keep), cwd, env_list,
p2cread, p2cwrite, c2pread, c2pwrite,
errread, errwrite,
errpipe_read, errpipe_write,
restore_signals, start_new_session, preexec_fn)
self._child_created = True
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
# self._devnull is not always defined.
devnull_fd = getattr(self, '_devnull', None)
if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd:
os.close(p2cread)
if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd:
os.close(c2pwrite)
if errwrite != -1 and errread != -1 and errwrite != devnull_fd:
os.close(errwrite)
if devnull_fd is not None:
os.close(devnull_fd)
# Prevent a double close of these fds from __init__ on error.
self._closed_child_pipe_fds = True
# Wait for exec to fail or succeed; possibly raising an
# exception (limited in size)
errpipe_data = bytearray()
while True:
part = os.read(errpipe_read, 50000)
errpipe_data += part
if not part or len(errpipe_data) > 50000:
break
finally:
# be sure the FD is closed no matter what
os.close(errpipe_read)
if errpipe_data:
try:
os.waitpid(self.pid, 0)
except ChildProcessError:
pass
try:
exception_name, hex_errno, err_msg = (
errpipe_data.split(b':', 2))
except ValueError:
exception_name = b'SubprocessError'
hex_errno = b'0'
err_msg = (b'Bad exception data from child: ' +
repr(errpipe_data))
child_exception_type = getattr(
builtins, exception_name.decode('ascii'),
SubprocessError)
err_msg = err_msg.decode(errors="surrogatepass")
if issubclass(child_exception_type, OSError) and hex_errno:
errno_num = int(hex_errno, 16)
child_exec_never_called = (err_msg == "noexec")
if child_exec_never_called:
err_msg = ""
if errno_num != 0:
err_msg = os.strerror(errno_num)
if errno_num == errno.ENOENT:
if child_exec_never_called:
# The error must be from chdir(cwd).
err_msg += ': ' + repr(cwd)
else:
err_msg += ': ' + repr(orig_executable)
> raise child_exception_type(errno_num, err_msg)
E FileNotFoundError: [Errno 2] No such file or directory: 'pvs'
/usr/lib/python3.5/subprocess.py:1551: FileNotFoundError
----------------------------- Captured stderr call -----------------------------
--> Executable pvs not in PATH: /home/jenkins-build/build/workspace/ceph-volume-pr/src/ceph-volume/.tox/py35/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
------------------------------ Captured log call -------------------------------
WARNING ceph_volume.util.system:terminal.py:170 Executable pvs not in PATH: /home/jenkins-build/build/workspace/ceph-volume-pr/src/ceph-volume/.tox/py35/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
INFO ceph_volume.process:process.py:191 Running command: pvs --no-heading --readonly --separator=";" -S lv_uuid=y -o pv_name,pv_tags,pv_uuid,vg_name,lv_uuid
__________ TestFullReport.test_physical_2nd_device_gets_reported[db] ___________
self = <ceph_volume.tests.devices.lvm.test_listing.TestFullReport object at 0x7fad40c44da0>
type_ = 'db'
monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7fad40c44748>
@pytest.mark.parametrize('type_', ['journal', 'db', 'wal'])
def test_physical_2nd_device_gets_reported(self, type_, monkeypatch):
tags = ('ceph.osd_id=0,ceph.{t}_uuid=x,ceph.type=data,'
'ceph.{t}_device=/dev/sda1').format(t=type_)
osd = api.Volume(lv_name='volume1', lv_uuid='y', lv_tags=tags,
vg_name='VolGroup', lv_path='/dev/VolGroup/lv')
monkeypatch.setattr(lvm.listing.api, 'get_lvs', lambda **kwargs:
[osd])
> result = lvm.listing.List([]).full_report()
ceph_volume/tests/devices/lvm/test_listing.py:165:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
ceph_volume/devices/lvm/listing.py:141: in full_report
return self.create_report(api.get_lvs())
ceph_volume/devices/lvm/listing.py:113: in create_report
pvs = api.get_pvs(filters={'lv_uuid': lv.lv_uuid})
ceph_volume/api/lvm.py:1466: in get_pvs
stdout, stderr, returncode = process.call(args, verbose_on_failure=False)
ceph_volume/process.py:201: in call
**kw
/usr/lib/python3.5/subprocess.py:947: in __init__
restore_signals, start_new_session)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <subprocess.Popen object at 0x7fad40bcfef0>
args = ['pvs', '--no-heading', '--readonly', '--separator=";"', '-S', 'lv_uuid=y', ...]
executable = b'pvs', preexec_fn = None, close_fds = True, pass_fds = ()
cwd = None, env = None, startupinfo = None, creationflags = 0, shell = False
p2cread = 12, p2cwrite = 13, c2pread = 14, c2pwrite = 15, errread = 16
errwrite = 17, restore_signals = True, start_new_session = False
def _execute_child(self, args, executable, preexec_fn, close_fds,
pass_fds, cwd, env,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
restore_signals, start_new_session):
"""Execute program (POSIX version)"""
if isinstance(args, (str, bytes)):
args = [args]
else:
args = list(args)
if shell:
args = ["/bin/sh", "-c"] + args
if executable:
args[0] = executable
if executable is None:
executable = args[0]
orig_executable = executable
# For transferring possible exec failure from child to parent.
# Data format: "exception name:hex errno:description"
# Pickle is not used; it is complex and involves memory allocation.
errpipe_read, errpipe_write = os.pipe()
# errpipe_write must not be in the standard io 0, 1, or 2 fd range.
low_fds_to_close = []
while errpipe_write < 3:
low_fds_to_close.append(errpipe_write)
errpipe_write = os.dup(errpipe_write)
for low_fd in low_fds_to_close:
os.close(low_fd)
try:
try:
# We must avoid complex work that could involve
# malloc or free in the child process to avoid
# potential deadlocks, thus we do all this here.
# and pass it to fork_exec()
if env is not None:
env_list = [os.fsencode(k) + b'=' + os.fsencode(v)
for k, v in env.items()]
else:
env_list = None # Use execv instead of execve.
executable = os.fsencode(executable)
if os.path.dirname(executable):
executable_list = (executable,)
else:
# This matches the behavior of os._execvpe().
executable_list = tuple(
os.path.join(os.fsencode(dir), executable)
for dir in os.get_exec_path(env))
fds_to_keep = set(pass_fds)
fds_to_keep.add(errpipe_write)
self.pid = _posixsubprocess.fork_exec(
args, executable_list,
close_fds, sorted(fds_to_keep), cwd, env_list,
p2cread, p2cwrite, c2pread, c2pwrite,
errread, errwrite,
errpipe_read, errpipe_write,
restore_signals, start_new_session, preexec_fn)
self._child_created = True
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
# self._devnull is not always defined.
devnull_fd = getattr(self, '_devnull', None)
if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd:
os.close(p2cread)
if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd:
os.close(c2pwrite)
if errwrite != -1 and errread != -1 and errwrite != devnull_fd:
os.close(errwrite)
if devnull_fd is not None:
os.close(devnull_fd)
# Prevent a double close of these fds from __init__ on error.
self._closed_child_pipe_fds = True
# Wait for exec to fail or succeed; possibly raising an
# exception (limited in size)
errpipe_data = bytearray()
while True:
part = os.read(errpipe_read, 50000)
errpipe_data += part
if not part or len(errpipe_data) > 50000:
break
finally:
# be sure the FD is closed no matter what
os.close(errpipe_read)
if errpipe_data:
try:
os.waitpid(self.pid, 0)
except ChildProcessError:
pass
try:
exception_name, hex_errno, err_msg = (
errpipe_data.split(b':', 2))
except ValueError:
exception_name = b'SubprocessError'
hex_errno = b'0'
err_msg = (b'Bad exception data from child: ' +
repr(errpipe_data))
child_exception_type = getattr(
builtins, exception_name.decode('ascii'),
SubprocessError)
err_msg = err_msg.decode(errors="surrogatepass")
if issubclass(child_exception_type, OSError) and hex_errno:
errno_num = int(hex_errno, 16)
child_exec_never_called = (err_msg == "noexec")
if child_exec_never_called:
err_msg = ""
if errno_num != 0:
err_msg = os.strerror(errno_num)
if errno_num == errno.ENOENT:
if child_exec_never_called:
# The error must be from chdir(cwd).
err_msg += ': ' + repr(cwd)
else:
err_msg += ': ' + repr(orig_executable)
> raise child_exception_type(errno_num, err_msg)
E FileNotFoundError: [Errno 2] No such file or directory: 'pvs'
/usr/lib/python3.5/subprocess.py:1551: FileNotFoundError
----------------------------- Captured stderr call -----------------------------
--> Executable pvs not in PATH: /home/jenkins-build/build/workspace/ceph-volume-pr/src/ceph-volume/.tox/py35/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
------------------------------ Captured log call -------------------------------
WARNING ceph_volume.util.system:terminal.py:170 Executable pvs not in PATH: /home/jenkins-build/build/workspace/ceph-volume-pr/src/ceph-volume/.tox/py35/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
INFO ceph_volume.process:process.py:191 Running command: pvs --no-heading --readonly --separator=";" -S lv_uuid=y -o pv_name,pv_tags,pv_uuid,vg_name,lv_uuid
__________ TestFullReport.test_physical_2nd_device_gets_reported[wal] __________
self = <ceph_volume.tests.devices.lvm.test_listing.TestFullReport object at 0x7fad40beb128>
type_ = 'wal'
monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7fad40beb550>
@pytest.mark.parametrize('type_', ['journal', 'db', 'wal'])
def test_physical_2nd_device_gets_reported(self, type_, monkeypatch):
tags = ('ceph.osd_id=0,ceph.{t}_uuid=x,ceph.type=data,'
'ceph.{t}_device=/dev/sda1').format(t=type_)
osd = api.Volume(lv_name='volume1', lv_uuid='y', lv_tags=tags,
vg_name='VolGroup', lv_path='/dev/VolGroup/lv')
monkeypatch.setattr(lvm.listing.api, 'get_lvs', lambda **kwargs:
[osd])
> result = lvm.listing.List([]).full_report()
ceph_volume/tests/devices/lvm/test_listing.py:165:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
ceph_volume/devices/lvm/listing.py:141: in full_report
return self.create_report(api.get_lvs())
ceph_volume/devices/lvm/listing.py:113: in create_report
pvs = api.get_pvs(filters={'lv_uuid': lv.lv_uuid})
ceph_volume/api/lvm.py:1466: in get_pvs
stdout, stderr, returncode = process.call(args, verbose_on_failure=False)
ceph_volume/process.py:201: in call
**kw
/usr/lib/python3.5/subprocess.py:947: in __init__
restore_signals, start_new_session)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <subprocess.Popen object at 0x7fad40beb710>
args = ['pvs', '--no-heading', '--readonly', '--separator=";"', '-S', 'lv_uuid=y', ...]
executable = b'pvs', preexec_fn = None, close_fds = True, pass_fds = ()
cwd = None, env = None, startupinfo = None, creationflags = 0, shell = False
p2cread = 12, p2cwrite = 13, c2pread = 14, c2pwrite = 15, errread = 16
errwrite = 17, restore_signals = True, start_new_session = False
def _execute_child(self, args, executable, preexec_fn, close_fds,
pass_fds, cwd, env,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
restore_signals, start_new_session):
"""Execute program (POSIX version)"""
if isinstance(args, (str, bytes)):
args = [args]
else:
args = list(args)
if shell:
args = ["/bin/sh", "-c"] + args
if executable:
args[0] = executable
if executable is None:
executable = args[0]
orig_executable = executable
# For transferring possible exec failure from child to parent.
# Data format: "exception name:hex errno:description"
# Pickle is not used; it is complex and involves memory allocation.
errpipe_read, errpipe_write = os.pipe()
# errpipe_write must not be in the standard io 0, 1, or 2 fd range.
low_fds_to_close = []
while errpipe_write < 3:
low_fds_to_close.append(errpipe_write)
errpipe_write = os.dup(errpipe_write)
for low_fd in low_fds_to_close:
os.close(low_fd)
try:
try:
# We must avoid complex work that could involve
# malloc or free in the child process to avoid
# potential deadlocks, thus we do all this here.
# and pass it to fork_exec()
if env is not None:
env_list = [os.fsencode(k) + b'=' + os.fsencode(v)
for k, v in env.items()]
else:
env_list = None # Use execv instead of execve.
executable = os.fsencode(executable)
if os.path.dirname(executable):
executable_list = (executable,)
else:
# This matches the behavior of os._execvpe().
executable_list = tuple(
os.path.join(os.fsencode(dir), executable)
for dir in os.get_exec_path(env))
fds_to_keep = set(pass_fds)
fds_to_keep.add(errpipe_write)
self.pid = _posixsubprocess.fork_exec(
args, executable_list,
close_fds, sorted(fds_to_keep), cwd, env_list,
p2cread, p2cwrite, c2pread, c2pwrite,
errread, errwrite,
errpipe_read, errpipe_write,
restore_signals, start_new_session, preexec_fn)
self._child_created = True
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
# self._devnull is not always defined.
devnull_fd = getattr(self, '_devnull', None)
if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd:
os.close(p2cread)
if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd:
os.close(c2pwrite)
if errwrite != -1 and errread != -1 and errwrite != devnull_fd:
os.close(errwrite)
if devnull_fd is not None:
os.close(devnull_fd)
# Prevent a double close of these fds from __init__ on error.
self._closed_child_pipe_fds = True
# Wait for exec to fail or succeed; possibly raising an
# exception (limited in size)
errpipe_data = bytearray()
while True:
part = os.read(errpipe_read, 50000)
errpipe_data += part
if not part or len(errpipe_data) > 50000:
break
finally:
# be sure the FD is closed no matter what
os.close(errpipe_read)
if errpipe_data:
try:
os.waitpid(self.pid, 0)
except ChildProcessError:
pass
try:
exception_name, hex_errno, err_msg = (
errpipe_data.split(b':', 2))
except ValueError:
exception_name = b'SubprocessError'
hex_errno = b'0'
err_msg = (b'Bad exception data from child: ' +
repr(errpipe_data))
child_exception_type = getattr(
builtins, exception_name.decode('ascii'),
SubprocessError)
err_msg = err_msg.decode(errors="surrogatepass")
if issubclass(child_exception_type, OSError) and hex_errno:
errno_num = int(hex_errno, 16)
child_exec_never_called = (err_msg == "noexec")
if child_exec_never_called:
err_msg = ""
if errno_num != 0:
err_msg = os.strerror(errno_num)
if errno_num == errno.ENOENT:
if child_exec_never_called:
# The error must be from chdir(cwd).
err_msg += ': ' + repr(cwd)
else:
err_msg += ': ' + repr(orig_executable)
> raise child_exception_type(errno_num, err_msg)
E FileNotFoundError: [Errno 2] No such file or directory: 'pvs'
/usr/lib/python3.5/subprocess.py:1551: FileNotFoundError
----------------------------- Captured stderr call -----------------------------
--> Executable pvs not in PATH: /home/jenkins-build/build/workspace/ceph-volume-pr/src/ceph-volume/.tox/py35/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
------------------------------ Captured log call -------------------------------
WARNING ceph_volume.util.system:terminal.py:170 Executable pvs not in PATH: /home/jenkins-build/build/workspace/ceph-volume-pr/src/ceph-volume/.tox/py35/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
INFO ceph_volume.process:process.py:191 Running command: pvs --no-heading --readonly --separator=";" -S lv_uuid=y -o pv_name,pv_tags,pv_uuid,vg_name,lv_uuid
______________ TestSingleReport.test_report_a_ceph_journal_device ______________
self = <ceph_volume.tests.devices.lvm.test_listing.TestSingleReport object at 0x7fad40f7b518>
monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7fad40ff2f60>
def test_report_a_ceph_journal_device(self, monkeypatch):
# ceph lvs are detected by looking into its tags
tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data,' + \
'ceph.journal_device=/dev/sda1'
lv = api.Volume(lv_name='lv', lv_uuid='aaa', lv_tags=tags,
lv_path='/dev/VolGroup/lv', vg_name='VolGroup')
monkeypatch.setattr(lvm.listing.api, 'get_lvs', lambda **kwargs:
[lv] if 'tags' in kwargs else [])
> result = lvm.listing.List([]).single_report('/dev/sda1')
ceph_volume/tests/devices/lvm/test_listing.py:209:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
ceph_volume/devices/lvm/listing.py:152: in single_report
lvs = api.get_device_lvs(device)
ceph_volume/api/lvm.py:1380: in get_device_lvs
verbose_on_failure=False
ceph_volume/process.py:201: in call
**kw
/usr/lib/python3.5/subprocess.py:947: in __init__
restore_signals, start_new_session)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <subprocess.Popen object at 0x7fad40b98ef0>
args = ['pvs', '--noheadings', '--readonly', '--separator=";"', '-a', '-o', ...]
executable = b'pvs', preexec_fn = None, close_fds = True, pass_fds = ()
cwd = None, env = None, startupinfo = None, creationflags = 0, shell = False
p2cread = 12, p2cwrite = 13, c2pread = 14, c2pwrite = 15, errread = 16
errwrite = 17, restore_signals = True, start_new_session = False
def _execute_child(self, args, executable, preexec_fn, close_fds,
pass_fds, cwd, env,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
restore_signals, start_new_session):
"""Execute program (POSIX version)"""
if isinstance(args, (str, bytes)):
args = [args]
else:
args = list(args)
if shell:
args = ["/bin/sh", "-c"] + args
if executable:
args[0] = executable
if executable is None:
executable = args[0]
orig_executable = executable
# For transferring possible exec failure from child to parent.
# Data format: "exception name:hex errno:description"
# Pickle is not used; it is complex and involves memory allocation.
errpipe_read, errpipe_write = os.pipe()
# errpipe_write must not be in the standard io 0, 1, or 2 fd range.
low_fds_to_close = []
while errpipe_write < 3:
low_fds_to_close.append(errpipe_write)
errpipe_write = os.dup(errpipe_write)
for low_fd in low_fds_to_close:
os.close(low_fd)
try:
try:
# We must avoid complex work that could involve
# malloc or free in the child process to avoid
# potential deadlocks, thus we do all this here.
# and pass it to fork_exec()
if env is not None:
env_list = [os.fsencode(k) + b'=' + os.fsencode(v)
for k, v in env.items()]
else:
env_list = None # Use execv instead of execve.
executable = os.fsencode(executable)
if os.path.dirname(executable):
executable_list = (executable,)
else:
# This matches the behavior of os._execvpe().
executable_list = tuple(
os.path.join(os.fsencode(dir), executable)
for dir in os.get_exec_path(env))
fds_to_keep = set(pass_fds)
fds_to_keep.add(errpipe_write)
self.pid = _posixsubprocess.fork_exec(
args, executable_list,
close_fds, sorted(fds_to_keep), cwd, env_list,
p2cread, p2cwrite, c2pread, c2pwrite,
errread, errwrite,
errpipe_read, errpipe_write,
restore_signals, start_new_session, preexec_fn)
self._child_created = True
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
# self._devnull is not always defined.
devnull_fd = getattr(self, '_devnull', None)
if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd:
os.close(p2cread)
if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd:
os.close(c2pwrite)
if errwrite != -1 and errread != -1 and errwrite != devnull_fd:
os.close(errwrite)
if devnull_fd is not None:
os.close(devnull_fd)
# Prevent a double close of these fds from __init__ on error.
self._closed_child_pipe_fds = True
# Wait for exec to fail or succeed; possibly raising an
# exception (limited in size)
errpipe_data = bytearray()
while True:
part = os.read(errpipe_read, 50000)
errpipe_data += part
if not part or len(errpipe_data) > 50000:
break
finally:
# be sure the FD is closed no matter what
os.close(errpipe_read)
if errpipe_data:
try:
os.waitpid(self.pid, 0)
except ChildProcessError:
pass
try:
exception_name, hex_errno, err_msg = (
errpipe_data.split(b':', 2))
except ValueError:
exception_name = b'SubprocessError'
hex_errno = b'0'
err_msg = (b'Bad exception data from child: ' +
repr(errpipe_data))
child_exception_type = getattr(
builtins, exception_name.decode('ascii'),
SubprocessError)
err_msg = err_msg.decode(errors="surrogatepass")
if issubclass(child_exception_type, OSError) and hex_errno:
errno_num = int(hex_errno, 16)
child_exec_never_called = (err_msg == "noexec")
if child_exec_never_called:
err_msg = ""
if errno_num != 0:
err_msg = os.strerror(errno_num)
if errno_num == errno.ENOENT:
if child_exec_never_called:
# The error must be from chdir(cwd).
err_msg += ': ' + repr(cwd)
else:
err_msg += ': ' + repr(orig_executable)
> raise child_exception_type(errno_num, err_msg)
E FileNotFoundError: [Errno 2] No such file or directory: 'pvs'
/usr/lib/python3.5/subprocess.py:1551: FileNotFoundError
----------------------------- Captured stderr call -----------------------------
--> Executable pvs not in PATH: /home/jenkins-build/build/workspace/ceph-volume-pr/src/ceph-volume/.tox/py35/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
------------------------------ Captured log call -------------------------------
WARNING ceph_volume.util.system:terminal.py:170 Executable pvs not in PATH: /home/jenkins-build/build/workspace/ceph-volume-pr/src/ceph-volume/.tox/py35/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
INFO ceph_volume.process:process.py:191 Running command: pvs --noheadings --readonly --separator=";" -a -o lv_tags,lv_path,lv_name,vg_name,lv_uuid,lv_size /dev/sda1
_______________________ TestDevice.test_vgs_is_not_empty _______________________
self = <test_device.TestDevice object at 0x7fad40c0ad30>
device_info = <function device_info.<locals>.apply at 0x7fad40af5950>
monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7fad40b4c048>
def test_vgs_is_not_empty(self, device_info, monkeypatch):
vg = api.VolumeGroup(vg_name='foo/bar', vg_free_count=6,
vg_extent_size=1073741824)
monkeypatch.setattr(api, 'get_device_vgs', lambda x: [vg])
lsblk = {"TYPE": "disk"}
device_info(lsblk=lsblk)
> disk = device.Device("/dev/nvme0n1")
ceph_volume/tests/util/test_device.py:50:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
ceph_volume/util/device.py:92: in __init__
self._parse()
ceph_volume/util/device.py:148: in _parse
self._set_lvm_membership()
ceph_volume/util/device.py:250: in _set_lvm_membership
self.lvs.extend(lvm.get_device_lvs(path))
ceph_volume/api/lvm.py:1380: in get_device_lvs
verbose_on_failure=False
ceph_volume/process.py:201: in call
**kw
/usr/lib/python3.5/subprocess.py:947: in __init__
restore_signals, start_new_session)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <subprocess.Popen object at 0x7fad40b4c978>
args = ['pvs', '--noheadings', '--readonly', '--separator=";"', '-a', '-o', ...]
executable = b'pvs', preexec_fn = None, close_fds = True, pass_fds = ()
cwd = None, env = None, startupinfo = None, creationflags = 0, shell = False
p2cread = 15, p2cwrite = 16, c2pread = 17, c2pwrite = 18, errread = 19
errwrite = 20, restore_signals = True, start_new_session = False
def _execute_child(self, args, executable, preexec_fn, close_fds,
pass_fds, cwd, env,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
restore_signals, start_new_session):
"""Execute program (POSIX version)"""
if isinstance(args, (str, bytes)):
args = [args]
else:
args = list(args)
if shell:
args = ["/bin/sh", "-c"] + args
if executable:
args[0] = executable
if executable is None:
executable = args[0]
orig_executable = executable
# For transferring possible exec failure from child to parent.
# Data format: "exception name:hex errno:description"
# Pickle is not used; it is complex and involves memory allocation.
errpipe_read, errpipe_write = os.pipe()
# errpipe_write must not be in the standard io 0, 1, or 2 fd range.
low_fds_to_close = []
while errpipe_write < 3:
low_fds_to_close.append(errpipe_write)
errpipe_write = os.dup(errpipe_write)
for low_fd in low_fds_to_close:
os.close(low_fd)
try:
try:
# We must avoid complex work that could involve
# malloc or free in the child process to avoid
# potential deadlocks, thus we do all this here.
# and pass it to fork_exec()
if env is not None:
env_list = [os.fsencode(k) + b'=' + os.fsencode(v)
for k, v in env.items()]
else:
env_list = None # Use execv instead of execve.
executable = os.fsencode(executable)
if os.path.dirname(executable):
executable_list = (executable,)
else:
# This matches the behavior of os._execvpe().
executable_list = tuple(
os.path.join(os.fsencode(dir), executable)
for dir in os.get_exec_path(env))
fds_to_keep = set(pass_fds)
fds_to_keep.add(errpipe_write)
self.pid = _posixsubprocess.fork_exec(
args, executable_list,
close_fds, sorted(fds_to_keep), cwd, env_list,
p2cread, p2cwrite, c2pread, c2pwrite,
errread, errwrite,
errpipe_read, errpipe_write,
restore_signals, start_new_session, preexec_fn)
self._child_created = True
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
# self._devnull is not always defined.
devnull_fd = getattr(self, '_devnull', None)
if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd:
os.close(p2cread)
if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd:
os.close(c2pwrite)
if errwrite != -1 and errread != -1 and errwrite != devnull_fd:
os.close(errwrite)
if devnull_fd is not None:
os.close(devnull_fd)
# Prevent a double close of these fds from __init__ on error.
self._closed_child_pipe_fds = True
# Wait for exec to fail or succeed; possibly raising an
# exception (limited in size)
errpipe_data = bytearray()
while True:
part = os.read(errpipe_read, 50000)
errpipe_data += part
if not part or len(errpipe_data) > 50000:
break
finally:
# be sure the FD is closed no matter what
os.close(errpipe_read)
if errpipe_data:
try:
os.waitpid(self.pid, 0)
except ChildProcessError:
pass
try:
exception_name, hex_errno, err_msg = (
errpipe_data.split(b':', 2))
except ValueError:
exception_name = b'SubprocessError'
hex_errno = b'0'
err_msg = (b'Bad exception data from child: ' +
repr(errpipe_data))
child_exception_type = getattr(
builtins, exception_name.decode('ascii'),
SubprocessError)
err_msg = err_msg.decode(errors="surrogatepass")
if issubclass(child_exception_type, OSError) and hex_errno:
errno_num = int(hex_errno, 16)
child_exec_never_called = (err_msg == "noexec")
if child_exec_never_called:
err_msg = ""
if errno_num != 0:
err_msg = os.strerror(errno_num)
if errno_num == errno.ENOENT:
if child_exec_never_called:
# The error must be from chdir(cwd).
err_msg += ': ' + repr(cwd)
else:
err_msg += ': ' + repr(orig_executable)
> raise child_exception_type(errno_num, err_msg)
E FileNotFoundError: [Errno 2] No such file or directory: 'pvs'
/usr/lib/python3.5/subprocess.py:1551: FileNotFoundError
----------------------------- Captured stderr call -----------------------------
--> Executable pvs not in PATH: /home/jenkins-build/build/workspace/ceph-volume-pr/src/ceph-volume/.tox/py35/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
------------------------------ Captured log call -------------------------------
WARNING ceph_volume.util.system:terminal.py:170 Executable pvs not in PATH: /home/jenkins-build/build/workspace/ceph-volume-pr/src/ceph-volume/.tox/py35/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
INFO ceph_volume.process:process.py:191 Running command: pvs --noheadings --readonly --separator=";" -a -o lv_tags,lv_path,lv_name,vg_name,lv_uuid,lv_size /dev/nvme0n1
___________________ TestDevice.test_reject_bluestore_device ____________________
self = <test_device.TestDevice object at 0x7fad40a83dd8>
monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7fad40a870b8>
patch_bluestore_label = <PropertyMock name='has_bluestore_label' id='140382090850992'>
def test_reject_bluestore_device(self, monkeypatch, patch_bluestore_label):
patch_bluestore_label.return_value = True
> disk = device.Device("/dev/sda")
ceph_volume/tests/util/test_device.py:207:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
ceph_volume/util/device.py:92: in __init__
self._parse()
ceph_volume/util/device.py:134: in _parse
lv = lvm.get_lv_from_argument(self.path)
ceph_volume/api/lvm.py:1325: in get_lv_from_argument
lv = get_lv(lv_path=argument)
ceph_volume/api/lvm.py:1311: in get_lv
lvs = Volumes()
ceph_volume/api/lvm.py:1035: in __init__
self._populate()
ceph_volume/api/lvm.py:1039: in _populate
for lv_item in get_api_lvs():
ceph_volume/api/lvm.py:898: in get_api_lvs
verbose_on_failure=False
ceph_volume/process.py:201: in call
**kw
/usr/lib/python3.5/subprocess.py:947: in __init__
restore_signals, start_new_session)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <subprocess.Popen object at 0x7fad40a872e8>
args = ['lvs', '--noheadings', '--readonly', '--separator=";"', '-a', '-o', ...]
executable = b'lvs', preexec_fn = None, close_fds = True, pass_fds = ()
cwd = None, env = None, startupinfo = None, creationflags = 0, shell = False
p2cread = 15, p2cwrite = 16, c2pread = 17, c2pwrite = 18, errread = 19
errwrite = 20, restore_signals = True, start_new_session = False
def _execute_child(self, args, executable, preexec_fn, close_fds,
pass_fds, cwd, env,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
restore_signals, start_new_session):
"""Execute program (POSIX version)"""
if isinstance(args, (str, bytes)):
args = [args]
else:
args = list(args)
if shell:
args = ["/bin/sh", "-c"] + args
if executable:
args[0] = executable
if executable is None:
executable = args[0]
orig_executable = executable
# For transferring possible exec failure from child to parent.
# Data format: "exception name:hex errno:description"
# Pickle is not used; it is complex and involves memory allocation.
errpipe_read, errpipe_write = os.pipe()
# errpipe_write must not be in the standard io 0, 1, or 2 fd range.
low_fds_to_close = []
while errpipe_write < 3:
low_fds_to_close.append(errpipe_write)
errpipe_write = os.dup(errpipe_write)
for low_fd in low_fds_to_close:
os.close(low_fd)
try:
try:
# We must avoid complex work that could involve
# malloc or free in the child process to avoid
# potential deadlocks, thus we do all this here.
# and pass it to fork_exec()
if env is not None:
env_list = [os.fsencode(k) + b'=' + os.fsencode(v)
for k, v in env.items()]
else:
env_list = None # Use execv instead of execve.
executable = os.fsencode(executable)
if os.path.dirname(executable):
executable_list = (executable,)
else:
# This matches the behavior of os._execvpe().
executable_list = tuple(
os.path.join(os.fsencode(dir), executable)
for dir in os.get_exec_path(env))
fds_to_keep = set(pass_fds)
fds_to_keep.add(errpipe_write)
self.pid = _posixsubprocess.fork_exec(
args, executable_list,
close_fds, sorted(fds_to_keep), cwd, env_list,
p2cread, p2cwrite, c2pread, c2pwrite,
errread, errwrite,
errpipe_read, errpipe_write,
restore_signals, start_new_session, preexec_fn)
self._child_created = True
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
# self._devnull is not always defined.
devnull_fd = getattr(self, '_devnull', None)
if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd:
os.close(p2cread)
if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd:
os.close(c2pwrite)
if errwrite != -1 and errread != -1 and errwrite != devnull_fd:
os.close(errwrite)
if devnull_fd is not None:
os.close(devnull_fd)
# Prevent a double close of these fds from __init__ on error.
self._closed_child_pipe_fds = True
# Wait for exec to fail or succeed; possibly raising an
# exception (limited in size)
errpipe_data = bytearray()
while True:
part = os.read(errpipe_read, 50000)
errpipe_data += part
if not part or len(errpipe_data) > 50000:
break
finally:
# be sure the FD is closed no matter what
os.close(errpipe_read)
if errpipe_data:
try:
os.waitpid(self.pid, 0)
except ChildProcessError:
pass
try:
exception_name, hex_errno, err_msg = (
errpipe_data.split(b':', 2))
except ValueError:
exception_name = b'SubprocessError'
hex_errno = b'0'
err_msg = (b'Bad exception data from child: ' +
repr(errpipe_data))
child_exception_type = getattr(
builtins, exception_name.decode('ascii'),
SubprocessError)
err_msg = err_msg.decode(errors="surrogatepass")
if issubclass(child_exception_type, OSError) and hex_errno:
errno_num = int(hex_errno, 16)
child_exec_never_called = (err_msg == "noexec")
if child_exec_never_called:
err_msg = ""
if errno_num != 0:
err_msg = os.strerror(errno_num)
if errno_num == errno.ENOENT:
if child_exec_never_called:
# The error must be from chdir(cwd).
err_msg += ': ' + repr(cwd)
else:
err_msg += ': ' + repr(orig_executable)
> raise child_exception_type(errno_num, err_msg)
E FileNotFoundError: [Errno 2] No such file or directory: 'lvs'
/usr/lib/python3.5/subprocess.py:1551: FileNotFoundError
----------------------------- Captured stderr call -----------------------------
--> Executable lvs not in PATH: /home/jenkins-build/build/workspace/ceph-volume-pr/src/ceph-volume/.tox/py35/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
------------------------------ Captured log call -------------------------------
INFO ceph_volume.process:process.py:191 Running command: /bin/lsblk -plno KNAME,NAME,TYPE
INFO ceph_volume.process:process.py:34 stdout /dev/sda /dev/sda disk
INFO ceph_volume.process:process.py:34 stdout /dev/sda1 /dev/sda1 part
INFO ceph_volume.process:process.py:34 stdout /dev/sr0 /dev/sr0 rom
WARNING ceph_volume.util.system:terminal.py:170 Executable lvs not in PATH: /home/jenkins-build/build/workspace/ceph-volume-pr/src/ceph-volume/.tox/py35/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
INFO ceph_volume.process:process.py:191 Running command: lvs --noheadings --readonly --separator=";" -a -o lv_tags,lv_path,lv_name,vg_name,lv_uuid,lv_size
=========================== short test summary info ============================
FAILED ceph_volume/tests/test_process.py::TestFunctionalCall::test_unicode_encoding
FAILED ceph_volume/tests/devices/lvm/test_listing.py::TestFullReport::test_physical_2nd_device_gets_reported[journal]
FAILED ceph_volume/tests/devices/lvm/test_listing.py::TestFullReport::test_physical_2nd_device_gets_reported[db]
FAILED ceph_volume/tests/devices/lvm/test_listing.py::TestFullReport::test_physical_2nd_device_gets_reported[wal]
FAILED ceph_volume/tests/devices/lvm/test_listing.py::TestSingleReport::test_report_a_ceph_journal_device
FAILED ceph_volume/tests/util/test_device.py::TestDevice::test_vgs_is_not_empty
FAILED ceph_volume/tests/util/test_device.py::TestDevice::test_reject_bluestore_device
================== 7 failed, 1885 passed, 1 skipped in 7.72s ===================
No data to display
Actions