Ceph » mgr » Dashboard

Description of problem¶

One of the dashboard Jenkins jobs is leaving behind libvirt VMs.

https://jenkins.ceph.com/job/ceph-pull-requests/99881/consoleFull

+ ps -ef
+ grep -v jnlp
+ grep ceph
jenkins+ 1390938  970823  0 13:56 ?        00:00:00 grep ceph
libvirt+ 2707949       1  0 Jul19 ?        00:00:00 /usr/sbin/dnsmasq --conf-file=/var/lib/libvirt/dnsmasq/ceph-dashboard.conf --leasefile-ro --dhcp-script=/usr/lib/libvirt/libvirt_leaseshelper
root     2707950 2707949  0 Jul19 ?        00:00:00 /usr/sbin/dnsmasq --conf-file=/var/lib/libvirt/dnsmasq/ceph-dashboard.conf --leasefile-ro --dhcp-script=/usr/lib/libvirt/libvirt_leaseshelper
libvirt+ 2708642       1 19 Jul19 ?        10:07:44 /usr/bin/qemu-system-x86_64 -name guest=ceph-node-00,debug-threads=on -S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-288-ceph-node-00/master-key.aes -machine pc-i440fx-4.2,accel=kvm,usb=off,dump-guest-core=off -cpu EPYC-Rome,x2apic=on,tsc-deadline=on,hypervisor=on,tsc-adjust=on,spec-ctrl=on,stibp=on,arch-capabilities=on,ssbd=on,xsaves=on,cmp-legacy=on,ibrs=on,amd-ssbd=on,virt-ssbd=on,rdctl-no=on,skip-l1dfl-vmentry=on,mds-no=on,pschange-mc-no=on -m 2048 -overcommit mem-lock=off -smp 1,sockets=1,cores=1,threads=1 -uuid 89c43223-4a12-4e8d-a507-3fc876758918 -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=31,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown -boot menu=on,splash-time=3000,strict=on -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device ahci,id=sata0,bus=pci.0,addr=0x4 -device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x5 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/Fedora-Cloud-Base-34-1.2.x86_64.qcow2","node-name":"libvirt-5-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-5-format","read-only":true,"driver":"qcow2","file":"libvirt-5-storage","backing":null} -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-00_0.img","node-name":"libvirt-4-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-4-format","read-only":false,"driver":"qcow2","file":"libvirt-4-storage","backing":"libvirt-5-format"} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x6,drive=libvirt-4-format,id=virtio-disk0,bootindex=1 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-00_1.img","node-name":"libvirt-3-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-3-format","read-only":false,"driver":"qcow2","file":"libvirt-3-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x7,drive=libvirt-3-format,id=virtio-disk1 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-00_2.img","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-2-format","read-only":false,"driver":"qcow2","file":"libvirt-2-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x8,drive=libvirt-2-format,id=virtio-disk2 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-00.ISO","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-1-format","read-only":true,"driver":"raw","file":"libvirt-1-storage"} -device ide-cd,bus=sata0.3,drive=libvirt-1-format,id=sata0-0-3 -fsdev local,security_model=mapped,id=fsdev-fs0,path=/home/jenkins-build/build/workspace/ceph-dashboard-cephadm-e2e -device virtio-9p-pci,id=fs0,fsdev=fsdev-fs0,mount_tag=ceph-dashboard-cephadm-e2e,bus=pci.0,addr=0x9 -netdev tap,fd=33,id=hostnet0,vhost=on,vhostfd=34 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:4d:8f:ec,bus=pci.0,addr=0x3 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev socket,id=charchannel0,fd=35,server,nowait -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 -device usb-tablet,id=input0,bus=usb.0,port=1 -spice port=5900,addr=127.0.0.1,disable-ticketing,seamless-migration=on -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0xa -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on
libvirt+ 2708773       1  6 Jul19 ?        03:24:14 /usr/bin/qemu-system-x86_64 -name guest=ceph-node-01,debug-threads=on -S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-289-ceph-node-01/master-key.aes -machine pc-i440fx-4.2,accel=kvm,usb=off,dump-guest-core=off -cpu EPYC-Rome,x2apic=on,tsc-deadline=on,hypervisor=on,tsc-adjust=on,spec-ctrl=on,stibp=on,arch-capabilities=on,ssbd=on,xsaves=on,cmp-legacy=on,ibrs=on,amd-ssbd=on,virt-ssbd=on,rdctl-no=on,skip-l1dfl-vmentry=on,mds-no=on,pschange-mc-no=on -m 2048 -overcommit mem-lock=off -smp 1,sockets=1,cores=1,threads=1 -uuid 37c9187d-ea8a-4cfe-9424-5f72910fc47d -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=32,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown -boot menu=on,splash-time=3000,strict=on -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device ahci,id=sata0,bus=pci.0,addr=0x4 -device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x5 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/Fedora-Cloud-Base-34-1.2.x86_64.qcow2","node-name":"libvirt-5-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-5-format","read-only":true,"driver":"qcow2","file":"libvirt-5-storage","backing":null} -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-01_0.img","node-name":"libvirt-4-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-4-format","read-only":false,"driver":"qcow2","file":"libvirt-4-storage","backing":"libvirt-5-format"} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x6,drive=libvirt-4-format,id=virtio-disk0,bootindex=1 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-01_1.img","node-name":"libvirt-3-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-3-format","read-only":false,"driver":"qcow2","file":"libvirt-3-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x7,drive=libvirt-3-format,id=virtio-disk1 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-01_2.img","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-2-format","read-only":false,"driver":"qcow2","file":"libvirt-2-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x8,drive=libvirt-2-format,id=virtio-disk2 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-01.ISO","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-1-format","read-only":true,"driver":"raw","file":"libvirt-1-storage"} -device ide-cd,bus=sata0.3,drive=libvirt-1-format,id=sata0-0-3 -fsdev local,security_model=mapped,id=fsdev-fs0,path=/home/jenkins-build/build/workspace/ceph-dashboard-cephadm-e2e -device virtio-9p-pci,id=fs0,fsdev=fsdev-fs0,mount_tag=ceph-dashboard-cephadm-e2e,bus=pci.0,addr=0x9 -netdev tap,fd=34,id=hostnet0,vhost=on,vhostfd=35 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:4c:e4:32,bus=pci.0,addr=0x3 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev socket,id=charchannel0,fd=36,server,nowait -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 -device usb-tablet,id=input0,bus=usb.0,port=1 -spice port=5901,addr=127.0.0.1,disable-ticketing,seamless-migration=on -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0xa -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on
libvirt+ 2708904       1 11 Jul19 ?        05:49:13 /usr/bin/qemu-system-x86_64 -name guest=ceph-node-02,debug-threads=on -S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-290-ceph-node-02/master-key.aes -machine pc-i440fx-4.2,accel=kvm,usb=off,dump-guest-core=off -cpu EPYC-Rome,x2apic=on,tsc-deadline=on,hypervisor=on,tsc-adjust=on,spec-ctrl=on,stibp=on,arch-capabilities=on,ssbd=on,xsaves=on,cmp-legacy=on,ibrs=on,amd-ssbd=on,virt-ssbd=on,rdctl-no=on,skip-l1dfl-vmentry=on,mds-no=on,pschange-mc-no=on -m 2048 -overcommit mem-lock=off -smp 1,sockets=1,cores=1,threads=1 -uuid 424e945b-7410-411a-ae22-b6ef7083f224 -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=33,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown -boot menu=on,splash-time=3000,strict=on -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device ahci,id=sata0,bus=pci.0,addr=0x4 -device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x5 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/Fedora-Cloud-Base-34-1.2.x86_64.qcow2","node-name":"libvirt-5-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-5-format","read-only":true,"driver":"qcow2","file":"libvirt-5-storage","backing":null} -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-02_0.img","node-name":"libvirt-4-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-4-format","read-only":false,"driver":"qcow2","file":"libvirt-4-storage","backing":"libvirt-5-format"} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x6,drive=libvirt-4-format,id=virtio-disk0,bootindex=1 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-02_1.img","node-name":"libvirt-3-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-3-format","read-only":false,"driver":"qcow2","file":"libvirt-3-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x7,drive=libvirt-3-format,id=virtio-disk1 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-02_2.img","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-2-format","read-only":false,"driver":"qcow2","file":"libvirt-2-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x8,drive=libvirt-2-format,id=virtio-disk2 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-02.ISO","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-1-format","read-only":true,"driver":"raw","file":"libvirt-1-storage"} -device ide-cd,bus=sata0.3,drive=libvirt-1-format,id=sata0-0-3 -fsdev local,security_model=mapped,id=fsdev-fs0,path=/home/jenkins-build/build/workspace/ceph-dashboard-cephadm-e2e -device virtio-9p-pci,id=fs0,fsdev=fsdev-fs0,mount_tag=ceph-dashboard-cephadm-e2e,bus=pci.0,addr=0x9 -netdev tap,fd=35,id=hostnet0,vhost=on,vhostfd=36 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:0d:26:a3,bus=pci.0,addr=0x3 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev socket,id=charchannel0,fd=37,server,nowait -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 -device usb-tablet,id=input0,bus=usb.0,port=1 -spice port=5902,addr=127.0.0.1,disable-ticketing,seamless-migration=on -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0xa -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on
libvirt+ 2709035       1  4 Jul19 ?        02:19:42 /usr/bin/qemu-system-x86_64 -name guest=ceph-node-03,debug-threads=on -S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-291-ceph-node-03/master-key.aes -machine pc-i440fx-4.2,accel=kvm,usb=off,dump-guest-core=off -cpu EPYC-Rome,x2apic=on,tsc-deadline=on,hypervisor=on,tsc-adjust=on,spec-ctrl=on,stibp=on,arch-capabilities=on,ssbd=on,xsaves=on,cmp-legacy=on,ibrs=on,amd-ssbd=on,virt-ssbd=on,rdctl-no=on,skip-l1dfl-vmentry=on,mds-no=on,pschange-mc-no=on -m 2048 -overcommit mem-lock=off -smp 1,sockets=1,cores=1,threads=1 -uuid 930ccaee-65cc-4dc7-a957-62c7457b9dbf -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=34,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown -boot menu=on,splash-time=3000,strict=on -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device ahci,id=sata0,bus=pci.0,addr=0x4 -device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x5 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/Fedora-Cloud-Base-34-1.2.x86_64.qcow2","node-name":"libvirt-5-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-5-format","read-only":true,"driver":"qcow2","file":"libvirt-5-storage","backing":null} -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-03_0.img","node-name":"libvirt-4-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-4-format","read-only":false,"driver":"qcow2","file":"libvirt-4-storage","backing":"libvirt-5-format"} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x6,drive=libvirt-4-format,id=virtio-disk0,bootindex=1 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-03_1.img","node-name":"libvirt-3-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-3-format","read-only":false,"driver":"qcow2","file":"libvirt-3-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x7,drive=libvirt-3-format,id=virtio-disk1 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-03_2.img","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-2-format","read-only":false,"driver":"qcow2","file":"libvirt-2-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x8,drive=libvirt-2-format,id=virtio-disk2 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/ceph-dashboard/ceph-node-03.ISO","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-1-format","read-only":true,"driver":"raw","file":"libvirt-1-storage"} -device ide-cd,bus=sata0.3,drive=libvirt-1-format,id=sata0-0-3 -fsdev local,security_model=mapped,id=fsdev-fs0,path=/home/jenkins-build/build/workspace/ceph-dashboard-cephadm-e2e -device virtio-9p-pci,id=fs0,fsdev=fsdev-fs0,mount_tag=ceph-dashboard-cephadm-e2e,bus=pci.0,addr=0x9 -netdev tap,fd=36,id=hostnet0,vhost=on,vhostfd=37 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:dd:69:af,bus=pci.0,addr=0x3 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev socket,id=charchannel0,fd=38,server,nowait -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 -device usb-tablet,id=input0,bus=usb.0,port=1 -spice port=5903,addr=127.0.0.1,disable-ticketing,seamless-migration=on -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0xa -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on

Note there are some functions that already exist to clean up libvirt networks and VMs that you can use: https://github.com/ceph/ceph-build/blob/main/scripts/build_utils.sh#L1061-L1074

These should probably run after a successful and failed job.