Bug #58398
closedceph-rbdnamer manpage shows incompatible/outdated udev rules
0%
Description
We stumbled over an issue where rbd map took around 3 minutes to process. We were able to track this down to the udev rules processing being stuck in a tight retry loop in systemd-udev:
```
[pid 21390] 09:58:08.785199 mkdir("/dev/rbd/rbd.hdd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.785262 symlink("../../rbd0", "/dev/rbd/rbd.hdd/test03.root.UNNEEDED/") = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.785325 stat("/dev/rbd/rbd.hdd/test03.root.UNNEEDED", 0x7ffcc3f23de0) = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.785385 mkdir("/dev", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.785440 mkdir("/dev/rbd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.785495 mkdir("/dev/rbd/rbd.hdd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.785551 symlink("../../rbd0", "/dev/rbd/rbd.hdd/test03.root.UNNEEDED/") = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.785620 stat("/dev/rbd/rbd.hdd/test03.root.UNNEEDED", 0x7ffcc3f23de0) = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.785679 mkdir("/dev", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.785734 mkdir("/dev/rbd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.785789 mkdir("/dev/rbd/rbd.hdd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.785843 symlink("../../rbd0", "/dev/rbd/rbd.hdd/test03.root.UNNEEDED/") = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.785911 stat("/dev/rbd/rbd.hdd/test03.root.UNNEEDED", 0x7ffcc3f23de0) = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.785969 mkdir("/dev", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.786024 mkdir("/dev/rbd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.786079 mkdir("/dev/rbd/rbd.hdd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.786134 symlink("../../rbd0", "/dev/rbd/rbd.hdd/test03.root.UNNEEDED/") = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.786195 stat("/dev/rbd/rbd.hdd/test03.root.UNNEEDED", 0x7ffcc3f23de0) = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.786269 mkdir("/dev", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.786326 mkdir("/dev/rbd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.786381 mkdir("/dev/rbd/rbd.hdd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.786437 symlink("../../rbd0", "/dev/rbd/rbd.hdd/test03.root.UNNEEDED/") = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.786498 stat("/dev/rbd/rbd.hdd/test03.root.UNNEEDED", 0x7ffcc3f23de0) = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.786563 mkdir("/dev", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.786617 mkdir("/dev/rbd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.786672 mkdir("/dev/rbd/rbd.hdd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.786727 symlink("../../rbd0", "/dev/rbd/rbd.hdd/test03.root.UNNEEDED/") = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.786787 stat("/dev/rbd/rbd.hdd/test03.root.UNNEEDED", 0x7ffcc3f23de0) = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.786846 mkdir("/dev", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.786899 mkdir("/dev/rbd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.786952 mkdir("/dev/rbd/rbd.hdd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.787009 symlink("../../rbd0", "/dev/rbd/rbd.hdd/test03.root.UNNEEDED/") = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.787069 stat("/dev/rbd/rbd.hdd/test03.root.UNNEEDED", 0x7ffcc3f23de0) = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.787127 mkdir("/dev", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.787183 mkdir("/dev/rbd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.787247 mkdir("/dev/rbd/rbd.hdd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.787305 symlink("../../rbd0", "/dev/rbd/rbd.hdd/test03.root.UNNEEDED/") = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.787366 stat("/dev/rbd/rbd.hdd/test03.root.UNNEEDED", 0x7ffcc3f23de0) = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.787425 mkdir("/dev", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.787481 mkdir("/dev/rbd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.787535 mkdir("/dev/rbd/rbd.hdd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.787590 symlink("../../rbd0", "/dev/rbd/rbd.hdd/test03.root.UNNEEDED/") = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.787658 stat("/dev/rbd/rbd.hdd/test03.root.UNNEEDED", 0x7ffcc3f23de0) = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.787717 mkdir("/dev", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.787769 mkdir("/dev/rbd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.787822 mkdir("/dev/rbd/rbd.hdd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.787877 symlink("../../rbd0", "/dev/rbd/rbd.hdd/test03.root.UNNEEDED/") = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.787938 stat("/dev/rbd/rbd.hdd/test03.root.UNNEEDED", 0x7ffcc3f23de0) = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.787997 mkdir("/dev", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.788051 mkdir("/dev/rbd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.788106 mkdir("/dev/rbd/rbd.hdd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.788162 symlink("../../rbd0", "/dev/rbd/rbd.hdd/test03.root.UNNEEDED/") = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.788221 stat("/dev/rbd/rbd.hdd/test03.root.UNNEEDED", 0x7ffcc3f23de0) = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.788291 mkdir("/dev", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.788346 mkdir("/dev/rbd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.788402 mkdir("/dev/rbd/rbd.hdd", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.788456 symlink("../../rbd0", "/dev/rbd/rbd.hdd/test03.root.UNNEEDED/") = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.788515 stat("/dev/rbd/rbd.hdd/test03.root.UNNEEDED", 0x7ffcc3f23de0) = -1 ENOENT (No such file or directory)
[pid 21390] 09:58:08.788581 mkdir("/dev", 0755) = -1 EEXIST (File exists)
[pid 21390] 09:58:08.788636 mkdir("/dev/rbd", 0755) = -1 EEXIST (File exists)
```
This takes around 3 minutes (likely due to `#define LINK_UPDATE_MAX_RETRIES 128` in udev) until it settles.
The issue here is a trailing slash causing udev trying to create a symlink for `/dev/rbd/pool/image/` instead of `/dev/rbd/pool/image` (note the trailing slash). This is caused by the incorrect usage of `%c{1}/%c{2}` - splitting is only done on spaces and not slashes. Maybe this has changed over time.
We're suggesting an updated ruleset, that also includes partition numbering that we've been using in the past (and now updated to the correct %c usage):
```
KERNEL=="rbd[0-9]*", ENV{DEVTYPE}=="disk", PROGRAM="/usr/bin/ceph-rbdnamer %k", SYMLINK+="rbd/%c"
KERNEL=="rbd[0-9]*", ENV{DEVTYPE}=="partition", PROGRAM="/usr//bin/ceph-rbdnamer %k", SYMLINK+="rbd/%c-part%n"
```
Updated by Ilya Dryomov over 1 year ago
- Project changed from Ceph to rbd
- Category deleted (
documentation) - Status changed from New to Need More Info
- Assignee set to Ilya Dryomov
Christian Theune wrote:
We're suggesting an updated ruleset, that also includes partition numbering that we've been using in the past (and now updated to the correct %c usage):
```
KERNEL=="rbd[0-9]*", ENV{DEVTYPE}=="disk", PROGRAM="/usr/bin/ceph-rbdnamer %k", SYMLINK+="rbd/%c"
KERNEL=="rbd[0-9]*", ENV{DEVTYPE}=="partition", PROGRAM="/usr//bin/ceph-rbdnamer %k", SYMLINK+="rbd/%c-part%n"
```
Hi Christian,
The suggested ruleset appears to match the one in the upstream Ceph repo:
$ cat udev/50-rbd.rules
KERNEL=="rbd[0-9]*", ENV{DEVTYPE}=="disk", PROGRAM="/usr/bin/ceph-rbdnamer %k", SYMLINK+="rbd/%c"
KERNEL=="rbd[0-9]*", ENV{DEVTYPE}=="partition", PROGRAM="/usr/bin/ceph-rbdnamer %k", SYMLINK+="rbd/%c-part%n"
# This is a placeholder, uncomment and edit as necessary
#KERNEL=="rbd[0-9]*", ENV{DEVTYPE}=="disk", ACTION=="add|change", ATTR{bdi/read_ahead_kb}="128"
Other than the two slashes in ceph-rbdnamer path in your suggestion (/usr//bin/ceph-rbdnamer) which I assume to be a typo, I don't see any difference.
Updated by Oliver Schmidt over 1 year ago
Ilya Dryomov wrote:
Hi Christian,
The suggested ruleset appears to match the one in the upstream Ceph repo:
You're correct that the bundled udev rule file indeed contains the correct rules matching our suggestion: https://github.com/ceph/ceph/blob/9939dcc40134c15145a004abfc6bb126f51e32c7/udev/50-rbd.rules
But what is still wrong is the manpage for `rbdnamer` which still contains an old and broken udev rule: https://github.com/ceph/ceph/blob/9939dcc40134c15145a004abfc6bb126f51e32c7/doc/man/8/ceph-rbdnamer.rst
This is the part of documentation that needs to be updated.
[context: I was also involved in discovering this issue alongside Christian]
Updated by Ilya Dryomov over 1 year ago
- Status changed from Need More Info to In Progress
- Target version deleted (
v18.0.0)
Oh, I see. We will remove that from the man page.
Updated by Ilya Dryomov over 1 year ago
- Status changed from In Progress to Fix Under Review
- Pull request ID set to 49690
Updated by Ilya Dryomov over 1 year ago
- Status changed from Fix Under Review to Pending Backport
- Backport set to pacific,quincy
Updated by Backport Bot over 1 year ago
- Copied to Backport #58413: pacific: ceph-rbdnamer manpage shows incompatible/outdated udev rules added
Updated by Backport Bot over 1 year ago
- Copied to Backport #58414: quincy: ceph-rbdnamer manpage shows incompatible/outdated udev rules added
Updated by Ilya Dryomov over 1 year ago
- Status changed from Pending Backport to Resolved