Ceph

Only issue that I found that could be similar was http://tracker.ceph.com/issues/20848

However, the key created looks valid, and deleting and recreating the client.bootstrap-mgr key does not fix.

root@mon1 ~ # ceph -s
    cluster f00d2bed-436a-4cd1-bb15-deaf6c4f4495
     health HEALTH_OK
     monmap e22: 3 mons at {mon1=172.28.19.7:6789/0,mon2=172.28.19.10:6789/0,mon3=172.28.19.21:6789/0}
            election epoch 1620, quorum 0,1,2 mon1,mon2,mon3
        mgr no daemons active 
     osdmap e46255: 16 osds: 16 up, 16 in
            flags sortbitwise,require_jewel_osds,require_kraken_osds
      pgmap v23318665: 6208 pgs, 3 pools, 1377 GB data, 212 Mobjects
            3374 GB used, 3539 GB / 6913 GB avail
                6208 active+clean
  client io 9958 B/s rd, 16823 kB/s wr, 201 op/s rd, 2298 op/s wr

ceph-deploy@mon1:~/deploy$ ceph-deploy mgr create mon1
[ceph_deploy.conf][DEBUG ] found configuration file at: /srv/ceph/.cephdeploy.conf
[ceph_deploy.cli][INFO  ] Invoked (1.5.38): /usr/bin/ceph-deploy --overwrite-conf mgr create mon1
[ceph_deploy.cli][INFO  ] ceph-deploy options:
[ceph_deploy.cli][INFO  ]  username                      : None
[ceph_deploy.cli][INFO  ]  verbose                       : False
[ceph_deploy.cli][INFO  ]  mgr                           : [('mon1', 'mon1')]
[ceph_deploy.cli][INFO  ]  overwrite_conf                : True
[ceph_deploy.cli][INFO  ]  subcommand                    : create
[ceph_deploy.cli][INFO  ]  quiet                         : False
[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f470ae69560>
[ceph_deploy.cli][INFO  ]  cluster                       : ceph
[ceph_deploy.cli][INFO  ]  func                          : <function mgr at 0x7f470b2bf500>
[ceph_deploy.cli][INFO  ]  ceph_conf                     : None
[ceph_deploy.cli][INFO  ]  default_release               : False
[ceph_deploy.mgr][DEBUG ] Deploying mgr, cluster ceph hosts mon1:mon1
[mon1][DEBUG ] connection detected need for sudo
[mon1][DEBUG ] connected to host: mon1 
[mon1][DEBUG ] detect platform information from remote host
[mon1][DEBUG ] detect machine type
[mon1][DEBUG ] find the location of an executable
[mon1][INFO  ] Running command: sudo /sbin/initctl version
[ceph_deploy.mgr][INFO  ] Distro info: Ubuntu 16.04 xenial
[ceph_deploy.mgr][DEBUG ] remote host will use upstart
[ceph_deploy.mgr][DEBUG ] deploying mgr bootstrap to mon1
[mon1][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[mon1][DEBUG ] create path if it doesn't exist
[mon1][INFO  ] Running command: sudo ceph --cluster ceph --name client.bootstrap-mgr --keyring /var/lib/ceph/bootstrap-mgr/ceph.keyring auth get-or-create mgr.mon1 mon allow profile mgr osd allow * mds allow * -o /var/lib/ceph/mgr/ceph-mon1/keyring
[mon1][ERROR ] Error EACCES: access denied
[mon1][ERROR ] exit code from command was: 13
[ceph_deploy.mgr][ERROR ] could not create mgr
[ceph_deploy][ERROR ] GenericError: Failed to create 1 MGRs

root@mon1 ~ # cat /var/lib/ceph/bootstrap-mgr/ceph.keyring 
[client.bootstrap-mgr]
    key = AQDV9JVZ/7elLhAA67celYbMvhyZezIqK3VDwQ==

root@mon1 ~ # ceph auth list
installed auth entries:

osd.0
    key: AQAG7CxYXSP6HBAAdwShVKq15j9aIa7yLVRMug==
    caps: [mon] allow profile osd
    caps: [osd] allow *
osd.1
    key: AQDEKC5Yus6aNhAAJiEgx1rE7ywZdEZPclBjDg==
    caps: [mon] allow profile osd
    caps: [osd] allow *
osd.10
    key: AQB0ZzRYtpxHLRAAtwKV3aqB1+XSrOBsOWTYVw==
    caps: [mon] allow profile osd
    caps: [osd] allow *
osd.11
    key: AQACaDRYSKsaEBAALb3+n1KIEJsMuwSzFO8UJQ==
    caps: [mon] allow profile osd
    caps: [osd] allow *
osd.12
    key: AQAtEzhYyVbEARAAGYDtAh54x79POP/sVeVjRQ==
    caps: [mon] allow profile osd
    caps: [osd] allow *
osd.13
    key: AQBXDDpYEMf4JhAAQU/duEChCfX82TQlI+6J1w==
    caps: [mon] allow profile osd
    caps: [osd] allow *
osd.14
    key: AQB8FThYHs8ILBAAOERUYUzP6/drqGh7TJ+5Xw==
    caps: [mon] allow profile osd
    caps: [osd] allow *
osd.15
    key: AQDtajhYEjphIxAAwWlpErZML54OWGcdA66Sgg==
    caps: [mon] allow profile osd
    caps: [osd] allow *
osd.2
    key: AQDexi1Ybu1VBhAASqRLp27uE3fWGOXc/ylhKQ==
    caps: [mon] allow profile osd
    caps: [osd] allow *
osd.3
    key: AQC65y1YQzZYEBAAZohQpHz9/Sh7sybqX8YYEw==
    caps: [mon] allow profile osd
    caps: [osd] allow *
osd.4
    key: AQCGYi1YVKizMhAA/6Pr/U9wmPMVQ9rbkoQwqg==
    caps: [mon] allow profile osd
    caps: [osd] allow *
osd.5
    key: AQBmFixYmNieBBAABx4JHL+lM+agwtmOV6uz3w==
    caps: [mon] allow profile osd
    caps: [osd] allow *
osd.6
    key: AQBF3yxYEmLzNBAAAFY1gGXW370+y+x2/dkx4Q==
    caps: [mon] allow profile osd
    caps: [osd] allow *
osd.7
    key: AQBJLitY+fiMLRAA1/0G0INJgdg5LZT4enXYKA==
    caps: [mon] allow profile osd
    caps: [osd] allow *
osd.8
    key: AQBqTDVYvWetJxAApXjAKeN8K8JQXDgKf16jaQ==
    caps: [mon] allow profile osd
    caps: [osd] allow *
osd.9
    key: AQABojRYL9PIExAAp6eJsRAYskofVjMD+MYFrA==
    caps: [mon] allow profile osd
    caps: [osd] allow *
client.admin
    key: AQCZer1XQa53NRAAnPYoldfBxA97FJAEpQuvbA==
    caps: [mds] allow *
    caps: [mon] allow *
    caps: [osd] allow *
client.bootstrap-mds
    key: AQCaer1XrWvHJhAAlZKjTFjASPVkX/2yJJYqyg==
    caps: [mon] allow profile bootstrap-mds
client.bootstrap-mgr
    key: AQDV9JVZ/7elLhAA67celYbMvhyZezIqK3VDwQ==
    caps: [mon] allow profile bootstrap-mgr
client.bootstrap-osd
    key: AQCaer1XuS7FCBAADm6llPoiHFCdu54ppb69jw==
    caps: [mon] allow profile bootstrap-osd
client.bootstrap-rgw
    key: AQCaer1XNYryGBAAZLGGSLZHjjG1l3uamU2KaA==
    caps: [mon] allow profile bootstrap-rgw