; ; Sample ceph_bup ceph_bup.conf file. ; ; This file defines cluster membership, the various locations ; that Ceph stores data, and any other runtime options. ; If a 'host' is defined for a daemon, the start/stop script will ; verify that it matches the hostname (or else ignore it). If it is ; not defined, it is assumed that the daemon is intended to start on ; the current host (e.g., in a setup with a startup.conf on each ; node). ; global [global] ; enable secure authentication ;auth supported = cephx max open files = 131072 log file = /scratch/ceph_bup/ceph.log ; monitors ; You need at least one. You need at least three if you want to ; tolerate any node failures. Always create an odd number. [mon] mon data = /scratch/ceph_bup/mon$id ; some minimal logging (just message traffic) to aid debugging debug ms = 1 debug mon = 20 [mon.0] host = bzt1 mon addr = 192.168.2.101:6789 ; mds ; You need at least one. Define two to get a standby. [mds] ; where the mds keeps it's secret encryption keys debug ms = 1 debug mds = 20 [mds.0] host = bzt1 ; osd ; You need at least one. Two if you want data to be replicated. ; Define as many as you like. [osd] debug ms = 1 debug osd = 10 debug journal = 20 debug filestore = 20 osd journal = /data/osd$id/journal osd journal size = 1000 ; journal size, in megabytes ; This is where the btrfs volume will be mounted. osd data = /data/osd$id ; Ideally, make this a separate disk or partition. A few GB ; is usually enough; more if you have fast disks. You can use ; a file under the osd data dir if need be ; (e.g. /data/osd$id/journal), but it will be slower than a ; separate disk or partition. ; if 'btrfs devs' is not specified, you're responsible for ; setting up the 'osd data' dir. if it is not btrfs, things ; will behave up until you try to recover from a crash (which ; usually fine for basic testing). ; Every bz data nodes have 3 disks partitioned into 3. sda1 - scratch, sda2 - lustre, sda3 - ceph_bup ; So every node with provide 3 OSD/OST one from each disk 3rd partition ; bz1 is designated MDS for ceph_bup now [osd.0] host = bzt2 btrfs devs = /dev/sda3 [osd.1] host = bzt2 btrfs devs = /dev/sdb3 [osd.2] host = bzt2 btrfs devs = /dev/sdc3 [osd.3] host = bzt2 btrfs devs = /dev/sdd3 [osd.4] host = bzt2 btrfs devs = /dev/sde3 [osd.5] host = bzt2 btrfs devs = /dev/sdf3 ; next host bz3 [osd.6] host = bzt3 btrfs devs = /dev/sda3 [osd.7] host = bzt3 btrfs devs = /dev/sdb3 [osd.8] host = bzt3 btrfs devs = /dev/sdc3 [osd.9] host = bzt3 btrfs devs = /dev/sdd3 [osd.10] host = bzt3 btrfs devs = /dev/sde3 [osd.11] host = bzt3 btrfs devs = /dev/sdf3 ; Next host bz4 [osd.12] host = bzt4 btrfs devs = /dev/sda3 [osd.13] host = bzt4 btrfs devs = /dev/sdb3 [osd.14] host = bzt4 btrfs devs = /dev/sdc3 [osd.15] host = bzt4 btrfs devs = /dev/sdd3 [osd.16] host = bzt4 btrfs devs = /dev/sde3 [osd.17] host = bzt4 btrfs devs = /dev/sdf3 ; bz5 3 disks 3 partition ; [osd.18] host = bzt5 btrfs devs = /dev/sda3 [osd.19] host = bzt5 btrfs devs = /dev/sdb3 [osd.20] host = bzt5 btrfs devs = /dev/sdc3 [osd.21] host = bzt5 btrfs devs = /dev/sdd3 [osd.22] host = bzt5 btrfs devs = /dev/sde3 [osd.23] host = bzt5 btrfs devs = /dev/sdf3 ; bz6 3 disks 3 partition [osd.24] host = bzt6 btrfs devs = /dev/sda3 [osd.25] host = bzt6 btrfs devs = /dev/sdb3 [osd.26] host = bzt6 btrfs devs = /dev/sdc3 [osd.27] host = bzt6 btrfs devs = /dev/sdd3 [osd.28] host = bzt6 btrfs devs = /dev/sde3 [osd.29] host = bzt6 btrfs devs = /dev/sdf3 ; bz7 3 disks 3 partition ; [osd.30] host = bzt7 btrfs devs = /dev/sda3 [osd.31] host = bzt7 btrfs devs = /dev/sdb3 [osd.32] host = bzt7 btrfs devs = /dev/sdc3 [osd.33] host = bzt7 btrfs devs = /dev/sdd3 [osd.34] host = bzt7 btrfs devs = /dev/sde3 [osd.35] host = bzt7 btrfs devs = /dev/sdf3 ; bz8 3 disks 3rd partition from each drive. [osd.36] host = bzt8 btrfs devs = /dev/sda3 [osd.37] host = bzt8 btrfs devs = /dev/sdb3 [osd.38] host = bzt8 btrfs devs = /dev/sdc3 [osd.39] host = bzt8 btrfs devs = /dev/sdd3 [osd.40] host = bzt8 btrfs devs = /dev/sde3 [osd.41] host = bzt8 btrfs devs = /dev/sdf3 ; bz9 host 6 osds [osd.42] host = bzt9 btrfs devs = /dev/sda3 [osd.43] host = bzt9 btrfs devs = /dev/sdb3 [osd.44] host = bzt9 btrfs devs = /dev/sdc3 [osd.45] host = bzt9 btrfs devs = /dev/sdd3 [osd.46] host = bzt9 btrfs devs = /dev/sde3 [osd.47] host = bzt9 btrfs devs = /dev/sdf3 ; last host bz10 [osd.48] host = bzt10 btrfs devs = /dev/sda3 [osd.49] host = bzt10 btrfs devs = /dev/sdb3 [osd.50] host = bzt10 btrfs devs = /dev/sdc3 [osd.51] host = bzt10 btrfs devs = /dev/sdd3 [osd.52] host = bzt10 btrfs devs = /dev/sde3 [osd.53] host = bzt10 btrfs devs = /dev/sdf3 [osd.54] host = bzt1 btrfs devs = /dev/sda3 [osd.55] host = bzt1 btrfs devs = /dev/sdb3 [osd.56] host = bzt1 btrfs devs = /dev/sdc3 [osd.57] host = bzt1 btrfs devs = /dev/sdd3 [osd.58] host = bzt1 btrfs devs = /dev/sde3 [osd.59] host = bzt1 btrfs devs = /dev/sdf3