Ceph 13.2.8 三节点部署

bs-k8s-ceph eth1 mon osd mgr deploy 2c2g sdb sdc sdd 各20Gbs-hk-hk01 eth1 mon osd mgr 2c2g sdb sdc sdd 各20Gbs-hk-hk02 eth0 mon osd mgr 2c2g sdb sdc sdd 各20Gbs-k8s-node01 eth1 客户端 2c2g sdb sdc sdd 各20Gbs-k8s-node02 eth1 客户端 2c2g sdb sdc sdd 各20Gbs-k8s-node03 eth1 客户端 2c2g sdb sdc sdd 各20G注：# 为bs-k8s-ceph bs-hk-hk01 bs-hk-hk02 上都操作# uname -aLinux bs-k8s-ceph -.el7.elrepo.x86_64 # SMP Sun Jul :: EDT x86_64 x86_64 x86_64 GNU/Linux# hostname -I20.0.0.208 10.0.0.208# ls /dev/sd*/dev/sda /dev/sda1 /dev/sda2 /dev/sda3 /dev/sdb /dev/sdc /dev/sdd# systemctl status firewalld● firewalld.service - firewalld - dynamic firewall daemon Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled) Active: inactive (dead) Docs: )# getenforceDisabled时间同步 采取内外双向同步法外同步# yum install -y ntpdate# ntpdate cn.pool.ntp.org内同步# yum -y install chrony[root@bs-k8s-ceph ~]# cp /etc/chrony.conf{,.bak}[root@bs-k8s-ceph ~]# vim /etc/chrony.conf[root@bs-k8s-ceph ~]# diff /etc/chrony.conf{,.bak}26c26< allow---> #allow[root@bs-hk-hk01 ~]# cp /etc/chrony.conf{,.bak}[root@bs-hk-hk01 ~]# vim /etc/chrony.conf[root@bs-hk-hk01 ~]# diff /etc/chrony.conf{,.bak}3c3,< server 10.0.0.208 iburst---> server .centos.pool.ntp.org iburst> server .centos.pool.ntp.org iburst> server .centos.pool.ntp.org iburst> server .centos.pool.ntp.org iburst[root@bs-hk-hk01 ~]# scp /etc/chrony.conf 20.0.0.207:/etc/chrony.conf# systemctl restart chronyd.service# chronyc sources -vssh-keygen信任[root@bs-k8s-ceph ~]# cat /etc/hosts127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4:: localhost localhost.localdomain localhost6 localhost6.localdomain610.0.0.206 bs-hk-hk0110.0.0.207 bs-hk-hk0210.0.0.208 bs-k8s-ceph[root@bs-k8s-ceph ~]# mkdir /service/scripts -p[root@bs-k8s-ceph ~]# vim /service/scripts/ssh-key.sh[root@bs-k8s-ceph ~]# vim /service/scripts/ssh-key.sh[root@bs-k8s-ceph ~]# cat /service/scripts/ssh-key.sh###########################################################################Author: zisefeizhu#QQ: ********#Date: --#FileName: /service/scripts/ssh-key.sh#URL: https://www.cnblogs.com/zisefeizhu/#Description: The test script#Copyright (C): All rights reserved###########################################################################!/bin/bashPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/binexport $PATH#目标主机列表IP="10.0.0.206bs-hk-hk0110.0.0.207bs-hk-hk0210.0.0.208bs-k8s-ceph"for node in ${IP};do sshpass -p ssh-copy-id ${node} -o StrictHostKeyChecking=no scp /etc/hosts ${node}:/etc/hosts ];then echo "${node} 秘钥copy完成" else echo "${node} 秘钥copy失败" fidone[root@bs-k8s-ceph ~]# yum install -y sshpass[root@bs-k8s-ceph ~]# ssh-keygen -t rsa准备数据盘# lsblkNAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINTsda : 20G disk├─sda1 : 1G part /boot├─sda2 : 1G part [SWAP]└─sda3 : 18G part /sdb : 20G disksdc : 20G disksdd : 20G disksr0 : 918M rom在每个节点上为Journal磁盘分区, 分别为 sdb1, sdb2, 各自对应本机的2个OSD, journal磁盘对应osd的大小为25%使用 parted 命令进行创建分区操作# parted /dev/sdbGNU Parted 3.1使用 /dev/sdbWelcome to GNU Parted! Type 'help' to view a list of commands.(parted) mklabel gpt(parted) mkpart primary xfs % %(parted) mkpart primary xfs % %(parted) q信息: You may need to update /etc/fstab.# lsblkNAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT......sdb : 20G disk├─sdb1 : 10G part└─sdb2 : 10G part......配置源# cat /etc/yum.repos.d/ceph.repo[Ceph]name=Ceph packages for $basearchbaseurl=https://mirrors.aliyun.com/ceph/rpm-mimic/el7/$basearchenabled=gpgcheck=type=rpm-mdgpgkey=https://download.ceph.com/keys/release.asc[Ceph-noarch]name=Ceph noarch packagesbaseurl=https://mirrors.aliyun.com/ceph/rpm-mimic/el7/noarchenabled=gpgcheck=type=rpm-mdgpgkey=https://download.ceph.com/keys/release.asc[ceph-source]name=Ceph source packagesbaseurl=https://mirrors.aliyun.com/ceph/rpm-mimic/el7/SRPMSenabled=gpgcheck=type=rpm-mdgpgkey=https://download.ceph.com/keys/release.asc# yum clean all && yum makecache# yum install snappy leveldb gdisk python-argparse gperftools-libs -y 部署ceph-deploy bs-k8s-ceph[root@bs-k8s-ceph ~]# yum install -y ceph-deploy python-pip[root@bs-k8s-ceph ~]# ceph-deploy --version管理节点创建工作目录 [root@bs-k8s-ceph ~]# mkdir /etc/ceph 创建三个mon[root@bs-k8s-ceph ceph]# ceph-deploy new bs-k8s-ceph bs-hk-hk01 bs-hk-hk02[root@bs-k8s-ceph ceph]# lsceph.conf ceph-deploy-ceph.log ceph.mon.keyring[root@bs-k8s-ceph ceph]# cp ceph.conf ceph.conf-`date +%F`[root@bs-k8s-ceph ceph]# --,9d7< public network =< cluster network = 安装ceph#yum install -y ceph ceph-radosgw初始monitor 并收集密钥分发密钥到其他节点[root@bs-k8s-ceph ceph]# ceph-deploy admin bs-k8s-ceph bs-hk-hk01 bs-hk-hk02[root@bs-k8s-ceph ceph]# ceph-deploy osd create bs-k8s-ceph --data /dev/sdc --journal /dev/sdb1[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf[ceph_deploy.cli][INFO ] Invoked (): /usr/bin/ceph-deploy osd create bs-k8s-ceph --data /dev/sdc --journal /dev/sdb1[ceph_deploy.cli][INFO ] ceph-deploy options:[ceph_deploy.cli][INFO ] verbose : False[ceph_deploy.cli][INFO ] bluestore : None[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7fa1152d8680>[ceph_deploy.cli][INFO ] cluster : ceph[ceph_deploy.cli][INFO ] fs_type : xfs[ceph_deploy.cli][INFO ] block_wal : None[ceph_deploy.cli][INFO ] default_release : False[ceph_deploy.cli][INFO ] username : None[ceph_deploy.cli][INFO ] journal : /dev/sdb1[ceph_deploy.cli][INFO ] subcommand : create[ceph_deploy.cli][INFO ] host : bs-k8s-ceph[ceph_deploy.cli][INFO ] filestore : None[ceph_deploy.cli][INFO ] func : <function osd at 0x7fa1155228c0>[ceph_deploy.cli][INFO ] ceph_conf : None[ceph_deploy.cli][INFO ] zap_disk : False[ceph_deploy.cli][INFO ] data : /dev/sdc[ceph_deploy.cli][INFO ] block_db : None[ceph_deploy.cli][INFO ] dmcrypt : False[ceph_deploy.cli][INFO ] overwrite_conf : False[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys[ceph_deploy.cli][INFO ] quiet : False[ceph_deploy.cli][INFO ] debug : False[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdc[bs-k8s-ceph][DEBUG ] connected to host: bs-k8s-ceph[bs-k8s-ceph][DEBUG ] detect platform information from remote host[bs-k8s-ceph][DEBUG ] detect machine type[bs-k8s-ceph][DEBUG ] find the location of an executable[ceph_deploy.osd][INFO ] Distro Core[ceph_deploy.osd][DEBUG ] Deploying osd to bs-k8s-ceph[bs-k8s-ceph][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf[bs-k8s-ceph][WARNIN] osd keyring does not exist yet, creating one[bs-k8s-ceph][DEBUG ] create a keyring file[bs-k8s-ceph][DEBUG ] find the location of an executable[bs-k8s-ceph][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdc[bs-k8s-ceph][WARNIN] Running command: /bin/ceph-authtool --gen-print-key[bs-k8s-ceph][WARNIN] Running command: /bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 37d3767e-4e21-448c-91f4-d3a1079bd8bd[bs-k8s-ceph][WARNIN] Running command: /usr/sbin/vgcreate -s 1G --force --yes ceph-9f219cda--4ea0-8b79-2899b68a94d9 /dev/sdc[bs-k8s-ceph][WARNIN] stdout: Physical volume "/dev/sdc" successfully created.[bs-k8s-ceph][WARNIN] stdout: Volume group "ceph-9f219cda-7297-4ea0-8b79-2899b68a94d9" successfully created[bs-k8s-ceph][WARNIN] Running command: /usr/sbin/lvcreate --yes -l %FREE -n osd-block-37d3767e-4e21-448c-91f4-d3a1079bd8bd ceph-9f219cda--4ea0-8b79-2899b68a94d9[bs-k8s-ceph][WARNIN] stdout: Logical volume "osd-block-37d3767e-4e21-448c-91f4-d3a1079bd8bd" created.[bs-k8s-ceph][WARNIN] Running command: /bin/ceph-authtool --gen-print-key[bs-k8s-ceph][WARNIN] Running command: /bin/[bs-k8s-ceph][WARNIN] Running command: /usr/sbin/restorecon /var/lib/ceph/osd/ceph-[bs-k8s-ceph][WARNIN] Running command: /bin/-4ea0-8b79-2899b68a94d9/osd-block-37d3767e-4e21-448c-91f4-d3a1079bd8bd[bs-k8s-ceph][WARNIN] Running command: /bin/[bs-k8s-ceph][WARNIN] Running command: /bin/-4ea0-8b79-2899b68a94d9/osd-block-37d3767e-4e21-448c-91f4-d3a1079bd8bd /var/lib/ceph/osd/ceph-/block[bs-k8s-ceph][WARNIN] Running command: /bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-/activate.monmap[bs-k8s-ceph][WARNIN] stderr: got monmap epoch[bs-k8s-ceph][WARNIN] Running command: /bin/ceph-authtool /var/lib/ceph/osd/ceph-/keyring --create-keyring --name osd. --add-key AQA4ckJerZKGGBAAK2c6hoeR8WkOAZMi3h7vjw==[bs-k8s-ceph][WARNIN] stdout: creating /var/lib/ceph/osd/ceph-/keyring[bs-k8s-ceph][WARNIN] added entity osd. auth auth(auid = key=AQA4ckJerZKGGBAAK2c6hoeR8WkOAZMi3h7vjw== with caps)[bs-k8s-ceph][WARNIN] Running command: /bin//keyring[bs-k8s-ceph][WARNIN] Running command: /bin//[bs-k8s-ceph][WARNIN] Running command: /bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i --monmap /var/lib/ceph/osd/ceph-/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-/ --osd-uuid 37d3767e-4e21-448c-91f4-d3a1079bd8bd --setuser ceph --setgroup ceph[bs-k8s-ceph][WARNIN] --> ceph-volume lvm prepare successful for: /dev/sdc[bs-k8s-ceph][WARNIN] Running command: /bin/[bs-k8s-ceph][WARNIN] Running command: /bin/ceph-bluestore-tool --cluster=ceph prime-osd--4ea0-8b79-2899b68a94d9/osd-block-37d3767e-4e21-448c-91f4-d3a1079bd8bd --path /var/lib/ceph/osd/ceph- --no-mon-config[bs-k8s-ceph][WARNIN] Running command: /bin/-4ea0-8b79-2899b68a94d9/osd-block-37d3767e-4e21-448c-91f4-d3a1079bd8bd /var/lib/ceph/osd/ceph-/block[bs-k8s-ceph][WARNIN] Running command: /bin//block[bs-k8s-ceph][WARNIN] Running command: /bin/[bs-k8s-ceph][WARNIN] Running command: /bin/[bs-k8s-ceph][WARNIN] Running command: /bin/systemctl enable ceph-volume@lvm--37d3767e-4e21-448c-91f4-d3a1079bd8bd[bs-k8s-ceph][WARNIN] stderr: Created symlink from /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm--37d3767e-4e21-448c-91f4-d3a1079bd8bd.service to /usr/lib/systemd/system/ceph-volume@.service.[bs-k8s-ceph][WARNIN] Running command: /bin/systemctl enable --runtime ceph-osd@[bs-k8s-ceph][WARNIN] stderr: Created symlink from /run/systemd/system/ceph-osd.target.wants/ceph-osd@.service to /usr/lib/systemd/system/ceph-osd@.service.[bs-k8s-ceph][WARNIN] Running command: /bin/systemctl start ceph-osd@[bs-k8s-ceph][WARNIN] --> ceph-volume lvm activate successful[bs-k8s-ceph][WARNIN] --> ceph-volume lvm create successful for: /dev/sdc[bs-k8s-ceph][INFO ] checking OSD status...[bs-k8s-ceph][DEBUG ] find the location of an executable[bs-k8s-ceph][INFO ] Running command: /bin/ceph --cluster=ceph osd stat --format=json[ceph_deploy.osd][DEBUG ] Host bs-k8s-ceph is now ready for osd use.[root@bs-k8s-ceph ceph]# ceph-deploy osd create bs-k8s-ceph --data /dev/sdd --journal /dev/sdb1[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf[ceph_deploy.cli][INFO ] Invoked (): /usr/bin/ceph-deploy osd create bs-k8s-ceph --data /dev/sdd --journal /dev/sdb1[ceph_deploy.cli][INFO ] ceph-deploy options:[ceph_deploy.cli][INFO ] verbose : False[ceph_deploy.cli][INFO ] bluestore : None[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f342f4d0680>[ceph_deploy.cli][INFO ] cluster : ceph[ceph_deploy.cli][INFO ] fs_type : xfs[ceph_deploy.cli][INFO ] block_wal : None[ceph_deploy.cli][INFO ] default_release : False[ceph_deploy.cli][INFO ] username : None[ceph_deploy.cli][INFO ] journal : /dev/sdb1[ceph_deploy.cli][INFO ] subcommand : create[ceph_deploy.cli][INFO ] host : bs-k8s-ceph[ceph_deploy.cli][INFO ] filestore : None[ceph_deploy.cli][INFO ] func : <function osd at 0x7f342f71a8c0>[ceph_deploy.cli][INFO ] ceph_conf : None[ceph_deploy.cli][INFO ] zap_disk : False[ceph_deploy.cli][INFO ] data : /dev/sdd[ceph_deploy.cli][INFO ] block_db : None[ceph_deploy.cli][INFO ] dmcrypt : False[ceph_deploy.cli][INFO ] overwrite_conf : False[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys[ceph_deploy.cli][INFO ] quiet : False[ceph_deploy.cli][INFO ] debug : False[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdd[bs-k8s-ceph][DEBUG ] connected to host: bs-k8s-ceph[bs-k8s-ceph][DEBUG ] detect platform information from remote host[bs-k8s-ceph][DEBUG ] detect machine type[bs-k8s-ceph][DEBUG ] find the location of an executable[ceph_deploy.osd][INFO ] Distro Core[ceph_deploy.osd][DEBUG ] Deploying osd to bs-k8s-ceph[bs-k8s-ceph][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf[bs-k8s-ceph][DEBUG ] find the location of an executable[bs-k8s-ceph][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdd[bs-k8s-ceph][WARNIN] Running command: /bin/ceph-authtool --gen-print-key[bs-k8s-ceph][WARNIN] Running command: /bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 0f718ba7-de6b-4d22-b74d-90657e220121[bs-k8s-ceph][WARNIN] Running command: /usr/sbin/vgcreate -s 1G --force --yes ceph-b722303f-5fee--84d1-057b1d05ca96 /dev/sdd[bs-k8s-ceph][WARNIN] stdout: Physical volume "/dev/sdd" successfully created.[bs-k8s-ceph][WARNIN] stdout: Volume group "ceph-b722303f-5fee-4425-84d1-057b1d05ca96" successfully created[bs-k8s-ceph][WARNIN] Running command: /usr/sbin/lvcreate --yes -l %FREE -n osd-block-0f718ba7-de6b-4d22-b74d-90657e220121 ceph-b722303f-5fee--84d1-057b1d05ca96[bs-k8s-ceph][WARNIN] stdout: Logical volume "osd-block-0f718ba7-de6b-4d22-b74d-90657e220121" created.[bs-k8s-ceph][WARNIN] Running command: /bin/ceph-authtool --gen-print-key[bs-k8s-ceph][WARNIN] Running command: /bin/[bs-k8s-ceph][WARNIN] Running command: /usr/sbin/restorecon /var/lib/ceph/osd/ceph-[bs-k8s-ceph][WARNIN] Running command: /bin/-84d1-057b1d05ca96/osd-block-0f718ba7-de6b-4d22-b74d-90657e220121[bs-k8s-ceph][WARNIN] Running command: /bin/[bs-k8s-ceph][WARNIN] Running command: /bin/-84d1-057b1d05ca96/osd-block-0f718ba7-de6b-4d22-b74d-90657e220121 /var/lib/ceph/osd/ceph-/block[bs-k8s-ceph][WARNIN] Running command: /bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-/activate.monmap[bs-k8s-ceph][WARNIN] stderr: got monmap epoch[bs-k8s-ceph][WARNIN] stderr:[bs-k8s-ceph][WARNIN] Running command: /bin/ceph-authtool /var/lib/ceph/osd/ceph-/keyring --create-keyring --name osd. --add-key AQBMckJeIKP3FhAAqOLDpdF+kDfZu5cQC0zmww==[bs-k8s-ceph][WARNIN] stdout: creating /var/lib/ceph/osd/ceph-/keyring[bs-k8s-ceph][WARNIN] added entity osd. auth auth(auid = key=AQBMckJeIKP3FhAAqOLDpdF+kDfZu5cQC0zmww== with caps)[bs-k8s-ceph][WARNIN] Running command: /bin//keyring[bs-k8s-ceph][WARNIN] Running command: /bin//[bs-k8s-ceph][WARNIN] Running command: /bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i --monmap /var/lib/ceph/osd/ceph-/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-/ --osd-uuid 0f718ba7-de6b-4d22-b74d-90657e220121 --setuser ceph --setgroup ceph[bs-k8s-ceph][WARNIN] --> ceph-volume lvm prepare successful for: /dev/sdd[bs-k8s-ceph][WARNIN] Running command: /bin/[bs-k8s-ceph][WARNIN] Running command: /bin/ceph-bluestore-tool --cluster=ceph prime-osd--84d1-057b1d05ca96/osd-block-0f718ba7-de6b-4d22-b74d-90657e220121 --path /var/lib/ceph/osd/ceph- --no-mon-config[bs-k8s-ceph][WARNIN] Running command: /bin/-84d1-057b1d05ca96/osd-block-0f718ba7-de6b-4d22-b74d-90657e220121 /var/lib/ceph/osd/ceph-/block[bs-k8s-ceph][WARNIN] Running command: /bin//block[bs-k8s-ceph][WARNIN] Running command: /bin/[bs-k8s-ceph][WARNIN] Running command: /bin/[bs-k8s-ceph][WARNIN] Running command: /bin/systemctl enable ceph-volume@lvm--0f718ba7-de6b-4d22-b74d-90657e220121[bs-k8s-ceph][WARNIN] stderr: Created symlink from /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm--0f718ba7-de6b-4d22-b74d-90657e220121.service to /usr/lib/systemd/system/ceph-volume@.service.[bs-k8s-ceph][WARNIN] Running command: /bin/systemctl enable --runtime ceph-osd@[bs-k8s-ceph][WARNIN] stderr: Created symlink from /run/systemd/system/ceph-osd.target.wants/ceph-osd@.service to /usr/lib/systemd/system/ceph-osd@.service.[bs-k8s-ceph][WARNIN] Running command: /bin/systemctl start ceph-osd@[bs-k8s-ceph][WARNIN] --> ceph-volume lvm activate successful[bs-k8s-ceph][WARNIN] --> ceph-volume lvm create successful for: /dev/sdd[bs-k8s-ceph][INFO ] checking OSD status...[bs-k8s-ceph][DEBUG ] find the location of an executable[bs-k8s-ceph][INFO ] Running command: /bin/ceph --cluster=ceph osd stat --format=json[ceph_deploy.osd][DEBUG ] Host bs-k8s-ceph is now ready for osd use.[root@bs-k8s-ceph ceph]# ceph-deploy osd create bs-hk-hk01 --data /dev/sdc --journal /dev/sdb1[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf[ceph_deploy.cli][INFO ] Invoked (): /usr/bin/ceph-deploy osd create bs-hk-hk01 --data /dev/sdc --journal /dev/sdb1[ceph_deploy.cli][INFO ] ceph-deploy options:[ceph_deploy.cli][INFO ] verbose : False[ceph_deploy.cli][INFO ] bluestore : None[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7fc87226c680>[ceph_deploy.cli][INFO ] cluster : ceph[ceph_deploy.cli][INFO ] fs_type : xfs[ceph_deploy.cli][INFO ] block_wal : None[ceph_deploy.cli][INFO ] default_release : False[ceph_deploy.cli][INFO ] username : None[ceph_deploy.cli][INFO ] journal : /dev/sdb1[ceph_deploy.cli][INFO ] subcommand : create[ceph_deploy.cli][INFO ] host : bs-hk-hk01[ceph_deploy.cli][INFO ] filestore : None[ceph_deploy.cli][INFO ] func : <function osd at 0x7fc8724b68c0>[ceph_deploy.cli][INFO ] ceph_conf : None[ceph_deploy.cli][INFO ] zap_disk : False[ceph_deploy.cli][INFO ] data : /dev/sdc[ceph_deploy.cli][INFO ] block_db : None[ceph_deploy.cli][INFO ] dmcrypt : False[ceph_deploy.cli][INFO ] overwrite_conf : False[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys[ceph_deploy.cli][INFO ] quiet : False[ceph_deploy.cli][INFO ] debug : False[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdc[bs-hk-hk01][DEBUG ] connected to host: bs-hk-hk01[bs-hk-hk01][DEBUG ] detect platform information from remote host[bs-hk-hk01][DEBUG ] detect machine type[bs-hk-hk01][DEBUG ] find the location of an executable[ceph_deploy.osd][INFO ] Distro Core[ceph_deploy.osd][DEBUG ] Deploying osd to bs-hk-hk01[bs-hk-hk01][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf[bs-hk-hk01][WARNIN] osd keyring does not exist yet, creating one[bs-hk-hk01][DEBUG ] create a keyring file[bs-hk-hk01][DEBUG ] find the location of an executable[bs-hk-hk01][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdc[bs-hk-hk01][WARNIN] Running command: /bin/ceph-authtool --gen-print-key[bs-hk-hk01][WARNIN] Running command: /bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 0f7e4e52-5e01-45bd-9ef6-55ec53db45e9[bs-hk-hk01][WARNIN] Running command: /usr/sbin/vgcreate -s 1G --force --yes ceph-8ee7795f-ac6c-42a8-b930-44f06f4d19b6 /dev/sdc[bs-hk-hk01][WARNIN] stdout: Physical volume "/dev/sdc" successfully created.[bs-hk-hk01][WARNIN] stdout: Volume group "ceph-8ee7795f-ac6c-42a8-b930-44f06f4d19b6" successfully created[bs-hk-hk01][WARNIN] Running command: /usr/sbin/lvcreate --yes -l %FREE -n osd-block-0f7e4e52-5e01-45bd-9ef6-55ec53db45e9 ceph-8ee7795f-ac6c-42a8-b930-44f06f4d19b6[bs-hk-hk01][WARNIN] stdout: Logical volume "osd-block-0f7e4e52-5e01-45bd-9ef6-55ec53db45e9" created.[bs-hk-hk01][WARNIN] Running command: /bin/ceph-authtool --gen-print-key[bs-hk-hk01][WARNIN] Running command: /bin/[bs-hk-hk01][WARNIN] Running command: /usr/sbin/restorecon /var/lib/ceph/osd/ceph-[bs-hk-hk01][WARNIN] Running command: /bin/chown -h ceph:ceph /dev/ceph-8ee7795f-ac6c-42a8-b930-44f06f4d19b6/osd-block-0f7e4e52-5e01-45bd-9ef6-55ec53db45e9[bs-hk-hk01][WARNIN] Running command: /bin/[bs-hk-hk01][WARNIN] Running command: /bin//block[bs-hk-hk01][WARNIN] Running command: /bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-/activate.monmap[bs-hk-hk01][WARNIN] stderr: got monmap epoch[bs-hk-hk01][WARNIN] Running command: /bin/ceph-authtool /var/lib/ceph/osd/ceph-/keyring --create-keyring --name osd. --add-key AQB0ckJeXwuZHRAA6nfA5V/kCZs9FjJYm6JOfw==[bs-hk-hk01][WARNIN] stdout: creating /var/lib/ceph/osd/ceph-/keyring[bs-hk-hk01][WARNIN] added entity osd. auth auth(auid = key=AQB0ckJeXwuZHRAA6nfA5V/kCZs9FjJYm6JOfw== with caps)[bs-hk-hk01][WARNIN] Running command: /bin//keyring[bs-hk-hk01][WARNIN] Running command: /bin//[bs-hk-hk01][WARNIN] Running command: /bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i --monmap /var/lib/ceph/osd/ceph-/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-/ --osd-uuid 0f7e4e52-5e01-45bd-9ef6-55ec53db45e9 --setuser ceph --setgroup ceph[bs-hk-hk01][WARNIN] --> ceph-volume lvm prepare successful for: /dev/sdc[bs-hk-hk01][WARNIN] Running command: /bin/[bs-hk-hk01][WARNIN] Running command: /bin/ceph-bluestore-tool --cluster=ceph prime-osd- --no-mon-config[bs-hk-hk01][WARNIN] Running command: /bin//block[bs-hk-hk01][WARNIN] Running command: /bin//block[bs-hk-hk01][WARNIN] Running command: /bin/[bs-hk-hk01][WARNIN] Running command: /bin/[bs-hk-hk01][WARNIN] Running command: /bin/systemctl enable ceph-volume@lvm--0f7e4e52-5e01-45bd-9ef6-55ec53db45e9[bs-hk-hk01][WARNIN] stderr: Created symlink from /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm--0f7e4e52-5e01-45bd-9ef6-55ec53db45e9.service to /usr/lib/systemd/system/ceph-volume@.service.[bs-hk-hk01][WARNIN] Running command: /bin/systemctl enable --runtime ceph-osd@[bs-hk-hk01][WARNIN] stderr: Created symlink from /run/systemd/system/ceph-osd.target.wants/ceph-osd@.service to /usr/lib/systemd/system/ceph-osd@.service.[bs-hk-hk01][WARNIN] Running command: /bin/systemctl start ceph-osd@[bs-hk-hk01][WARNIN] --> ceph-volume lvm activate successful[bs-hk-hk01][WARNIN] --> ceph-volume lvm create successful for: /dev/sdc[bs-hk-hk01][INFO ] checking OSD status...[bs-hk-hk01][DEBUG ] find the location of an executable[bs-hk-hk01][INFO ] Running command: /bin/ceph --cluster=ceph osd stat --format=json[ceph_deploy.osd][DEBUG ] Host bs-hk-hk01 is now ready for osd use.[root@bs-k8s-ceph ceph]# ceph-deploy osd create bs-hk-hk01 --data /dev/sdd --journal /dev/sdb1[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf[ceph_deploy.cli][INFO ] Invoked (): /usr/bin/ceph-deploy osd create bs-hk-hk01 --data /dev/sdd --journal /dev/sdb1[ceph_deploy.cli][INFO ] ceph-deploy options:[ceph_deploy.cli][INFO ] verbose : False[ceph_deploy.cli][INFO ] bluestore : None[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f90903ad680>[ceph_deploy.cli][INFO ] cluster : ceph[ceph_deploy.cli][INFO ] fs_type : xfs[ceph_deploy.cli][INFO ] block_wal : None[ceph_deploy.cli][INFO ] default_release : False[ceph_deploy.cli][INFO ] username : None[ceph_deploy.cli][INFO ] journal : /dev/sdb1[ceph_deploy.cli][INFO ] subcommand : create[ceph_deploy.cli][INFO ] host : bs-hk-hk01[ceph_deploy.cli][INFO ] filestore : None[ceph_deploy.cli][INFO ] func : <function osd at 0x7f90905f78c0>[ceph_deploy.cli][INFO ] ceph_conf : None[ceph_deploy.cli][INFO ] zap_disk : False[ceph_deploy.cli][INFO ] data : /dev/sdd[ceph_deploy.cli][INFO ] block_db : None[ceph_deploy.cli][INFO ] dmcrypt : False[ceph_deploy.cli][INFO ] overwrite_conf : False[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys[ceph_deploy.cli][INFO ] quiet : False[ceph_deploy.cli][INFO ] debug : False[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdd[bs-hk-hk01][DEBUG ] connected to host: bs-hk-hk01[bs-hk-hk01][DEBUG ] detect platform information from remote host[bs-hk-hk01][DEBUG ] detect machine type[bs-hk-hk01][DEBUG ] find the location of an executable[ceph_deploy.osd][INFO ] Distro Core[ceph_deploy.osd][DEBUG ] Deploying osd to bs-hk-hk01[bs-hk-hk01][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf[bs-hk-hk01][DEBUG ] find the location of an executable[bs-hk-hk01][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdd[bs-hk-hk01][WARNIN] Running command: /bin/ceph-authtool --gen-print-key[bs-hk-hk01][WARNIN] Running command: /bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 7d03f8ee-8ada-485e-a20b-8677a7e57c84[bs-hk-hk01][WARNIN] Running command: /usr/sbin/vgcreate -s 1G --force --yes ceph-4d157a1d-e7e4-4c41-80c9-2596c35345ff /dev/sdd[bs-hk-hk01][WARNIN] stdout: Physical volume "/dev/sdd" successfully created.[bs-hk-hk01][WARNIN] stdout: Volume group "ceph-4d157a1d-e7e4-4c41-80c9-2596c35345ff" successfully created[bs-hk-hk01][WARNIN] Running command: /usr/sbin/lvcreate --yes -l %FREE -n osd-block-7d03f8ee-8ada-485e-a20b-8677a7e57c84 ceph-4d157a1d-e7e4-4c41-80c9-2596c35345ff[bs-hk-hk01][WARNIN] stdout: Logical volume "osd-block-7d03f8ee-8ada-485e-a20b-8677a7e57c84" created.[bs-hk-hk01][WARNIN] Running command: /bin/ceph-authtool --gen-print-key[bs-hk-hk01][WARNIN] Running command: /bin/[bs-hk-hk01][WARNIN] Running command: /usr/sbin/restorecon /var/lib/ceph/osd/ceph-[bs-hk-hk01][WARNIN] Running command: /bin/chown -h ceph:ceph /dev/ceph-4d157a1d-e7e4-4c41-80c9-2596c35345ff/osd-block-7d03f8ee-8ada-485e-a20b-8677a7e57c84[bs-hk-hk01][WARNIN] Running command: /bin/[bs-hk-hk01][WARNIN] Running command: /bin//block[bs-hk-hk01][WARNIN] Running command: /bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-/activate.monmap[bs-hk-hk01][WARNIN] stderr: got monmap epoch[bs-hk-hk01][WARNIN] Running command: /bin/ceph-authtool /var/lib/ceph/osd/ceph-/keyring --create-keyring --name osd. --add-key AQCCckJeTm5AMBAAQnCk+nGkKQifYDHOGVRy/w==[bs-hk-hk01][WARNIN] stdout: creating /var/lib/ceph/osd/ceph-/keyring[bs-hk-hk01][WARNIN] added entity osd. auth auth(auid = key=AQCCckJeTm5AMBAAQnCk+nGkKQifYDHOGVRy/ caps)[bs-hk-hk01][WARNIN] Running command: /bin//keyring[bs-hk-hk01][WARNIN] Running command: /bin//[bs-hk-hk01][WARNIN] Running command: /bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i --monmap /var/lib/ceph/osd/ceph-/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-/ --osd-uuid 7d03f8ee-8ada-485e-a20b-8677a7e57c84 --setuser ceph --setgroup ceph[bs-hk-hk01][WARNIN] --> ceph-volume lvm prepare successful for: /dev/sdd[bs-hk-hk01][WARNIN] Running command: /bin/[bs-hk-hk01][WARNIN] Running command: /bin/ceph-bluestore-tool --cluster=ceph prime-osd- --no-mon-config[bs-hk-hk01][WARNIN] Running command: /bin//block[bs-hk-hk01][WARNIN] Running command: /bin//block[bs-hk-hk01][WARNIN] Running command: /bin/[bs-hk-hk01][WARNIN] Running command: /bin/[bs-hk-hk01][WARNIN] Running command: /bin/systemctl enable ceph-volume@lvm--7d03f8ee-8ada-485e-a20b-8677a7e57c84[bs-hk-hk01][WARNIN] stderr: Created symlink from /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm--7d03f8ee-8ada-485e-a20b-8677a7e57c84.service to /usr/lib/systemd/system/ceph-volume@.service.[bs-hk-hk01][WARNIN] Running command: /bin/systemctl enable --runtime ceph-osd@[bs-hk-hk01][WARNIN] stderr: Created symlink from /run/systemd/system/ceph-osd.target.wants/ceph-osd@.service to /usr/lib/systemd/system/ceph-osd@.service.[bs-hk-hk01][WARNIN] Running command: /bin/systemctl start ceph-osd@[bs-hk-hk01][WARNIN] --> ceph-volume lvm activate successful[bs-hk-hk01][WARNIN] --> ceph-volume lvm create successful for: /dev/sdd[bs-hk-hk01][INFO ] checking OSD status...[bs-hk-hk01][DEBUG ] find the location of an executable[bs-hk-hk01][INFO ] Running command: /bin/ceph --cluster=ceph osd stat --format=json[ceph_deploy.osd][DEBUG ] Host bs-hk-hk01 is now ready for osd use.[root@bs-k8s-ceph ceph]# ceph-deploy osd create bs-hk-hk02 --data /dev/sdc --journal /dev/sdb1[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf[ceph_deploy.cli][INFO ] Invoked (): /usr/bin/ceph-deploy osd create bs-hk-hk02 --data /dev/sdc --journal /dev/sdb1[ceph_deploy.cli][INFO ] ceph-deploy options:[ceph_deploy.cli][INFO ] verbose : False[ceph_deploy.cli][INFO ] bluestore : None[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f90371d0680>[ceph_deploy.cli][INFO ] cluster : ceph[ceph_deploy.cli][INFO ] fs_type : xfs[ceph_deploy.cli][INFO ] block_wal : None[ceph_deploy.cli][INFO ] default_release : False[ceph_deploy.cli][INFO ] username : None[ceph_deploy.cli][INFO ] journal : /dev/sdb1[ceph_deploy.cli][INFO ] subcommand : create[ceph_deploy.cli][INFO ] host : bs-hk-hk02[ceph_deploy.cli][INFO ] filestore : None[ceph_deploy.cli][INFO ] func : <function osd at 0x7f903741a8c0>[ceph_deploy.cli][INFO ] ceph_conf : None[ceph_deploy.cli][INFO ] zap_disk : False[ceph_deploy.cli][INFO ] data : /dev/sdc[ceph_deploy.cli][INFO ] block_db : None[ceph_deploy.cli][INFO ] dmcrypt : False[ceph_deploy.cli][INFO ] overwrite_conf : False[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys[ceph_deploy.cli][INFO ] quiet : False[ceph_deploy.cli][INFO ] debug : False[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdc[bs-hk-hk02][DEBUG ] connected to host: bs-hk-hk02[bs-hk-hk02][DEBUG ] detect platform information from remote host[bs-hk-hk02][DEBUG ] detect machine type[bs-hk-hk02][DEBUG ] find the location of an executable[ceph_deploy.osd][INFO ] Distro Core[ceph_deploy.osd][DEBUG ] Deploying osd to bs-hk-hk02[bs-hk-hk02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf[bs-hk-hk02][WARNIN] osd keyring does not exist yet, creating one[bs-hk-hk02][DEBUG ] create a keyring file[bs-hk-hk02][DEBUG ] find the location of an executable[bs-hk-hk02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdc[bs-hk-hk02][WARNIN] Running command: /bin/ceph-authtool --gen-print-key[bs-hk-hk02][WARNIN] Running command: /bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 115e2aab--489d-8a8b-d8d6f6351ce8[bs-hk-hk02][WARNIN] Running command: /usr/sbin/vgcreate -s 1G --force --yes ceph-66b51d23--4ecd-a82b-c552eb4e5767 /dev/sdc[bs-hk-hk02][WARNIN] stdout: Physical volume "/dev/sdc" successfully created.[bs-hk-hk02][WARNIN] stdout: Volume group "ceph-66b51d23-8490-4ecd-a82b-c552eb4e5767" successfully created[bs-hk-hk02][WARNIN] Running command: /usr/sbin/lvcreate --yes -l %FREE -n osd-block-115e2aab--489d-8a8b-d8d6f6351ce8 ceph-66b51d23--4ecd-a82b-c552eb4e5767[bs-hk-hk02][WARNIN] stdout: Logical volume "osd-block-115e2aab-5560-489d-8a8b-d8d6f6351ce8" created.[bs-hk-hk02][WARNIN] Running command: /bin/ceph-authtool --gen-print-key[bs-hk-hk02][WARNIN] Running command: /bin/[bs-hk-hk02][WARNIN] Running command: /usr/sbin/restorecon /var/lib/ceph/osd/ceph-[bs-hk-hk02][WARNIN] Running command: /bin/-4ecd-a82b-c552eb4e5767/osd-block-115e2aab--489d-8a8b-d8d6f6351ce8[bs-hk-hk02][WARNIN] Running command: /bin/[bs-hk-hk02][WARNIN] Running command: /bin/-4ecd-a82b-c552eb4e5767/osd-block-115e2aab--489d-8a8b-d8d6f6351ce8 /var/lib/ceph/osd/ceph-/block[bs-hk-hk02][WARNIN] Running command: /bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-/activate.monmap[bs-hk-hk02][WARNIN] stderr: got monmap epoch[bs-hk-hk02][WARNIN] Running command: /bin/ceph-authtool /var/lib/ceph/osd/ceph-/keyring --create-keyring --name osd. --add-key AQCcckJeNXjDMBAAOhS0lvDoLHjPaCAuylecsg==[bs-hk-hk02][WARNIN] stdout: creating /var/lib/ceph/osd/ceph-/keyring[bs-hk-hk02][WARNIN] added entity osd. auth auth(auid = key=AQCcckJeNXjDMBAAOhS0lvDoLHjPaCAuylecsg== with caps)[bs-hk-hk02][WARNIN] Running command: /bin//keyring[bs-hk-hk02][WARNIN] Running command: /bin//[bs-hk-hk02][WARNIN] Running command: /bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i --monmap /var/lib/ceph/osd/ceph-/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-/ --osd-uuid 115e2aab--489d-8a8b-d8d6f6351ce8 --setuser ceph --setgroup ceph[bs-hk-hk02][WARNIN] --> ceph-volume lvm prepare successful for: /dev/sdc[bs-hk-hk02][WARNIN] Running command: /bin/[bs-hk-hk02][WARNIN] Running command: /bin/ceph-bluestore-tool --cluster=ceph prime-osd--4ecd-a82b-c552eb4e5767/osd-block-115e2aab--489d-8a8b-d8d6f6351ce8 --path /var/lib/ceph/osd/ceph- --no-mon-config[bs-hk-hk02][WARNIN] Running command: /bin/-4ecd-a82b-c552eb4e5767/osd-block-115e2aab--489d-8a8b-d8d6f6351ce8 /var/lib/ceph/osd/ceph-/block[bs-hk-hk02][WARNIN] Running command: /bin//block[bs-hk-hk02][WARNIN] Running command: /bin/[bs-hk-hk02][WARNIN] Running command: /bin/[bs-hk-hk02][WARNIN] Running command: /bin/systemctl enable ceph-volume@lvm--115e2aab--489d-8a8b-d8d6f6351ce8[bs-hk-hk02][WARNIN] stderr: Created symlink from /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm--115e2aab--489d-8a8b-d8d6f6351ce8.service to /usr/lib/systemd/system/ceph-volume@.service.[bs-hk-hk02][WARNIN] Running command: /bin/systemctl enable --runtime ceph-osd@[bs-hk-hk02][WARNIN] stderr: Created symlink from /run/systemd/system/ceph-osd.target.wants/ceph-osd@.service to /usr/lib/systemd/system/ceph-osd@.service.[bs-hk-hk02][WARNIN] Running command: /bin/systemctl start ceph-osd@[bs-hk-hk02][WARNIN] --> ceph-volume lvm activate successful[bs-hk-hk02][WARNIN] --> ceph-volume lvm create successful for: /dev/sdc[bs-hk-hk02][INFO ] checking OSD status...[bs-hk-hk02][DEBUG ] find the location of an executable[bs-hk-hk02][INFO ] Running command: /bin/ceph --cluster=ceph osd stat --format=json[ceph_deploy.osd][DEBUG ] Host bs-hk-hk02 is now ready for osd use.[root@bs-k8s-ceph ceph]# ceph-deploy osd create bs-hk-hk02 --data /dev/sdd --journal /dev/sdb1[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf[ceph_deploy.cli][INFO ] Invoked (): /usr/bin/ceph-deploy osd create bs-hk-hk02 --data /dev/sdd --journal /dev/sdb1[ceph_deploy.cli][INFO ] ceph-deploy options:[ceph_deploy.cli][INFO ] verbose : False[ceph_deploy.cli][INFO ] bluestore : None[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7feaf4a7a680>[ceph_deploy.cli][INFO ] cluster : ceph[ceph_deploy.cli][INFO ] fs_type : xfs[ceph_deploy.cli][INFO ] block_wal : None[ceph_deploy.cli][INFO ] default_release : False[ceph_deploy.cli][INFO ] username : None[ceph_deploy.cli][INFO ] journal : /dev/sdb1[ceph_deploy.cli][INFO ] subcommand : create[ceph_deploy.cli][INFO ] host : bs-hk-hk02[ceph_deploy.cli][INFO ] filestore : None[ceph_deploy.cli][INFO ] func : <function osd at 0x7feaf4cc48c0>[ceph_deploy.cli][INFO ] ceph_conf : None[ceph_deploy.cli][INFO ] zap_disk : False[ceph_deploy.cli][INFO ] data : /dev/sdd[ceph_deploy.cli][INFO ] block_db : None[ceph_deploy.cli][INFO ] dmcrypt : False[ceph_deploy.cli][INFO ] overwrite_conf : False[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys[ceph_deploy.cli][INFO ] quiet : False[ceph_deploy.cli][INFO ] debug : False[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdd[bs-hk-hk02][DEBUG ] connected to host: bs-hk-hk02[bs-hk-hk02][DEBUG ] detect platform information from remote host[bs-hk-hk02][DEBUG ] detect machine type[bs-hk-hk02][DEBUG ] find the location of an executable[ceph_deploy.osd][INFO ] Distro Core[ceph_deploy.osd][DEBUG ] Deploying osd to bs-hk-hk02[bs-hk-hk02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf[bs-hk-hk02][DEBUG ] find the location of an executable[bs-hk-hk02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdd[bs-hk-hk02][WARNIN] Running command: /bin/ceph-authtool --gen-print-key[bs-hk-hk02][WARNIN] Running command: /bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 191064b8-92ae-4fa9-895a-671f3d134af2[bs-hk-hk02][WARNIN] Running command: /usr/sbin/vgcreate -s 1G --force --yes ceph-1ff9394c-24bd-4f6c-9f0d-80d7e6a8c316 /dev/sdd[bs-hk-hk02][WARNIN] stdout: Physical volume "/dev/sdd" successfully created.[bs-hk-hk02][WARNIN] stdout: Volume group "ceph-1ff9394c-24bd-4f6c-9f0d-80d7e6a8c316" successfully created[bs-hk-hk02][WARNIN] Running command: /usr/sbin/lvcreate --yes -l %FREE -n osd-block-191064b8-92ae-4fa9-895a-671f3d134af2 ceph-1ff9394c-24bd-4f6c-9f0d-80d7e6a8c316[bs-hk-hk02][WARNIN] stdout: Logical volume "osd-block-191064b8-92ae-4fa9-895a-671f3d134af2" created.[bs-hk-hk02][WARNIN] Running command: /bin/ceph-authtool --gen-print-key[bs-hk-hk02][WARNIN] Running command: /bin/[bs-hk-hk02][WARNIN] Running command: /usr/sbin/restorecon /var/lib/ceph/osd/ceph-[bs-hk-hk02][WARNIN] Running command: /bin/chown -h ceph:ceph /dev/ceph-1ff9394c-24bd-4f6c-9f0d-80d7e6a8c316/osd-block-191064b8-92ae-4fa9-895a-671f3d134af2[bs-hk-hk02][WARNIN] Running command: /bin/[bs-hk-hk02][WARNIN] Running command: /bin//block[bs-hk-hk02][WARNIN] Running command: /bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-/activate.monmap[bs-hk-hk02][WARNIN] stderr: got monmap epoch[bs-hk-hk02][WARNIN] Running command: /bin/ceph-authtool /var/lib/ceph/osd/ceph-/keyring --create-keyring --name osd. --add-key AQCtckJeShPOBhAAdZ2Jd9o2oAIXBWmLPb4pBA==[bs-hk-hk02][WARNIN] stdout: creating /var/lib/ceph/osd/ceph-/keyring[bs-hk-hk02][WARNIN] added entity osd. auth auth(auid = key=AQCtckJeShPOBhAAdZ2Jd9o2oAIXBWmLPb4pBA== with caps)[bs-hk-hk02][WARNIN] Running command: /bin//keyring[bs-hk-hk02][WARNIN] Running command: /bin//[bs-hk-hk02][WARNIN] Running command: /bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i --monmap /var/lib/ceph/osd/ceph-/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-/ --osd-uuid 191064b8-92ae-4fa9-895a-671f3d134af2 --setuser ceph --setgroup ceph[bs-hk-hk02][WARNIN] --> ceph-volume lvm prepare successful for: /dev/sdd[bs-hk-hk02][WARNIN] Running command: /bin/[bs-hk-hk02][WARNIN] Running command: /bin/ceph-bluestore-tool --cluster=ceph prime-osd- --no-mon-config[bs-hk-hk02][WARNIN] Running command: /bin//block[bs-hk-hk02][WARNIN] Running command: /bin//block[bs-hk-hk02][WARNIN] Running command: /bin/[bs-hk-hk02][WARNIN] Running command: /bin/[bs-hk-hk02][WARNIN] Running command: /bin/systemctl enable ceph-volume@lvm--191064b8-92ae-4fa9-895a-671f3d134af2[bs-hk-hk02][WARNIN] stderr: Created symlink from /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm--191064b8-92ae-4fa9-895a-671f3d134af2.service to /usr/lib/systemd/system/ceph-volume@.service.[bs-hk-hk02][WARNIN] Running command: /bin/systemctl enable --runtime ceph-osd@[bs-hk-hk02][WARNIN] stderr: Created symlink from /run/systemd/system/ceph-osd.target.wants/ceph-osd@.service to /usr/lib/systemd/system/ceph-osd@.service.[bs-hk-hk02][WARNIN] Running command: /bin/systemctl start ceph-osd@[bs-hk-hk02][WARNIN] --> ceph-volume lvm activate successful[bs-hk-hk02][WARNIN] --> ceph-volume lvm create successful for: /dev/sdd[bs-hk-hk02][INFO ] checking OSD status...[bs-hk-hk02][DEBUG ] find the location of an executable[bs-hk-hk02][INFO ] Running command: /bin/ceph --cluster=ceph osd stat --format=json[ceph_deploy.osd][DEBUG ] Host bs-hk-hk02 is now ready for osd use.[root@bs-k8s-ceph ceph]# ceph -s cluster: -1a9a-4b55-a353-4b141e2199d8 health: HEALTH_WARN no active mgr services: mon: daemons, quorum bs-hk-hk01,bs-hk-hk02,bs-k8s-ceph mgr: no daemons active osd: osds: up, in data: pools: pools, pgs objects: objects, B usage: B used, B / B avail pgs: [root@bs-k8s-ceph ceph]# chmod +r ceph.client.admin.keyring[root@bs-k8s-ceph ceph]# ceph-deploy mgr create bs-k8s-ceph bs-hk-hk01 bs-hk-hk02[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf[ceph_deploy.cli][INFO ] Invoked (): /usr/bin/ceph-deploy mgr create bs-k8s-ceph bs-hk-hk01 bs-hk-hk02[ceph_deploy.cli][INFO ] ceph-deploy options:[ceph_deploy.cli][INFO ] username : None[ceph_deploy.cli][INFO ] verbose : False[ceph_deploy.cli][INFO ] mgr : [('bs-k8s-ceph', 'bs-k8s-ceph'), ('bs-hk-hk01', 'bs-hk-hk01'), ('bs-hk-hk02', 'bs-hk-hk02')][ceph_deploy.cli][INFO ] overwrite_conf : False[ceph_deploy.cli][INFO ] subcommand : create[ceph_deploy.cli][INFO ] quiet : False[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f1bc292c908>[ceph_deploy.cli][INFO ] cluster : ceph[ceph_deploy.cli][INFO ] func : <function mgr at 0x7f1bc321c140>[ceph_deploy.cli][INFO ] ceph_conf : None[ceph_deploy.cli][INFO ] default_release : False[ceph_deploy.mgr][DEBUG ] Deploying mgr, cluster ceph hosts bs-k8s-ceph:bs-k8s-ceph bs-hk-hk01:bs-hk-hk01 bs-hk-hk02:bs-hk-hk02[bs-k8s-ceph][DEBUG ] connected to host: bs-k8s-ceph[bs-k8s-ceph][DEBUG ] detect platform information from remote host[bs-k8s-ceph][DEBUG ] detect machine type[ceph_deploy.mgr][INFO ] Distro Core[ceph_deploy.mgr][DEBUG ] remote host will use systemd[ceph_deploy.mgr][DEBUG ] deploying mgr bootstrap to bs-k8s-ceph[bs-k8s-ceph][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf[bs-k8s-ceph][WARNIN] mgr keyring does not exist yet, creating one[bs-k8s-ceph][DEBUG ] create a keyring file[bs-k8s-ceph][DEBUG ] create path recursively if it doesn't exist[bs-k8s-ceph][INFO ] Running command: ceph --cluster ceph --name client.bootstrap-mgr --keyring /var/lib/ceph/bootstrap-mgr/ceph.keyring auth get-or-create mgr.bs-k8s-ceph mon allow profile mgr osd allow * mds allow * -o /var/lib/ceph/mgr/ceph-bs-k8s-ceph/keyring[bs-k8s-ceph][INFO ] Running command: systemctl enable ceph-mgr@bs-k8s-ceph[bs-k8s-ceph][WARNIN] Created symlink from /etc/systemd/system/ceph-mgr.target.wants/ceph-mgr@bs-k8s-ceph.service to /usr/lib/systemd/system/ceph-mgr@.service.[bs-k8s-ceph][INFO ] Running command: systemctl start ceph-mgr@bs-k8s-ceph[bs-k8s-ceph][INFO ] Running command: systemctl enable ceph.target[bs-hk-hk01][DEBUG ] connected to host: bs-hk-hk01[bs-hk-hk01][DEBUG ] detect platform information from remote host[bs-hk-hk01][DEBUG ] detect machine type[ceph_deploy.mgr][INFO ] Distro Core[ceph_deploy.mgr][DEBUG ] remote host will use systemd[ceph_deploy.mgr][DEBUG ] deploying mgr bootstrap to bs-hk-hk01[bs-hk-hk01][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf[bs-hk-hk01][WARNIN] mgr keyring does not exist yet, creating one[bs-hk-hk01][DEBUG ] create a keyring file[bs-hk-hk01][DEBUG ] create path recursively if it doesn't exist[bs-hk-hk01][INFO ] Running command: ceph --cluster ceph --name client.bootstrap-mgr --keyring /var/lib/ceph/bootstrap-mgr/ceph.keyring auth get-or-create mgr.bs-hk-hk01 mon allow profile mgr osd allow * mds allow * -o /var/lib/ceph/mgr/ceph-bs-hk-hk01/keyring[bs-hk-hk01][INFO ] Running command: systemctl enable ceph-mgr@bs-hk-hk01[bs-hk-hk01][WARNIN] Created symlink from /etc/systemd/system/ceph-mgr.target.wants/ceph-mgr@bs-hk-hk01.service to /usr/lib/systemd/system/ceph-mgr@.service.[bs-hk-hk01][INFO ] Running command: systemctl start ceph-mgr@bs-hk-hk01[bs-hk-hk01][INFO ] Running command: systemctl enable ceph.target[bs-hk-hk02][DEBUG ] connected to host: bs-hk-hk02[bs-hk-hk02][DEBUG ] detect platform information from remote host[bs-hk-hk02][DEBUG ] detect machine type[ceph_deploy.mgr][INFO ] Distro Core[ceph_deploy.mgr][DEBUG ] remote host will use systemd[ceph_deploy.mgr][DEBUG ] deploying mgr bootstrap to bs-hk-hk02[bs-hk-hk02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf[bs-hk-hk02][WARNIN] mgr keyring does not exist yet, creating one[bs-hk-hk02][DEBUG ] create a keyring file[bs-hk-hk02][DEBUG ] create path recursively if it doesn't exist[bs-hk-hk02][INFO ] Running command: ceph --cluster ceph --name client.bootstrap-mgr --keyring /var/lib/ceph/bootstrap-mgr/ceph.keyring auth get-or-create mgr.bs-hk-hk02 mon allow profile mgr osd allow * mds allow * -o /var/lib/ceph/mgr/ceph-bs-hk-hk02/keyring[bs-hk-hk02][INFO ] Running command: systemctl enable ceph-mgr@bs-hk-hk02[bs-hk-hk02][WARNIN] Created symlink from /etc/systemd/system/ceph-mgr.target.wants/ceph-mgr@bs-hk-hk02.service to /usr/lib/systemd/system/ceph-mgr@.service.[bs-hk-hk02][INFO ] Running command: systemctl start ceph-mgr@bs-hk-hk02[bs-hk-hk02][INFO ] Running command: systemctl enable ceph.target[root@bs-k8s-ceph ceph]# ceph -s cluster: -1a9a-4b55-a353-4b141e2199d8 health: HEALTH_OK services: mon: daemons, quorum bs-hk-hk01,bs-hk-hk02,bs-k8s-ceph mgr: bs-k8s-ceph(active), standbys: bs-hk-hk01, bs-hk-hk02 osd: osds: up, in data: pools: pools, pgs objects: objects, B usage: GiB / GiB avail pgs: [root@bs-k8s-ceph ceph]# vim /etc/ceph/ceph.conf[root@bs-k8s-ceph ceph]# ceph mgr module enable dashboard[root@bs-k8s-ceph ceph]# ceph dashboard create-self-signed-certSelf-signed certificate created[root@bs-k8s-ceph ceph]# openssl req -new -nodes -x509 -subj -keyout dashboard.key -out dashboard.crt -extensions v3_caGenerating a bit RSA private key.............................................................+++..........+++writing new private key to 'dashboard.key'-----[root@bs-k8s-ceph ceph]# lsceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.client.admin.keyring ceph-deploy-ceph.log dashboard.crt rbdmapceph.bootstrap-mgr.keyring ceph.bootstrap-rgw.keyring ceph.conf ceph.mon.keyring dashboard.key[root@bs-k8s-ceph ceph]# ceph -s cluster: -1a9a-4b55-a353-4b141e2199d8 health: HEALTH_OK services: mon: daemons, quorum bs-hk-hk01,bs-hk-hk02,bs-k8s-ceph mgr: bs-k8s-ceph(active), standbys: bs-hk-hk02, bs-hk-hk01 osd: osds: up, in data: pools: pools, pgs objects: objects, B usage: GiB / GiB avail pgs: [root@bs-k8s-ceph ceph]# ceph config set mgr mgr/dashboard/server_addr 0.0.0.0[root@bs-k8s-ceph ceph]# ceph config set mgr mgr/dashboard/server_port[root@bs-k8s-ceph ceph]# ceph -s cluster: -1a9a-4b55-a353-4b141e2199d8 health: HEALTH_OK services: mon: daemons, quorum bs-hk-hk01,bs-hk-hk02,bs-k8s-ceph mgr: bs-k8s-ceph(active), standbys: bs-hk-hk02, bs-hk-hk01 osd: osds: up, in data: pools: pools, pgs objects: objects, B usage: GiB / GiB avail pgs: [root@bs-k8s-ceph ceph]# ceph dashboard set-login-credentials admin zisefeizhuUsername and password updated[root@bs-k8s-ceph ceph]# ceph -s cluster: -1a9a-4b55-a353-4b141e2199d8 health: HEALTH_OK services: mon: daemons, quorum bs-hk-hk01,bs-hk-hk02,bs-k8s-ceph mgr: bs-k8s-ceph(active), standbys: bs-hk-hk02, bs-hk-hk01 osd: osds: up, in data: pools: pools, pgs objects: objects, B usage: GiB / GiB avail pgs: [root@bs-k8s-ceph ceph]# ceph mgr services{ "dashboard": "https://bs-k8s-ceph:8443/"}[root@bs-k8s-ceph ceph]# ceph-deploy --overwrite-conf config push bs-hk-hk01[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf[ceph_deploy.cli][INFO ] Invoked (): /usr/bin/ceph-deploy --overwrite-conf config push bs-hk-hk01[ceph_deploy.cli][INFO ] ceph-deploy options:[ceph_deploy.cli][INFO ] username : None[ceph_deploy.cli][INFO ] verbose : False[ceph_deploy.cli][INFO ] overwrite_conf : True[ceph_deploy.cli][INFO ] subcommand : push[ceph_deploy.cli][INFO ] quiet : False[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f1907c07638>[ceph_deploy.cli][INFO ] cluster : ceph[ceph_deploy.cli][INFO ] client : ['bs-hk-hk01'][ceph_deploy.cli][INFO ] func : <function config at 0x7f1907e3ac08>[ceph_deploy.cli][INFO ] ceph_conf : None[ceph_deploy.cli][INFO ] default_release : False[ceph_deploy.config][DEBUG ] Pushing config to bs-hk-hk01[bs-hk-hk01][DEBUG ] connected to host: bs-hk-hk01[bs-hk-hk01][DEBUG ] detect platform information from remote host[bs-hk-hk01][DEBUG ] detect machine type[bs-hk-hk01][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf[root@bs-k8s-ceph ceph]# ceph-deploy --overwrite-conf config push bs-hk-hk02[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf[ceph_deploy.cli][INFO ] Invoked (): /usr/bin/ceph-deploy --overwrite-conf config push bs-hk-hk02[ceph_deploy.cli][INFO ] ceph-deploy options:[ceph_deploy.cli][INFO ] username : None[ceph_deploy.cli][INFO ] verbose : False[ceph_deploy.cli][INFO ] overwrite_conf : True[ceph_deploy.cli][INFO ] subcommand : push[ceph_deploy.cli][INFO ] quiet : False[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7fb720986638>[ceph_deploy.cli][INFO ] cluster : ceph[ceph_deploy.cli][INFO ] client : ['bs-hk-hk02'][ceph_deploy.cli][INFO ] func : <function config at 0x7fb720bb9c08>[ceph_deploy.cli][INFO ] ceph_conf : None[ceph_deploy.cli][INFO ] default_release : False[ceph_deploy.config][DEBUG ] Pushing config to bs-hk-hk02[bs-hk-hk02][DEBUG ] connected to host: bs-hk-hk02[bs-hk-hk02][DEBUG ] detect platform information from remote host[bs-hk-hk02][DEBUG ] detect machine type[bs-hk-hk02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf[root@bs-k8s-ceph ceph]# ceph -s cluster: -1a9a-4b55-a353-4b141e2199d8 health: HEALTH_OK services: mon: daemons, quorum bs-hk-hk01,bs-hk-hk02,bs-k8s-ceph mgr: bs-k8s-ceph(active), standbys: bs-hk-hk02, bs-hk-hk01 osd: osds: up, in data: pools: pools, pgs objects: objects, B usage: GiB / GiB avail pgs: [root@bs-k8s-ceph ceph]# ceph osd treeID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF- 0.11151 root default- 0.03717 host bs-hk-hk01 hdd up 1.00000 1.00000 hdd up 1.00000 1.00000- 0.03717 host bs-hk-hk02 hdd up 1.00000 1.00000 hdd up 1.00000 1.00000- 0.03717 host bs-k8s-ceph hdd up 1.00000 1.00000 hdd up 1.00000 1.00000[root@bs-k8s-ceph ceph]# systemctl status ceph-ceph-crash.service ceph-osd@.serviceceph-mds.target ceph-osd@.serviceceph-mgr@bs-k8s-ceph.service ceph-osd.targetceph-mgr.target ceph-volume@lvm--37d3767e-4e21-448c-91f4-d3a1079bd8bd.serviceceph-mon@bs-k8s-ceph.service ceph-volume@lvm--0f718ba7-de6b-4d22-b74d-90657e220121.serviceceph-mon.target[root@bs-k8s-ceph ceph]# systemctl status ceph-osd.target● ceph-osd.target - ceph target allowing to start/stop all ceph-osd@.service instances at once Loaded: loaded (/usr/lib/systemd/system/ceph-osd.target; enabled; vendor preset: enabled) Active: active since 二 -- :: CST; 12min ago2月 :: bs-k8s-ceph systemd[]: Reached target ceph target allowing to start/stop all ceph-osd@.service instances at once.[root@bs-k8s-ceph ceph]# ceph -s cluster: -1a9a-4b55-a353-4b141e2199d8 health: HEALTH_OK services: mon: daemons, quorum bs-hk-hk01,bs-hk-hk02,bs-k8s-ceph mgr: bs-k8s-ceph(active), standbys: bs-hk-hk02, bs-hk-hk01 osd: osds: up, in data: pools: pools, pgs objects: objects, B usage: GiB / GiB avail pgs: [root@bs-k8s-ceph ceph]# ceph[root@bs-k8s-ceph ceph]# systemctl restart ceph-mgr@bs-k8s-ceph.service[root@bs-k8s-ceph ceph]# ceph mgr services{ "dashboard": "https://0.0.0.0:7000/"}[root@bs-k8s-ceph ceph]# ceph osd treeID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF- 0.11151 root default- 0.03717 host bs-hk-hk01 hdd up 1.00000 1.00000 hdd up 1.00000 1.00000- 0.03717 host bs-hk-hk02 hdd up 1.00000 1.00000 hdd up 1.00000 1.00000- 0.03717 host bs-k8s-ceph hdd up 1.00000 1.00000 hdd up 1.00000 1.00000[root@bs-k8s-ceph ceph]# ceph -s cluster: -1a9a-4b55-a353-4b141e2199d8 health: HEALTH_OK services: mon: daemons, quorum bs-hk-hk01,bs-hk-hk02,bs-k8s-ceph mgr: bs-k8s-ceph(active), standbys: bs-hk-hk02, bs-hk-hk01 osd: osds: up, in data: pools: pools, pgs objects: objects, B usage: GiB / GiB avail pgs: [root@bs-k8s-ceph ceph]# ceph osd pool create rbdpool 'rbd' created[root@bs-k8s-ceph ceph]# ceph osd pool get rbd pg_numpg_num:[root@bs-k8s-ceph ceph]# ceph -s cluster: -1a9a-4b55-a353-4b141e2199d8 health: HEALTH_OK services: mon: daemons, quorum bs-hk-hk01,bs-hk-hk02,bs-k8s-ceph mgr: bs-k8s-ceph(active), standbys: bs-hk-hk02, bs-hk-hk01 osd: osds: up, in data: pools: pools, pgs objects: objects, B usage: GiB / GiB avail pgs: active+clean