配置环境

timedatectl set-ntp true
hostnamectl set-hostname ceph-mon1
wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
echo "deb https://download.ceph.com/debian-quincy/ $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/ceph.list
apt update
apt install -y ceph ceph-mds ceph-mon ceph-mgr ceph-osd ceph-common

cat <<EOF | sudo tee -a /etc/hosts
192.168.234.150 ceph-mon1
192.168.234.151 ceph-mon2
192.168.234.152 ceph-mon3
EOF

ufw disable

启动mon（3台服务器一样）

mkdir -p /etc/ceph

ceph-authtool --create-keyring /etc/ceph/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mgr 'allow *' --cap mds 'allow *'
ceph-authtool /etc/ceph/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
chown ceph:ceph /etc/ceph/ceph.mon.keyring /etc/ceph/ceph.client.admin.keyring


mkdir -p /var/lib/ceph/mon/ceph-ceph-mon1
uuidgen  # 生成fsid

monmaptool --create --fsid 5b845f9d-a9ed-4b85-867d-5bf751ac6f20 monmap
monmaptool --add mon.ceph-mon1 v2:192.168.234.150:3300/0,v1:192.168.234.150:6789/0 monmap
monmaptool --add mon.ceph-mon2 v2:192.168.234.151:3300/0,v1:192.168.234.151:6789/0 monmap
monmaptool --add mon.ceph-mon3 v2:192.168.234.152:3300/0,v1:192.168.234.152:6789/0 monmap

ceph-mon --mkfs -i ceph-mon1 --monmap /root/monmap --keyring /etc/ceph/ceph.mon.keyring
ceph-mon --mkfs -i ceph-mon2 --monmap /root/monmap --keyring /etc/ceph/ceph.mon.keyring
ceph-mon --mkfs -i ceph-mon3 --monmap /root/monmap --keyring /etc/ceph/ceph.mon.keyring
chown -R ceph:ceph /var/lib/ceph/mon/ceph-ceph-mon1

sudo tee /etc/ceph/ceph.conf <<EOF
[global]
fsid = 123e4567-e89b-12d3-a456-426614174000
mon_initial_members = ceph-mon1,ceph-mon2,ceph-mon3
mon_host = 192.168.234.150,192.168.234.151,192.168.234.152
public_network = 192.168.234.0/24
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
osd_journal_size = 1024
osd_pool_default_size = 3
osd_pool_default_min_size = 2
osd_pool_default_pg_num = 128
osd_pool_default_pgp_num = 128
[mon.ceph-mon1]
host = ceph-mon1
[mon.ceph-mon2]
host = ceph-mon2
[mon.ceph-mon3]
host = ceph-mon3
EOF

scp /etc/ceph/ceph.conf ceph-mon2:/etc/ceph/
scp /etc/ceph/ceph.mon.keyring ceph-mon2:/etc/ceph/
scp /etc/ceph/ceph.client.admin.keyring ceph-mon2:/etc/ceph/
scp /etc/ceph/ceph.conf ceph-mon3:/etc/ceph/
scp /etc/ceph/ceph.mon.keyring ceph-mon3:/etc/ceph/
scp /etc/ceph/ceph.client.admin.keyring ceph-mon3:/etc/ceph/

vim /etc/systemd/system/ceph-mon@.service

[Unit]
Description=Ceph Monitor daemon
After=network.target

[Service]
ExecStart=/usr/bin/ceph-mon -f -i %i --keyring /etc/ceph/ceph.mon.keyring
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target



ceph -s

ceph -s 报错：
## 3 monitors have not enabled msgr2
ceph mon enable-msgr2
ceph config set mon auth_allow_insecure_global_id_reclaim false

添加mgr

ceph auth get-or-create mgr.ceph-mon1 mon 'allow *' osd 'allow *' > /etc/ceph/ceph.mgr.keyring
mkdir -p /var/lib/ceph/mgr/ceph-ceph-mon1
cp /etc/ceph/ceph.mgr.keyring /var/lib/ceph/mgr/ceph-ceph-mon1/keyring
chown -R ceph:ceph /var/lib/ceph/mgr/ceph-ceph-mon1
systemctl enable ceph-mgr@ceph-mon1
systemctl start ceph-mgr@ceph-mon1

## 安装dashboard模块
ceph mgr module enable dashboard
echo '123456' > /tmp/dashboard_pass.txt
ceph dashboard set-login-credentials admin -i /tmp/dashboard_pass.txt
ceph mgr module enable dashboard
ceph mgr module enable restful
systemctl restart ceph-mgr@ceph-mon1

# 查看dashboard访问
ceph mgr services

部署osd

## 清空磁盘
sgdisk --zap-all /dev/sdb
sgdisk --zap-all /dev/sdc
sgdisk --zap-all /dev/sdd

# 以 admin 权限生成 bootstrap-osd key 并输出到 /tmp
ceph auth get-or-create client.bootstrap-osd mon 'allow profile bootstrap-osd' -o /tmp/bootstrap-osd.keyring
mkdir -p /var/lib/ceph/bootstrap-osd
mv /tmp/bootstrap-osd.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
chown -R root:root /var/lib/ceph/bootstrap-osd
chmod 0640 /var/lib/ceph/bootstrap-osd/ceph.keyring
# 其他机器
mkdir -p /var/lib/ceph/bootstrap-osd
scp ceph-mon1:/var/lib/ceph/bootstrap-osd/ceph.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
chown -R root:root /var/lib/ceph/bootstrap-osd
chown ceph:ceph /var/lib/ceph/bootstrap-osd/ceph.keyring
chmod 0640 /var/lib/ceph/bootstrap-osd/ceph.keyring


### 创建osd
ceph-volume lvm create --data /dev/sdb
ceph-volume lvm create --data /dev/sdc
ceph-volume lvm create --data /dev/sdd

部署mds

ceph auth get-or-create mds.cephfs-mds0 mon 'allow profile mds' osd 'allow rwx' -o /etc/ceph/ceph.mds.cephfs-mds0.keyring
ceph auth get-or-create mds.cephfs-mds1 mon 'allow profile mds' osd 'allow rwx' -o /etc/ceph/ceph.mds.cephfs-mds1.keyring
ceph auth get-or-create mds.cephfs-mds2 mon 'allow profile mds' osd 'allow rwx' -o /etc/ceph/ceph.mds.cephfs-mds2.keyring

# 编辑ceph.conf
[mds.cephfs-mds0]
host = ceph-mon1

[mds.cephfs-mds1]
host = ceph-mon2

[mds.cephfs-mds2]
host = ceph-mon3

scp ceph-mon1:/etc/ceph/ceph.mds.cephfs-mds0.keyring /etc/ceph/ceph.mds.cephfs-mds1.keyring
scp ceph-mon1:/etc/ceph/ceph.mds.cephfs-mds0.keyring /etc/ceph/ceph.mds.cephfs-mds2.keyring

vim /etc/systemd/system/ceph-mds@.service
[Unit]
Description=Ceph MDS Daemon %i
After=network.target

[Service]
ExecStart=/usr/bin/ceph-mds -i %i --keyring /etc/ceph/ceph.mds.%i.keyring -f
Restart=on-failure
User=ceph
Group=ceph

[Install]
WantedBy=multi-user.target


systemctl daemon-reload
systemctl start ceph-mds@cephfs-mds0
systemctl enable ceph-mds@cephfs-mds0

部署cephfs

chown ceph:ceph /etc/ceph/ceph.mds.cephfs-mds0.keyring
chmod 600 /etc/ceph/ceph.mds.cephfs-mds0.keyring
### 创建元数据池和数据池
ceph osd pool create cephfs_metadata 128 128
ceph osd pool create cephfs_data 128 128
# CephFS 文件系统
ceph fs new cephfs cephfs_metadata cephfs_data

systemctl start ceph-mds@cephfs-mds0

systemctl status ceph-mds@cephfs-mds0
ceph fs status