Ceph 集群安装
# 此博客基于 Centos8 + Ceph17 安装
特别注意
此博客为在线 yum 源 + 离线 Ceph 容器安装
按照此博客,请确保已经有自己的 docker 仓库并以下载 Ceph 相关容器镜像
hub.instack.cloud 为自有仓库镜像
# 1. 环境准备
主机名 | IP | 磁盘 | CPU | memory |
---|---|---|---|---|
ceph1 | 10.10.10.10(public),10.10.20.10(cluster-network) | sda:100G, sdb:50G,sdc:50G | 2C | 4G |
ceph2 | 10.10.10.11(public),10.10.20.10(cluster-network) | sda:100G ,sdb:50G,sdc:50G | 2C | 4G |
ceph3 | 10.10.10.12(public),10.10.20.10(cluster-network) | sda:100G ,sdb:50G,sdc:50G | 2C | 4G |
# 1.1 配置网络 IP
# 1.2 更改主机名
- ceph1 节点
hostnamectl set-hostname ceph1
bash
- ceph2 节点
hostnamectl set-hostname ceph2
bash
- ceph3 节点
hostnamectl set-hostname ceph3
bash
# 2. 配置hosts解析(所有节点)
cat >> /etc/hosts <<EOF
192.168.1.200 hub.instack.cloud
10.10.10.10 ceph1
10.10.10.11 ceph2
10.10.10.12 ceph3
EOF
# 3. 添加源
- docker 源
cat >> /etc/yum.repos.d/docker-ce.repo <<EOF
[docker]
async = 1
baseurl = https://download.docker.com/linux/centos/\$releasever/\$basearch/stable
gpgcheck = 1
gpgkey = https://download.docker.com/linux/centos/gpg
module_hotfixes = 1
name = Docker main Repository
EOF
- epel 源
cat >> /etc/yum.repos.d/epel.repo <<EOF
[Ceph]
name=Ceph $basearch
baseurl=https://mirrors.aliyun.com/ceph/rpm-quincy/el8/\$basearch
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/ceph/keys/release.gpg
[Ceph-noarch]
name=Ceph noarch
baseurl=https://mirrors.aliyun.com/ceph/rpm-quincy/el8/noarch
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/ceph/keys/release.gpg
[Ceph-source]
name=Ceph SRPMS
baseurl=https://mirrors.aliyun.com/ceph/rpm-quincy/el8/SRPMS
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/ceph/keys/release.gpg
EOF
- centos8 源
cat >> /etc/yum.repos.d/centos-base.repo <<EOF
# CentOS-Base.repo
#
# The mirror system uses the connecting IP address of the client and the
# update status of each mirror to pick mirrors that are updated to and
# geographically close to the client. You should use this for CentOS updates
# unless you are manually picking other mirrors.
#
# If the mirrorlist= does not work for you, as a fall back you can try the
# remarked out baseurl= line instead.
#
#
[base]
name=CentOS-$releasever - Base - mirrors.aliyun.com
#failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/\$releasever/BaseOS/\$basearch/os/
http://mirrors.aliyuncs.com/centos/\$releasever/BaseOS/\$basearch/os/
http://mirrors.cloud.aliyuncs.com/centos/\$releasever/BaseOS/\$basearch/os/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-Official
#additional packages that may be useful
[extras]
name=CentOS-$releasever - Extras - mirrors.aliyun.com
#failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/\$releasever/extras/\$basearch/os/
http://mirrors.aliyuncs.com/centos/\$releasever/extras/\$basearch/os/
http://mirrors.cloud.aliyuncs.com/centos/\$releasever/extras/\$basearch/os/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-Official
#additional packages that extend functionality of existing packages
[centosplus]
name=CentOS-$releasever - Plus - mirrors.aliyun.com
#failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/\$releasever/centosplus/\$basearch/os/
http://mirrors.aliyuncs.com/centos/\$releasever/centosplus/\basearch/os/
http://mirrors.cloud.aliyuncs.com/centos/\$releasever/centosplus/\$basearch/os/
gpgcheck=1
enabled=0
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-Official
[PowerTools]
name=CentOS-$releasever - PowerTools - mirrors.aliyun.com
#failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/\$releasever/PowerTools/\$basearch/os/
http://mirrors.aliyuncs.com/centos/\$releasever/PowerTools/\$basearch/os/
http://mirrors.cloud.aliyuncs.com/centos/\$releasever/PowerTools/\$basearch/os/
gpgcheck=1
enabled=0
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-Official
[AppStream]
name=CentOS-$releasever - AppStream - mirrors.aliyun.com
#failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/\$releasever/AppStream/\$basearch/os/
http://mirrors.aliyuncs.com/centos/\$releasever/AppStream/\$basearch/os/
http://mirrors.cloud.aliyuncs.com/centos/\$releasever/AppStream/\$basearch/os/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-Official
EOF
- ceph 源
cat >> /etc/yum.repos.d/ceph.repo <<EOF
[Ceph]
name=Ceph $basearch
baseurl=https://mirrors.aliyun.com/ceph/rpm-quincy/el8/\$basearch
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/ceph/keys/release.gpg
[Ceph-noarch]
name=Ceph noarch
baseurl=https://mirrors.aliyun.com/ceph/rpm-quincy/el8/noarch
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/ceph/keys/release.gpg
[Ceph-source]
name=Ceph SRPMS
baseurl=https://mirrors.aliyun.com/ceph/rpm-quincy/el8/SRPMS
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/ceph/keys/release.gpg
EOF
# 4. 配置时间同步
- 所有节点更改时区
# 可配置开启
timedatectl set-ntp true
# 配置上海时区
timedatectl set-timezone Asia/Shanghai
# 系统时钟与硬件时钟同步
hwclock --systohc
- ceph1 节点
# 安装服务
yum install -y chrony
# 配置文件
vim /etc/chrony/chrony.conf
20 server ceph1 iburst maxsources 2
61 allow all
63 local stratum 10
# 重启服务
systemctl restart chronyd
- ceph2、ceph3节点
# 安装服务
apt install -y chrony
# 配置文件
vim /etc/chrony/chrony.conf
20 pool ceph1 iburst maxsources 4
# 重启服务
systemctl restart chronyd
# 5. 安装docker(所有节点)
yum install -y docker-ce
# 6. 安装cephadm(ceph1) 和 ceph 工具包
yum install -y ceph-common
curl --silent --remote-name --location https://download.ceph.com/rpm-17.2.6/el8/noarch/cephadm
chmod +x cephadm && mv cephadm /usr/bin
# 7.2 配置私有仓库
cat >> /etc/docker/daemon.json << EOF
{
"exec-opts": [
"native.cgroupdriver=systemd"
],
"insecure-registry": "hub.instack.cloud"
}
EOF
# 7.3 开启 Docker 远程连接
vi /usr/lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock -H tcp://0.0.0.0:2375
# 7.4 配置 hub.instack.cloud 证书
自己为 harbor 签发的证书复制到 /etc/docker/ 目录下
.
├── certs.d
│ └── hub.instack.cloud
│ ├── ca.crt
│ ├── hub.instack.cloud.cert
│ └── hub.instack.cloud.key
├── daemon.json
└── key.json
# 7.5 开启或重启容器
systemctl status docker
systemctl daemon-reload
systemctl enable docker && systemctl start docker
# 8. 引导集群(node1)
断网
vi /etc/reslove.conf
#nameserver 223.5.5.5
mkdir -p /etc/ceph
cephadm --image hub.instack.cloud/ceph/ceph:v17 bootstrap --mon-ip 10.10.10.10 --cluster-network 10.10.20.0/24 --initial-dashboard-user admin --initial-dashboard-password 000000
ps:
# 要部署其他监视器
ceph orch apply mon "test01,test02,test03"
# 删除集群
ceph orch pause; cephadm rm-cluster --force --zap-osds --fsid $(ceph fsid); docker rmi $(docker images -q);
# 8.1 设置 mgr 镜像为本地镜像
ceph config set mgr mgr/cephadm/container_image_prometheus hub.instack.cloud/prometheus/prometheus:v2.33.4
ceph config set mgr mgr/cephadm/container_image_grafana hub.instack.cloud/ceph/ceph-grafana:8.3.5
ceph config set mgr mgr/cephadm/container_image_alertmanager hub.instack.cloud/prometheus/alertmanager:v0.23.0
ceph config set mgr mgr/cephadm/container_image_node_exporter hub.instack.cloud/prometheus/node-exporter:v1.3.1
ceph config set mgr mgr/cephadm/container_image_loki hub.instack.cloud/grafana/loki:2.4.0
ceph config set mgr mgr/cephadm/container_image_promtail hub.instack.cloud/grafana/promtail:2.4.0
ceph config set mgr mgr/cephadm/container_image_haproxy hub.instack.cloud/ceph/haproxy:2.3
ceph config set mgr mgr/cephadm/container_image_keepalived hub.instack.cloud/ceph/keepalived:2.1.5
ceph config set mgr mgr/cephadm/container_image_snmp_gateway hub.instack.cloud/ceph/snmp-notifier:v1.2.1
# ceph config set mgr mgr/cephadm/container_image_elasticsearch
# ceph config set mgr mgr/cephadm/container_image_jaeger_agent
# ceph config set mgr mgr/cephadm/container_image_jaeger_collector
# ceph config set mgr mgr/cephadm/container_image_jaeger_query
# 9. 添加主机到集群(node1)
- 传输ceph密钥(免提示方式)
yum -y install sshpass
ssh-keygen # Always Enter
sshpass -p 'root' ssh-copy-id -o StrictHostKeyChecking=no ceph2
sshpass -p 'root' ssh-copy-id ceph3
sshpass -p 'root' ssh-copy-id -o StrictHostKeyChecking=no -f -i /etc/ceph/ceph.pub ceph2
sshpass -p 'root' ssh-copy-id -o StrictHostKeyChecking=no -f -i /etc/ceph/ceph.pub ceph3
- 集群机器发现
ceph orch host add ceph2
ceph orch host add ceph3
- 添加新的节点(初始化之后)
# ceph1 节点
echo "ceph5 10.10.10.14" >> /etc/hosts
ssh-copy-id -f -i /etc/ceph/ceph.pub ceph5
ceph orch host add ceph5 10.10.10.14 # 一定要 ceph orch host add <hostname> <address>
# 11. 部署OSD
- ceph1
# 查看可用的磁盘设备 - 确保全部识别到 OSD
ceph orch device ls
# 添加到ceph集群中,在未使用的设备上自动创建osd
ceph orch apply osd --all-available-devices
PS:
# 从特定主机上的特定设备创建OSD:
ceph orch daemon add osd ceph1:/dev/sdb
ceph orch daemon add osd ceph2:/dev/sdb
ceph orch daemon add osd ceph3:/dev/sdb
# 查看osd磁盘
ceph -s
ceph df
# 12. 访问仪表盘查看状态
https://10.10.10.10:8443/ (opens new window)
cat /etc/ceph/ceph.conf
# 13. 简单使用
# 13.1 开启对象网关
- dashboard >> Cluster >> Service >> Create 选择 rgw, ID 随意,点击创建
- 回到 Object Gateway Daemons 查看状态
这里开启了 Ceph2,Ceph3 为对象网关地址,端口为 80
# 13.2 创建用户
创建用户
获取密钥(用于访问对象存储)
# 13.3 创建容器
# 13.4 s3cmd
# 13.4.1 安装
yum install s3cmd
# 13.4.2 配置 s3cmd
s3cmd --configure
Enter new values or accept defaults in brackets with Enter.
Refer to user manual for detailed description of all options.
Access key and Secret key are your identifiers for Amazon S3. Leave them empty for using the env variables.
Access Key: 04XUIEYRYTDUXC332R7H
Secret Key: uysQEmdYcp9UCv56UHimnMNKQwdiGFfuv4TsMPWy
Default Region [US]: #回车
Use "s3.amazonaws.com" for S3 Endpoint and not modify it to the target Amazon S3.
S3 Endpoint [s3.amazonaws.com]: ceph2:80
Use "%(bucket)s.s3.amazonaws.com" to the target Amazon S3. "%(bucket)s" and "%(location)s" vars can be used
if the target S3 system supports dns based buckets.
DNS-style bucket+hostname:port template for accessing a bucket [%(bucket)s.s3.amazonaws.com]: ceph2:80/%(bucket)
Encryption password is used to protect your files from reading
by unauthorized persons while in transfer to S3
Encryption password: #回车
Path to GPG program [/usr/bin/gpg]: #回车
When using secure HTTPS protocol all communication with Amazon S3
servers is protected from 3rd party eavesdropping. This method is
slower than plain HTTP, and can only be proxied with Python 2.7 or newer
Use HTTPS protocol [Yes]: No
On some networks all internet access must go through a HTTP proxy.
Try setting it here if you can't connect to S3 directly
HTTP Proxy server name: #回车
New settings: #最终配置
Access Key: 04XUIEYRYTDUXC332R7H
Secret Key: uysQEmdYcp9UCv56UHimnMNKQwdiGFfuv4TsMPWy
Default Region: US
S3 Endpoint: cep2:80
DNS-style bucket+hostname:port template for accessing a bucket: cep2:80/%(bucket)
Encryption password: redhat
Path to GPG program: /usr/bin/gpg
Use HTTPS protocol: False
HTTP Proxy server name:
HTTP Proxy server port: 0
Test access with supplied credentials? [Y/n] Y #是否保存以上配置
Please wait, attempting to list all buckets...
WARNING: Retrying failed request: /?delimiter=%2F (Remote end closed connection without response)
WARNING: Waiting 3 sec...
Success. Your access key and secret key worked fine :-)
Now verifying that encryption works...
Success. Encryption and decryption worked fine :-)
Save settings? [y/N] Y
Configuration saved to '/root/.s3cfg' #默认文件保存路径
cat /root/.s3cfg
[default]
access_key = 04XUIEYRYTDUXC332R7H
access_token =
add_encoding_exts =
add_headers =
bucket_location = US
ca_certs_file =
cache_file =
check_ssl_certificate = True
check_ssl_hostname = True
cloudfront_host = cloudfront.amazonaws.com
content_disposition =
content_type =
default_mime_type = binary/octet-stream
delay_updates = False
delete_after = False
delete_after_fetch = False
delete_removed = False
dry_run = False
enable_multipart = True
encoding = UTF-8
encrypt = False
expiry_date =
expiry_days =
expiry_prefix =
follow_symlinks = False
force = False
get_continue = False
gpg_command = /usr/bin/gpg
gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_passphrase = redhat
guess_mime_type = True
host_base = ceph2:80
host_bucket = ceph2:80/%(bucket)
human_readable_sizes = False
invalidate_default_index_on_cf = False
invalidate_default_index_root_on_cf = True
invalidate_on_cf = False
kms_key =
limit = -1
limitrate = 0
list_md5 = False
log_target_prefix =
long_listing = False
max_delete = -1
mime_type =
multipart_chunk_size_mb = 15
multipart_max_chunks = 10000
preserve_attrs = True
progress_meter = True
proxy_host =
proxy_port = 0
put_continue = False
recursive = False
recv_chunk = 65536
reduced_redundancy = False
requester_pays = False
restore_days = 1
restore_priority = Standard
secret_key = uysQEmdYcp9UCv56UHimnMNKQwdiGFfuv4TsMPWy
send_chunk = 65536
server_side_encryption = False
signature_v2 = False
signurl_use_https = False
simpledb_host = sdb.amazonaws.com
skip_existing = False
socket_timeout = 300
stats = False
stop_on_error = False
storage_class =
throttle_max = 100
upload_id =
urlencoding_mode = normal
use_http_expect = False
use_https = False
use_mime_magic = True
verbosity = WARNING
website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/
website_error =
website_index = index.html
# 13.4.3 操作 bucket
# 创建桶
s3cmd mb s3://mybucket
# Bucket 's3://mybucket/' created
s3cmd mb s3://css
# Bucket 's3://css/' created
# 上传文件
s3cmd put test.txt s3://images
# 上传文件夹
s3cmd put --recursive rgw-test s3://images
# 查看桶文件列表
s3cmd ls s3://images
# 查看桶文件夹列表
s3cmd ls s3://images/rgw-test/
# 下载文件
s3cmd get s3://myserver/v2.1.0.tar.gz /tmp
# 删除文件
s3cmd rm s3://images/jpg/nqtgls2.jpg
# 13.4.4 通过 bucket 桶策略控制访问权限(浏览器匿名访问文件)
# 策略内容
cat examplepol
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": "*",
"Action": "s3:GetObject",
"Resource": [
"arn:aws:s3:::testbucket1/*"
]
}]
}
# 设置策略
s3cmd setpolicy examplepol s3://testbucket2
s3cmd setpolicy examplepol s3://testbucket2 # 设置桶策略
s3cmd delpolicy s3://testbucket1 # 删除桶策略
s3cmd info s3://testbucket1 # 查看桶策略
# 与 OpenStack 集成
# 引用
CSDN://wsxier (opens new window)
上次更新: 2023/10/07, 19:55:59