Ubuntu 24.04
Sponsored Link

Ceph Squid : Use File System2024/05/03

 
Configure a Client Host [dlp] to use Ceph Storage like follows.
                                         |
        +--------------------+           |
        |   [dlp.srv.world]  |10.0.0.30  |
        |     Ceph Client    +-----------+
        |                    |           |
        +--------------------+           |
            +----------------------------+----------------------------+
            |                            |                            |
            |10.0.0.51                   |10.0.0.52                   |10.0.0.53 
+-----------+-----------+    +-----------+-----------+    +-----------+-----------+
|   [node01.srv.world]  |    |   [node02.srv.world]  |    |   [node03.srv.world]  |
|     Object Storage    +----+     Object Storage    +----+     Object Storage    |
|     Monitor Daemon    |    |                       |    |                       |
|     Manager Daemon    |    |                       |    |                       |
+-----------------------+    +-----------------------+    +-----------------------+

 
For example, mount as Filesystem on a Client Host.
[1] Transfer SSH public key to Client Host and Configure it from Admin Node.
# transfer public key

root@node01:~#
ssh-copy-id dlp

# install required packages

root@node01:~#
ssh dlp "apt -y install ceph-fuse"
# transfer required files to Client Host

root@node01:~#
scp /etc/ceph/ceph.conf dlp:/etc/ceph/

ceph.conf                                     100%  435   252.9KB/s   00:00
root@node01:~#
scp /etc/ceph/ceph.client.admin.keyring dlp:/etc/ceph/

ceph.client.admin.keyring                     100%  151    88.2KB/s   00:00
root@node01:~#
ssh dlp "chown ceph:ceph /etc/ceph/ceph.*"

[2] Configure MDS (MetaData Server) on a Node.
Configure it on [node01] Node on this example.
# create directory
# directory name ⇒ (Cluster Name)-(Node Name)

root@node01:~#
mkdir -p /var/lib/ceph/mds/ceph-node01

root@node01:~#
ceph-authtool --create-keyring /var/lib/ceph/mds/ceph-node01/keyring --gen-key -n mds.node01

creating /var/lib/ceph/mds/ceph-node01/keyring
root@node01:~#
chown -R ceph:ceph /var/lib/ceph/mds/ceph-node01

root@node01:~#
ceph auth add mds.node01 osd "allow rwx" mds "allow" mon "allow profile mds" -i /var/lib/ceph/mds/ceph-node01/keyring

added key for mds.node01
root@node01:~#
systemctl enable --now ceph-mds@node01

[3] Create 2 RADOS pools for Data and MeataData on MDS Node.
Refer to the official documents to specify the end number (64 on the example below)
⇒ http://docs.ceph.com/docs/master/rados/operations/placement-groups/
root@node01:~#
ceph osd pool create cephfs_data 32

pool 'cephfs_data' created
root@node01:~#
ceph osd pool create cephfs_metadata 32

pool 'cephfs_metadata' created
root@node01:~#
ceph osd pool set cephfs_data bulk true

set pool 5 bulk to true
root@node01:~#
ceph fs new cephfs cephfs_metadata cephfs_data

new fs with metadata pool 6 and data pool 5
root@node01:~#
ceph fs ls

ame: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ]
root@node01:~#
ceph mds stat

cephfs:1 {0=node01=up:active}
root@node01:~#
ceph fs status cephfs

cephfs - 2 clients
======
RANK  STATE    MDS       ACTIVITY     DNS    INOS   DIRS   CAPS
 0    active  node01  Reqs:    0 /s    10     13     12      2
      POOL         TYPE     USED  AVAIL
cephfs_metadata  metadata   102k   151G
  cephfs_data      data       0    151G
MDS version: ceph version 19.2.0~git20240301.4c76c50 (4c76c50a73f63ba48ccdf0adccce03b00d1d80c7) squid (dev)
[4] Mount CephFS on a Client Host.
# Base64 encode client key

root@dlp:~#
ceph-authtool -p /etc/ceph/ceph.client.admin.keyring > admin.key

root@dlp:~#
chmod 600 admin.key
root@dlp:~#
mount -t ceph node01.srv.world:6789:/ /mnt -o name=admin,secretfile=admin.key

root@dlp:~#
df -hT

Filesystem                        Type   Size  Used Avail Use% Mounted on
tmpfs                             tmpfs  392M  1.1M  391M   1% /run
/dev/mapper/ubuntu--vg-ubuntu--lv ext4    28G  6.4G   20G  25% /
tmpfs                             tmpfs  2.0G     0  2.0G   0% /dev/shm
tmpfs                             tmpfs  5.0M     0  5.0M   0% /run/lock
/dev/vda2                         ext4   2.0G   95M  1.7G   6% /boot
tmpfs                             tmpfs  392M   12K  392M   1% /run/user/0
10.0.0.51:6789:/                  ceph   152G     0  152G   0% /mnt
[5] For delete CephFS or Pools you created, run commands like follows.
For deleting Pools, it needs to set [mon allow pool delete = true] on [Monitor Daemon] configuration.
# stop MDS

root@node01:~#
systemctl stop ceph-mds@node01
# delete CephFS

root@node01:~#
ceph fs rm cephfs --yes-i-really-mean-it
# delete pools
# ceph osd pool delete [Pool Name] [Pool Name] ***

root@node01:~#
ceph osd pool delete cephfs_data cephfs_data --yes-i-really-really-mean-it

root@node01:~#
ceph osd pool delete cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it

Matched Content