事前導讀及參考資訊
https://xrcd2.blogspot.com/2022/04/glusterfs-samba.html
https://www.server-world.info/en/note?os=CentOS_Stream_9&p=glusterfs11&f=1
https://core.vmware.com/resource/nfs-iscsi-multipathing-vsphere
測試架構
192.168.100.201 Rocky Linux nfs1 glusterfs-server+nfs-ganesha
192.168.100.202 Rocky Linux nfs2 glusterfs-server+nfs-ganesha
192.168.100.x Oracle Linux Oracle9 glusterfs/nfsclient test
192.168.100.111 ESXi NFS Client
192.168.100.222 PVE Glusterfs Client + Client
===========================
操作及設定簡要記錄
fdisk /dev/nvme0n2
mkfs.xfs /dev/nvme0n2p1
mkdir /syncdisk
blkid
vi /etc/fstab
UUID=33548bbf-a5be-4d08-ab25-28e68dc438f5 /syncdisk xfs defaults 0 0
vi /etc/hosts
192.168.100.201 nfs1
192.168.100.202 nfs2
[root@nfs1 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
devtmpfs 4.0M 0 4.0M 0% /dev
tmpfs 3.8G 0 3.8G 0% /dev/shm
tmpfs 1.5G 9.4M 1.5G 1% /run
/dev/nvme0n1p2 44G 6.8G 38G 16% /
/dev/nvme0n1p1 960M 469M 492M 49% /boot
/dev/nvme0n2p1 20G 175M 20G 1% /syncdisk
tmpfs 766M 36K 766M 1% /run/user/1000
[root@nfs1 ~]#
dnf install centos-release-gluster11.noarch
dnf install centos-release-nfs-ganesha5.noarch
dnf --enablerepo=crb install python3-pyxattr
dnf install glusterfs-server
[root@nfs1 yum.repos.d]# systemctl enable --now glusterd
[root@nfs1 yum.repos.d]# gluster --version
glusterfs 11.1
Repository revision: git://git.gluster.org/glusterfs.git
Copyright (c) 2006-2016 Red Hat, Inc. <https://www.gluster.org/>
GlusterFS comes with ABSOLUTELY NO WARRANTY.
It is licensed to you under your choice of the GNU Lesser
General Public License, version 3 or any later version (LGPLv3
or later), or the GNU General Public License, version 2 (GPLv2),
in all cases as published by the Free Software Foundation.
[root@nfs1 yum.repos.d]# gluster peer probe nfs2
peer probe: success
[root@nfs1 yum.repos.d]# gluster peer status
Number of Peers: 1
Hostname: nfs2
Uuid: bc6daa49-a8ca-438c-871c-45a978ea4251
State: Peer in Cluster (Connected)
[root@nfs1 yum.repos.d]# gluster volume create nfsvolume replica 2 transport tcp nfs1:/syncdisk/nfs nfs2:/syncdisk/nfs
Replica 2 volumes are prone to split-brain. Use Arbiter or Replica 3 to avoid this. See: http://docs.gluster.org/en/latest/Administrator-Guide/Split-brain-and-ways-to-deal-with-it/.
Do you still want to continue?
(y/n) y
volume create: nfsvolume: success: please start the volume to access data
[root@nfs1 yum.repos.d]# gluster volume info
Volume Name: nfsvolume
Type: Replicate
Volume ID: 776a642f-53fe-47ee-924a-20a45a911304
Status: Created
Snapshot Count: 0
Number of Bricks: 1 x 2 = 2
Transport-type: tcp
Bricks:
Brick1: nfs1:/syncdisk/nfs
Brick2: nfs2:/syncdisk/nfs
Options Reconfigured:
cluster.granular-entry-heal: on
storage.fips-mode-rchecksum: on
transport.address-family: inet
nfs.disable: on
performance.client-io-threads: off
[root@nfs1 yum.repos.d]#
[root@nfs1 yum.repos.d]# gluster volume start nfsvolume
volume start: nfsvolume: success
[root@nfs1 yum.repos.d]#
[root@nfs2 ~]# mount -t glusterfs nfs1:/nfsvolume /mnt
[root@nfs2 ~]# df
Filesystem 1K-blocks Used Available Use% Mounted on
devtmpfs 4096 0 4096 0% /dev
tmpfs 3918632 0 3918632 0% /dev/shm
tmpfs 1567456 9600 1557856 1% /run
/dev/nvme0n1p2 46064640 7125096 38939544 16% /
/dev/nvme0n2p1 20904960 179044 20725916 1% /syncdisk
/dev/nvme0n1p1 983040 479448 503592 49% /boot
tmpfs 783724 36 783688 1% /run/user/1000
nfs1:/nfsvolume 20904960 388092 20516868 2% /mnt
[root@nfs2 ~]# echo "GlusterFS write test from nfs2" > /mnt/test.txt
[root@nfs2 ~]# ls -la /syncdisk/nfs/*
-rw-r--r-- 2 root root 21 May 11 12:57 /syncdisk/nfs/test.txt
[root@nfs2 ~]#
[root@nfs1 yum.repos.d]# ls -la /syncdisk/nfs/*
-rw-r--r-- 2 root root 21 May 11 12:57 /syncdisk/nfs/test.txt
[root@nfs1 yum.repos.d]#
mkdir /syncgfs
vi /etc/fstab
[root@nfs1 nfs]# cat /etc/fstab
#
# /etc/fstab
# Created by anaconda on Tue Feb 20 23:34:32 2024
#
# Accessible filesystems, by reference, are maintained under '/dev/disk/'.
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info.
#
# After editing this file, run 'systemctl daemon-reload' to update systemd
# units generated from this file.
#
UUID=c5d392e3-edbd-44e3-bd92-0053aec8bc10 / xfs defaults 0 0
UUID=6fe1dba6-5999-491c-9679-c32ee4c84490 /boot xfs defaults 0 0
UUID=d631d923-aa38-475f-9609-8283d3a598dc none swap defaults 0 0
UUID=33548bbf-a5be-4d08-ab25-28e68dc438f5 /syncdisk xfs defaults 0 0
nfs1:/nfsvolume /syncgfs glusterfs defaults,_netdev 0 0
[root@nfs1 nfs]#
[root@nfs2 syncgfs]# cat /etc/fstab
#
# /etc/fstab
# Created by anaconda on Tue Feb 20 23:34:32 2024
#
# Accessible filesystems, by reference, are maintained under '/dev/disk/'.
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info.
#
# After editing this file, run 'systemctl daemon-reload' to update systemd
# units generated from this file.
#
UUID=c5d392e3-edbd-44e3-bd92-0053aec8bc10 / xfs defaults 0 0
UUID=6fe1dba6-5999-491c-9679-c32ee4c84490 /boot xfs defaults 0 0
UUID=d631d923-aa38-475f-9609-8283d3a598dc none swap defaults 0 0
UUID=575c8913-d8df-4d79-857e-4b586c439b52 /syncdisk xfs defaults 0 0
nfs2:/nfsvolume /syncgfs glusterfs defaults,_netdev 0 0
[root@nfs2 syncgfs]#
dnf install nfs-ganesha-vfs nfs-ganesha nfs-ganesha-gluster
[root@nfs1 nfs]# cp /etc/ganesha/ganesha.conf /etc/ganesha/ganesha.conf.org
[root@nfs1 nfs]# vi /etc/ganesha/ganesha.conf
###################################################
#
# Ganesha Config Example
#
# This is a commented example configuration file for Ganesha. It is not
# complete, but only has some common configuration options. See the man pages
# for complete documentation.
#
###################################################
## These are core parameters that affect Ganesha as a whole.
NFS_CORE_PARAM {
## Allow NFSv3 to mount paths with the Pseudo path, the same as NFSv4,
## instead of using the physical paths.
mount_path_pseudo = true;
## Configure the protocols that Ganesha will listen for. This is a hard
## limit, as this list determines which sockets are opened. This list
## can be restricted per export, but cannot be expanded.
Protocols = 3,4;
}
## These are defaults for exports. They can be overridden per-export.
EXPORT_DEFAULTS {
## Access type for clients. Default is None, so some access must be
## given either here or in the export itself.
Access_Type = RW;
}
## Configure settings for the object handle cache
#MDCACHE {
## The point at which object cache entries will start being reused.
#Entries_HWMark = 100000;
#}
## Configure an export for some file tree
EXPORT
{
## Export Id (mandatory, each EXPORT must have a unique Export_Id)
Export_Id = 12345;
## Exported path (mandatory)
Path = /nfsha;
## Pseudo Path (required for NFSv4 or if mount_path_pseudo = true)
Pseudo = /nfsha;
## Restrict the protocols that may use this export. This cannot allow
## access that is denied in NFS_CORE_PARAM.
Protocols = 3,4;
## Access type for clients. Default is None, so some access must be
## given. It can be here, in the EXPORT_DEFAULTS, or in a CLIENT block
Access_Type = RW;
## Whether to squash various users.
Squash = No_root_squash;
## Allowed security types for this export
Sectype = sys;
## Exporting FSAL
FSAL {
Name = GLUSTER;
hostname="192.168.100.201";
volume="nfsvolume";
}
}
## Configure logging. Default is to log to Syslog. Basic logging can also be
## configured from the command line
LOG {
## Default log level for all components
Default_Log_Level = WARN;
## Configure per-component log levels.
#Components {
#FSAL = INFO;
#NFS4 = EVENT;
#}
## Where to log
#Facility {
#name = FILE;
#destination = "/var/log/ganesha.log";
#enable = active;
#}
}
[root@nfs1 nfs]# systemctl disable --now nfs-server
[root@nfs1 nfs]# systemctl enable --now nfs-ganesha
Created symlink /etc/systemd/system/multi-user.target.wants/nfs-ganesha.service → /usr/lib/systemd/system/nfs-ganesha.service.
Created symlink /etc/systemd/system/nfs-ganesha.service.wants/nfs-ganesha-lock.service → /usr/lib/systemd/system/nfs-ganesha-lock.service.
[root@nfs1 nfs]# showmount -e localhost
Export list for localhost:
/nfsha (everyone)
[root@nfs1 nfs]#
[root@Oracle9 ~]# yum install nfsv4-client-utils.x86_64 nfs-utils.x86_64
[root@Oracle9 mnt]#
vi /ets/hosts
192.168.100.201 nfs1
192.168.100.202 nfs2
mkdir /mnt/HA-NFS
[root@Oracle9 mnt]# mount -t nfs4 nfs1:/nfsha /mnt/HA-NFS/
[root@Oracle9 mnt]# df -h
Filesystem Size Used Avail Use% Mounted on
devtmpfs 4.0M 0 4.0M 0% /dev
tmpfs 3.7G 0 3.7G 0% /dev/shm
tmpfs 1.5G 9.2M 1.5G 1% /run
/dev/nvme0n1p3 45G 11G 35G 23% /
/dev/nvme0n1p1 960M 707M 254M 74% /boot
tmpfs 749M 36K 749M 1% /run/user/0
tmpfs 749M 36K 749M 1% /run/user/1001
nfs1:/nfsha 20G 379M 20G 2% /mnt/HA-NFS
[root@Oracle9 mnt]#
[root@Oracle9 mnt]# cat /mnt/HA-NFS/test.txt
GlusterFS write test from nfs 1
GlusterFS write test from nfs 2
[root@Oracle9 mnt]# echo "GlusterFS write test from Oracle-Client " >> /mnt/HA-NFS/test.txt
[root@Oracle9 mnt]#
[root@Oracle9 mnt]# cat /mnt/HA-NFS/test.txt
GlusterFS write test from nfs 1
GlusterFS write test from nfs 2
GlusterFS write test from Oracle-Client
[root@Oracle9 mnt]#
[root@nfs1 nfs]# cat /syncdisk/nfs/test.txt
GlusterFS write test from nfs 1
GlusterFS write test from nfs 2
GlusterFS write test from Oracle-Client
[root@nfs1 nfs]#
[root@nfs2 syncgfs]# cat /syncdisk/nfs/test.txt
GlusterFS write test from nfs 1
GlusterFS write test from nfs 2
GlusterFS write test from Oracle-Client
[root@nfs2 syncgfs]#
[root@Oracle9 mnt]# umount /mnt/HA-NFS
[root@Oracle9 mnt]# mount -t nfs4 nfs2:/nfsha /mnt/HA-NFS/
[root@Oracle9 mnt]# echo "GlusterFS write test2 from Oracle-Client " >> /mnt/HA-NFS/test.txt
[root@Oracle9 mnt]#
[root@nfs1 nfs]# cat /syncdisk/nfs/test.txt
GlusterFS write test from nfs 1
GlusterFS write test from nfs 2
GlusterFS write test from Oracle-Client
GlusterFS write test2 from Oracle-Client
[root@nfs1 nfs]#
[root@nfs2 syncgfs]# cat /syncdisk/nfs/test.txt
GlusterFS write test from nfs 1
GlusterFS write test from nfs 2
GlusterFS write test from Oracle-Client
GlusterFS write test2 from Oracle-Client
[root@nfs2 syncgfs]#
ESXi
[root@localhost:~] esxcli storage nfs41 list
Volume Name Host(s) Share Accessible Mounted Read-Only Security isPE Hardware Acceleration
----------- ------------------------------- ------ ---------- ------- --------- -------- ----- ---------------------
gfs-storage 192.168.100.201,192.168.100.202 /nfsha true true false AUTH_SYS false Not Supported
[root@localhost:~]
[root@localhost:~] vim-cmd hostsvc/datastore/info gfs-storage
(vim.host.NasDatastoreInfo) {
name = "gfs-storage",
url = "/vmfs/volumes/d3b21ae2-01cd6b50-0000-000000000000",
freeSpace = 14566449152,
maxFileSize = 70368744177664,
maxVirtualDiskCapacity = 68169720922112,
maxMemoryFileSize = 70368744177664,
timestamp = "2024-05-11T09:42:04.121385Z",
containerId = <unset>,
aliasOf = <unset>,
datastoreFormat = <unset>,
logicalSectorSize = <unset>,
physicalSectorSize = <unset>,
nas = (vim.host.NasVolume) {
type = "NFS41",
name = "gfs-storage",
capacity = 21406679040,
remoteHost = "192.168.100.201",
remotePath = "/nfsha",
userName = <unset>,
remoteHostNames = (string) [
"192.168.100.201",
"192.168.100.202"
],
securityType = "AUTH_SYS",
protocolEndpoint = false
}
}
(vim.Datastore.HostMount) [
(vim.Datastore.HostMount) {
key = 'vim.HostSystem:ha-host',
mountInfo = (vim.host.MountInfo) {
path = "/vmfs/volumes/d3b21ae2-01cd6b50-0000-000000000000",
accessMode = "readWrite",
mounted = true,
accessible = true,
inaccessibleReason = <unset>,
vmknicName = "None",
vmknicActive = false,
mountFailedReason = <unset>,
numTcpConnections = 1
}
}
]
[root@localhost:~]
[root@localhost:~] esxcli storage filesystem list
Mount Point Volume Name UUID Mounted Type Size Free
------------------------------------------------- ------------------------------------------ ----------------------------------- ------- ------ ------------ ------------
/vmfs/volumes/d3b21ae2-01cd6b50-0000-000000000000 gfs-storage d3b21ae2-01cd6b50-0000-000000000000 true NFS41 21406679040 14566449152
/vmfs/volumes/654e469c-44d19cd7-8715-000c2921ee83 datastore1 654e469c-44d19cd7-8715-000c2921ee83 true VMFS-6 137170518016 99963895808
/vmfs/volumes/654e469c-378e9fe6-73ec-000c2921ee83 OSDATA-654e469c-378e9fe6-73ec-000c2921ee83 654e469c-378e9fe6-73ec-000c2921ee83 true VMFSOS 128580583424 125292249088
/vmfs/volumes/507161ab-d84dd9c3-c4a7-a5422027a503 BOOTBANK1 507161ab-d84dd9c3-c4a7-a5422027a503 true vfat 4293591040 4000317440
/vmfs/volumes/d9628348-1545f094-5e85-a3d31d093e2e BOOTBANK2 d9628348-1545f094-5e85-a3d31d093e2e true vfat 4293591040 4293525504
[root@localhost:~]
PVE
ESXi NFS
不同的儲存供應商有不同的方法來啟用此功能，
但通常 NAS 伺服器使用 no_root_squash 選項。
如果 NAS 伺服器不授予 root 存取權限，
仍然可以在主機上掛載 NFS 資料儲存。
但是，您無法在資料儲存上建立任何虛擬機器
