{"id":6953,"date":"2017-09-06T11:39:07","date_gmt":"2017-09-06T03:39:07","guid":{"rendered":"http:\/\/rmohan.com\/?p=6953"},"modified":"2017-09-06T11:39:07","modified_gmt":"2017-09-06T03:39:07","slug":"glusterfs-servers-on-rhel-7-and-centos-7-2","status":"publish","type":"post","link":"https:\/\/mohan.sg\/?p=6953","title":{"rendered":"GlusterFS servers on RHEL 7 and Centos 7"},"content":{"rendered":"

GlusterFS servers on RHEL 7 and Centos 7
\n<\/strong><\/p>\n

vi \/etc\/hosts<\/p>\n

192.168.1.71 apache1.rmohan.com apache1
\n192.168.1.72 apache2.rmohan.com apache2
\n192.168.1.74 apache3.rmohan.com apache3<\/p>\n

For this lab, we disabled both firewall and SELinux<\/p>\n

vi \/etc\/selinux\/config
\negrep “^SELINUX=” \/etc\/sysconfig\/selinux
\nSELINUX=disabled
\nsystemctl is-enabled firewalld
\ndisabled<\/p>\n

systemctl disable firewalld
\nyum update
\nyum -y install wget openssh* net-tools ntp
\nntpdate -u 0.centos.pool.ntp.org<\/p>\n

Before proceeding with the installation, we need to enable both EPEL (Exta Packages for Enterprise Linux) and GlusterFS repositories on both the servers to resolve the external dependencies at the time of installation. If you have enabled only GlusterFS repository and not enabled EPEL repository then it is likely that you will hit the following error message while installing glusterfs-server:
\n–> Finished Dependency Resolution
\nError: Package: glusterfs-server-3.7.11-1.el7.x86_64 (glusterfs-epel)
\nRequires: liburcu-cds.so.1()(64bit)
\nError: Package: glusterfs-server-3.7.11-1.el7.x86_64 (glusterfs-epel)
\nRequires: liburcu-bp.so.1()(64bit)
\nYou could try using –skip-broken to work around the problem
\nYou could try running: rpm -Va –nofiles –nodigest<\/p>\n

EPEL installation
\nRetrieve EPEL repository : all 3 servers
\nwget http:\/\/dl.fedoraproject.org\/pub\/epel\/7\/x86_64\/e\/epel-release-7-6.noarch.rpm<\/p>\n

Gluster installation :
\nRetrieve GlusterFS repository : all 3 servers
\nwget -P \/etc\/yum.repos.d http:\/\/download.gluster.org\/pub\/gluster\/glusterfs\/LATEST\/RHEL\/glusterfs-epel.repo<\/p>\n

Download GlusterFS packages : all 3 servers
\nyum -y install glusterfs-server<\/p>\n

Now, start the service and keep it started across reboot :
\n[root@apache3 ~]# systemctl start glusterd && systemctl enable glusterd
\nCreated symlink from \/etc\/systemd\/system\/multi-user.target.wants\/glusterd.service to \/usr\/lib\/systemd\/system\/glusterd.service.
\n[root@apache3 ~]#<\/p>\n

Update \u201c\/etc\/hosts\u201d :
\n192.168.1.71 apache1.rmohan.com apache1
\n192.168.1.72 apache2.rmohan.com apache2
\n192.168.1.74 apache3.rmohan.com apache3<\/p>\n

Check the 2 GlusterFS servers<\/p>\n

[root@apache1 ~]# gluster peer probe apache2
\npeer probe: success.
\n[root@apache1 ~]# gluster peer probe apache3
\npeer probe: success.
\n[root@apache1 ~]#<\/p>\n

[root@apache2 ~]# gluster peer probe apache3
\npeer probe: success. Host apache3 port 24007 already in peer list
\n[root@apache2 ~]# gluster peer probe apache1
\npeer probe: success. Host apache1 port 24007 already in peer list
\n[root@apache2 ~]#<\/p>\n

[root@apache3 ~]# gluster peer probe apache1
\npeer probe: success. Host apache1 port 24007 already in peer list
\n[root@apache3 ~]# gluster peer probe apache2
\npeer probe: success. Host apache2 port 24007 already in peer list
\n[root@apache3 ~]#<\/p>\n

Check the Peers :
\nroot@apache1 ~]# gluster peer status
\nNumber of Peers: 2<\/p>\n

Hostname: apache2
\nUuid: 09db776a-ab96-47a3-ad77-6bc77906566b
\nState: Peer in Cluster (Connected)<\/p>\n

Hostname: apache3
\nUuid: 14db2875-378d-481d-968c-552a10f2fd7e
\nState: Peer in Cluster (Connected)
\n[root@apache1 ~]#<\/p>\n

[root@apache2 ~]# gluster peer status
\nNumber of Peers: 2<\/p>\n

Hostname: apache1.rmohan.com
\nUuid: 16612fd4-8b94-4006-8ed7-24ac33c564e0
\nState: Peer in Cluster (Connected)
\nOther names:
\napache1<\/p>\n

Hostname: apache3
\nUuid: 14db2875-378d-481d-968c-552a10f2fd7e
\nState: Peer in Cluster (Connected)<\/p>\n

[root@apache3 ~]# gluster peer status
\nNumber of Peers: 2<\/p>\n

Hostname: apache1.rmohan.com
\nUuid: 16612fd4-8b94-4006-8ed7-24ac33c564e0
\nState: Peer in Cluster (Connected)
\nOther names:
\napache1<\/p>\n

Hostname: apache2
\nUuid: 09db776a-ab96-47a3-ad77-6bc77906566b
\nState: Peer in Cluster (Connected)<\/p>\n

Create GlusterFS volume
\nWe add 2 new drives from VMware ESXi (in fact, the 3 GlusterFS servers are 3 virtual machines) : do the below commands on all 3 servers<\/p>\n

[root@apache1 ~]# for HOST in $(ls \/sys\/class\/scsi_host) ; do echo ‘- – -‘>”\/sys\/class\/scsi_host\/$HOST\/scan” ; done
\n[root@apache1 ~]# lsblk -f
\nNAME FSTYPE LABEL UUID MOUNTPOINT
\nsda
\n??sda1 xfs be8c0a37-1ed9-4743-b4ef-03d59ba3d06d \/boot
\n??sda2 LVM2_member YXOIJV-EPlD-dXwg-ePQX-D7av-jPdr-Grb4rp
\n??centos-root xfs d2c46dc9-6365-4345-9b60-021aabfd0525 \/
\n??centos-swap swap 74c70324-6875-40e6-9d8f-591d3325c06e [SWAP]
\n??centos-home xfs bd8e9210-bc95-4af9-be87-a33da31f8dbb \/home
\nsdb
\nsr0 iso9660 CentOS 7 x86_64 2016-04-01-13-21-01-00
\n[root@apache1 ~]#<\/p>\n

[root@apache1 ~]# parted -s \/dev\/sdb mklabel gpt && parted -s \/dev\/sdb mkpart primary xfs 0% 100% && mkfs.xfs -f \/dev\/sdb
\nmeta-data=\/dev\/sdb isize=256 agcount=4, agsize=1310720 blks
\n= sectsz=512 attr=2, projid32bit=1
\n= crc=0 finobt=0
\ndata = bsize=4096 blocks=5242880, imaxpct=25
\n= sunit=0 swidth=0 blks
\nnaming =version 2 bsize=4096 ascii-ci=0 ftype=0
\nlog =internal log bsize=4096 blocks=2560, version=2
\n= sectsz=512 sunit=0 blks, lazy-count=1
\nrealtime =none extsz=4096 blocks=0, rtextents=0
\n[root@apache1 ~]#<\/p>\n

Create gluster and mount on all 3 nodes<\/p>\n

[root@apache1 ~]# mkdir \/gluster
\n[root@apache1 ~]# vi \/etc\/fstab
\n[root@apache1 ~]# grep -i gluster \/etc\/fstab
\n\/dev\/sdb \/gluster xfs defaults 0 0
\n[root@apache1 ~]#<\/p>\n

[root@apache1 ~]# mount -a
\n[root@apache1 ~]# df -Th
\nFilesystem Type Size Used Avail Use% Mounted on
\n\/dev\/mapper\/centos-root xfs 38G 1.2G 37G 3% \/
\ndevtmpfs devtmpfs 1.8G 0 1.8G 0% \/dev
\ntmpfs tmpfs 1.9G 0 1.9G 0% \/dev\/shm
\ntmpfs tmpfs 1.9G 8.6M 1.9G 1% \/run
\ntmpfs tmpfs 1.9G 0 1.9G 0% \/sys\/fs\/cgroup
\n\/dev\/sda1 xfs 497M 186M 312M 38% \/boot
\n\/dev\/mapper\/centos-home xfs 19G 33M 19G 1% \/home
\ntmpfs tmpfs 371M 0 371M 0% \/run\/user\/0
\n\/dev\/sdb xfs 20G 33M 20G 1% \/gluster
\n[root@apache1 ~]#<\/p>\n

Create a directory within the file system on the servers :
\nmkdir -p \/gluster\/briks<\/p>\n

Create the GlusterFS volume based on those 3 replicas :<\/p>\n

gluster volume create appgluster_vol replica 3 apache1:\/gluster\/briks apache2:\/gluster\/briks apache3:\/gluster\/briks<\/p>\n

[root@apache1 ~]# gluster volume create appgluster_vol replica 3 apache1:\/gluster\/briks apache2:\/gluster\/briks apache3:\/gluster\/briks
\nvolume create: appgluster_vol: success: please start the volume to access data
\n[root@apache1 ~]#<\/p>\n

Then, start this volume :<\/p>\n

[root@apache1 ~]# gluster volume start appgluster_vol
\nvolume start: appgluster_vol: success
\n[root@apache1 ~]#<\/p>\n

[root@apache1 ~]# gluster volume info<\/p>\n

Volume Name: appgluster_vol
\nType: Replicate
\nVolume ID: 856f1052-9d19-49ee-b0c1-1e53d32b86f4
\nStatus: Started
\nNumber of Bricks: 1 x 3 = 3
\nTransport-type: tcp
\nBricks:
\nBrick1: apache1:\/gluster\/briks
\nBrick2: apache2:\/gluster\/briks
\nBrick3: apache3:\/gluster\/briks
\nOptions Reconfigured:
\nperformance.readdir-ahead: on
\n[root@apache1 ~]#<\/p>\n

[root@apache1 ~]# gluster volume status
\nStatus of volume: appgluster_vol
\nGluster process TCP Port RDMA Port Online Pid
\n——————————————————————————
\nBrick apache1:\/gluster\/briks 49152 0 Y 3108
\nBrick apache2:\/gluster\/briks 49152 0 Y 2923
\nBrick apache3:\/gluster\/briks 49152 0 Y 2892
\nNFS Server on localhost 2049 0 Y 3128
\nSelf-heal Daemon on localhost N\/A N\/A Y 3136
\nNFS Server on apache2 2049 0 Y 2943
\nSelf-heal Daemon on apache2 N\/A N\/A Y 2951
\nNFS Server on apache3 2049 0 Y 2912
\nSelf-heal Daemon on apache3 N\/A N\/A Y 2920<\/p>\n

Task Status of Volume appgluster_vol
\n——————————————————————————
\nThere are no active volume tasks<\/p>\n

[root@apache1 ~]#<\/p>\n

Client side
\nBoth firewall and SELinux are disabled :<\/p>\n

192.168.1.21 clusterserver2.rmohan.com clusterserver2
\n192.168.1.26 clusterserver3.rmohan.com clusterserver3<\/p>\n

one the clinet side add the host file
\n192.168.1.21 clusterserver2.rmohan.com clusterserver2
\n192.168.1.26 clusterserver3.rmohan.com clusterserver3
\n192.168.1.71 apache1.rmohan.com apache1
\n192.168.1.72 apache2.rmohan.com apache2
\n192.168.1.74 apache3.rmohan.com apache3<\/p>\n

[root@clusterserver3 ~]# mount.glusterfs apache1:\/appgluster_vol \/mnt\/
\n[root@clusterserver3 ~]# df -Th
\nFilesystem Type Size Used Avail Use% Mounted on
\n\/dev\/mapper\/centos-root xfs 50G 3.6G 47G 8% \/
\ndevtmpfs devtmpfs 1.8G 0 1.8G 0% \/dev
\ntmpfs tmpfs 1.9G 16K 1.9G 1% \/dev\/shm
\ntmpfs tmpfs 1.9G 17M 1.8G 1% \/run
\ntmpfs tmpfs 1.9G 0 1.9G 0% \/sys\/fs\/cgroup
\n\/dev\/loop0 ext4 1.9G 6.1M 1.7G 1% \/srv\/node\/swiftloopback
\n\/dev\/mapper\/centos-home xfs 46G 33M 46G 1% \/home
\n\/dev\/sda1 xfs 497M 213M 285M 43% \/boot
\ntmpfs tmpfs 371M 0 371M 0% \/run\/user\/0
\napache1:\/appgluster_vol fuse.glusterfs 20G 33M 20G 1% \/mnt
\n[root@clusterserver3 ~]#<\/p>\n

[root@clusterserver3 ~]# echo “here is a GlusterFS file” > \/mnt\/file<\/p>\n

[root@apache1 ~]# cat \/gluster\/briks\/file
\nhere is a GlusterFS file<\/p>\n

[root@apache2 ~]# cat \/gluster\/briks\/file
\nhere is a GlusterFS file
\n[root@apache2 ~]#<\/p>\n

[root@apache3 ~]# cat \/gluster\/briks\/file
\nhere is a GlusterFS file
\n[root@apache3 ~]#<\/p>\n","protected":false},"excerpt":{"rendered":"

GlusterFS servers on RHEL 7 and Centos 7 <\/p>\n

vi \/etc\/hosts<\/p>\n

192.168.1.71 apache1.rmohan.com apache1 192.168.1.72 apache2.rmohan.com apache2 192.168.1.74 apache3.rmohan.com apache3<\/p>\n

For this lab, we disabled both firewall and SELinux<\/p>\n

vi \/etc\/selinux\/config egrep “^SELINUX=” \/etc\/sysconfig\/selinux SELINUX=disabled systemctl is-enabled firewalld disabled<\/p>\n

systemctl disable firewalld yum update yum -y install wget openssh* net-tools ntp ntpdate -u 0.centos.pool.ntp.org<\/p>\n

[…]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[73],"tags":[],"_links":{"self":[{"href":"https:\/\/mohan.sg\/index.php?rest_route=\/wp\/v2\/posts\/6953"}],"collection":[{"href":"https:\/\/mohan.sg\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/mohan.sg\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/mohan.sg\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/mohan.sg\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=6953"}],"version-history":[{"count":1,"href":"https:\/\/mohan.sg\/index.php?rest_route=\/wp\/v2\/posts\/6953\/revisions"}],"predecessor-version":[{"id":6954,"href":"https:\/\/mohan.sg\/index.php?rest_route=\/wp\/v2\/posts\/6953\/revisions\/6954"}],"wp:attachment":[{"href":"https:\/\/mohan.sg\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=6953"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/mohan.sg\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=6953"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/mohan.sg\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=6953"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}