ＩＴ技術発信ブログ

I tried the OCFS2 + DRBD8 on CentOS 5.3. I’ve written the document below.
Please give me advice/suggestions on it. I would appreciate any kind of advice/suggestion!

= OCFS2

download rpm pacakge from ocfs2 site.

> yum -y install vte
> rpm -ivh ocfs2-tools-1.4.2-1.el5.x86_64.rpm
> rpm -ivh ocfs2console-1.4.2-1.el5.x86_64.rpm
> rpm -ivh ocfs2-2.6.18-128.2.1.el5-1.4.2-1.el5.x86_64.rpm

> vi /etc/sysconfig/o2cb
O2CB_ENABLED=true

> vi /etc/ocfs2/cluster.conf

node:
ip_port = 7777
ip_address = 10.0.0.74
number = 0
name = ocfs21
cluster = ocfs2

node:
ip_port = 7777
ip_address = 10.0.0.75
number = 1
name = ocfs22
cluster = ocfs2

cluster:
node_count = 2
name = ocfs2

== DRBD8
> yum -y install drbd82 kmod-drbd82

> vim /etc/drbd.conf

global {
usage-count yes;
}

common {
syncer {
rate 100M;
al-extents 257;
}
}

resource r0 {

protocol C;

startup {
become-primary-on both; ### For Primary/Primary ###
degr-wfc-timeout 60;
wfc-timeout 30;
}

disk {
on-io-error detach;
}

net {
allow-two-primaries; ### For Primary/Primary ###
cram-hmac-alg sha1;
shared-secret "FooFunFactory";
after-sb-0pri discard-zero-changes;
after-sb-1pri violently-as0p;
after-sb-2pri violently-as0p;
}

on ocfs21 {
device /dev/drbd0;
disk /dev/sdc;
address 10.0.0.81:7788;
meta-disk /dev/sdb[0];
}

on ocfs22 {
device /dev/drbd0;
disk /dev/sdc;
address 10.0.0.82:7788;
meta-disk /dev/sdb[0];
}
}

=Create Metadata

> dd if=/dev/zero of=/dev/sda3 bs=1M count=1024
> drbdadm create-md r0
> /etc/init.d/drbd stop
> /etc/init.d/drbd start

=Make them Primary/Primary

> drbdsetup /dev/drbd0 primary -o
> cat /proc/drbd
version: 8.3.0 (api:88/proto:86-89)
GIT-hash: 9ba8b93e24d842f0dd3fb1f9b90e8348ddb95829 build by ivoks@ubuntu, 2009-01-17 07:49:56
0: cs:Connected ro:Primary/Primary ds:UpToDate/Diskless C r---
ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:4883760

> mkfs.ocfs2 /dev/drbd0

> /etc/init.d/o2cb start

> mount -t ocfs2 /dev/drbd0 /data

ラベル: Cluster

# RINA さんの投稿 : 2:58 0 件のコメント

I installed GFS2 and DRBD. I've written the installation document below. Please give me advice/suggestions on it. I would appreciate any kind of advice/suggestion!

== on GFS1 and GFS2

> vim /etc/hosts
10.0.0.71 GFS1
10.0.0.72 GFS2

= install DRBD8 from source

> aptitude install drbd8-utils
> aptitude install build-essential autoconf automake libtool flex libncurses-dev linux-source
> aptitude install linux-source-2.6.28
> cd /usr/src/
> tar jxvf linux-source-2.6.28.tar.bz2
> cd linux-source-2.6.28
> make mrproper
> cp /boot/config-2.6.28-13-server /usr/src/linux-source-2.6.28/.config
> make menuconfig
exit
> make prepare
> wget http://oss.linbit.com/drbd/8.3/drbd-8.3.1.tar.gz
> tar xf drbd-8.3.1.tar.gz
> cd drbd-8.3.1
> make KDIR=/usr/src/linux-source-2.6.28
> make install
> mv /lib/modules/2.6.28.9/kernel/drivers/block/drbd.ko /lib/modules/2.6.28-13-server/kernel/drivers/block/
> modprobe drbd
> echo 'drbd' >> /etc/modules
> update-rc.d drbd defaults
> lsmod | grep drbd

= Edit drbd.conf
> vim /etc/drbd.conf
global {
usage-count yes;
}

common {
syncer {
rate 100M;
al-extents 257;
}
}

resource r0 {

protocol C;

startup {
become-primary-on both; ### For Primary/Primary ###
degr-wfc-timeout 60;
wfc-timeout 30;
}

disk {
on-io-error detach;
}

net {
allow-two-primaries; ### For Primary/Primary ###
cram-hmac-alg sha1;
shared-secret "FooFunFactory";
after-sb-0pri discard-zero-changes;
after-sb-1pri violently-as0p;
after-sb-2pri violently-as0p;
}

on GFS1 {
device /dev/drbd0;
disk /dev/sda4;
address 216.69.70.71:7788;
meta-disk /dev/sda3[0];
}

on GFS2 {
device /dev/drbd0;
disk /dev/sda4;
address 216.69.70.72:7788;
meta-disk /dev/sda3[0];
}
}

=Create Metadata
> dd if=/dev/zero of=/dev/sda3 bs=1M count=256
> drbdadm create-md r0
> /etc/init.d/drbd stop
> /etc/init.d/drbd start

=Make them Primary/Primary
> drbdsetup /dev/drbd0 primary -o
> cat /proc/drbd
version: 8.3.0 (api:88/proto:86-89)
GIT-hash: 9ba8b93e24d842f0dd3fb1f9b90e8348ddb95829 build by ivoks@ubuntu, 2009-01-17 07:49:56
0: cs:Connected ro:Primary/Primary ds:UpToDate/Diskless C r---
ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:4883760

= Install GFS package

> aptitude install gfs-tools cman clvm

= Configure GFS cluster. Change name= and IPADDR=.

> vim /etc/cluster/cluster.conf

<?xml version="1.0"?>
<cluster name="cluster1" config_version="3">

<cman two_node="1" expected_votes="1"/>

<?xml version="1.0"?>
<cluster name="cluster1" config_version="3">

<cman two_node="1" expected_votes="1"/>

<clusternodes>
<clusternode name="GFS1" votes="1" nodeid="1">
<fence>
<method name="single">
<device name="manual" ipaddr="10.0.0.71"/>
</method>
</fence>
</clusternode>

<clusternode name="GFS2" votes="1" nodeid="2">
<fence>
<method name="single">
<device name="manual" ipaddr="10.0.0.72"/>
</method>
</fence>
</clusternode>
</clusternodes>

<fence_daemon clean_start="1" post_fail_delay="0" post_join_delay="3"/>

<fencedevices>
<fencedevice name="manual" agent="fence_manual"/>
</fencedevices>

</cluster>

= Change Lock type, depend on system enviroments.

> vim /etc/lvm/lvm.conf

#locking_type = 1
#locking_dir = "/var/lock/lvm"
#library_dir = "/lib/lvm2"
locking_library = "liblvm2clusterlock.so"
locking_type = 2
library_dir = "/lib/lvm2"

= restart service

> /etc/init.d/cman stop
> /etc/init.d/cman start
> /etc/init.d/clvm stop
> /etc/init.d/clvm start

= Format drbd0 to GFS
> gfs2_mkfs -p lock_dlm -t cluster1:gfs -j 2 /dev/drbd0
> fs2_fsck /dev/drbd0

= mount GFS

> mkdir /data
> mount -t gfs2 /dev/drbd0 /data
> mount -t gfs2 /dev/drbd0 /data

> vim /etc/rc.local
sleep 5
mount -t gfs2 /dev/drbd0 /data

== GFS check

= GFS1
> i=0;while true; do echo aaaaaaaa,$i >> test.log ;i=`expr $i + 1`;done
> tail -f test.log

= GFS2
> i=0;while true; do echo bbbbbbbb,$i >> test.log ;i=`expr $i + 1`;done
> tail -f test.log

= If Split brian happen and did not sysnc.

#host 1 (main side)
> drbdadm connect resource

#host 2 (adjust side)
>umount /data
>drbdadm secondary resource
>drbdadm -- --discard-my-data connect resource
>drbdsetup /dev/drbd0 primary -o

ラベル: Cluster

# RINA さんの投稿 : 2:54 0 件のコメント

I tried the OCFS2 + DRBD8 on CentOS 5.3. I’ve written the document below.
Please give me advice/suggestions on it. I would appreciate any kind of advice/suggestion!

= OCFS2

download rpm pacakge from ocfs2 site.

> yum -y install vte
> rpm -ivh ocfs2-tools-1.4.2-1.el5.x86_64.rpm
> rpm -ivh ocfs2console-1.4.2-1.el5.x86_64.rpm
> rpm -ivh ocfs2-2.6.18-128.2.1.el5-1.4.2-1.el5.x86_64.rpm

> vi /etc/sysconfig/o2cb
O2CB_ENABLED=true

> vi /etc/ocfs2/cluster.conf

node:
ip_port = 7777
ip_address = 10.0.0.74
number = 0
name = ocfs21
cluster = ocfs2

node:
ip_port = 7777
ip_address = 10.0.0.75
number = 1
name = ocfs22
cluster = ocfs2

cluster:
node_count = 2
name = ocfs2

== DRBD8
> yum -y install drbd82 kmod-drbd82

> vim /etc/drbd.conf

global {
usage-count yes;
}

common {
syncer {
rate 100M;
al-extents 257;
}
}

resource r0 {

protocol C;

startup {
become-primary-on both; ### For Primary/Primary ###
degr-wfc-timeout 60;
wfc-timeout 30;
}

disk {
on-io-error detach;
}

net {
allow-two-primaries; ### For Primary/Primary ###
cram-hmac-alg sha1;
shared-secret "FooFunFactory";
after-sb-0pri discard-zero-changes;
after-sb-1pri violently-as0p;
after-sb-2pri violently-as0p;
}

on ocfs21 {
device /dev/drbd0;
disk /dev/sdc;
address 10.0.0.81:7788;
meta-disk /dev/sdb[0];
}

on ocfs22 {
device /dev/drbd0;
disk /dev/sdc;
address 10.0.0.82:7788;
meta-disk /dev/sdb[0];
}
}

=Create Metadata

> dd if=/dev/zero of=/dev/sda3 bs=1M count=1024
> drbdadm create-md r0
> /etc/init.d/drbd stop
> /etc/init.d/drbd start

=Make them Primary/Primary

> drbdsetup /dev/drbd0 primary -o
> cat /proc/drbd
version: 8.3.0 (api:88/proto:86-89)
GIT-hash: 9ba8b93e24d842f0dd3fb1f9b90e8348ddb95829 build by ivoks@ubuntu, 2009-01-17 07:49:56
0: cs:Connected ro:Primary/Primary ds:UpToDate/Diskless C r---
ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:4883760

> mkfs.ocfs2 /dev/drbd0

> /etc/init.d/o2cb start

> mount -t ocfs2 /dev/drbd0 /data

ラベル: Cluster

# RINA さんの投稿 : 2:53 0 件のコメント

I tried the CTDB, Samba with Luster on CentOS 5.3. I’ve written the document below.
Please give me advice/suggestions on it. I would appreciate any kind of advice/suggestion!

=== CTDB SAMBA Lustre on CetnOS 5.3

VIP 192.168.0.21   VIP 192.168.0.22
      192.168.0.11       192.168.0.12
         [cTDB1]           [CTDB2]
            |                |
            |                |
            |                |
     ----------------------------
     |            |             |
     |            |             |
   [MDT]        [OST1]        [OST2]
192.168.0.31  192.168.0.32  192.168.0.33

== Lustre

> rpm -ivh kernel-lustre-smp-2.6.18-128.1.6.el5_lustre.1.8.0.1.x86_64.rpm
> rpm -ivh lustre-modules-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-client-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-client-modules-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-ldiskfs-3.0.8-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm

> vim /etc/grub.conf

default=0 ########## Change to 0 ###########
timeout=5
splashimage=(hd0,0)/grub/splash.xpm.gz
hiddenmenu
title CentOS (2.6.18-128.1.6.el5_lustre.1.8.0.1smp)
root (hd0,0)
kernel /vmlinuz-2.6.18-128.1.6.el5_lustre.1.8.0.1smp ro root=/dev/VolGroup00/LogVol00
initrd /initrd-2.6.18-128.1.6.el5_lustre.1.8.0.1smp.img
title CentOS (2.6.18-128.2.1.el5)
root (hd0,0)
kernel /vmlinuz-2.6.18-128.2.1.el5 ro root=/dev/VolGroup00/LogVol00
initrd /initrd-2.6.18-128.2.1.el5.img
title CentOS (2.6.18-128.el5)
root (hd0,0)
kernel /vmlinuz-2.6.18-128.el5 ro root=/dev/VolGroup00/LogVol00
initrd /initrd-2.6.18-128.el5.img

> reboot

> vim /etc/hosts
192.168.0.11 ctdb1
192.168.0.12 ctdb2
192.168.0.31 lustre1
192.168.0.32 lustre2
192.168.0.33 lustre3

= MDT Server

> mkfs.lustre --fsname=test --mgs --mdt --device-size=262144 /tmp/mdt0
> mkdir -p /mnt/mdt0
> mount -t lustre -o loop /tmp/mdt0 /mnt/mdt0

= OST Server 1

> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost0
> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost1
> mkdir -p /mnt/ost0 /mnt/ost1
> mount -t lustre -o loop /tmp/ost0 /mnt/ost0
> mount -t lustre -o loop /tmp/ost1 /mnt/ost1

= OST Server 2

> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost2
> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost3
> mkdir -p /mnt/ost2 /mnt/ost3
> mount -t lustre -o loop /tmp/ost2 /mnt/ost2
> mount -t lustre -o loop /tmp/ost3 /mnt/ost3

= on ctdb1, ctdb2

> mkdir -p /mnt/test
> mount -t lustre lustre1:/test /mnt/test

== samba

> yum -y install samba
> vi /etc/samba/smb.conf

[global]

clustering = yes
idmap backend = tdb2
private dir=/data/ctdb
fileid:mapping = fsname
use mmap = no
nt acl support = yes
ea support = yes

[share]

comment = Public Stuff
path = /data/share
public = yes
writeable = yes
only guest = yes

> smbpasswd -a root
> mkdir /data/share
> chmod 777 /data/share

== CTDB

> mkdir /data/ctdb
> rsync -avz samba.org::ftp/unpacked/ctdb .
> ./autogen.sh
> ./configure
> make
> make install
> cp config/ctdb.sysconfig /etc/sysconfig/ctdb
> cp config/ctdb.init /etc/rc.d/init.d/ctdb
> vim /etc/sysconfig/ctdb

CTDB_RECOVERY_LOCK="/data/ctdb/storage"
CTDB_PUBLIC_INTERFACE=eth0
CTDB_PUBLIC_ADDRESSES=/etc/ctdb/public_addresses
CTDB_MANAGES_SAMBA=yes
ulimit -n 10000
CTDB_NODES=/etc/ctdb/nodes
CTDB_LOGFILE=/var/log/log.ctdb
CTDB_DEBUGLEVEL=2
CTDB_PUBLIC_NETWORK="192.168.0.0/24"
CTDB_PUBLIC_GATEWAY="192.168.0.1"

> ln -s /usr/local/etc/ctdb/ /etc/ctdb
> ln -s /usr/local/bin/ctdb /usr/bin/ctdb
> ln -s /usr/local/sbin/ctdbd /usr/sbin/ctdbd

> vi /etc/ctdb/public_addresses
192.168.0.21/24
192.168.0.22/24

> vi cat /etc/ctdb/nodes
192.168.0.11
192.168.0.12

> vim /etc/ctdb/events.d/11.route
#!/bin/sh

. /etc/ctdb/functions
loadconfig ctdb

cmd="$1"
shift

case $cmd in
takeip)
# we ignore errors from this, as the route might be up already when we're grabbing
# a 2nd IP on this interface
/sbin/ip route add $CTDB_PUBLIC_NETWORK via $CTDB_PUBLIC_GATEWAY dev $1 2> /dev/null
;;
esac

exit 0

> chmod 755 /etc/ctdb/events.d/11.route
> /etc/rc.d/init.d/ctdb start

ラベル: Cluster

# RINA さんの投稿 : 2:52 0 件のコメント

I tried the CTDB, Samba with Luster on CentOS 5.3. I’ve written the document below.
Please give me advice/suggestions on it. I would appreciate any kind of advice/suggestion!

=== CTDB SAMBA Lustre on CetnOS 5.3

VIP 192.168.0.21   VIP 192.168.0.22
      192.168.0.11       192.168.0.12
         [cTDB1]           [CTDB2]
            |                |
            |                |
            |                |
     ----------------------------
     |            |             |
     |            |             |
   [MDT]        [OST1]        [OST2]
192.168.0.31  192.168.0.32  192.168.0.33

== Lustre

> rpm -ivh kernel-lustre-smp-2.6.18-128.1.6.el5_lustre.1.8.0.1.x86_64.rpm
> rpm -ivh lustre-modules-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-client-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-client-modules-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-ldiskfs-3.0.8-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm

> vim /etc/grub.conf

default=0 ########## Change to 0 ###########
timeout=5
splashimage=(hd0,0)/grub/splash.xpm.gz
hiddenmenu
title CentOS (2.6.18-128.1.6.el5_lustre.1.8.0.1smp)
root (hd0,0)
kernel /vmlinuz-2.6.18-128.1.6.el5_lustre.1.8.0.1smp ro root=/dev/VolGroup00/LogVol00
initrd /initrd-2.6.18-128.1.6.el5_lustre.1.8.0.1smp.img
title CentOS (2.6.18-128.2.1.el5)
root (hd0,0)
kernel /vmlinuz-2.6.18-128.2.1.el5 ro root=/dev/VolGroup00/LogVol00
initrd /initrd-2.6.18-128.2.1.el5.img
title CentOS (2.6.18-128.el5)
root (hd0,0)
kernel /vmlinuz-2.6.18-128.el5 ro root=/dev/VolGroup00/LogVol00
initrd /initrd-2.6.18-128.el5.img

> reboot

> vim /etc/hosts
192.168.0.11 ctdb1
192.168.0.12 ctdb2
192.168.0.31 lustre1
192.168.0.32 lustre2
192.168.0.33 lustre3

= MDT Server

> mkfs.lustre --fsname=test --mgs --mdt --device-size=262144 /tmp/mdt0
> mkdir -p /mnt/mdt0
> mount -t lustre -o loop /tmp/mdt0 /mnt/mdt0

= OST Server 1

> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost0
> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost1
> mkdir -p /mnt/ost0 /mnt/ost1
> mount -t lustre -o loop /tmp/ost0 /mnt/ost0
> mount -t lustre -o loop /tmp/ost1 /mnt/ost1

= OST Server 2

> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost2
> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost3
> mkdir -p /mnt/ost2 /mnt/ost3
> mount -t lustre -o loop /tmp/ost2 /mnt/ost2
> mount -t lustre -o loop /tmp/ost3 /mnt/ost3

= on ctdb1, ctdb2

> mkdir -p /mnt/test
> mount -t lustre lustre1:/test /mnt/test

== samba

> yum -y install samba
> vi /etc/samba/smb.conf

[global]

clustering = yes
idmap backend = tdb2
private dir=/data/ctdb
fileid:mapping = fsname
use mmap = no
nt acl support = yes
ea support = yes

[share]

comment = Public Stuff
path = /data/share
public = yes
writeable = yes
only guest = yes

> smbpasswd -a root
> mkdir /data/share
> chmod 777 /data/share

== CTDB

> mkdir /data/ctdb
> rsync -avz samba.org::ftp/unpacked/ctdb .
> ./autogen.sh
> ./configure
> make
> make install
> cp config/ctdb.sysconfig /etc/sysconfig/ctdb
> cp config/ctdb.init /etc/rc.d/init.d/ctdb
> vim /etc/sysconfig/ctdb

CTDB_RECOVERY_LOCK="/data/ctdb/storage"
CTDB_PUBLIC_INTERFACE=eth0
CTDB_PUBLIC_ADDRESSES=/etc/ctdb/public_addresses
CTDB_MANAGES_SAMBA=yes
ulimit -n 10000
CTDB_NODES=/etc/ctdb/nodes
CTDB_LOGFILE=/var/log/log.ctdb
CTDB_DEBUGLEVEL=2
CTDB_PUBLIC_NETWORK="192.168.0.0/24"
CTDB_PUBLIC_GATEWAY="192.168.0.1"

> ln -s /usr/local/etc/ctdb/ /etc/ctdb
> ln -s /usr/local/bin/ctdb /usr/bin/ctdb
> ln -s /usr/local/sbin/ctdbd /usr/sbin/ctdbd

> vi /etc/ctdb/public_addresses
192.168.0.21/24
192.168.0.22/24

> vi cat /etc/ctdb/nodes
192.168.0.11
192.168.0.12

> vim /etc/ctdb/events.d/11.route
#!/bin/sh

. /etc/ctdb/functions
loadconfig ctdb

cmd="$1"
shift

case $cmd in
takeip)
# we ignore errors from this, as the route might be up already when we're grabbing
# a 2nd IP on this interface
/sbin/ip route add $CTDB_PUBLIC_NETWORK via $CTDB_PUBLIC_GATEWAY dev $1 2> /dev/null
;;
esac

exit 0

> chmod 755 /etc/ctdb/events.d/11.route
> /etc/rc.d/init.d/ctdb start

ラベル: Cluster

# RINA さんの投稿 : 2:51 0 件のコメント

ＩＴ技術発信ブログ

2010年12月6日月曜日

OCFS2 + DRBD8 on CentOS 5.3

GFS2+DRBD on Ubuntu 9.04 64Bit

OCFS2 + DRBD8 on CentOS 5.3

CTDB, Samba, Luster on CentOS 5.3

CTDB, Samba, Luster on CentOS 5.3

自己紹介

リンク

アーカイブファイル

ＩＴ技術発信ブログ

2010年12月6日月曜日

OCFS2 + DRBD8 on CentOS 5.3

GFS2+DRBD on Ubuntu 9.04 64Bit

OCFS2 + DRBD8 on CentOS 5.3

CTDB, Samba, Luster on CentOS 5.3

CTDB, Samba, Luster on CentOS 5.3

自己紹介

リンク

アーカイブ ファイル

アーカイブファイル