2010年12月6日月曜日
Lustre
I installed Lustre. I've written the installation document below. Please give me advice/suggestions on it. I would appreciate any kind of advice/suggestion!
= Lustre on CetnOS 5.3
> rpm -ivh kernel-lustre-smp-2.6.18-128.1.6.el5_lustre.1.8.0.1.x86_64.rpm
> rpm -ivh lustre-modules-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-client-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-client-modules-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-ldiskfs-3.0.8-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> vim /etc/grub.conf
default=0 ########## Change to 0 ###########
timeout=5
splashimage=(hd0,0)/grub/splash.xpm.gz
hiddenmenu
title CentOS (2.6.18-128.1.6.el5_lustre.1.8.0.1smp)
root (hd0,0)
kernel /vmlinuz-2.6.18-128.1.6.el5_lustre.1.8.0.1smp ro root=/dev/VolGroup00/LogVol00
initrd /initrd-2.6.18-128.1.6.el5_lustre.1.8.0.1smp.img
title CentOS (2.6.18-128.2.1.el5)
root (hd0,0)
kernel /vmlinuz-2.6.18-128.2.1.el5 ro root=/dev/VolGroup00/LogVol00
initrd /initrd-2.6.18-128.2.1.el5.img
title CentOS (2.6.18-128.el5)
root (hd0,0)
kernel /vmlinuz-2.6.18-128.el5 ro root=/dev/VolGroup00/LogVol00
initrd /initrd-2.6.18-128.el5.img
> reboot
> vim /etc/hosts
192.168.0.11 lustre1
192.168.0.12 lustre2
192.168.0.13 lustre3
= MDT Server
> mkfs.lustre --fsname=test --mgs --mdt --device-size=262144 /tmp/mdt0
> mkdir -p /mnt/mdt0
> mount -t lustre -o loop /tmp/mdt0 /mnt/mdt0
= OST Server 1
> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost0
> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost1
> mkdir -p /mnt/ost0 /mnt/ost1
> mount -t lustre -o loop /tmp/ost0 /mnt/ost0
> mount -t lustre -o loop /tmp/ost1 /mnt/ost1
= OST Server 2
> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost2
> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost3
> mkdir -p /mnt/ost2 /mnt/ost3
> mount -t lustre -o loop /tmp/ost2 /mnt/ost2
> mount -t lustre -o loop /tmp/ost3 /mnt/ost3
= Client
> mkdir -p /mnt/test
> mount -t lustre lustre1:/test /mnt/test
= Lustre on CetnOS 5.3
192.168.0.10
[client]
|
|
|
----------------------------
| | |
| | |
[MDT] [OST1] [OST2]
192.168.0.11 192.168.0.12 192.168.0.13
> rpm -ivh kernel-lustre-smp-2.6.18-128.1.6.el5_lustre.1.8.0.1.x86_64.rpm
> rpm -ivh lustre-modules-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-client-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-client-modules-1.8.0.1-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> rpm -ivh lustre-ldiskfs-3.0.8-2.6.18_128.1.6.el5_lustre.1.8.0.1smp.x86_64.rpm
> vim /etc/grub.conf
default=0 ########## Change to 0 ###########
timeout=5
splashimage=(hd0,0)/grub/splash.xpm.gz
hiddenmenu
title CentOS (2.6.18-128.1.6.el5_lustre.1.8.0.1smp)
root (hd0,0)
kernel /vmlinuz-2.6.18-128.1.6.el5_lustre.1.8.0.1smp ro root=/dev/VolGroup00/LogVol00
initrd /initrd-2.6.18-128.1.6.el5_lustre.1.8.0.1smp.img
title CentOS (2.6.18-128.2.1.el5)
root (hd0,0)
kernel /vmlinuz-2.6.18-128.2.1.el5 ro root=/dev/VolGroup00/LogVol00
initrd /initrd-2.6.18-128.2.1.el5.img
title CentOS (2.6.18-128.el5)
root (hd0,0)
kernel /vmlinuz-2.6.18-128.el5 ro root=/dev/VolGroup00/LogVol00
initrd /initrd-2.6.18-128.el5.img
> reboot
> vim /etc/hosts
192.168.0.11 lustre1
192.168.0.12 lustre2
192.168.0.13 lustre3
= MDT Server
> mkfs.lustre --fsname=test --mgs --mdt --device-size=262144 /tmp/mdt0
> mkdir -p /mnt/mdt0
> mount -t lustre -o loop /tmp/mdt0 /mnt/mdt0
= OST Server 1
> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost0
> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost1
> mkdir -p /mnt/ost0 /mnt/ost1
> mount -t lustre -o loop /tmp/ost0 /mnt/ost0
> mount -t lustre -o loop /tmp/ost1 /mnt/ost1
= OST Server 2
> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost2
> mkfs.lustre --fsname=test --ost --mgsnode=lustre1 --device-size=1048576 /tmp/ost3
> mkdir -p /mnt/ost2 /mnt/ost3
> mount -t lustre -o loop /tmp/ost2 /mnt/ost2
> mount -t lustre -o loop /tmp/ost3 /mnt/ost3
= Client
> mkdir -p /mnt/test
> mount -t lustre lustre1:/test /mnt/test
ラベル: Distribute Storage
Parascale for huge storage
I installed Parascale. I’ve written the installation document below. Please give me advice/suggestions on it. I would appreciate any kind of advice/suggestion!
Console server : 192.168.0.10, 172.16.0.10 + 1LUN
Storage server1 : 192.168.0.1, 172.16.0.1 + 1LUN
Storage server2 : 192.168.0.2, 172.16.0.2 + 1LUN
Storage server3 : 192.168.0.3, 172.16.0.3 + 1LUN
Virtual ip: 192.168.0.10
External pool: 192.168.10.20-23
Internal Pool: 172.16.0.
= install console server
> mkdir /tmp/pscl
> cd /tmp/pscl
> tar zxvf pcs_v1.3.0-r4788-64bit.tgz
> ./pcsinstall -c
> vi /root/.bash_profile
export PATH=$PATH:/opt/pscl/vsn/bin
export MANPATH=$MANPATH:/opt/pscl/vsn/man
> source /root/.bash_profile
= install storage server
> mkdir /tmp/pscl
> mv pcs_v1.3.0-r4788-64bit.tgz /tmp/pscl/
> cd /tmp/pscl
> tar zxvf pcs_v1.3.0-r4788-64bit.tgz
> ./pcsinstall -s
= metenance
access http:///admin/
Console server : 192.168.0.10, 172.16.0.10 + 1LUN
Storage server1 : 192.168.0.1, 172.16.0.1 + 1LUN
Storage server2 : 192.168.0.2, 172.16.0.2 + 1LUN
Storage server3 : 192.168.0.3, 172.16.0.3 + 1LUN
Virtual ip: 192.168.0.10
External pool: 192.168.10.20-23
Internal Pool: 172.16.0.
= install console server
> mkdir /tmp/pscl
> cd /tmp/pscl
> tar zxvf pcs_v1.3.0-r4788-64bit.tgz
> ./pcsinstall -c
> vi /root/.bash_profile
export PATH=$PATH:/opt/pscl/vsn/bin
export MANPATH=$MANPATH:/opt/pscl/vsn/man
> source /root/.bash_profile
= install storage server
> mkdir /tmp/pscl
> mv pcs_v1.3.0-r4788-64bit.tgz /tmp/pscl/
> cd /tmp/pscl
> tar zxvf pcs_v1.3.0-r4788-64bit.tgz
> ./pcsinstall -s
= metenance
access http://
ラベル: Distribute Storage
Cloudstore on CentOS5.3
== CloudStore on CentOS5.3
== Install Package on All server
> yum -y install gcc gcc-++ zlib-devel e2fsprogs-devel openssl-devel xfsprogs-devel boost boost-devel
> yum -y groupinstall "Java" "Java Development"
> rpm -ivh http://dag.wieers.com/rpm/packages/log4cpp/log4cpp-0.3.5-0.rc3.el5.rf.x86_64.rpm
> rpm -ivh http://dag.wieers.com/rpm/packages/log4cpp/log4cpp-devel-0.3.5-0.rc3.el5.rf.x86_64.rpm
> rpm -ivh http://dl.atrpms.net/all/cmake-2.6.4-7.el5.x86_64.rpm
> wget http://downloads.sourceforge.net/project/kosmosfs/kosmosfs/kfs-0.3/kfs-0.3.tar.gz
> mkdir /home/kfs
> tar xzf kfs-0.3.tar.gz
> cd kfs-0.3
> mkdir build
> cd build
> cmake -D CMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=/home/kfs/cloudstore ../
> gmake
> gmake install
> vi /etc/hosts
192.168.0.1 kfs1
192.168.0.2 kfs2
192.168.0.3 kfs3
192.168.0.4 kfs4
== On meta server
=Send public key to other server
> ssh-keygen -t rsa
> scp /root/.ssh/id_rsa.pub kfs1:/root/.ssh/authorized_keys
> scp /root/.ssh/id_rsa.pub kfs2:/root/.ssh/authorized_keys
> scp /root/.ssh/id_rsa.pub kfs3:/root/.ssh/authorized_keys
> scp /root/.ssh/id_rsa.pub kfs4:/root/.ssh/authorized_keys
=COnfiguration
> vi /root/work/kfs-0.3/script/smachines.cfg
# KFS Machine configuration file
[metaserver]
node: kfs1
clusterkey: kfs-test-cluster
rundir: /home/kfs/meta
baseport: 20000
loglevel: INFO
numservers: 2
[chunkserver_defaults]
rundir: /home/kfs/chunk
chunkDir: /home/kfs/chunk/bin/kfschunk
baseport: 30000
space: 5 G
loglevel: INFO
= Configure for Chunk Server's info
> /root/work/kfs-0.3/scripts/machines.txt
kfs2
kfs3
kfs4
> cp machines.txt ../webui/all-machines.txt
= Distribute setting file to other server
> python kfssetup.py -f machines.cfg -m machines.txt -b /home/kfs/cloudstore -w ../webui
= Start all of service from Meta Server
> cd /home/kfs/meta/scripts/
> python kfslaunch.py -f machines.cfg -m machines.txt -s
= Check the status
> /home/kfs/cloudstore/bin/tools/kfsping -m -s kfs1 -p 20000
= Stop Service
> python kfslaunch.py -f machines.cfg -m machines.txt -S
= Put Data
> echo "jun" > /root/test/jun.txt
> /home/kfs/cloudstore/bin/tools/cptokfs -s kfs1 -p 20000 -d /root/test/jun.txt -k jun.txt
= Check
> /home/kfs/cloudstore/bin/tools/kfsshell -s kfs1 -p 20000
KfsShell> ls -al
dumpster/ Aug 24 07:57 (dir)
jun.txt Aug 24 08:57 4
= Read Data
> cd /tmp
> /home/kfs/cloudstore/bin/tools/cpfromkfs -s kfs1 -p 20000 -d ./jun.txt -k jun.txt
> cat jun.txt
jun
= Confirm chunk data
> /home/kfs/cloudstore/bin/tools/kfsfileenum -s kfs1 -p 20000 -f jun.txt
0 6
216.69.70.84 30000 4
216.69.70.82 30000 4
216.69.70.83 30000 4
File size computed from chunksizes: 4
=Mount Cloudstore as FUSE
> /etc/yum.repos.d/CentOS-Base.repo
[dag]
name=Dag RPM Repository for Red Hat Enterprise Linux
baseurl=http://ftp.riken.jp/Linux/dag/redhat/el$releasever/en/$basearch/dag
> wget http://dag.wieers.com/packages/RPM-GPG-KEY.dag.txt
> rpm --import RPM-GPG-KEY.dag.txt
> yum -y install fuse
> vi /root/work/kfs-0.3/CMakeLists.txt
SET(Fuse_LIBRARY_DIR "/usr/lib64")
SET(Fuse_INCLUDE_DIR "/usr/include")
> cd /root/work/kfs-0.3/build
> make install
> vi /home/kfs/cloudstore/bin/kfs.prp
metaServer.name = kfs1
metaServer.port = 20000
> mkdir /tmp/kfs-fuse
> cd /home/kfs/cloudstore/bin/
> ./kfs_fuse /tmp/kfs-fuse -f &
> cd /tmp/kfs-fuse
> echo "jun2" > jun2.txt
> cat jun2.txt
jun2
|
-----------------------------------------
| | | |
192.168.0.1 192.168.0.2 192.168.0.3 192.168.0.4
[Mata Server] [Chunk] [Chunk] [Chunk]
== Install Package on All server
> yum -y install gcc gcc-++ zlib-devel e2fsprogs-devel openssl-devel xfsprogs-devel boost boost-devel
> yum -y groupinstall "Java" "Java Development"
> rpm -ivh http://dag.wieers.com/rpm/packages/log4cpp/log4cpp-0.3.5-0.rc3.el5.rf.x86_64.rpm
> rpm -ivh http://dag.wieers.com/rpm/packages/log4cpp/log4cpp-devel-0.3.5-0.rc3.el5.rf.x86_64.rpm
> rpm -ivh http://dl.atrpms.net/all/cmake-2.6.4-7.el5.x86_64.rpm
> wget http://downloads.sourceforge.net/project/kosmosfs/kosmosfs/kfs-0.3/kfs-0.3.tar.gz
> mkdir /home/kfs
> tar xzf kfs-0.3.tar.gz
> cd kfs-0.3
> mkdir build
> cd build
> cmake -D CMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=/home/kfs/cloudstore ../
> gmake
> gmake install
> vi /etc/hosts
192.168.0.1 kfs1
192.168.0.2 kfs2
192.168.0.3 kfs3
192.168.0.4 kfs4
== On meta server
=Send public key to other server
> ssh-keygen -t rsa
> scp /root/.ssh/id_rsa.pub kfs1:/root/.ssh/authorized_keys
> scp /root/.ssh/id_rsa.pub kfs2:/root/.ssh/authorized_keys
> scp /root/.ssh/id_rsa.pub kfs3:/root/.ssh/authorized_keys
> scp /root/.ssh/id_rsa.pub kfs4:/root/.ssh/authorized_keys
=COnfiguration
> vi /root/work/kfs-0.3/script/smachines.cfg
# KFS Machine configuration file
[metaserver]
node: kfs1
clusterkey: kfs-test-cluster
rundir: /home/kfs/meta
baseport: 20000
loglevel: INFO
numservers: 2
[chunkserver_defaults]
rundir: /home/kfs/chunk
chunkDir: /home/kfs/chunk/bin/kfschunk
baseport: 30000
space: 5 G
loglevel: INFO
= Configure for Chunk Server's info
> /root/work/kfs-0.3/scripts/machines.txt
kfs2
kfs3
kfs4
> cp machines.txt ../webui/all-machines.txt
= Distribute setting file to other server
> python kfssetup.py -f machines.cfg -m machines.txt -b /home/kfs/cloudstore -w ../webui
= Start all of service from Meta Server
> cd /home/kfs/meta/scripts/
> python kfslaunch.py -f machines.cfg -m machines.txt -s
= Check the status
> /home/kfs/cloudstore/bin/tools/kfsping -m -s kfs1 -p 20000
= Stop Service
> python kfslaunch.py -f machines.cfg -m machines.txt -S
= Put Data
> echo "jun" > /root/test/jun.txt
> /home/kfs/cloudstore/bin/tools/cptokfs -s kfs1 -p 20000 -d /root/test/jun.txt -k jun.txt
= Check
> /home/kfs/cloudstore/bin/tools/kfsshell -s kfs1 -p 20000
KfsShell> ls -al
dumpster/ Aug 24 07:57 (dir)
jun.txt Aug 24 08:57 4
= Read Data
> cd /tmp
> /home/kfs/cloudstore/bin/tools/cpfromkfs -s kfs1 -p 20000 -d ./jun.txt -k jun.txt
> cat jun.txt
jun
= Confirm chunk data
> /home/kfs/cloudstore/bin/tools/kfsfileenum -s kfs1 -p 20000 -f jun.txt
0 6
216.69.70.84 30000 4
216.69.70.82 30000 4
216.69.70.83 30000 4
File size computed from chunksizes: 4
=Mount Cloudstore as FUSE
> /etc/yum.repos.d/CentOS-Base.repo
[dag]
name=Dag RPM Repository for Red Hat Enterprise Linux
baseurl=http://ftp.riken.jp/Linux/dag/redhat/el$releasever/en/$basearch/dag
> wget http://dag.wieers.com/packages/RPM-GPG-KEY.dag.txt
> rpm --import RPM-GPG-KEY.dag.txt
> yum -y install fuse
> vi /root/work/kfs-0.3/CMakeLists.txt
SET(Fuse_LIBRARY_DIR "/usr/lib64")
SET(Fuse_INCLUDE_DIR "/usr/include")
> cd /root/work/kfs-0.3/build
> make install
> vi /home/kfs/cloudstore/bin/kfs.prp
metaServer.name = kfs1
metaServer.port = 20000
> mkdir /tmp/kfs-fuse
> cd /home/kfs/cloudstore/bin/
> ./kfs_fuse /tmp/kfs-fuse -f &
> cd /tmp/kfs-fuse
> echo "jun2" > jun2.txt
> cat jun2.txt
jun2
ラベル: Distribute Storage
登録 投稿 [Atom]