OrangeFS

From HP-SEE Wiki

Making of

Installation of prerequisites:

 yum -y install gcc flex bison openssl-devel db4-devel kernel-devel perl make

Update BerkleyDB

  wget http://download.oracle.com/berkeley-db/db-5.0.32.tar.gz
  tar -xzf db-5.0.32.tar.gz
  cd db-5.0.32/build_unix
  ../dist/configure --prefix=/usr/local/db5.0.32
  make
  make install

Add BerkleyDB lib to system libraries by adding to /etc/profile this line:

  export LD_LIBRARY_PATH=/usr/local/db5.0.32/lib:${LD_LIBRARY_PATH}

Build OrangeFS

 wget http://www.orangefs.org/downloads/2.8.6/source/orangefs-2.8.6.tar.gz
 tar -xzf orangefs-2.8.6.tar.gz
 cd orangefs-2.8.6
 ./configure --build=x86_64 --prefix=/opt/orangefs_bdb --with-kernel=/usr/src/kernels/2.6.18-308.24.1.el5-x86_64/ --with-db=/usr/local/db5.0.32/
 make
 make install
 make kmod
 make kmod_prefix=/opt/orangefs_bdb kmod_install
 cd /opt/orangefs_bdb
 mkdir -p etc/
 /opt/orangefs_bdb/bin/pvfs2-genconfig
 /opt/orangefs_bdb/etc/orangefs-server.conf

cat orangefs-server.conf

 <Defaults>
       UnexpectedRequests 50
       EventLogging none
       EnableTracing no
       LogStamp datetime
       BMIModules bmi_tcp
       FlowModules flowproto_multiqueue
       PerfUpdateInterval 1000
       ServerJobBMITimeoutSecs 30
       ServerJobFlowTimeoutSecs 30
       ClientJobBMITimeoutSecs 300
       ClientJobFlowTimeoutSecs 300
       ClientRetryLimit 5
       ClientRetryDelayMilliSecs 2000
       PrecreateBatchSize 0,32,512,32,32,32,0
       PrecreateLowThreshold 0,16,256,16,16,16,0
       DataStorageSpace /opt/orangefs/storage/data
       MetadataStorageSpace /opt/orangefs/storage/meta
       LogFile /opt/orangefs/log/orangefs-server.log
 </Defaults>
 <Aliases>
       Alias testhpc02 tcp://194.102.58.204:3334
       Alias testhpc03 tcp://194.102.58.205:3334
       Alias niifidata1 tcp://193.225.36.47:3334
       Alias niifidata2 tcp://193.225.36.52:3334
       Alias par03 tcp://194.102.58.168:3334
       Alias niifimd1 tcp://193.225.36.51:3334
 </Aliases>
 <Filesystem>
       Name pvfs2-fs
       ID 2145471292
       RootHandle 1048576
       FileStuffing no
       <MetaHandleRanges>
               Range par03 3-230584300921369397
               Range niifimd1 230584300921369398-461168601842738792
       </MetaHandleRanges>
       <DataHandleRanges>
               Range testhpc02 1152921504606846978-1383505805528216372
               Range testhpc03 1383505805528216373-1614090106449585767
               Range niifidata1 1614090106449585768-1844674407370955162
               Range niifidata2 1844674407370955163-2075258708292324557
       </DataHandleRanges>
       <StorageHints>
               TroveSyncMeta yes
               TroveSyncData no
               TroveMethod alt-aio
       </StorageHints>
 </Filesystem>

Add this rule on firewall:

 iptables -A RH-Firewall-1-INPUT -m state --state NEW -m tcp -p tcp --dport 3334 -j ACCEPT

Inititialize the storage space on each server

 /opt/orangefs_bdb/sbin/pvfs2-server -f -a par03 /opt/orangefs_bdb/etc/orangefs-server.conf

Start the server process on each server

 /opt/orangefs_bdb/sbin/pvfs2-server -a par03 /opt/orangefs_bdb/etc/orangefs-server.conf

Stopping The Server Process

 killall pvfs2-server

If you want start/stop script you can copy from

/root/orangefs-2.8.6/examples/pvfs2-server.rc to /etc/init.d/

Add Linux Clients

 scp -r /opt/orangefs_bdb clientIP:/opt/
 mkdir /mnt/orangefs
 cat > /etc/fstab <<EOF
    tcp://par03:3334/pvfs2-fs       /mnt/orangefs   pvfs2   defaults        0 0
 EOF

Insert The Kernel Module

 insmod /opt/orangefs_bdb/lib/modules/2.6.18-308.24.1.el5/kernel/fs/pvfs2/pvfs2.ko

Start The Client Process

 /opt/orangefs_bdb/sbin/pvfs2-client

Mount OrangeFS

 mount -a

Check connection with the OrangeFS server:

 /opt/orangefs_bdb/bin/pvfs2-check-server -h 194.102.58.205 -f pvfs2-fs -n tcp -p 3334

 /opt/orangefs_bdb/bin/pvfs2-ping -m /mnt/orangefs/

 /opt/orangefs_bdb/bin/pvfs2-ls /mnt/orangefs/dltest

 /opt/orangefs_bdb/bin/pvfs2-cp -t /mnt/orangefs/dltest/1000MB /home/dragos/

 /opt/orangefs_bdb/bin/pvfs2-viewdist -f /mnt/orangefs/dltest/1000MB