//
you're reading...
Centos, Systems

Infiniband on Centos 6.3

These are notes from setting up Centos 6.3 against Violin memory storage using SRPT
yum -y install srptools rdma

# This is just a bonus to make it easier to find our local info if iinfiniband-diags is installed.
echo -n `uname -n` > /sys/class/infiniband/mlx4_0/node_desc

vi /etc/rdma/rdma.conf
SRP_LOAD=yes

chkconfig rdma on
chkconfig srpd on

service rdma start
service srpd start

#---- Using srp_daemon ---#
srp_daemon -e -o
# or
srp_daemon -c -a -o -i mlx4_0 -p 1 -e
srp_daemon -c -a -o -i mlx4_0 -p 2 -e
#-------------------------#

#---- Manually ---#
ibsrpdm -c
id_ext=xxxxxxxxxxxxxxxxxx,ioc_guid=xxxxxxxxxxxxxxxxxx,dgid=xxxxxxxxxxxxxxxxx,pkey=ffff,service_id=xxxxxxxxxxxxxxxxxx

## Do this twice, once to get the session to show on the violin memory, and again after exporting the lun to the group to get the LUN to show up
echo -n id_ext=xxxxxxxxxxxxxxxxxx,ioc_guid=xxxxxxxxxxxxxxxxxx,dgid=xxxxxxxxxxxxxxxxx,pkey=ffff,service_id=xxxxxxxxxxxxxxxxxx > /sys/class/infiniband_srp/srp-mlx4_0-1/add_target
#-----------------#

## Setup multipath alias
# get wwid
multipath -ll

vi /etc/multipath.conf
multipaths {
    multipath {
        wwid 12345678901234567890(Use your WWID)
        alias lun01
    }
}

# Flush and reload multipath
service multipathd stop
multipath -F
service multipathd start
multipath -v2
multipath -ll

mount /dev/mapper/lun01p1 /mnt/lun01

# If LUN has an existing partition and is not showing up, run partprobe
partprobe /dev/mapper/lun01

Advertisements

Discussion

2 thoughts on “Infiniband on Centos 6.3

  1. Thank you, Koaps. I was strugging in setting up my DDN storage system via SRP. Your post saved my day. This approach works magically well in Cent OS 7. Finally I can get rid of the tedious hard-to-download DDN software.

    A tip may be helpful to other guys is that, if `multipath -ll` displays nothing, reboot the server (initiator here) and tru `mutlipath -ll` again.

    Posted by weijianwen | January 1, 2017, 6:46 am

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s