RAC_多路径配置
多路径配置
http://blog.itpub.net/31397003/viewspace-2143390/
- 挂盘/配置好yum源
2.程序包的安装
device-mapper-1.02.95-2.el6.x86_64.rpm
device-mapper-event-1.02.95-2.el6.x86_64.rpm
device-mapper-event-libs-1.02.95-2.el6.x86_64.rpm
device-mapper-libs-1.02.95-2.el6.x86_64.rpm
device-mapper-multipath-0.4.9-87.el6.x86_64.rpm
device-mapper-multipath-libs-0.4.9-87.el6.x86_64.rpm
device-mapper-persistent-data-0.3.2-1.el6.x86_64.rpm
yum install -y device-mapper*
3.启动multipath，并修改为自启动。
/etc/init.d/multipathd start，执行这个命令，启动multipath。
chkconfig --level 345 multipathd on，执行这个命令修为自启动。
4.配置/etc/multipath.conf文件。
该文件的模板，我们可以在/usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf这里找到，然后把这个文件复制到/etc目录下，可以执行如下命令，
cp /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf /etc/multipath.conf
然后编辑/etc/mulitpath.conf文件，进行如下的修改:
[root@dbrac1 etc]# vi multipath.conf
# This is a basic configuration file with some examples, for device mapper
# multipath.
# For a complete list of the default configuration values, see
# /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf.defaults
# For a list of configuration options with descriptions, see
# /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf.annotated
#
# REMEMBER: After updating multipath.conf, you must run
#
# service multipathd reload
#
# for the changes to take effect in multipathd
## By default, devices with vendor = "IBM" and product = "S/390.*" are
## blacklisted. To enable mulitpathing on these devies, uncomment the
## following lines.
#blacklist_exceptions {
# device {
# vendor "IBM"
# product "S/390.*"
# }
#}
## Use user friendly names, instead of using WWIDs as names.
#defaults {
# user_friendly_names yes
#}
##
## Here is an example of how to configure some standard options.
##
#
defaults {
udev_dir /dev
polling_interval 10
path_selector "round-robin 0"
path_grouping_policy multibus
getuid_callout "/lib/udev/scsi_id --whitelisted --device=/dev/%n"
prio alua
path_checker readsector0
rr_min_io 100
max_fds 8192
rr_weight priorities
failback immediate
no_path_retry fail
user_friendly_names yes
}
##
## The wwid line in the following blacklist section is shown as an example
## of how to blacklist devices by wwid. The 2 devnode lines are the
## compiled in default blacklist. If you want to blacklist entire types
## of devices, such as all scsi devices, you should use a devnode line.
## However, if you want to blacklist specific devices, you should use
## a wwid line. Since there is no guarantee that a specific device will
## not change names on reboot (from /dev/sda to /dev/sdb for example)
## devnode lines are not recommended for blacklisting specific devices.
##
blacklist {
wwid 26353900f02796769
devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
devnode "^hd[a-z]"
}
#multipaths {
# multipath {
# wwid 3600508b4000156d700012000000b0000
# alias yellow
# path_grouping_policy multibus
# path_checker readsector0
# path_selector "round-robin 0"
# failback manual
# rr_weight priorities
# no_path_retry 5
# }
# multipath {
# wwid 1DEC_____321816758474
# alias red
# }
#}
#devices {
# device {
# vendor "COMPAQ "
# product "HSV110 (C)COMPAQ"
# path_grouping_policy multibus
# getuid_callout "/lib/udev/scsi_id --whitelisted --device=/dev/%n"
# path_checker readsector0
# path_selector "round-robin 0"
# hardware_handler "0"
# failback 15
# rr_weight priorities
# no_path_retry queue
# }
# device {
# vendor "COMPAQ "
# product "MSA1000 "
# path_grouping_policy multibus
# }
#}
"multipath.conf" 96L, 2722C written
其中，blacklist用来添加设备的黑名单，也就是说，设置某些设备，不让它多路径化。比如，我们可以把某些设备的wwid加入在这里
5.执行如下命令
modprobe dm-multipath
multipath -F
multipath dm-multipath
multipath dm-round-robin
/etc/init.d/multipathd restart
执行完成上述命令之后，会在/etc/multipath目录下生成一个bindings这的一个文件，如下:
[root@dbrac1 etc]# modprobe dm-multipath
WARNING: Deprecated config file /etc/modprobe.conf, all config files belong into /etc/modprobe.d/.
[root@dbrac1 etc]# multipath -F
[root@dbrac1 etc]# multipath dm-multipath
Aug 10 22:39:40 | sda: alua not supported
Aug 10 22:39:40 | sdb: alua not supported
Aug 10 22:39:40 | sdc: alua not supported
Aug 10 22:39:40 | sdd: alua not supported
Aug 10 22:39:40 | sde: alua not supported
Aug 10 22:39:40 | sdf: alua not supported
Aug 10 22:39:40 | sdg: alua not supported
Aug 10 22:39:40 | sdh: alua not supported
[root@dbrac1 etc]#
[root@dbrac1 etc]# multipath dm-round-robin
Aug 10 22:39:49 | sda: alua not supported
Aug 10 22:39:49 | sdb: alua not supported
Aug 10 22:39:49 | sdc: alua not supported
Aug 10 22:39:49 | sdd: alua not supported
Aug 10 22:39:49 | sde: alua not supported
Aug 10 22:39:49 | sdf: alua not supported
Aug 10 22:39:49 | sdg: alua not supported
Aug 10 22:39:49 | sdh: alua not supported
[root@dbrac1 etc]# /etc/init.d/multipathd restart
ok
Stopping multipathd daemon: [ OK ]
Starting multipathd daemon: [ OK ]
[root@dbrac1 etc]#
[root@dbrac1 etc]# cd /etc/multipath
[root@dbrac1 multipath]# ls
bindings
[root@dbrac1 multipath]# cat bindings
# Multipath bindings, Version : 1.0
# NOTE: this file is automatically maintained by the multipath program.
# You should not need to edit this file in normal circumstances.
#
# Format:
# alias wwid
#
mpatha 1ATA VBOX HARDDISK VBebc3bf35-371818d2
mpathb 1ATA VBOX HARDDISK VB403dd68a-b8a5a8be
mpathc 1ATA VBOX HARDDISK VBcb1b0d8b-1ee73ac2
mpathd 1ATA VBOX HARDDISK VB5eb6b908-65677cda
mpathe 1ATA VBOX HARDDISK VBb020940b-4a482e15
mpathf 1ATA VBOX HARDDISK VBd60d065d-37d1710d
mpathg 1ATA VBOX HARDDISK VBf71eae83-8c806c3b
mpathh 1ATA VBOX HARDDISK VBd46e23c6-7ece6a6e
[root@dbrac1 multipath]#
这里存放的是多路径的磁盘名称和对应的ID。这时，我们可以使用fdisk -l这个命令查看到这些设备，比如/dev/mapper/mpatha。
6.再次修改 vi /etc/multipath.conf 文件，对应ID对应设备的别名。
由于上述的设备名称的可读性不是很好，所以我们要加入方便我们快速读的名称。修改/etc/multipath.conf文件，内容如下:
multipaths {
multipath {
wwid VB403dd68a-b8a5a8be
alias ocrvotedisk1
path_grouping_policy multibus
}
multipath {
wwid VBcb1b0d8b-1ee73ac2
alias ocrvotedisk2
path_grouping_policy multibus
}
multipath {
wwid VB5eb6b908-65677cda
alias ocrvotedisk3
path_grouping_policy multibus
}
multipath {
wwid VBb020940b-4a482e15
alias fra1
path_grouping_policy multibus
}
multipath {
wwid VBd60d065d-37d1710d
alias fra2
path_grouping_policy multibus
}
multipath {
wwid VBf71eae83-8c806c3b
alias data1
path_grouping_policy multibus
}
multipath {
wwid VBd46e23c6-7ece6a6e
alias data2
path_grouping_policy multibus
}
}
[root@dbrac1 multipath]# cat /etc/multipath.conf
# This is a basic configuration file with some examples, for device mapper
# multipath.
# For a complete list of the default configuration values, see
# /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf.defaults
# For a list of configuration options with descriptions, see
# /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf.annotated
#
# REMEMBER: After updating multipath.conf, you must run
#
# service multipathd reload
#
# for the changes to take effect in multipathd
## By default, devices with vendor = "IBM" and product = "S/390.*" are
## blacklisted. To enable mulitpathing on these devies, uncomment the
## following lines.
#blacklist_exceptions {
# device {
# vendor "IBM"
# product "S/390.*"
# }
#}
## Use user friendly names, instead of using WWIDs as names.
#defaults {
# user_friendly_names yes
#}
##
## Here is an example of how to configure some standard options.
##
#
defaults {
udev_dir /dev
polling_interval 10
path_selector "round-robin 0"
path_grouping_policy multibus
getuid_callout "/lib/udev/scsi_id --whitelisted --device=/dev/%n"
prio alua
path_checker readsector0
rr_min_io 100
max_fds 8192
rr_weight priorities
failback immediate
no_path_retry fail
user_friendly_names yes
}
##
## The wwid line in the following blacklist section is shown as an example
## of how to blacklist devices by wwid. The 2 devnode lines are the
## compiled in default blacklist. If you want to blacklist entire types
## of devices, such as all scsi devices, you should use a devnode line.
## However, if you want to blacklist specific devices, you should use
## a wwid line. Since there is no guarantee that a specific device will
## not change names on reboot (from /dev/sda to /dev/sdb for example)
## devnode lines are not recommended for blacklisting specific devices.
##
blacklist {
wwid 26353900f02796769
devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
devnode "^hd[a-z]"
}
multipaths {
multipath {
wwid VB403dd68a-b8a5a8be
alias ocrvotedisk1
path_grouping_policy multibus
}
multipath {
wwid VBcb1b0d8b-1ee73ac2
alias ocrvotedisk2
path_grouping_policy multibus
}
multipath {
wwid VB5eb6b908-65677cda
alias ocrvotedisk3
path_grouping_policy multibus
}
multipath {
wwid VBb020940b-4a482e15
alias fra1
path_grouping_policy multibus
}
multipath {
wwid VBd60d065d-37d1710d
alias fra2
path_grouping_policy multibus
}
multipath {
wwid VBf71eae83-8c806c3b
alias data1
path_grouping_policy multibus
}
multipath {
wwid VBd46e23c6-7ece6a6e
alias data2
path_grouping_policy multibus
}
}
#multipaths {
# multipath {
# wwid 3600508b4000156d700012000000b0000
# alias yellow
# path_grouping_policy multibus
# path_checker readsector0
# path_selector "round-robin 0"
# failback manual
# rr_weight priorities
# no_path_retry 5
# }
# multipath {
# wwid 1DEC_____321816758474
# alias red
# }
#}
#devices {
# device {
# vendor "COMPAQ "
# product "HSV110 (C)COMPAQ"
# path_grouping_policy multibus
# getuid_callout "/lib/udev/scsi_id --whitelisted --device=/dev/%n"
# path_checker readsector0
# path_selector "round-robin 0"
# hardware_handler "0"
# failback 15
# rr_weight priorities
# no_path_retry queue
# }
# device {
# vendor "COMPAQ "
# product "MSA1000 "
# path_grouping_policy multibus
# }
#}
[root@dbrac1 multipath]#
修改完成以后，重启机器即可（注意，在FC存储环境下不需要重新启动机器，直接执行service multipathd reload命令即可）。
7.重启完成以后查看对应的设备。
重启完成以后，使用fdisk -l命令再次查看，会看到如下的形式的设备。
fdisk -l
8.完成多路径的配置以后，我们还不能直接安装grid，因为多路径对应的设备的权限需要更改。这个时候，我们还需要借助udev来完成
下面，我们来修改多路径的权限,对于多路径操作的有两个位置，/dev/mapper目录对应的文件与/dev/dm-*开头对应的文件。我们只需要修改/dev/dm-*应对的权限就可以了.
第一种方法：我们需要用root修改 disk为grid asmadmin。直接修改，grid还是无法识别，所以，可以通过udev来完成。
在/etc/udev/rules.d目录下，加入如下的文件与内容，
[root@dbrac1 rules.d]# cat 99-asm.rules
KERNEL=="dm-[0-6]", OWNER="grid", GROUP="asmadmin", MODE="0660"
[root@dbrac1 rules.d]#
第二种方法：编辑另外一个udev的配置文件，为12-.rules这样的一个udev的配置文件，比如文件名为12-mulitpath-privs.rules，其内容如下：
[root@dbrac1 rules.d]# vi /etc/udev/rules.d/12-mulitpath-privs.rules
ENV{DM_NAME}=="ocr*", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="fra*", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="data*", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
第三种方法：以上是使用name来进行的，也可以使用uuid，比如如下：
[root@dbrac1 rules.d]# cat 12-mulitpath-privs.rules
ENV{DM_UUID}=="mpath-?*", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
................................
[root@dbrac1 rules.d]#
每个磁盘的uuid可以在/dev/disk/by-id目录下找到，用ls -l命令就可以详细看到。
该udev的配置文件，可以在/usr/share/doc/device-mapper-/12-dm-permissions.rules中找到，这里有详细的模板。
推荐使用第二种方法来修改多路径设备的权限。
至此多路径配置完成了！
----------补充------------------------
#查看是否安装了multipath
rpm -qa | grep device-mapper
#查看本地存储wwid
/lib/udev/scsi_id --whitelisted --device=/dev/sda
36141877030bcca001d9c4d52106b9d90
#查看存储参数
cat /sys/block/sdb/device/vendor
3PARdata
cat /sys/block/sdb/device/model
VV
#清空已有的multipath记录
multipath -F
#打印诊断信息
multipath -v3
#启用多路径守护程序以在引导时启动
mpathconf --enable
#启动多路径服务
service multipathd start
#如果在启动 multipath 守护程序后更改多路径配置文件，请运行以下命令以使更改生效。
service multipathd reload
systemctl reload multipath
#重启系统测试
init 6
#查看多路径当前状态
multipath -ll
ls -l /dev/dm-*
cd /dev/mapper
ll
QQ: 774269490 【欢迎加我讨论，问题处理，在线解疑~~】
微信：17372230632 【Oracle故障处理，数据恢复~~】
