测试主机的整机磁盘带宽

前言

为什么需要有整机带宽的概念，因为我们很多时候去计算性能的时候，上去都会计算你有多少块盘，每块盘有多大的带宽，那么一乘就算出来了，其实没有那么简单，整个IO路径上面任何一个地方的瓶颈都会降低总体带宽的

本篇的来源，很多年前，做机器测试的时候，总觉得机器上面的带宽不对，加起来不对，然后搜资料看到了，一个盘一个盘测试的时候带宽没问题，但是整机的磁盘IO同时下发的时候，阵列卡，背板都有可能带不动

而这个整机测试就能发现问题，最后定位到是背板有问题，刷行一下固件，总性能提升了300MB/s

测试方法

测试很简单，就算模拟所有磁盘的同时写，而fio工具是最好的测试工具

配置文件如下

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
[global]
bs=4M
iodepth=16
direct=1
ioengine=libaio
randrepeat=0
group_reporting
time_based
runtime=60
filesize=20G

[job1]
rw=write
filename=/dev/sdb:/dev/sdc:/dev/sdd:/dev/sde:/dev/sdf:/dev/sdg:/dev/sdh:/dev/sdi
name=write

上面测试的是8个磁盘的并行写入的带宽

vdbench测试整机带宽

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
hd=default,vdbench=/root/vdbench,user=root,shell=ssh
hd=hd1,system=66.66.66.60
sd=sd1,lun=/dev/sdl,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd2,lun=/dev/sdc,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd3,lun=/dev/sdd,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd4,lun=/dev/sde,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd5,lun=/dev/sdf,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd6,lun=/dev/sdg,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd7,lun=/dev/sdh,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd8,lun=/dev/sdi,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd9,lun=/dev/sdj,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd10,lun=/dev/sdk,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd11,lun=/dev/sdl,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd12,lun=/dev/sdm,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd13,lun=/dev/sdn,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd14,lun=/dev/sdo,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd15,lun=/dev/sdp,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd16,lun=/dev/sdq,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd17,lun=/dev/sdr,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd18,lun=/dev/sds,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd19,lun=/dev/sdt,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd20,lun=/dev/sdu,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd21,lun=/dev/sdv,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd22,lun=/dev/sdw,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd23,lun=/dev/sdx,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd24,lun=/dev/sdy,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd25,lun=/dev/sdz,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd26,lun=/dev/sdaa,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd27,lun=/dev/sdab,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd28,lun=/dev/sdac,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd29,lun=/dev/sdad,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd30,lun=/dev/sdae,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd31,lun=/dev/sdaf,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd32,lun=/dev/sdag,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd33,lun=/dev/sdah,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd34,lun=/dev/sdai,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd35,lun=/dev/sdaj,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
sd=sd36,lun=/dev/sdak,host=hd1,openflags=o_direct,hitarea=0,range=(0,100),threads=2
wd=wd1,sd=(sd1,sd2,sd3,sd4,sd5,sd6,sd7,sd8,sd9,sd10,sd11,sd12,sd13,sd14,sd15,sd16,sd17,sd18,sd19,sd20,sd21,sd22,sd23,sd24,sd25,sd26,sd27,sd28,sd29,sd30,sd31,sd32,sd33,sd34,sd35,sd36),xfersize=(1048576,100),rdpct=0,seekpct=0
rd=run1,wd=wd1,iorate=max,elapsed=600,warmup=300

文件系统主机带宽

下面的是测试文件系统的带宽

1
2
3
4
5
6
7
[root@lab101 vdbench]# cat file.conf
hd=default,vdbench=/root/vdbench,user=root,shell=ssh
hd=hd1,system=192.168.19.101
fsd=fsd1,anchor=/a/,depth=2,width=2,files=10000,size=64k
fsd=fsd2,anchor=/b/,depth=2,width=2,files=10000,size=64k
fwd=fwd1,fsd=(fsd1,fsd2),operation=write,xfersize=64k,fileio=sequential,fileselect=random,threads=2
rd=rd1,fwd=fwd1,fwdrate=max,format=yes,elapsed=60,interval=1