查询osd上的pg数

本文中的命令的第一版来源于国外的一个博客，后面的版本为我自己修改的版本

查询的命令如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
ceph pg dump | awk '
 /^pg_stat/ { col=1; while($col!="up") {col++}; col++ }
 /^[0-9a-f]+\.[0-9a-f]+/ { match($0,/^[0-9a-f]+/); pool=substr($0, RSTART, RLENGTH); poollist[pool]=0;
 up=$col; i=0; RSTART=0; RLENGTH=0; delete osds; while(match(up,/[0-9]+/)>0) { osds[++i]=substr(up,RSTART,RLENGTH); up = substr(up, RSTART+RLENGTH) }
 for(i in osds) {array[osds[i],pool]++; osdlist[osds[i]];}
}
END {
 printf("\n");
 printf("pool :\t"); for (i in poollist) printf("%s\t",i); printf("| SUM \n");
 for (i in poollist) printf("--------"); printf("----------------\n");
 for (i in osdlist) { printf("osd.%i\t", i); sum=0;
 for (j in poollist) { printf("%i\t", array[i,j]); sum+=array[i,j]; poollist[j]+=array[i,j] }; printf("| %i\n",sum) }
 for (i in poollist) printf("--------"); printf("----------------\n");
 printf("SUM :\t"); for (i in poollist) printf("%s\t",poollist[i]); printf("|\n");
}'

默认的输出如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

pool :	0	1	2	| SUM 
----------------------------------------
osd.4	54	133	79	| 266
osd.5	57	104	88	| 249
osd.6	61	132	86	| 279
osd.7	54	114	85	| 253
osd.8	63	123	85	| 271
osd.0	62	120	87	| 269
osd.1	52	126	81	| 259
osd.2	52	103	88	| 243
osd.3	57	125	89	| 271
----------------------------------------
SUM :	512	1080	768	|

这个有个问题就是osd是乱序的，并且对于一个存储池来说不清楚哪个osd的pg是最多的

重构第一版：

跟上面的相比按顺序来排列

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
ceph pg dump | awk '
 /^pg_stat/ { col=1; while($col!="up") {col++}; col++ }
 /^[0-9a-f]+\.[0-9a-f]+/ { match($0,/^[0-9a-f]+/); pool=substr($0, RSTART, RLENGTH); poollist[pool]=0;
 up=$col; i=0; RSTART=0; RLENGTH=0; delete osds; while(match(up,/[0-9]+/)>0) { osds[++i]=substr(up,RSTART,RLENGTH); up = substr(up, RSTART+RLENGTH) }
 for(i in osds) {array[osds[i],pool]++; osdlist[osds[i]];}
}
END {
 printf("\n");
 slen=asorti(poollist,newpoollist);
 printf("pool :\t");for (i=1;i<=slen;i++) {printf("%s\t", newpoollist[i])}; printf("| SUM \n");
 for (i in poollist) printf("--------"); printf("----------------\n");
 slen1=asorti(osdlist,newosdlist)
 delete poollist;
 for (i=1;i<=slen1;i++) { printf("osd.%i\t", newosdlist[i]); sum=0; 
 for (j=1;j<=slen;j++)  { printf("%i\t", array[newosdlist[i],newpoollist[j]]); sum+=array[newosdlist[i],newpoollist[j]]; poollist[j]+=array[newosdlist[i],newpoollist[j]] }; printf("| %i\n",sum)
} 
for (i in poollist) printf("--------"); printf("----------------\n");
 printf("SUM :\t"); for (i=1;i<=slen;i++) printf("%s\t",poollist[i]); printf("|\n");
}'

输出结果为下面的，可以看到现在是按顺序来的，存储池是顺序的，osd编号也是顺序的

1
2
3
4
5
6
7
8
9
10
11
12
13
14

pool :	0	1	2	| SUM 
----------------------------------------
osd.0	62	120	87	| 269
osd.1	52	126	81	| 259
osd.2	52	103	88	| 243
osd.3	57	125	89	| 271
osd.4	54	133	79	| 266
osd.5	57	104	88	| 249
osd.6	61	132	86	| 279
osd.7	54	114	85	| 253
osd.8	63	123	85	| 271
----------------------------------------
SUM :	512	1080	768	|

重构第二版：

包含osd pool的排序，包含osd的排序，输出平均pg数目，输出最大的osd编号，输出超过平均值的百分比

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
ceph pg dump | awk '
 /^pg_stat/ { col=1; while($col!="up") {col++}; col++ }
 /^[0-9a-f]+\.[0-9a-f]+/ { match($0,/^[0-9a-f]+/); pool=substr($0, RSTART, RLENGTH); poollist[pool]=0;
 up=$col; i=0; RSTART=0; RLENGTH=0; delete osds; while(match(up,/[0-9]+/)>0) { osds[++i]=substr(up,RSTART,RLENGTH); up = substr(up, RSTART+RLENGTH) }
 for(i in osds) {array[osds[i],pool]++; osdlist[osds[i]];}
}
END {
 printf("\n");
 slen=asorti(poollist,newpoollist);
 printf("pool :\t");for (i=1;i<=slen;i++) {printf("%s\t", newpoollist[i])}; printf("| SUM \n");
 for (i in poollist) printf("--------"); printf("----------------\n");
 slen1=asorti(osdlist,newosdlist)
 delete poollist;
 for (i=1;i<=slen1;i++) { printf("osd.%i\t", newosdlist[i]); sum=0; 
 for (j=1;j<=slen;j++)  { printf("%i\t", array[newosdlist[i],newpoollist[j]]); sum+=array[newosdlist[i],newpoollist[j]]; poollist[j]+=array[newosdlist[i],newpoollist[j]];if(array[newosdlist[i],newpoollist[j]] != 0){poolhasid[j]+=1 };if(array[newosdlist[i],newpoollist[j]]>maxpoolosd[j]){maxpoolosd[j]=array[newosdlist[i],newpoollist[j]];maxosdid[j]=newosdlist[i]}}; printf("| %i\n",sum)} 
for (i in poollist) printf("--------"); printf("----------------\n");
 printf("SUM :\t"); for (i=1;i<=slen;i++) printf("%s\t",poollist[i]); printf("|\n");
 printf("AVE :\t"); for (i=1;i<=slen;i++) printf("%d\t",poollist[i]/poolhasid[i]); printf("|\n");
 printf("max :\t"); for (i=1;i<=slen;i++) printf("%s\t",maxpoolosd[i]); printf("|\n");
 printf("osdid :\t"); for (i=1;i<=slen;i++) printf("osd.%s\t",maxosdid[i]); printf("|\n");
 printf("per:\t"); for (i=1;i<=slen;i++) printf("%.1f%\t",100*(maxpoolosd[i]-poollist[i]/poolhasid[i])/(poollist[i]/poolhasid[i])); printf("|\n");
}'

输出如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
pool :	0	1	2	| SUM 
----------------------------------------
osd.0	62	120	87	| 269
osd.1	52	126	81	| 259
osd.2	52	103	88	| 243
osd.3	57	125	89	| 271
osd.4	54	133	79	| 266
osd.5	57	104	88	| 249
osd.6	61	132	86	| 279
osd.7	54	114	85	| 253
osd.8	63	123	85	| 271
----------------------------------------
SUM :	512	1080	768	|
AVE :	56	120	85	|
max :	63	133	89	|
osdid :	osd.8	osd.4	osd.3	|
per:	10.7%	10.8%	4.3%	|

重构第三版：

包含osd pool的排序，包含osd的排序，输出平均pg数目，输出最大的osd编号，输出最大超过平均值的百分比，输出最少pg的osd编号，输出最小低于平均值的百分比

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
ceph pg dump | awk '
 /^pg_stat/ { col=1; while($col!="up") {col++}; col++ }
 /^[0-9a-f]+\.[0-9a-f]+/ { match($0,/^[0-9a-f]+/); pool=substr($0, RSTART, RLENGTH); poollist[pool]=0;
 up=$col; i=0; RSTART=0; RLENGTH=0; delete osds; while(match(up,/[0-9]+/)>0) { osds[++i]=substr(up,RSTART,RLENGTH); up = substr(up, RSTART+RLENGTH) }
 for(i in osds) {array[osds[i],pool]++; osdlist[osds[i]];}
}
END {
 printf("\n");
 slen=asorti(poollist,newpoollist);
 printf("pool :\t");for (i=1;i<=slen;i++) {printf("%s\t", newpoollist[i])}; printf("| SUM \n");
 for (i in poollist) printf("--------"); printf("----------------\n");
 slen1=asorti(osdlist,newosdlist)
 delete poollist;
 for (j=1;j<=slen;j++) {maxpoolosd[j]=0};
 for (j=1;j<=slen;j++) {for (i=1;i<=slen1;i++){if (array[newosdlist[i],newpoollist[j]] >0  ){minpoolosd[j]=array[newosdlist[i],newpoollist[j]] ;break } }};	
 for (i=1;i<=slen1;i++) { printf("osd.%i\t", newosdlist[i]); sum=0; 
 for (j=1;j<=slen;j++)  { printf("%i\t", array[newosdlist[i],newpoollist[j]]); sum+=array[newosdlist[i],newpoollist[j]]; poollist[j]+=array[newosdlist[i],newpoollist[j]];if(array[newosdlist[i],newpoollist[j]] != 0){poolhasid[j]+=1 };if(array[newosdlist[i],newpoollist[j]]>maxpoolosd[j]){maxpoolosd[j]=array[newosdlist[i],newpoollist[j]];maxosdid[j]=newosdlist[i]};if(array[newosdlist[i],newpoollist[j]] != 0){if(array[newosdlist[i],newpoollist[j]]<=minpoolosd[j]){minpoolosd[j]=array[newosdlist[i],newpoollist[j]];minosdid[j]=newosdlist[i]}}}; printf("| %i\n",sum)} for (i in poollist) printf("--------"); printf("----------------\n");
 slen2=asorti(poollist,newpoollist);
 printf("SUM :\t"); for (i=1;i<=slen;i++) printf("%s\t",poollist[i]); printf("|\n");
 printf("Osd :\t"); for (i=1;i<=slen;i++) printf("%s\t",poolhasid[i]); printf("|\n");
 printf("AVE :\t"); for (i=1;i<=slen;i++) printf("%.2f\t",poollist[i]/poolhasid[i]); printf("|\n");
 printf("Max :\t"); for (i=1;i<=slen;i++) printf("%s\t",maxpoolosd[i]); printf("|\n");
 printf("Osdid :\t"); for (i=1;i<=slen;i++) printf("osd.%s\t",maxosdid[i]); printf("|\n");
 printf("per:\t"); for (i=1;i<=slen;i++) printf("%.1f%\t",100*(maxpoolosd[i]-poollist[i]/poolhasid[i])/(poollist[i]/poolhasid[i])); printf("|\n");
 for (i=1;i<=slen2;i++) printf("--------");printf("----------------\n");
 printf("min :\t"); for (i=1;i<=slen;i++) printf("%s\t",minpoolosd[i]); printf("|\n");
 printf("osdid :\t"); for (i=1;i<=slen;i++) printf("osd.%s\t",minosdid[i]); printf("|\n");
 printf("per:\t"); for (i=1;i<=slen;i++) printf("%.1f%\t",100*(minpoolosd[i]-poollist[i]/poolhasid[i])/(poollist[i]/poolhasid[i])); printf("|\n");
}'

输出如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
dumped all in format plain

pool :	0	1	2	| SUM 
----------------------------------------
osd.0	206	206	53	| 465
osd.1	22	19	5	| 46
osd.2	202	196	49	| 447
osd.3	19	25	6	| 50
osd.4	29	35	9	| 73
osd.5	34	31	6	| 71
----------------------------------------
SUM :	512	512	128	|
AVE :	85	85	21	|
max :	206	206	53	|
osdid :	osd.0	osd.0	osd.0	|
per:	141.4%	141.4%	148.4%	|
---------------------------------------
min :	19	19	5	|
osdid :	osd.3	osd.1	osd.1	|
per:	-77.7%	-77.7%	-76.6%	|

Luminous以及之后的版本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
ceph pg dump | awk '
 /^PG_STAT/ { col=1; while($col!="UP") {col++}; col++ }
 /^[0-9a-f]+\.[0-9a-f]+/ { match($0,/^[0-9a-f]+/); pool=substr($0, RSTART, RLENGTH); poollist[pool]=0;
 up=$col; i=0; RSTART=0; RLENGTH=0; delete osds; while(match(up,/[0-9]+/)>0) { osds[++i]=substr(up,RSTART,RLENGTH); up = substr(up, RSTART+RLENGTH) }
 for(i in osds) {array[osds[i],pool]++; osdlist[osds[i]];}
}
END {
 printf("\n");
 slen=asorti(poollist,newpoollist);
 printf("pool :\t");for (i=1;i<=slen;i++) {printf("%s\t", newpoollist[i])}; printf("| SUM \n");
 for (i in poollist) printf("--------"); printf("----------------\n");
 slen1=asorti(osdlist,newosdlist)
 delete poollist;
 for (j=1;j<=slen;j++) {maxpoolosd[j]=0};
 for (j=1;j<=slen;j++) {for (i=1;i<=slen1;i++){if (array[newosdlist[i],newpoollist[j]] >0  ){minpoolosd[j]=array[newosdlist[i],newpoollist[j]] ;break } }}; 
 for (i=1;i<=slen1;i++) { printf("osd.%i\t", newosdlist[i]); sum=0; 
 for (j=1;j<=slen;j++)  { printf("%i\t", array[newosdlist[i],newpoollist[j]]); sum+=array[newosdlist[i],newpoollist[j]]; poollist[j]+=array[newosdlist[i],newpoollist[j]];if(array[newosdlist[i],newpoollist[j]] != 0){poolhasid[j]+=1 };if(array[newosdlist[i],newpoollist[j]]>maxpoolosd[j]){maxpoolosd[j]=array[newosdlist[i],newpoollist[j]];maxosdid[j]=newosdlist[i]};if(array[newosdlist[i],newpoollist[j]] != 0){if(array[newosdlist[i],newpoollist[j]]<=minpoolosd[j]){minpoolosd[j]=array[newosdlist[i],newpoollist[j]];minosdid[j]=newosdlist[i]}}}; printf("| %i\n",sum)} for (i in poollist) printf("--------"); printf("----------------\n");
 slen2=asorti(poollist,newpoollist);
 printf("SUM :\t"); for (i=1;i<=slen;i++) printf("%s\t",poollist[i]); printf("|\n");
 printf("Osd :\t"); for (i=1;i<=slen;i++) printf("%s\t",poolhasid[i]); printf("|\n");
 printf("AVE :\t"); for (i=1;i<=slen;i++) printf("%.2f\t",poollist[i]/poolhasid[i]); printf("|\n");
 printf("Max :\t"); for (i=1;i<=slen;i++) printf("%s\t",maxpoolosd[i]); printf("|\n");
 printf("Osdid :\t"); for (i=1;i<=slen;i++) printf("osd.%s\t",maxosdid[i]); printf("|\n");
 printf("per:\t"); for (i=1;i<=slen;i++) printf("%.1f%\t",100*(maxpoolosd[i]-poollist[i]/poolhasid[i])/(poollist[i]/poolhasid[i])); printf("|\n");
 for (i=1;i<=slen2;i++) printf("--------");printf("----------------\n");
 printf("min :\t"); for (i=1;i<=slen;i++) printf("%s\t",minpoolosd[i]); printf("|\n");
 printf("osdid :\t"); for (i=1;i<=slen;i++) printf("osd.%s\t",minosdid[i]); printf("|\n");
 printf("per:\t"); for (i=1;i<=slen;i++) printf("%.1f%\t",100*(minpoolosd[i]-poollist[i]/poolhasid[i])/(poollist[i]/poolhasid[i])); printf("|\n");
}'

luminous之后的版本json的输出小写改成大写了，需要替换几个字段，上面的已经增加了

上面的处理使用的是awk处理，开始的时候看不懂什么意思，然后就去看了这本书《The AWK Programming Language》

语法的解释

/^pg_stat/ { col=1; while($col!=”up”) {col++}; col++ }

这个是匹配pg dump 的输出结果里面pg_stat那个字段，开始计数为1，不是up值就将col的值加1，这个匹配到的就是我们经常看到的[1,10]这个值最后的col++是将col值+1,因为字段里面有up,up_primary,我们需要的是up_primary

/^[0-9a-f]+.[0-9a-f]+/ { match($0,/^[0-9a-f]+/); pool=substr($0, RSTART, RLENGTH); poollist[pool]=0;

这个是匹配前面的 1.17a pg号 ，使用自带的match函数 做字符串的过滤统计匹配.号前面的存储池ID， 并得到 RSTART, RLENGTH 值，这个是取到前面的存储池ID，使用substr 函数，就可以得到pool的值了，poollist[pool]=0，是将数组的值置为0

up=$col; i=0; RSTART=0; RLENGTH=0; delete osds; while(match(up,/[0-9]+/)>0) { osds[++i]=substr(up,RSTART,RLENGTH); up = substr(up, RSTART+RLENGTH) }

先将变量置0，然后将osd编号一个个输入到osds[i]的数组当中去

for(i in osds) {array[osds[i],pool]++; osdlist[osds[i]];}

将osds数组中的值输入到数组当中去，并且记录成osdlist，和数组array[osd[i],pool]

printf(“\n”);
printf(“pool :\t”); for (i in poollist) printf(“%s\t”,i); printf(“| SUM \n”);

打印osd pool的编号

for (i in poollist) printf(“——–”); printf(“—————-\n”);

根据osd pool的长度打印—-

for (i in osdlist) { printf(“osd.%i\t”, i); sum=0;

打印osd的编号

for (j in poollist) { printf(“%i\t”, array[i,j]); sum+=array[i,j]; poollist[j]+=array[i,j] }; printf(“| %i\n”,sum) }
打印对应的osd的pg数目，并做求和的统计

for (i in poollist) printf(“——–”); printf(“—————-\n”);
printf(“SUM :\t”); for (i in poollist) printf(“%s\t”,poollist[i]); printf(“|\n”);

打印新的poollist里面的求和的值

修改版本里面用到的函数

slen1=asorti(osdlist,newosdlist)

这个是将数组里面的下标进行排序，这里是对osd和poollist的编号进行排序 slen1是拿到数组的长度，使用for进行遍历输出

脚本的逻辑

  • 匹配到pg的id和pg对应的osd，
  • 使用数组的方式，将统计到的osd id存储起来，
  • 然后打印数组

其他资源：
pg设置的计算器：
http://ceph.com/pgcalc/
pg的查询和设置：
http://ceph.com/docs/master/rados/operations/placement-groups/

变更记录

Why Who When
创建 武汉-运维-磨渣 2015-10-04
修改 武汉-运维-磨渣 2016-08-24
修改有0的统计BUG 武汉-运维-磨渣 2016-09-08
增加luminous版本脚本 武汉-运维-磨渣 2020-06-17

引用博客地址如下：

http://cephnotes.ksperis.com/blog/2015/02/23/get-the-number-of-placement-groups-per-osd/