一个KADB测试实践
- 测试结果
本文档描述xxxx测试中6个典型测试场景的测试结果及背景，旨在对不同数据量，不同存储方式，不同优化器三者的组合优化进行探索，进而为未来的类似测试提供组合优化参考。
| 数据插入（500万） | 5进程批量插入（500万） | 单表增量与数据合并去重（1000万） | 简单查询（3亿数据） | 综合统计（3亿数据） | 排序过滤（3亿数据） | |
| xxx | 18s | 25s | 28s | 0.049s | 41s | 144s |
| 金仓 | 31.808s | 36.998s | 19.888s | 0.463s | 8.496s | 37.311s |
经测试发现在目前的测试数据量和计算实例部署的情况下，大多数场景使用pg优化器+heap表+索引的方式得到比AO表+9级zlib压缩+GPORA的组合更好的测试结果。
在简单查询模式测试（三个in查询子嵌套）的测试场景，使用heap表+索引+GPORCA得到比pg优化器更好的测试结果
思考：选用heap或者ao表和数据量，部署的计算实例及查询语句条件相关，目前没有探索到一个选择定量标准，需要在更多的测试环境中验证
- 测试背景
- 测试数据量
tb_mz_sfmxb_source_r：25401708，行存
tb_mz_sfmxb_p_r：304820496，行存按字段sfjssj月分区表
tb_mz_ghmxb:24000000，行存
tb_dic_wd_yljgzd：2000w，行存，复制分布
tb_dic_wd_wsxzjgzdb：2000w，行存，复制分布
- 测试环境
cpu：arm 16c
内存：128GB
磁盘：1T SSD
三个虚拟机节点，每个节点部署12个计算实例
- 资源测试结果
gpcheckperf -f /home/mppadmin/hostfile -r N -D -d /dbdata
====================
== RESULT 2025-02-22T16:21:24.741047
====================
Netperf bisection bandwidth test
mdw -> sdw1 = 984.920000
sdw2 -> mdw = 894.770000
sdw1 -> mdw = 851.700000
mdw -> sdw2 = 985.110000
Summary:
sum = 3716.50 MB/sec
min = 851.70 MB/sec
max = 985.11 MB/sec
avg = 929.13 MB/sec
median = 984.92 MB/sec
gpcheckperf -f /home/mppadmin/hostfile -d /dbdata -r ds -D -v -S 256GB
====================
== RESULT 2025-02-22T16:30:31.592169
====================
disk write avg time (sec): 152.18
disk write tot bytes: 824633720832
disk write tot bandwidth (MB/s): 5167.87
disk write min bandwidth (MB/s): 1712.91 [ mdw]
disk write max bandwidth (MB/s): 1731.35 [sdw1]
-- per host bandwidth --
disk write bandwidth (MB/s): 1712.91 [ mdw]
disk write bandwidth (MB/s): 1731.35 [sdw1]
disk write bandwidth (MB/s): 1723.61 [sdw2]
disk read avg time (sec): 292.12
disk read tot bytes: 824633720832
disk read tot bandwidth (MB/s): 2694.91
disk read min bandwidth (MB/s): 860.84 [ mdw]
disk read max bandwidth (MB/s): 929.59 [sdw1]
-- per host bandwidth --
disk read bandwidth (MB/s): 860.84 [ mdw]
disk read bandwidth (MB/s): 929.59 [sdw1]
disk read bandwidth (MB/s): 904.48 [sdw2]
stream tot bandwidth (MB/s): 28555.80
stream min bandwidth (MB/s): 8279.00 [sdw2]
stream max bandwidth (MB/s): 10142.00 [sdw1]
-- per host bandwidth --
stream bandwidth (MB/s): 10134.80 [ mdw]
stream bandwidth (MB/s): 10142.00 [sdw1
- 测试场景及SQL
测试共6个场景，各个场景sql如下（后缀’_c’表示列存表，‘_r’表示行存表,’_p’为分区表），除特别注明默认优化器使用pg
- 批量数据插入测试
INSERT INTO tb_mz_sfmxb_500_c select * from tb_mz_sfmxb_source_r limit 5000000;
- 多进程批量数据插入测试
INSERT INTO CENYLFW.tb_mz_sfmxb_500_bf_r select * from CENYLFW.tb_mz_sfmxb_source_r limit 5000000;
- 单表增量数据与历史数据去重合并测试
Insert into tb_mz_sfmxb_union_1000_r select yljgdm,sfmxid from (select yljgdm,sfmxid,row_number() over(partition by yljgdm,sfmxid order by yljgdm,sfmxid desc) row_num from (select yljgdm,sfmxid,jlgxsj from tb_mz_sfmxb_1000_r a
union all
select yljgdm,sfmxid,jlgxsj from tb_mz_sfmxb_p where SFJSSJ>='20220101' and SFJSSJ<'20220211') c
group by yljgdm,sfmxid) d
where row_num=1 ;
- 简单模式测试
使用GPORA优化器
set optimizer to on;
select * from tb_mz_sfmxb_p_r where brwyid in (select pid from tb_his_grbsxx where daid in (select daid from tb_chss_grjbxx where sfzh='16b550deef4679c4faff44901310b676'));
- 统计测试
select to_char(a.sfjssj,'yyyy') f_year,
extract(quarter from a.sfjssj) f_quarter,
to_char(a.sfjssj,'yyyymm') f_month,
to_char(a.sfjssj,'yyyy')||extract(week from now()) f_week,
to_char(a.sfjssj,'yyyymm') f_day,
qh.sjjgid f_sjqh,
jg.xnxzqhdm f_qxqh,
jg.yljgid f_jg_id,
substr(jg.wsjglbdm, 1, 4) f_jg_lx,
jg.jglsgxdm f_jg_lsgxdm,
a.kdksptdm f_ks,
a.kdysid f_ys_id,
a.kdysbh f_ys_gh,
a.kdysxm f_ys_mc,
sum(case
when a.tfbz = '2' then
-abs(mxxmssje)
else
abs(mxxmssje)
end) as ylsr,
sum(case
when a.tfbz = '2' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
abs(mxxmssje)
end) as ylsr_zy,
sum(case
when a.tfbz = '2' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
abs(mxxmssje)
end) as ylsr_xy,
sum(case
when a.tfbz = '2' then
-abs(mxxmssje)
else
abs(mxxmssje)
end) as mzylsr,
sum(case
when a.tfbz = '2' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
abs(mxxmssje)
end) as mzylsr_zy,
sum(case
when a.tfbz = '2' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
abs(mxxmssje)
end) as mzylsr_xy,
0 as zyylsr,
0 as zyylsr_zy,
0 as zyylsr_xy,
sum(case
when a.tfbz = '2' and substr(a.fysrglbm, 1, 2) = '03' then
-abs(mxxmssje)
when a.tfbz <> '2' and substr(a.fysrglbm, 1, 2) = '03' then
abs(mxxmssje)
end) zlsr,
sum(case
when a.tfbz = '2' and substr(a.fysrglbm, 1, 2) = '03' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and substr(a.fysrglbm, 1, 2) = '03' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
abs(mxxmssje)
end) zlsr_zy,
sum(case
when a.tfbz = '2' and substr(a.fysrglbm, 1, 2) = '03' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and substr(a.fysrglbm, 1, 2) = '03' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
abs(mxxmssje)
end) zlsr_xy,
sum(case
when a.tfbz = '2' and substr(a.fysrglbm, 1, 2) = '03' then
-abs(mxxmssje)
when a.tfbz <> '2' and substr(a.fysrglbm, 1, 2) = '03' then
abs(mxxmssje)
end) mzzlsr,
sum(case
when a.tfbz = '2' and substr(a.fysrglbm, 1, 2) = '03' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and substr(a.fysrglbm, 1, 2) = '03' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
abs(mxxmssje)
end) mzzlsr_zy,
sum(case
when a.tfbz = '2' and substr(a.fysrglbm, 1, 2) = '03' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and substr(a.fysrglbm, 1, 2) = '03' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
abs(mxxmssje)
end) mzzlsr_xy,
0 zyzlsr,
0 zyzlsr_zy,
0 zyzlsr_xy,
sum(case
when a.tfbz = '2' and a.fysrglbm in ('01', '02', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and
a.fysrglbm in ('01', '02', '03') then
abs(mxxmssje)
end) ypsr,
sum(case
when a.tfbz = '2' and a.fysrglbm in ('01', '02', '03') and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and
a.fysrglbm in ('01', '02', '03') and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
abs(mxxmssje)
end) ypsr_zy,
sum(case
when a.tfbz = '2' and a.fysrglbm in ('01', '02', '03') and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and
a.fysrglbm in ('01', '02', '03') and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
abs(mxxmssje)
end) ypsr_xy,
sum(case
when a.tfbz = '2' and a.fysrglbm in ('01', '02', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and
a.fysrglbm in ('01', '02', '03') then
abs(mxxmssje)
end) mzypsr,
sum(case
when a.tfbz = '2' and a.fysrglbm in ('01', '02', '03') and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and
a.fysrglbm in ('01', '02', '03') and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
abs(mxxmssje)
end) mzypsr_zy,
sum(case
when a.tfbz = '2' and a.fysrglbm in ('01', '02', '03') and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and
a.fysrglbm in ('01', '02', '03') and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
abs(mxxmssje)
end) mzypsr_xy,
0 zyypsr,
0 zyypsr_zy,
0 zyypsr_xy,
sum(case
when a.tfbz = '2' and a.fysrglbm = '05' then
-abs(mxxmssje)
when a.tfbz <> '2' and a.fysrglbm = '05' then
abs(mxxmssje)
end) mzjcsr,
sum(case
when a.tfbz = '2' and a.fysrglbm = '05' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and a.fysrglbm = '05' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
abs(mxxmssje)
end) mzjcsr_zy,
sum(case
when a.tfbz = '2' and a.fysrglbm = '05' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and a.fysrglbm = '05' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
abs(mxxmssje)
end) mzjcsr_xy,
sum(case
when a.tfbz = '2' and a.fysrglbm = '08' then
-abs(mxxmssje)
when a.tfbz <> '2' and a.fysrglbm = '08' then
abs(mxxmssje)
end) mzzhilsr,
sum(case
when a.tfbz = '2' and a.fysrglbm = '08' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and a.fysrglbm = '08' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
abs(mxxmssje)
end) mzzhilsr_zy,
sum(case
when a.tfbz = '2' and a.fysrglbm = '08' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and a.fysrglbm = '08' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
abs(mxxmssje)
end) mzzhilsr_xy,
0 zyjcsr,
0 zyjcsr_zy,
0 zyjcsr_xy,
0 zyzhilsr,
0 zyzhilsr_zy,
0 zyzhilsr_xy,
0 zycwsr,
0 zycwsr_zy,
0 zycwsr_xy,
sum(case
when a.tfbz = '2' and a.fysrglbm = '0301' then
-abs(mxxmssje)
when a.tfbz <> '2' and a.fysrglbm = '0301' then
abs(mxxmssje)
end) mzghsr,
sum(case
when a.tfbz = '2' and a.fysrglbm = '0301' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and a.fysrglbm = '0301' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
abs(mxxmssje)
end) mzghsr_zy,
sum(case
when a.tfbz = '2' and a.fysrglbm = '0301' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and a.fysrglbm = '0301' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
abs(mxxmssje)
end) mzghsr_xy,
sum(case
when a.tfbz = '2' and a.fysrglbm = '04' then
-abs(mxxmssje)
when a.tfbz <> '2' and a.fysrglbm = '04' then
abs(mxxmssje)
end) mzzcsr,
sum(case
when a.tfbz = '2' and a.fysrglbm = '04' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and a.fysrglbm = '04' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
abs(mxxmssje)
end) mzzcsr_zy,
sum(case
when a.tfbz = '2' and a.fysrglbm = '04' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and a.fysrglbm = '04' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
abs(mxxmssje)
end) mzzcsr_xy,
sum(case
when a.tfbz = '2' and a.fysrglbm = '09' then
-abs(mxxmssje)
when a.tfbz <> '2' and a.fysrglbm = '09' then
abs(mxxmssje)
end) mzsssr,
sum(case
when a.tfbz = '2' and a.fysrglbm = '06' then
-abs(mxxmssje)
when a.tfbz <> '2' and a.fysrglbm = '06' then
abs(mxxmssje)
end) mzhysr,
sum(case
when a.tfbz = '2' and substr(a.fysrglbm, 1, 2) = '99' then
-abs(mxxmssje)
when a.tfbz <> '2' and substr(a.fysrglbm, 1, 2) = '99' then
abs(mxxmssje)
end) mzqtsr,
0 zysssr,
0 zyhysr,
0 zyqtsr,
0 zyhlsr,
0 zyhlsr_zy,
0 zyhlsr_xy,
0 zyzcsr,
0 zyzcsr_zy,
0 zyzcsr_xy,
sum(case
when a.tfbz = '2' and substr(a.fysrglbm, 1, 2) = '90' then
-abs(mxxmssje)
when a.tfbz <> '2' and substr(a.fysrglbm, 1, 2) = '90' then
abs(mxxmssje)
end) as clfsr,
sum(case
when a.tfbz = '2' and substr(a.fysrglbm, 1, 2) = '90' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and substr(a.fysrglbm, 1, 2) = '90' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
abs(mxxmssje)
end) as clfsr_xy,
sum(case
when a.tfbz = '2' and substr(a.fysrglbm, 1, 2) = '90' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and substr(a.fysrglbm, 1, 2) = '90' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
abs(mxxmssje)
end) as clfsr_zy,
sum(case
when a.tfbz = '2' and substr(a.fysrglbm, 1, 2) = '90' then
-abs(mxxmssje)
when a.tfbz <> '2' and substr(a.fysrglbm, 1, 2) = '90' then
abs(mxxmssje)
end) as mzclfsr,
0 as zyclfsr,
sum(case
when a.tfbz = '2' and substr(a.fysrglbm, 1, 2) = '90' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and substr(a.fysrglbm, 1, 2) = '90' and
coalesce(b.lcyxlxbm, '01') in ('01', '03') then
abs(mxxmssje)
end) as mzclfsr_xy,
sum(case
when a.tfbz = '2' and substr(a.fysrglbm, 1, 2) = '90' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
-abs(mxxmssje)
when a.tfbz <> '2' and substr(a.fysrglbm, 1, 2) = '90' and
coalesce(b.lcyxlxbm, '01') not in ('01', '03') then
abs(mxxmssje)
end) as mzclfsr_zy,
0 as zyclfsr_xy,
0 as zyclfsr_zy
from tb_mz_sfmxb_p_r a
inner join (select t.ysksptdm,
t.ysksptmc,
t.ghysgh,
t.ghysxm,
t.yljgdm,
t.jzlsh,
t.ysid,
t.lcyxlxbm
from tb_mz_ghmxb t
) b
on a.yljgdm = b.yljgdm
and a.jzlsh = b.jzlsh
inner join tb_dic_wd_yljgzd jg
on a.yljgdm= jg.yljgdm
inner join tb_dic_wd_wsxzjgzdb qh
on jg.xnxzqhdm = qh.xzqhdm
where a.sfjssj >='20220101'
and a.sfjssj <'20220201'
group by to_char(a.sfjssj,'yyyy') ,
extract(quarter from a.sfjssj) ,
to_char(a.sfjssj,'yyyymm') ,
to_char(a.sfjssj,'yyyy')||extract(week from now()),
to_char(a.sfjssj,'yyyymm'),
qh.sjjgid,
jg.xnxzqhdm,
jg.yljgid,
jg.wsjglbdm,
jg.jglsgxdm,
a.kdksptdm,
a.kdysid,
a.kdysbh,
a.kdysxm;
- 排序过滤取数测试
insert into tb_mz_sfmxb_500_sort(BRWYID,YLJGDMID,YLJGDM,JZLSH,SFMXID,BTFMXID,TFBZ,SFCJBM,YZMXID,FYSRGLBM,FYFSSJ,SYJSID,SFJSSJ,SFXMBZBM,MXXMBM,MXXMMC,MXXMDW,XMFLBM,XMFLMC,MXXMDJ,MXXMSL,MXXMYSJE,MXXMSSJE,RKBZ,YLYL1,YLYL2,XGBZ,JLGXSJ,KDKSID,KDKSMC,KDKSPTID,KDKSPTDM,KDKSPTMC,KDYSID,KDYSBH,KDYSXM,ZXKSID,ZXKSBM,ZXKSMC,ZXKSPTID,ZXKSPTDM,ZXKSPTMC,ZXRYID,ZXRYBH,ZXRYXM,SFXMLBBM,KDKSBM,FYSRGLMC)
select BRWYID,YLJGDMID,YLJGDM,JZLSH,SFMXID,BTFMXID,TFBZ,SFCJBM,YZMXID,FYSRGLBM,FYFSSJ,SYJSID,SFJSSJ,SFXMBZBM,MXXMBM,MXXMMC,MXXMDW,XMFLBM,XMFLMC,MXXMDJ,MXXMSL,MXXMYSJE,MXXMSSJE,RKBZ,YLYL1,YLYL2,XGBZ,JLGXSJ,KDKSID,KDKSMC,KDKSPTID,KDKSPTDM,KDKSPTMC,KDYSID,KDYSBH,KDYSXM,ZXKSID,ZXKSBM,ZXKSMC,ZXKSPTID,ZXKSPTDM,ZXKSPTMC,ZXRYID,ZXRYBH,ZXRYXM,SFXMLBBM,KDKSBM,FYSRGLMC
from (
SELECT BRWYID,YLJGDMID,YLJGDM,JZLSH,SFMXID,BTFMXID,TFBZ,SFCJBM,YZMXID,FYSRGLBM,FYFSSJ,SYJSID,SFJSSJ,SFXMBZBM,MXXMBM,MXXMMC,MXXMDW,XMFLBM,XMFLMC,MXXMDJ,MXXMSL,MXXMYSJE,MXXMSSJE,RKBZ,YLYL1,YLYL2,XGBZ,JLGXSJ,KDKSID,KDKSMC,KDKSPTID,KDKSPTDM,KDKSPTMC,KDYSID,KDYSBH,KDYSXM,ZXKSID,ZXKSBM,ZXKSMC,ZXKSPTID,ZXKSPTDM,ZXKSPTMC,ZXRYID,ZXRYBH,ZXRYXM,SFXMLBBM,KDKSBM,FYSRGLMC ,row_number() over(partition by yljgdm,sfmxid order by jlgxsj desc) row_num from ( SELECT * from tb_mz_sfmxb_p_r a where a.sfjssj>='2022-01-01'
and a.sfjssj<'2022-02-06'
) max GROUP BY BRWYID,YLJGDMID,YLJGDM,JZLSH,SFMXID,BTFMXID,TFBZ,SFCJBM,YZMXID,FYSRGLBM,FYFSSJ,SYJSID,SFJSSJ,SFXMBZBM,MXXMBM,MXXMMC,MXXMDW,XMFLBM,XMFLMC,MXXMDJ,MXXMSL,MXXMYSJE,MXXMSSJE,RKBZ,YLYL1,YLYL2,XGBZ,JLGXSJ,KDKSID,KDKSMC,KDKSPTID,KDKSPTDM,KDKSPTMC,KDYSID,KDYSBH,KDYSXM,ZXKSID,ZXKSBM,ZXKSMC,ZXKSPTID,ZXKSPTDM,ZXKSPTMC,ZXRYID,ZXRYBH,ZXRYXM,SFXMLBBM,KDKSBM,FYSRGLMC
) s where row_num=1 ;
本文来自互联网用户投稿，该文观点仅代表作者本人，不代表本站立场。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如若转载，请注明出处：http://www.kler.cn/a/598419.html 如若内容造成侵权/违法违规/事实不符，请联系邮箱：809451989@qq.com进行投诉反馈，一经查实，立即删除！