基于Flink的JDBC SINK插入Nested结构数据到Clickhouse

Jayce-Ye

已于 2022-08-01 17:02:36 修改

阅读量765

点赞数

CC 4.0 BY-SA版权

文章标签：大数据

于 2022-07-10 17:23:46 首次发布

本文链接：https://blog.csdn.net/u014258693/article/details/125708229

本文详细介绍了如何利用Apache Flink的JDBC SINK将包含嵌套结构的数据高效地插入到Clickhouse数据库中，探讨了数据转换和优化的策略，并分享了在实际操作中遇到的问题及解决方案。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

package com.msyc.bi.sync.utils;

import cn.hutool.core.util.ReflectUtil;
import com.msyc.bi.sync.annotation.NestedSink;
import com.msyc.bi.sync.annotation.TableSink;
import com.msyc.bi.sync.annotation.TransientSink;
import com.msyc.bi.sync.bean.ods.GmpSodPartnerRec;
import com.mysql.cj.jdbc.ClientPreparedStatement;
import lombok.SneakyThrows;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.connector.jdbc.JdbcConnectionOptions;
import org.apache.flink.connector.jdbc.JdbcExecutionOptions;
import org.apache.flink.connector.jdbc.JdbcSink;
import org.apache.flink.connector.jdbc.JdbcStatementBuilder;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ru.yandex.clickhouse.ClickHousePreparedStatement;

import java.beans.BeanInfo;
import java.beans.IntrospectionException;
import java.beans.Introspector;
import java.beans.PropertyDescriptor;
import java.lang.reflect.*;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.Array;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.*;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

public class ClickHouseUtil {
    /**
     * Ck bean转sql 工具类
     * @param param
     * @param clazz
     * @param <T>
     * @return
     * @throws NoSuchMethodException
     * @throws ClassNotFoundException
     */


    public static <T> SinkFunction<T> getSink(ParameterTool param,Class<?> clazz) throws NoSuchMethodException, ClassNotFoundException {

        String sql = createInsertSql(clazz);



        return JdbcSink.<T>sink(sql,
                new JdbcStatementBuilder<T>() {
                    @SneakyThrows
                    @Override
                    public void accept(PreparedStatement preparedStatement, T t) throws SQLException {
                        
                        //获取所有的属性信息
                        Field[] fields = t.getClass().getDeclaredFields();

                        //遍历字段
//                            int offset = 0;
                        int num = 1;
//                        int t = 0;
                        for (Field field : fields) {

                            //设置私有属性可访问
                            if (!field.isAccessible()) {
                                field