flink 测试demo

最新推荐文章于 2024-11-06 09:59:26 发布

老鼠扛刀满街找猫@

最新推荐文章于 2024-11-06 09:59:26 发布

阅读量298

点赞数

CC 4.0 BY-SA版权

文章标签： flink java kafka

本文链接：https://blog.csdn.net/qq_27242695/article/details/130007514

文章展示了如何搭建一个基于Flink1.13.1的流处理测试环境，包括Maven项目配置、prop.json参数设置，以及FlinkTest1类中DataStream的使用和测试。代码中包含了从Kafka读取数据并进行转换的操作，同时配置了checkpoint参数。此外，还使用JUnit进行了测试，并读取prop.json文件内容作为程序参数。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

flink 测试demo

以flink1.13.1为例

1. maven

<project>
  <groupId>com.example</groupId>
  <artifactId>flink-example</artifactId>
  <version>1.0-SNAPSHOT</version>
  <dependencies>
    <!-- Flink依赖 -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-java</artifactId>
      <version>1.13.1</version>
    </dependency>
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-streaming-java_2.12</artifactId>
      <version>1.13.1</version>
    </dependency>
    <!-- JUnit依赖 -->
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.13.2</version>
      <scope>test</scope>
    </dependency>
  </dependencies>
  <build>
    <plugins>
      <!-- Flink插件 -->
      <plugin>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-maven-plugin</artifactId>
        <version>1.13.1</version>
        <executions>
          <execution>
            <id>package</id>
            <goals>
              <goal>package</goal>
            </goals>
          </execution>
        </executions>
      </plugin>
    </plugins>
  </build>
</project>

2. prop.json 参数

{
  "topic":"pupu_main",
  "kafka_brokers":"127.0.0.1:9092",
  "group_id":"pHistTask",
  "s3_path":"/Users/Desktop/code/test",
  "checkpoint.interval":"60000",
  "checkpoint.timeout": "1200000"
}

3. code

3.1 FlinkTest1.class

package demo.test;

import com.alibaba.fastjson.JSON;
import lombok.Builder;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;


@Builder
public class FlinkTest1 {

    public void start(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
        ParameterTool params = ParameterTool.fromArgs(args);
        System.out.println(JSON.toJSON(params));
        env.setParallelism(1);
        // 构建测试数据
        DataStream<String> dataStream = env.fromElements("Hello", "World");
        // 执行测试操作
        DataStream<String> resultStream = dataStream.map(String::toUpperCase);
        // 验证测试结果
        resultStream.print();
        env.execute();
    }
}

3.1 FlinkTest1.class

package demo.test;

import cn.hutool.core.io.FileUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.apache.flink.api.java.utils.ParameterTool;
import org.junit.Test;

import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;

public class FlinkDemoTest {

    @Test
    public void FlinkTest1() throws Exception {
        String[] args = readFile("prop.json");
        FlinkTest1.builder().build().start(args);
    }

    private String[] readFile(String propFileName) throws Exception {
        List<String> sList = new ArrayList<>();
        File file = new File(Objects.requireNonNull(FlinkDemoTest.class.getClassLoader().getResource(propFileName)).getPath());
        JSONObject jsonObject = JSON.parseObject(FileUtil.readString(file, "UTF-8"));
        for (String key : jsonObject.keySet()) {
            String value = jsonObject.getString(key);
            sList.add("--" + key);
            sList.add(value);
        }
        return sList.toArray(new String[0]);
    }

    public static void main(String[] args) throws Exception {
        List<String> sList = new ArrayList<>();
        File file = new File(Objects.requireNonNull(FlinkDemoTest.class.getClassLoader().getResource("prop.json")).getPath());
        JSONObject jsonObject = JSON.parseObject(FileUtil.readString(file, "UTF-8"));
        for (String key : jsonObject.keySet()) {
            String value = jsonObject.getString(key);
            sList.add("--" + key);
            sList.add(value);
        }
        String[] result = sList.toArray(new String[0]);
        ParameterTool params = ParameterTool.fromArgs(result);
        System.out.println(JSON.toJSONString(params.toMap()));
    }

}

3.3 结果

在这里插入图片描述