当前位置：首页 > article >正文

docker compose部署flink集群

article 2025/2/13 21:52:45

本次部署2个jobmanager和3个taskmanager

一、部署zookeeper集群

flink使用zookeeper用作高可用

部署集群参考：docker compose部署zookeeper集群-CSDN博客

二、创建目录及配置文件

创建timezone文件，内容填写Asia/Shanghai

手动创建目录：data、conf、tmp、log

conf文件夹内容如下：

需要为每个节点单独准备配置文件，log4j和logback配置可以共用

这些配置文件，可以从官网下载flink包解压后获取，或启动flink容器后docker cp复制出来。

逐个修改flink-conf-xxx.yaml配置

以flink-conf-jobmanager1.yaml举例：配置如下

其他配置文件，根据各配置项的备注描述进行对应修改

三、创建docker compose文件

vim docker-compose.yaml

networks:
net:
external: true

services:
jobmanager1:
restart: always
image: apache/flink:1.16.3
container_name: jobmanager1
hostname: jobmanager1
ports:
- '8081:8081'
volumes:
- /etc/localtime:/etc/localtime
- /home/sumengnan/apache/flink/timezone:/etc/timezone
- /home/sumengnan/apache/flink/conf/flink-conf-jobmanager1.yaml:/opt/flink/conf/flink-conf.yaml
- /home/sumengnan/apache/flink/conf/log4j.properties:/opt/flink/conf/log4j.properties
- /home/sumengnan/apache/flink/conf/logback.xml:/opt/flink/conf/logback.xml
- /home/sumengnan/apache/flink/conf/log4j-console.properties:/opt/flink/conf/log4j-console.properties
- /home/sumengnan/apache/flink/conf/logback-console.xml:/opt/flink/conf/logback-console.xml
- /home/sumengnan/apache/flink/data:/opt/flink/data
- /home/sumengnan/apache/flink/log:/opt/flink/log
- /home/sumengnan/apache/flink/tmp:/opt/flink/tmp
networks:
- net
jobmanager2:
restart: always
image: apache/flink:1.16.3
container_name: jobmanager2
hostname: jobmanager2
ports:
- '8082:8081'
volumes:
- /etc/localtime:/etc/localtime
- /home/sumengnan/apache/flink/timezone:/etc/timezone
- /home/sumengnan/apache/flink/conf/flink-conf-jobmanager2.yaml:/opt/flink/conf/flink-conf.yaml
- /home/sumengnan/apache/flink/conf/log4j.properties:/opt/flink/conf/log4j.properties
- /home/sumengnan/apache/flink/conf/logback.xml:/opt/flink/conf/logback.xml
- /home/sumengnan/apache/flink/conf/log4j-console.properties:/opt/flink/conf/log4j-console.properties
- /home/sumengnan/apache/flink/conf/logback-console.xml:/opt/flink/conf/logback-console.xml
- /home/sumengnan/apache/flink/data:/opt/flink/data
- /home/sumengnan/apache/flink/log:/opt/flink/log
- /home/sumengnan/apache/flink/tmp:/opt/flink/tmp
networks:
- net
depenes_on:
- jobmanager1

taskmanager1:
restart: always
image: apache/flink:1.16.3
container_name: taskmanager1
hostname: taskmanager1
command: taskmanager
volumes:
- /etc/localtime:/etc/localtime
- /home/sumengnan/apache/flink/timezone:/etc/timezone
- /home/sumengnan/apache/flink/conf/flink-conf-taskmanager1.yaml:/opt/flink/conf/flink-conf.yaml
- /home/sumengnan/apache/flink/conf/log4j.properties:/opt/flink/conf/log4j.properties
- /home/sumengnan/apache/flink/conf/logback.xml:/opt/flink/conf/logback.xml
- /home/sumengnan/apache/flink/conf/log4j-console.properties:/opt/flink/conf/log4j-console.properties
- /home/sumengnan/apache/flink/conf/logback-console.xml:/opt/flink/conf/logback-console.xml
- /home/sumengnan/apache/flink/data:/opt/flink/data
- /home/sumengnan/apache/flink/log:/opt/flink/log
- /home/sumengnan/apache/flink/tmp:/opt/flink/tmp
networks:
- net
depenes_on:
- jobmanager1
- jobmanager2
taskmanager2:
restart: always
image: apache/flink:1.16.3
container_name: taskmanager2
hostname: taskmanager2
command: taskmanager
volumes:
- /etc/localtime:/etc/localtime
- /home/sumengnan/apache/flink/timezone:/etc/timezone
- /home/sumengnan/apache/flink/conf/flink-conf-taskmanager2.yaml:/opt/flink/conf/flink-conf.yaml
- /home/sumengnan/apache/flink/conf/log4j.properties:/opt/flink/conf/log4j.properties
- /home/sumengnan/apache/flink/conf/logback.xml:/opt/flink/conf/logback.xml
- /home/sumengnan/apache/flink/conf/log4j-console.properties:/opt/flink/conf/log4j-console.properties
- /home/sumengnan/apache/flink/conf/logback-console.xml:/opt/flink/conf/logback-console.xml
- /home/sumengnan/apache/flink/data:/opt/flink/data
- /home/sumengnan/apache/flink/log:/opt/flink/log
- /home/sumengnan/apache/flink/tmp:/opt/flink/tmp
networks:
- net
depenes_on:
- jobmanager1
- jobmanager2
taskmanager3:
restart: always
image: apache/flink:1.16.3
container_name: taskmanager3
hostname: taskmanager3
command: taskmanager
volumes:
- /etc/localtime:/etc/localtime
- /home/sumengnan/apache/flink/timezone:/etc/timezone
- /home/sumengnan/apache/flink/conf/flink-conf-taskmanager3.yaml:/opt/flink/conf/flink-conf.yaml
- /home/sumengnan/apache/flink/conf/log4j.properties:/opt/flink/conf/log4j.properties
- /home/sumengnan/apache/flink/conf/logback.xml:/opt/flink/conf/logback.xml
- /home/sumengnan/apache/flink/conf/log4j-console.properties:/opt/flink/conf/log4j-console.properties
- /home/sumengnan/apache/flink/conf/logback-console.xml:/opt/flink/conf/logback-console.xml
- /home/sumengnan/apache/flink/data:/opt/flink/data
- /home/sumengnan/apache/flink/log:/opt/flink/log
- /home/sumengnan/apache/flink/tmp:/opt/flink/tmp
networks:
- net
depenes_on:
- jobmanager1
- jobmanager2