当前位置：首页 > article >正文

Linux 上使用 Docker 部署 Kafka 集群

article 2025/3/31 19:27:32

在 Linux 上使用 Docker 部署 Kafka 集群的步骤如下

1. 准备工作

确保已安装：
Docker
Docker Compose

2. 创建 Docker Compose 文件 (docker-compose.yml)

version: '3.8'

services:
  zookeeper:
    image: wurstmeister/zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"
    networks:
      - kafka-net
    volumes:
      - ./zookeeper/data:/data
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181

  kafka1:
    image: wurstmeister/kafka
    container_name: kafka1
    ports:
      - "9092:9092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka1:9093,OUTSIDE://宿主机IP:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_LISTENERS: INSIDE://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
    volumes:
      - ./kafka1/data:/kafka
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - kafka-net
    depends_on:
      - zookeeper

  kafka2:
    image: wurstmeister/kafka
    container_name: kafka2
    ports:
      - "9093:9093"
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka2:9094,OUTSIDE://宿主机IP:9093
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_LISTENERS: INSIDE://0.0.0.0:9094,OUTSIDE://0.0.0.0:9093
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
    volumes:
      - ./kafka2/data:/kafka
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - kafka-net
    depends_on:
      - zookeeper

  kafka3:
    image: wurstmeister/kafka
    container_name: kafka3
    ports:
      - "9094:9094"
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka3:9095,OUTSIDE://宿主机IP:9094
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_LISTENERS: INSIDE://0.0.0.0:9095,OUTSIDE://0.0.0.0:9094
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
    volumes:
      - ./kafka3/data:/kafka
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - kafka-net
    depends_on:
      - zookeeper

networks:
  kafka-net:
    driver: bridge

关键配置说明

网络架构

networks:
  kafka-net:
    driver: bridge

所有服务使用同一个自定义网络 kafka-net，确保容器间通过服务名称通信

Zookeeper 配置

volumes:
  - ./zookeeper/data:/data  # 数据持久化路径
environment:
  ZOOKEEPER_CLIENT_PORT: 2181

Kafka 节点配置

environment:
  # 监听器配置（核心）
  # 如果外部连接监听消费的话 
  # KAFKA_ADVERTISED_LISTENERS : INSIDE://kafka1:9093,OUTSIDE://localhost:9092需要修改为
  # KAFKA_ADVERTISED_LISTENERS : INSIDE://kafka1:9093,OUTSIDE://宿主机IP:9092
  KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka1:9093,OUTSIDE://宿主机IP:9092
  KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
  KAFKA_LISTENERS: INSIDE://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092
  KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE

  # 其他重要参数
  KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"  # 禁止自动创建主题
  KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3  # 确保高可用

端口映射规则

节点	节点	宿主机端口	宿主机端口
kafka1	9093	9092	集群内部通信
kafka1	9092	9092	外部客户端访问
kafka2	9094	9093	集群内部通信
kafka2	9093	9093	外部客户端访问
kafka3	9095	9094	集群内部通信
kafka3	9094	9094	外部客户端访问

操作流程

启动集群

mkdir -p {zookeeper,kafka1,kafka2,kafka3}/data
docker-compose up -d

验证集群状态

# 查看 Kafka 日志
docker logs kafka1

# 进入容器检查 brokers
docker exec -it kafka1 bash
/opt/kafka/bin/kafka-broker-api-versions --bootstrap-server kafka1:9093

创建测试主题

docker exec -it kafka1 bash
/opt/kafka/bin/kafka-topics.sh --create \
  --topic test-topic \
  --partitions 3 \
  --replication-factor 3 \
  --bootstrap-server kafka1:9093

生产消费测试

# 生产者（使用外部端口）
docker exec -it kafka1 bash
/opt/kafka/bin/kafka-console-producer.sh \
  --topic test-topic \
  --bootstrap-server localhost:9092

# 消费者（使用另一个节点）
docker exec -it kafka2 bash
/opt/kafka/bin/kafka-console-consumer.sh \
  --topic test-topic \
  --from-beginning \
  --bootstrap-server localhost:9093

常见问题排查

Kafka 节点无法启动
检查 KAFKA_ADVERTISED_LISTENERS 配置
验证 Zookeeper 连接：

docker exec -it kafka1 bash
nc -zv zookeeper 2181

外部客户端无法连接
检查宿主机防火墙设置
确认使用外部监听器：

kafka-topics.sh --list --bootstrap-server localhost:9092

chmod -R a+rw ./kafka1/data ./kafka2/data ./kafka3/data

高级配置建议

调整 JVM 参数
在环境变量中添加：

KAFKA_HEAP_OPTS: "-Xmx2G -Xms2G"

启用 SSL 加密

KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:SSL,OUTSIDE:SSL
KAFKA_SSL_KEYSTORE_LOCATION: /kafka/keystore/kafka.jks
KAFKA_SSL_KEYSTORE_PASSWORD: yourpassword

集成监控

# 新增 Prometheus 服务
kafka-exporter:
  image: danielqsj/kafka-exporter
  ports:
    - "9308:9308"
  environment:
    KAFKA_BROKERS: "kafka1:9093,kafka2:9094,kafka3:9095"
  networks:
    - kafka-net