使用 Docker Compose 通过 KRaft 启动多个 Kafka 代理时出错：元数据和检查点文件问题

Question

尝试使用 Docker Compose 和 KRaft 共识协议部署多个 Kafka 代理时，设置无法正确初始化。每个 Kafka 代理都配置为使用 KRaft (KAFKA_CFG_PROCESS_ROLES=controller,broker)，并为每个节点（kafka-1、kafka-2、kafka-3）定义监听器和广告监听器。尽管配置了这些设置，代理在启动期间仍会遇到与元数据文件创建相关的错误（写入 meta.properties 文件时出错）。这些错误会阻止代理（kafka-2 和 kafka-3）完全初始化，并导致它们以代码 1 退出。该设置在部署单个代理（kafka-1）时有效，表明该问题特别与配置多个代理有关使用 Docker Compose 与 KRaft 一起使用。

yml

version: '3.8'

services:
  kafka-1:
    image: bitnami/kafka:latest
    user: 'root'
    ports: 
      - "9092:9092"
    environment:
      - KAFKA_CFG_NODE_ID=1
      - KAFKA_KRAFT_CLUSTER_ID=WnLkTHhk
      - KAFKA_CFG_PROCESS_ROLES=controller,broker
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka-1:9091,2@kafka-2:9091,3@kafka-3:9091
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9090,CONTROLLER://:9091,EXTERNAL://:9092
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka-1:9090,EXTERNAL://${HOSTNAME:-localhost}:9092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_INTER_BROKER_LISTENERS_NAME=PLAINTEXT
    volumes:
      - kafka-data:/bitnami/kafka

  kafka-2:
    image: bitnami/kafka:latest
    user: 'root'
    ports: 
      - "9094:9094"
    environment:
      - KAFKA_CFG_NODE_ID=2
      - KAFKA_KRAFT_CLUSTER_ID=WnLkTHhk
      - KAFKA_CFG_PROCESS_ROLES=controller,broker
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka-1:9091,2@kafka-2:9091,3@kafka-3:9091
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9090,CONTROLLER://:9091,EXTERNAL://:9094
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka-2:9090,EXTERNAL://${HOSTNAME:-localhost}:9094
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_INTER_BROKER_LISTENERS_NAME=PLAINTEXT
    volumes:
      - kafka-data:/bitnami/kafka

  kafka-3:
    image: bitnami/kafka:latest
    user: 'root'
    ports: 
      - "9096:9096"
    environment:
      - KAFKA_CFG_NODE_ID=3
      - KAFKA_KRAFT_CLUSTER_ID=WnLkTHhk
      - KAFKA_CFG_PROCESS_ROLES=controller,broker
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka-1:9091,2@kafka-2:9091,3@kafka-3:9091
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9090,CONTROLLER://:9091,EXTERNAL://:9096
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka-3:9090,EXTERNAL://${HOSTNAME:-localhost}:9096
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_INTER_BROKER_LISTENERS_NAME=PLAINTEXT
    volumes:
      - kafka-data:/bitnami/kafka

volumes:
  kafka-data:

环境文件：

HOSTNAME=主机.docker.internal

错误：

kafka-1_1  | kafka 22:33:12.35 INFO  ==> Formatting storage directories to add metadata...
kafka-3_1  | metaPropertiesEnsemble=MetaPropertiesEnsemble(metadataLogDir=Optional.empty, dirs={/bitnami/kafka/data: EMPTY})
kafka-3_1  | Formatting /bitnami/kafka/data with metadata.version 3.7-IV4.
kafka-2_1  | metaPropertiesEnsemble=MetaPropertiesEnsemble(metadataLogDir=Optional.empty, dirs={/bitnami/kafka/data: EMPTY})
kafka-2_1  | Formatting /bitnami/kafka/data with metadata.version 3.7-IV4.
kafka-1_1  | metaPropertiesEnsemble=MetaPropertiesEnsemble(metadataLogDir=Optional.empty, dirs={/bitnami/kafka/data: EMPTY})
kafka-1_1  | Formatting /bitnami/kafka/data with metadata.version 3.7-IV4.
kafka-3_1  | Error while writing meta.properties file /bitnami/kafka/data: /bitnami/kafka/data/bootstrap.checkpoint.tmp
kafka-2_1  | Error while writing meta.properties file /bitnami/kafka/data: /bitnami/kafka/data/bootstrap.checkpoint.tmp
kafka-1_1  | kafka 22:33:14.22 INFO  ==> ** Kafka setup finished! **
kafka-1_1  | 
kafka-1_1  | kafka 22:33:14.24 INFO  ==> ** Starting Kafka **
architecture-experiments_kafka-2_1 exited with code 1
architecture-experiments_kafka-3_1 exited with code 1

执行时

docker-compose --env-file environment.env up

配置调整，为每个代理提供唯一的 KAFKA_CFG_NODE_ID、KAFKA_CFG_PROCESS_ROLES 和 KAFKA_CFG_CONTROLLER_QUORUM_VOTERS 环境变量。 使用 Docker Compose 中的 /etc/hosts 成功配置具有适当 IP 定义的单个 Kafka 容器 (kafka-1)

Answer 1

我遇到了同样的问题，错误是您为两个经纪人安装了相同的卷。