sarama Docker & Self-Hosted Deployment

GitHub

Sarama is a Go library for Apache Kafka.

12,501 stars

Go 1 file · ~2,807 tokens #go #kafka #kafka-client

RAW Compose

1. Pull Image (DOCKERHUB) docker pull docker.io/library/zookeeper

2. Docker Run (1-Click Launch) docker run -d --name sarama -p 29091:29091 --restart always docker.io/library/zookeeper:3.7.2

3. Podman Run podman run -d --name sarama -p 29091:29091 --restart always docker.io/library/zookeeper:3.7.2

4. Skopeo Inspect (Zero-Pull Manifest) skopeo inspect docker://docker.io/docker.io/library/zookeeper:3.7.2

5. ORAS Pull (OCI Artifact) oras pull docker.io/library/zookeeper:3.7.2

6. Compose Deployment docker compose up -d

docker-compose.yml

Download .yml

x-zookeeper-base: &zookeeper-base
  image: 'docker.io/library/zookeeper:3.7.2'
  init: true
  restart: always
  profiles:
    - zookeeper
  environment: &zookeeper-base-env
    ZOO_SERVERS: 'server.1=zookeeper-1:2888:3888 server.2=zookeeper-2:2888:3888 server.3=zookeeper-3:2888:3888'
    ZOO_CFG_EXTRA: 'clientPort=2181 peerPort=2888 leaderPort=3888'
    ZOO_INIT_LIMIT: '10'
    ZOO_SYNC_LIMIT: '5'
    ZOO_MAX_CLIENT_CNXNS: '0'
    ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'

x-kafka-base: &kafka-base
  image: 'sarama/fv-kafka-${KAFKA_VERSION:-4.3.1}'
  init: true
  build:
    context: .
    dockerfile: Dockerfile.kafka
    args:
      KAFKA_VERSION: ${KAFKA_VERSION:-4.3.1}
      SCALA_VERSION: ${SCALA_VERSION:-2.13}
  depends_on:
    - toxiproxy
  restart: always
  environment: &kafka-base-env
    KAFKA_VERSION: ${KAFKA_VERSION:-4.3.1}
    KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
    KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: '2'
    KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '2'
    KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '6000'
    KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '6000'
    KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
    KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
    KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
    KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
    KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
    KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
    KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT,CONTROLLER:PLAINTEXT'
    # ZooKeeper-specific
    KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
    # KRaft-specific
    KAFKA_CFG_PROCESS_ROLES: 'broker,controller'
    KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: '1@kafka-1:9093,2@kafka-2:9093,3@kafka-3:9093,4@kafka-4:9093,5@kafka-5:9093'
    KAFKA_CFG_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
    KAFKA_CFG_CLUSTER_ID: 'cDZEekk4T3hTNGVlNzB3LUtUbkxaQQo='

services:
  zookeeper-1:
    <<: *zookeeper-base
    container_name: 'zookeeper-1'
    environment:
      <<: *zookeeper-base-env
      ZOO_MY_ID: '1'

  zookeeper-2:
    <<: *zookeeper-base
    container_name: 'zookeeper-2'
    environment:
      <<: *zookeeper-base-env
      ZOO_MY_ID: '2'

  zookeeper-3:
    <<: *zookeeper-base
    container_name: 'zookeeper-3'
    environment:
      <<: *zookeeper-base-env
      ZOO_MY_ID: '3'

  kafka-1:
    <<: *kafka-base
    container_name: 'kafka-1'
    healthcheck:
      test: ['CMD', '/opt/kafka-${KAFKA_VERSION:-4.3.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-1:9091']
      interval: 5s
      timeout: 15s
      retries: 30
      start_period: 360s
    environment:
      <<: *kafka-base-env
      KAFKA_CFG_BROKER_ID: '1'
      KAFKA_CFG_NODE_ID: '1'
      KAFKA_CFG_BROKER_RACK: '1'
      KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091,CONTROLLER://:9093'
      KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'

  kafka-2:
    <<: *kafka-base
    container_name: 'kafka-2'
    healthcheck:
      test: ['CMD', '/opt/kafka-${KAFKA_VERSION:-4.3.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-2:9091']
      interval: 5s
      timeout: 15s
      retries: 30
      start_period: 360s
    environment:
      <<: *kafka-base-env
      KAFKA_CFG_BROKER_ID: '2'
      KAFKA_CFG_NODE_ID: '2'
      KAFKA_CFG_BROKER_RACK: '2'
      KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092,CONTROLLER://:9093'
      KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'

  kafka-3:
    <<: *kafka-base
    container_name: 'kafka-3'
    healthcheck:
      test: ['CMD', '/opt/kafka-${KAFKA_VERSION:-4.3.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-3:9091']
      interval: 5s
      timeout: 15s
      retries: 30
      start_period: 360s
    environment:
      <<: *kafka-base-env
      KAFKA_CFG_BROKER_ID: '3'
      KAFKA_CFG_NODE_ID: '3'
      KAFKA_CFG_BROKER_RACK: '3'
      KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093,CONTROLLER://:9093'
      KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'

  kafka-4:
    <<: *kafka-base
    container_name: 'kafka-4'
    healthcheck:
      test: ['CMD', '/opt/kafka-${KAFKA_VERSION:-4.3.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-4:9091']
      interval: 5s
      timeout: 15s
      retries: 30
      start_period: 360s
    environment:
      <<: *kafka-base-env
      KAFKA_CFG_BROKER_ID: '4'
      KAFKA_CFG_NODE_ID: '4'
      KAFKA_CFG_BROKER_RACK: '4'
      KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094,CONTROLLER://:9093'
      KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'

  kafka-5:
    <<: *kafka-base
    container_name: 'kafka-5'
    healthcheck:
      test: ['CMD', '/opt/kafka-${KAFKA_VERSION:-4.3.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-5:9091']
      interval: 5s
      timeout: 15s
      retries: 30
      start_period: 360s
    environment:
      <<: *kafka-base-env
      KAFKA_CFG_BROKER_ID: '5'
      KAFKA_CFG_NODE_ID: '5'
      KAFKA_CFG_BROKER_RACK: '5'
      KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095,CONTROLLER://:9093'
      KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'

  toxiproxy:
    container_name: 'toxiproxy'
    image: 'ghcr.io/shopify/toxiproxy:2.12.0'
    init: true
    healthcheck:
      test: ['CMD', '/toxiproxy-cli', 'l']
      interval: 15s
      timeout: 15s
      retries: 3
      start_period: 30s
    ports:
      # The tests themselves actually start the proxies on these ports
      - '29091:29091'
      - '29092:29092'
      - '29093:29093'
      - '29094:29094'
      - '29095:29095'

      # This is the toxiproxy API port
      - '8474:8474'

Container Topology & Configured Services