Commit 64f01583 authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Merge branch 'dev' into 'master' 

Actualiza tipo y tamaño de volúmenes de producción

See merge request redmic-project/confluent/confluent-platform!1
parents 689dfc0d a3934f08

.gitlab-ci.yml

+47 −39
Original line number Diff line number Diff line 
image: redmic/docker-gitlab

image: registry.gitlab.com/redmic-project/docker/docker-deploy



stages:

  - package
@@ -6,11 +6,11 @@ stages: 
  - deploy



variables:

  CONFLUENT_VERSION: latest

  CONFLUENT_VERSION: 4.1.2



docker-build-dev:

docker-build-commit-non-master-branches:

  stage: package

  image: redmic/docker-gitlab

  image: docker:stable

  variables:

    DOCKER_DRIVER: overlay2

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}/connector-supplier
@@ -25,9 +25,9 @@ docker-build-dev: 
    - docker build -t ${IMAGE_NAME}:${CI_COMMIT_SHA} -t ${IMAGE_NAME}:${CI_COMMIT_REF_NAME}-latest .

    - docker push ${IMAGE_NAME}



docker-build-pro:

docker-build-commit-master-branch:

  stage: package

  image: redmic/docker-gitlab

  image: docker:stable

  variables:

    DOCKER_DRIVER: overlay2

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}/connector-supplier
@@ -40,7 +40,7 @@ docker-build-pro: 
    - docker build -t ${IMAGE_NAME}:${CI_COMMIT_SHA} -t ${IMAGE_NAME}:latest .

    - docker push ${IMAGE_NAME}



container_scanning:

container-scanning:

  stage: test-package

  image: docker:stable

  variables:
@@ -51,8 +51,6 @@ container_scanning: 
    - docker:stable-dind

  only:

    - branches

  except:

    - master

  script:

    - docker run -d --name db arminc/clair-db:latest

    - docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.1
@@ -63,7 +61,9 @@ container_scanning: 
    - mv clair-scanner_linux_amd64 clair-scanner

    - chmod +x clair-scanner

    - touch clair-whitelist.yml

    - ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-sast-container-report.json -l clair.log -w clair-whitelist.yml ${IMAGE_NAME}:${CI_COMMIT_SHA} || true

    - >

      ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-sast-container-report.json -l clair.log

      -w clair-whitelist.yml ${IMAGE_NAME}:${CI_COMMIT_SHA} || true

  artifacts:

    paths: [gl-sast-container-report.json]
@@ -71,6 +71,7 @@ deploy-zookeeper-dev: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: zookeeper

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    IMAGE_TAG: ${CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.tmpl.yml:docker-compose.dev.yml
@@ -78,7 +79,6 @@ deploy-zookeeper-dev: 
    - docker:dind

  script:

    - create-nets.sh kafka-net

    - cd zookeeper

    - export SERVICE=zk-1

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

    - export SERVICE=zk-2
@@ -97,6 +97,7 @@ deploy-zookeeper-pro: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: zookeeper

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    IMAGE_TAG: ${CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.tmpl.yml:docker-compose.prod.yml
@@ -104,7 +105,6 @@ deploy-zookeeper-pro: 
    - docker:dind

  script:

    - create-nets.sh kafka-net

    - cd zookeeper

    - export SERVICE=zk-1

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

    - export SERVICE=zk-2
@@ -121,6 +121,7 @@ deploy-kafka-dev: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: kafka

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    IMAGE_TAG: ${CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.tmpl.yml:docker-compose.dev.yml
@@ -128,7 +129,6 @@ deploy-kafka-dev: 
  services:

    - docker:dind

  script:

    - cd kafka

    - export SERVICE=kf-1

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

    - export SERVICE=kf-2
@@ -147,13 +147,13 @@ deploy-kafka-pro: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: kafka

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    IMAGE_TAG: ${CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.tmpl.yml:docker-compose.prod.yml

  services:

    - docker:dind

  script:

    - cd kafka

    - export SERVICE=kf-1

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

    - export SERVICE=kf-2
@@ -170,6 +170,7 @@ deploy-connect-dev: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: workers

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICE: connect

    IMAGE_TAG: ${CONFLUENT_VERSION}
@@ -177,7 +178,6 @@ deploy-connect-dev: 
  services:

    - docker:dind

  script:

    - cd workers

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: dev/${SERVICE}
@@ -191,6 +191,7 @@ deploy-connect-pro: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: workers

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICE: connect

    IMAGE_TAG: ${CONFLUENT_VERSION}
@@ -198,7 +199,6 @@ deploy-connect-pro: 
  services:

    - docker:dind

  script:

    - cd workers

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: pro/${SERVICE}
@@ -208,20 +208,21 @@ deploy-connect-pro: 


deploy-connector-supplier-dev:

  stage: deploy

  image: redmic/docker-gitlab

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: workers

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICE: connector-supplier

    IMAGE_NAME: ${DOCKER_ENV_CI_REGISTRY_IMAGE}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}/connector-supplier

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.${SERVICE}.tmpl.yml:docker-compose.${SERVICE}.dev.yml

    CONNECT_ADDRS: connect:8083

  services:

    - docker:dind

  script:

    - cd workers

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} CONNECT_ADDRS=${CONNECT_ADDRS} POSTGRES_USER=${POSTGRES_USER} POSTGRES_PASS=${POSTGRES_PASS}

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      CONNECT_ADDRS=${CONNECT_ADDRS} POSTGRES_USER=${POSTGRES_USER} POSTGRES_PASS=${POSTGRES_PASS}

  environment:

    name: dev/${SERVICE}

  only:
@@ -232,20 +233,21 @@ deploy-connector-supplier-dev: 


deploy-connector-supplier-pro:

  stage: deploy

  image: redmic/docker-gitlab

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: workers

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICE: connector-supplier

    IMAGE_NAME: ${DOCKER_ENV_CI_REGISTRY_IMAGE}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}/connector-supplier

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.${SERVICE}.tmpl.yml:docker-compose.${SERVICE}.prod.yml

    CONNECT_ADDRS: connect:8083

  services:

    - docker:dind

  script:

    - cd workers

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} CONNECT_ADDRS=${CONNECT_ADDRS} POSTGRES_USER=${POSTGRES_USER} POSTGRES_PASS=${POSTGRES_PASS}

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      CONNECT_ADDRS=${CONNECT_ADDRS} POSTGRES_USER=${POSTGRES_USER} POSTGRES_PASS=${POSTGRES_PASS}

  environment:

    name: pro/${SERVICE}

  only:
@@ -256,6 +258,7 @@ deploy-schema-registry-dev: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: workers

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICE: schema-registry

    IMAGE_TAG: ${CONFLUENT_VERSION}
@@ -263,7 +266,6 @@ deploy-schema-registry-dev: 
  services:

    - docker:dind

  script:

    - cd workers

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: dev/${SERVICE}
@@ -277,6 +279,7 @@ deploy-schema-registry-pro: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: workers

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICE: schema-registry

    IMAGE_TAG: ${CONFLUENT_VERSION}
@@ -284,7 +287,6 @@ deploy-schema-registry-pro: 
  services:

    - docker:dind

  script:

    - cd workers

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: pro/${SERVICE}
@@ -296,6 +298,7 @@ deploy-kafka-rest-dev: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: workers

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICE: kafka-rest

    IMAGE_TAG: ${CONFLUENT_VERSION}
@@ -304,7 +307,6 @@ deploy-kafka-rest-dev: 
  services:

    - docker:dind

  script:

    - cd workers

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev/${SERVICE}
@@ -318,6 +320,7 @@ deploy-kafka-rest-pro: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: workers

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICE: kafka-rest

    IMAGE_TAG: ${CONFLUENT_VERSION}
@@ -326,7 +329,6 @@ deploy-kafka-rest-pro: 
  services:

    - docker:dind

  script:

    - cd workers

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: pro/${SERVICE}
@@ -338,6 +340,7 @@ deploy-kafka-manager-dev: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: uis

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    IMAGE_TAG: latest

    SERVICE: kafka-manager
@@ -346,7 +349,6 @@ deploy-kafka-manager-dev: 
  services:

    - docker:dind

  script:

    - cd uis

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: dev/${SERVICE}
@@ -361,6 +363,7 @@ deploy-kafka-manager-pro: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: uis

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    IMAGE_TAG: latest

    SERVICE: kafka-manager
@@ -369,7 +372,6 @@ deploy-kafka-manager-pro: 
  services:

    - docker:dind

  script:

    - cd uis

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME} UI_AUTH=${UI_AUTH}

  environment:

    name: pro/${SERVICE}
@@ -382,6 +384,7 @@ deploy-kafka-topics-ui-dev: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: uis

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    IMAGE_TAG: latest

    SERVICE: kafka-topics-ui
@@ -390,7 +393,6 @@ deploy-kafka-topics-ui-dev: 
  services:

    - docker:dind

  script:

    - cd uis

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: dev/${SERVICE}
@@ -405,6 +407,7 @@ deploy-kafka-topics-ui-pro: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: uis

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    IMAGE_TAG: latest

    SERVICE: kafka-topics-ui
@@ -413,8 +416,9 @@ deploy-kafka-topics-ui-pro: 
  services:

    - docker:dind

  script:

    - cd uis

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME} UI_AUTH=${UI_AUTH}

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

      UI_AUTH=${UI_AUTH}

  environment:

    name: pro/${SERVICE}

    url: https://kafkatopicsui.${PUBLIC_HOSTNAME}
@@ -426,6 +430,7 @@ deploy-kafka-connect-ui-dev: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: uis

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    IMAGE_TAG: latest

    SERVICE: kafka-connect-ui
@@ -434,7 +439,6 @@ deploy-kafka-connect-ui-dev: 
  services:

    - docker:dind

  script:

    - cd uis

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev/${SERVICE}
@@ -449,6 +453,7 @@ deploy-kafka-connect-ui-pro: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: uis

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    IMAGE_TAG: latest

    SERVICE: kafka-connect-ui
@@ -457,8 +462,9 @@ deploy-kafka-connect-ui-pro: 
  services:

    - docker:dind

  script:

    - cd uis

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME} UI_AUTH=${UI_AUTH}

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

      UI_AUTH=${UI_AUTH}

  environment:

    name: pro/${SERVICE}

    url: https://kafkaconnectui.${PUBLIC_HOSTNAME}
@@ -470,6 +476,7 @@ deploy-schema-registry-ui-dev: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: uis

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    IMAGE_TAG: latest

    SERVICE: schema-registry-ui
@@ -478,7 +485,6 @@ deploy-schema-registry-ui-dev: 
  services:

    - docker:dind

  script:

    - cd uis

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: dev/${SERVICE}
@@ -493,6 +499,7 @@ deploy-schema-registry-ui-pro: 
  stage: deploy

  variables:

    DOCKER_DRIVER: overlay2

    DEPLOY_DIR_NAME: uis

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    IMAGE_TAG: latest

    SERVICE: schema-registry-ui
@@ -501,8 +508,9 @@ deploy-schema-registry-ui-pro: 
  services:

    - docker:dind

  script:

    - cd uis

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME} UI_AUTH=${UI_AUTH}

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

      UI_AUTH=${UI_AUTH}

  environment:

    name: pro/${SERVICE}

    url: https://schemaregistryui.${PUBLIC_HOSTNAME}

kafka/.env

+9 −0
Original line number Diff line number Diff line 
ZK_PORT=2181

KF_PORT=9092

KAFKA_LOG_CLEANER_ENABLE=false

KAFKA_LOG_CLEANUP_POLICY=compact

KAFKA_LOG_CLEANER_MIN_CLEANABLE_RATIO=0.5

KAFKA_LOG_CLEANER_MIN_COMPACTION_LAG_MS=3600000

KAFKA_LOG_ROLL_HOURS=168

KAFKA_NUM_PARTITIONS=5

KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=3

KAFKA_DEFAULT_REPLICATION_FACTOR=3

KAFKA_MIN_INSYNC_REPLICAS=2

kafka/docker-compose.dev.yml

+12 −12
Original line number Diff line number Diff line
@@ -5,31 +5,31 @@ services: 
    container_name: kf-1

    restart: on-failure:3

    ports:

      - 1${KF_PORT}:${KF_PORT}

      - 9997:9997

      - '1${KF_PORT}:${KF_PORT}'

      - '9997:9997'

    environment:

      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${PUBLIC_HOSTNAME}:1${KF_PORT}

      KAFKA_HEAP_OPTS: '-Xmx256m -Xms256m'

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://${PUBLIC_HOSTNAME}:1${KF_PORT}

      - KAFKA_HEAP_OPTS=-Xmx256m -Xms256m



  kf-2:

    container_name: kf-2

    restart: on-failure:3

    ports:

      - 2${KF_PORT}:${KF_PORT}

      - 9998:9998

      - '2${KF_PORT}:${KF_PORT}'

      - '9998:9998'

    environment:

      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${PUBLIC_HOSTNAME}:2${KF_PORT}

      KAFKA_HEAP_OPTS: '-Xmx256m -Xms256m'

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://${PUBLIC_HOSTNAME}:2${KF_PORT}

      - KAFKA_HEAP_OPTS=-Xmx256m -Xms256m



  kf-3:

    container_name: kf-3

    restart: on-failure:3

    ports:

      - 3${KF_PORT}:${KF_PORT}

      - 9999:9999

      - '3${KF_PORT}:${KF_PORT}'

      - '9999:9999'

    environment:

      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${PUBLIC_HOSTNAME}:3${KF_PORT}

      KAFKA_HEAP_OPTS: '-Xmx256m -Xms256m'

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://${PUBLIC_HOSTNAME}:3${KF_PORT}

      - KAFKA_HEAP_OPTS=-Xmx256m -Xms256m



volumes:

  kf-1-data-vol:

kafka/docker-compose.prod.yml

+18 −21
Original line number Diff line number Diff line
@@ -3,11 +3,11 @@ version: '3.5' 
services:

  kf-1:

    ports:

      - ${KF_PORT}

      - 9997

      - '${KF_PORT}'

      - '9997'

    environment:

      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kf-1:${KF_PORT}

      KAFKA_HEAP_OPTS: '-Xmx4g -Xms4g'

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kf-1:${KF_PORT}

      - KAFKA_HEAP_OPTS=-Xmx4g -Xms4g

    deploy:

      mode: replicated

      replicas: 1
@@ -21,11 +21,11 @@ services: 


  kf-2:

    ports:

      - ${KF_PORT}

      - 9998

      - '${KF_PORT}'

      - '9998'

    environment:

      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kf-2:${KF_PORT}

      KAFKA_HEAP_OPTS: '-Xmx4g -Xms4g'

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kf-2:${KF_PORT}

      - KAFKA_HEAP_OPTS=-Xmx4g -Xms4g

    deploy:

      mode: replicated

      replicas: 1
@@ -39,11 +39,11 @@ services: 


  kf-3:

    ports:

      - ${KF_PORT}

      - 9999

      - '${KF_PORT}'

      - '9999'

    environment:

      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kf-3:${KF_PORT}

      KAFKA_HEAP_OPTS: '-Xmx4g -Xms4g'

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kf-3:${KF_PORT}

      - KAFKA_HEAP_OPTS=-Xmx4g -Xms4g

    deploy:

      mode: replicated

      replicas: 1
@@ -61,24 +61,21 @@ volumes: 
    driver: "cloudstor:aws"

    driver_opts:

      backing: relocatable

      size: 25

      ebstype: io1

      iops: 100

      size: 500

      ebstype: st1



  kf-2-data-vol:

    name: kf-2-data-vol

    driver: "cloudstor:aws"

    driver_opts:

      backing: relocatable

      size: 25

      ebstype: io1

      iops: 100

      size: 500

      ebstype: st1



  kf-3-data-vol:

    name: kf-3-data-vol

    driver: "cloudstor:aws"

    driver_opts:

      backing: relocatable

      size: 25

      ebstype: io1

      iops: 100
 
      size: 500

      ebstype: st1

kafka/docker-compose.tmpl.yml

+39 −30
Original line number Diff line number Diff line
@@ -9,16 +9,19 @@ services: 
    volumes:

      - kf-1-data-vol:/var/lib/kafka/data

    environment:

      KAFKA_BROKER_ID: 1

      KAFKA_ZOOKEEPER_CONNECT: zk-1:${ZK_PORT},zk-2:${ZK_PORT},zk-3:${ZK_PORT}

      KAFKA_LISTENERS: PLAINTEXT://kf-1:${KF_PORT}

      KAFKA_LOG_CLEANUP_POLICY: compact

      KAFKA_SEGMENT_MS: 60000

      KAFKA_NUM_PARTITIONS: 1

      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2

      KAFKA_DEFAULT_REPLICATION_FACTOR: 3

      KAFKA_MIN_INSYNC_REPLICAS: 2

      KAFKA_JMX_PORT: 9997

      - KAFKA_BROKER_ID=1

      - KAFKA_ZOOKEEPER_CONNECT=zk-1:${ZK_PORT},zk-2:${ZK_PORT},zk-3:${ZK_PORT}

      - KAFKA_LISTENERS=PLAINTEXT://kf-1:${KF_PORT}

      - KAFKA_JMX_PORT=9997

      - KAFKA_LOG_CLEANER_ENABLE

      - KAFKA_LOG_CLEANUP_POLICY

      - KAFKA_LOG_CLEANER_MIN_CLEANABLE_RATIO

      - KAFKA_LOG_CLEANER_MIN_COMPACTION_LAG_MS

      - KAFKA_LOG_ROLL_HOURS

      - KAFKA_NUM_PARTITIONS

      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR

      - KAFKA_DEFAULT_REPLICATION_FACTOR

      - KAFKA_MIN_INSYNC_REPLICAS

    healthcheck:

      test: kafka-topics --list -zookeeper $${KAFKA_ZOOKEEPER_CONNECT}

      interval: 30s
@@ -34,16 +37,19 @@ services: 
    volumes:

      - kf-2-data-vol:/var/lib/kafka/data

    environment:

      KAFKA_BROKER_ID: 2

      KAFKA_ZOOKEEPER_CONNECT: zk-1:${ZK_PORT},zk-2:${ZK_PORT},zk-3:${ZK_PORT}

      KAFKA_LISTENERS: PLAINTEXT://kf-2:${KF_PORT}

      KAFKA_LOG_CLEANUP_POLICY: compact

      KAFKA_SEGMENT_MS: 60000

      KAFKA_NUM_PARTITIONS: 1

      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2

      KAFKA_DEFAULT_REPLICATION_FACTOR: 3

      KAFKA_MIN_INSYNC_REPLICAS: 2

      KAFKA_JMX_PORT: 9998

      - KAFKA_BROKER_ID=2

      - KAFKA_ZOOKEEPER_CONNECT=zk-1:${ZK_PORT},zk-2:${ZK_PORT},zk-3:${ZK_PORT}

      - KAFKA_LISTENERS=PLAINTEXT://kf-2:${KF_PORT}

      - KAFKA_JMX_PORT=9998

      - KAFKA_LOG_CLEANER_ENABLE

      - KAFKA_LOG_CLEANUP_POLICY

      - KAFKA_LOG_CLEANER_MIN_CLEANABLE_RATIO

      - KAFKA_LOG_CLEANER_MIN_COMPACTION_LAG_MS

      - KAFKA_LOG_ROLL_HOURS

      - KAFKA_NUM_PARTITIONS

      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR

      - KAFKA_DEFAULT_REPLICATION_FACTOR

      - KAFKA_MIN_INSYNC_REPLICAS

    healthcheck:

      test: kafka-topics --list -zookeeper $${KAFKA_ZOOKEEPER_CONNECT}

      interval: 30s
@@ -59,16 +65,19 @@ services: 
    volumes:

      - kf-3-data-vol:/var/lib/kafka/data

    environment:

      KAFKA_BROKER_ID: 3

      KAFKA_ZOOKEEPER_CONNECT: zk-1:${ZK_PORT},zk-2:${ZK_PORT},zk-3:${ZK_PORT}

      KAFKA_LISTENERS: PLAINTEXT://kf-3:${KF_PORT}

      KAFKA_LOG_CLEANUP_POLICY: compact

      KAFKA_SEGMENT_MS: 60000

      KAFKA_NUM_PARTITIONS: 1

      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2

      KAFKA_DEFAULT_REPLICATION_FACTOR: 3

      KAFKA_MIN_INSYNC_REPLICAS: 2

      KAFKA_JMX_PORT: 9999

      - KAFKA_BROKER_ID=3

      - KAFKA_ZOOKEEPER_CONNECT=zk-1:${ZK_PORT},zk-2:${ZK_PORT},zk-3:${ZK_PORT}

      - KAFKA_LISTENERS=PLAINTEXT://kf-3:${KF_PORT}

      - KAFKA_JMX_PORT=9999

      - KAFKA_LOG_CLEANER_ENABLE

      - KAFKA_LOG_CLEANUP_POLICY

      - KAFKA_LOG_CLEANER_MIN_CLEANABLE_RATIO

      - KAFKA_LOG_CLEANER_MIN_COMPACTION_LAG_MS

      - KAFKA_LOG_ROLL_HOURS

      - KAFKA_NUM_PARTITIONS

      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR

      - KAFKA_DEFAULT_REPLICATION_FACTOR

      - KAFKA_MIN_INSYNC_REPLICAS

    healthcheck:

      test: kafka-topics --list -zookeeper $${KAFKA_ZOOKEEPER_CONNECT}

      interval: 30s