Commit 2f2ddd27 authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Externaliza zookeeper y kafka

parent ee9d5e96

.gitlab-ci.yml

+0 −71
Original line number Diff line number Diff line
@@ -63,77 +63,6 @@ container-scanning: 
  artifacts:

    paths: [gl-sast-container-report.json]



deploy-zookeeper-dev:

  stage: deploy

  variables:

    DEPLOY_DIR_NAME: zookeeper

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: confluent_zk-1 confluent_zk-2 confluent_zk-3

    IMAGE_TAG: ${CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.tmpl.yml:docker-compose.dev.yml

  script:

    - create-nets.sh kafka-net

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: dev/zookeeper

  only:

    - branches

  except:

    - master

  when: manual



deploy-zookeeper-pro:

  stage: deploy

  variables:

    DEPLOY_DIR_NAME: zookeeper

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: confluent_zk-1 confluent_zk-2 confluent_zk-3

    IMAGE_TAG: ${CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.tmpl.yml:docker-compose.prod.yml

  script:

    - create-nets.sh kafka-net

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro/zookeeper

  only:

    - master

  when: manual



deploy-kafka-dev:

  stage: deploy

  variables:

    DEPLOY_DIR_NAME: kafka

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: confluent_kf-1 confluent_kf-2 confluent_kf-3

    IMAGE_TAG: ${CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.tmpl.yml:docker-compose.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev/kafka

  only:

    - branches

  except:

    - master

  when: manual



deploy-kafka-pro:

  stage: deploy

  variables:

    DEPLOY_DIR_NAME: kafka

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: confluent_kf-1 confluent_kf-2 confluent_kf-3

    IMAGE_TAG: ${CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.tmpl.yml:docker-compose.prod.yml

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro/kafka

  only:

    - master

  when: manual



deploy-connect-dev:

  stage: deploy

  variables:

kafka/.env

deleted100644 → 0
+0 −18
Original line number Diff line number Diff line 
ZK_PORT=2181

KF_PORT=9092

KF_PORT_EXT=9093

JMX_PORT=9999

JMX_HTTP_PORT=9998

KAFKA_LOG_DIRS=/var/lib/kafka/data/data

KAFKA_LOG_CLEANER_ENABLE=false

KAFKA_LOG_CLEANUP_POLICY=compact

KAFKA_LOG_CLEANER_MIN_CLEANABLE_RATIO=0.5

KAFKA_LOG_CLEANER_MIN_COMPACTION_LAG_MS=3600000

KAFKA_LOG_ROLL_HOURS=168

KAFKA_NUM_PARTITIONS=5

KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=2

KAFKA_DEFAULT_REPLICATION_FACTOR=2

KAFKA_MIN_INSYNC_REPLICAS=1

KAFKA_OFFSETS_RETENTION_MINUTES=10080

KAFKA_SOCKET_RECEIVE_BUFFER_BYTES=-1

KAFKA_MESSAGE_MAX_BYTES=15728640

kafka/config/config.yaml

deleted100644 → 0
+0 −2
Original line number Diff line number Diff line 
startDelaySeconds: 10

ssl: false

kafka/config/jmx_prometheus_javaagent-0.3.1.jar

deleted100644 → 0
−359 KiB

File deleted.

View file

kafka/docker-compose.dev.yml

deleted100644 → 0
+0 −63
Original line number Diff line number Diff line 
version: '3.5'



x-dev-environment:

  &dev-environment

  KAFKA_LISTENERS: 'PLAINTEXT://:${KF_PORT},EXT://0.0.0.0:${KF_PORT_EXT}'

  KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,EXT:PLAINTEXT

  KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT

  KAFKA_HEAP_OPTS: '-Xmx1g -Xms1g'



x-dev-deploy:

  &dev-deploy

  mode: replicated

  replicas: 1

  restart_policy:

    delay: 30s

    window: 2m

  resources:

    limits:

      cpus: '1'

      memory: 2G

    reservations:

      memory: 1639M



services:

  kf-1:

    environment:

      << : *dev-environment

      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://:${KF_PORT},EXT://${PUBLIC_HOSTNAME}:1${KF_PORT}'

    ports:

      - target: ${KF_PORT_EXT}

        published: 1${KF_PORT}

        mode: host

    deploy: *dev-deploy



  kf-2:

    environment:

      << : *dev-environment

      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://:${KF_PORT},EXT://${PUBLIC_HOSTNAME}:2${KF_PORT}'

    ports:

      - target: ${KF_PORT_EXT}

        published: 2${KF_PORT}

        mode: host

    deploy: *dev-deploy



  kf-3:

    environment:

      << : *dev-environment

      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://:${KF_PORT},EXT://${PUBLIC_HOSTNAME}:3${KF_PORT}'

    ports:

      - target: ${KF_PORT_EXT}

        published: 3${KF_PORT}

        mode: host

    deploy: *dev-deploy



volumes:

  kf-1-data-vol:

    name: kf-1-data-vol



  kf-2-data-vol:

    name: kf-2-data-vol



  kf-3-data-vol:

    name: kf-3-data-vol