Commit ff6fdbbe authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Commit inicial

parents

.gitignore

0 → 100644
+0 −0

Empty file added.

.gitlab-ci.yml

0 → 100644
+227 −0
Original line number Diff line number Diff line 
stages:

  - deploy



image: registry.gitlab.com/redmic-project/docker/docker-deploy:latest



services:

  - docker:dind



variables:

  DOCKER_DRIVER: overlay2

  STACK: confluent



deploy-kf-1-supporting-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '1'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev

  only:

    - branches

  except:

    - master

  when: manual



deploy-kf-2-supporting-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '2'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev

  only:

    - branches

  except:

    - master

  when: manual



deploy-kf-3-supporting-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '3'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev

  only:

    - branches

  except:

    - master

  when: manual



deploy-kf-1-stable-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '1'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev

  only:

    - master

  when: manual



deploy-kf-2-stable-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '2'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev

  only:

    - master

  when: manual



deploy-kf-3-stable-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '3'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev

  only:

    - master

  when: manual



deploy-kf-1-supporting-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '1'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro

  only:

    - branches

  except:

    - master

  when: manual



deploy-kf-2-supporting-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '2'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro

  only:

    - branches

  except:

    - master

  when: manual



deploy-kf-3-supporting-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '3'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro

  only:

    - branches

  except:

    - master

  when: manual



deploy-kf-1-stable-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '1'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro

  only:

    - master

  when: manual



deploy-kf-2-stable-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '2'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro

  only:

    - master

  when: manual



deploy-kf-3-stable-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '3'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro

  only:

    - master

  when: manual

README.md

0 → 100644
+1 −0
Original line number Diff line number Diff line 
# Kafka

deploy/.env

0 → 100644
+18 −0
Original line number Diff line number Diff line 
ZK_PORT=2181

KF_PORT=9092

KF_PORT_EXT=9093

JMX_HTTP_PORT=9998

KAFKA_JMX_PORT=9999

KAFKA_LOG_DIRS=/var/lib/kafka/data/data

KAFKA_LOG_CLEANER_ENABLE=false

KAFKA_LOG_CLEANUP_POLICY=compact

KAFKA_LOG_CLEANER_MIN_CLEANABLE_RATIO=0.5

KAFKA_LOG_CLEANER_MIN_COMPACTION_LAG_MS=3600000

KAFKA_LOG_ROLL_HOURS=168

KAFKA_NUM_PARTITIONS=5

KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=2

KAFKA_DEFAULT_REPLICATION_FACTOR=2

KAFKA_MIN_INSYNC_REPLICAS=1

KAFKA_OFFSETS_RETENTION_MINUTES=10080

KAFKA_SOCKET_RECEIVE_BUFFER_BYTES=-1

KAFKA_MESSAGE_MAX_BYTES=15728640

deploy/docker-compose.kafka-1.dev.yml

0 → 100644
+30 −0
Original line number Diff line number Diff line 
version: '3.5'



services:

  kafka-1:

    environment:

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://:${KF_PORT},EXT://${PUBLIC_HOSTNAME}:1${KF_PORT}

      - KAFKA_LISTENERS=PLAINTEXT://:${KF_PORT},EXT://0.0.0.0:${KF_PORT_EXT}

      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,EXT:PLAINTEXT

      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT

      - KAFKA_HEAP_OPTS=-Xmx1g -Xms1g

    ports:

      - target: ${KF_PORT_EXT}

        published: 1${KF_PORT}

        mode: host

    deploy:

      mode: replicated

      replicas: 1

      restart_policy:

        delay: 30s

        window: 2m

      resources:

        limits:

          cpus: '1'

          memory: 2G

        reservations:

          memory: 1639M



volumes:

  kf-data-vol:

    name: ${KF_DATA_VOL_NAME:-kf-1-data-vol}