Commit bcefa1b8 authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Merge branch 'dev' into 'master' 

Actualiza CI/CD y configs, limpia basura

See merge request redmic-project/confluent/kafka!3
parents 88dfb1b2 d1149650

.gitignore

deleted100644 → 0
+0 −0

Empty file deleted.

.gitlab-ci.yml

+136 −206
Original line number Diff line number Diff line 
include:

  - project: 'redmic-project/gitlab-ci-templates'

    ref: master

    file: '/_deployment.yml'



stages:

  - deploy



image: registry.gitlab.com/redmic-project/docker/docker-deploy:${DOCKER_DEPLOY_IMAGE_TAG}



services:

  - docker:dind



.deploy:

  variables:

  DOCKER_DRIVER: overlay2

    STACK: confluent

    DD_AWS_REGION: ${AWS_REGION}



deploy-kf-1-supporting-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '1'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

.deploy-development:

  environment:

    name: dev/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - branches

  except:

    - master

  when: manual

    name: dev/${SERVICE_NAME}



deploy-kf-2-supporting-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '2'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

.deploy-production:

  environment:

    name: dev/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - branches

  except:

    - master

  when: manual

    name: pro/${SERVICE_NAME}



.deploy-kf-1:

  variables: &deploy-kf-1-variables

    SERVICE_NAME: kf-1



.deploy-kf-2:

  variables: &deploy-kf-2-variables

    SERVICE_NAME: kf-2



deploy-kf-3-supporting-branch-develop:

  stage: deploy

.deploy-kf-3:

  variables: &deploy-kf-3-variables

    SERVICE_NAME: kf-3



.deploy-kf-1-development:

  extends: .deploy-development

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '3'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - branches

  except:

    - master

  when: manual

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-1.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-1.dev.yml

    <<: *deploy-kf-1-variables



deploy-kf-1-stable-branch-develop:

  stage: deploy

.deploy-kf-2-development:

  extends: .deploy-development

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '1'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - master

  when: manual

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-2.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-2.dev.yml

    <<: *deploy-kf-2-variables



deploy-kf-2-stable-branch-develop:

  stage: deploy

.deploy-kf-3-development:

  extends: .deploy-development

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '2'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - master

  when: manual

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-3.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-3.dev.yml

    <<: *deploy-kf-3-variables



deploy-kf-3-stable-branch-develop:

  stage: deploy

.deploy-kf-1-production:

  extends: .deploy-production

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '3'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - master

  when: manual

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-1.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-1.prod.yml

    <<: *deploy-kf-1-variables



deploy-kf-1-supporting-branch-production:

  stage: deploy

.deploy-kf-2-production:

  extends: .deploy-production

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '1'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - branches

  except:

    - master

  when: manual

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-2.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-2.prod.yml

    <<: *deploy-kf-2-variables



deploy-kf-2-supporting-branch-production:

  stage: deploy

.deploy-kf-3-production:

  extends: .deploy-production

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '2'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - branches

  except:

    - master

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-3.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-3.prod.yml

    <<: *deploy-kf-3-variables



.deploy-support-branch: &deploy-support-branch

  rules:

    - if: $CI_MERGE_REQUEST_ID ||

          $CI_COMMIT_TAG ||

          $CI_PIPELINE_SOURCE == "schedule" ||

          $CI_COMMIT_BRANCH == "master" ||

          $CI_COMMIT_BRANCH == "main"

      when: never

    - if: $CI_COMMIT_BRANCH

      when: manual

      allow_failure: true



deploy-kf-3-supporting-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '3'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - branches

  except:

    - master

deploy-kf-1-support-branch-development:

  extends: .deploy-kf-1-development

  <<: *deploy-support-branch



deploy-kf-2-support-branch-development:

  extends: .deploy-kf-2-development

  <<: *deploy-support-branch



deploy-kf-3-support-branch-development:

  extends: .deploy-kf-3-development

  <<: *deploy-support-branch



.deploy-stable-branch: &deploy-stable-branch

  rules:

    - if: $CI_MERGE_REQUEST_ID ||

          $CI_COMMIT_TAG ||

          $CI_PIPELINE_SOURCE == "schedule"

      when: never

    - if: $CI_COMMIT_BRANCH == "master" ||

          $CI_COMMIT_BRANCH == "main"

      when: manual

      allow_failure: true



deploy-kf-1-stable-branch-development:

  extends: .deploy-kf-1-development

  <<: *deploy-stable-branch



deploy-kf-2-stable-branch-development:

  extends: .deploy-kf-2-development

  <<: *deploy-stable-branch



deploy-kf-3-stable-branch-development:

  extends: .deploy-kf-3-development

  <<: *deploy-stable-branch



deploy-kf-1-stable-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '1'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - master

  when: manual

  extends: .deploy-kf-1-production

  <<: *deploy-stable-branch



deploy-kf-2-stable-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '2'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - master

  when: manual

  extends: .deploy-kf-2-production

  <<: *deploy-stable-branch



deploy-kf-3-stable-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '3'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - master

  extends: .deploy-kf-3-production

  <<: *deploy-stable-branch



.deploy-tag: &deploy-tag

  rules:

    - if: $CI_COMMIT_TAG

      when: manual

      allow_failure: true



deploy-kf-1-tag-development:

  extends: .deploy-kf-1-development

  <<: *deploy-tag



deploy-kf-2-tag-development:

  extends: .deploy-kf-2-development

  <<: *deploy-tag



deploy-kf-3-tag-development:

  extends: .deploy-kf-3-development

  <<: *deploy-tag



deploy-kf-1-tag-production:

  extends: .deploy-kf-1-production

  <<: *deploy-tag



deploy-kf-2-tag-production:

  extends: .deploy-kf-2-production

  <<: *deploy-tag



deploy-kf-3-tag-production:

  extends: .deploy-kf-3-production

  <<: *deploy-tag

LICENSE

0 → 100644
+21 −0
Original line number Diff line number Diff line 
MIT License



Copyright (c) 2020 REDMIC Project / Confluent



Permission is hereby granted, free of charge, to any person obtaining a copy

of this software and associated documentation files (the "Software"), to deal

in the Software without restriction, including without limitation the rights

to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

copies of the Software, and to permit persons to whom the Software is

furnished to do so, subject to the following conditions:



The above copyright notice and this permission notice shall be included in all

copies or substantial portions of the Software.



THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

SOFTWARE.

deploy/.env

+30 −7
Original line number Diff line number Diff line 
IMAGE_NAME=confluentinc/cp-kafka

IMAGE_TAG=latest



ZK_PORT=2181

KF_PORT=9092

KF_PORT_EXT=9093

JMX_HTTP_PORT=9998

KAFKA_JMX_PORT=9999



JMX_PROMETHEUS_JAVAAGENT_PATH=/jmx_prometheus_javaagent.jar

JMX_PROMETHEUS_CONFIG_PATH=/config.yaml



KAFKA_LOG_DIRS=/var/lib/kafka/data/data

KAFKA_LOG_CLEANER_ENABLE=false

KAFKA_LOG_CLEANER_ENABLE=true

KAFKA_LOG_CLEANUP_POLICY=compact

KAFKA_LOG_CLEANER_MIN_CLEANABLE_RATIO=0.5

KAFKA_LOG_CLEANER_MIN_COMPACTION_LAG_MS=3600000

KAFKA_LOG_ROLL_HOURS=168



KAFKA_NUM_PARTITIONS=5

KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=2

KAFKA_DEFAULT_REPLICATION_FACTOR=2

KAFKA_MIN_INSYNC_REPLICAS=1

KAFKA_OFFSETS_RETENTION_MINUTES=10080

KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=3

KAFKA_DEFAULT_REPLICATION_FACTOR=3

KAFKA_MIN_INSYNC_REPLICAS=2

KAFKA_OFFSETS_RETENTION_MINUTES=43200

KAFKA_SOCKET_RECEIVE_BUFFER_BYTES=-1

KAFKA_MESSAGE_MAX_BYTES=15728640

LOG_LEVEL=ERROR
 
KAFKA_LOG4J_ROOT_LOGLEVEL=ERROR

KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT

KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT

KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false



JMX_PROMETHEUS_JAVA_AGENT_NAME=jmx-prometheus-javaagent-0.11.0

JMX_PROMETHEUS_CONFIG_NAME=jmx-prometheus-config-0.11.0



KF_1_DATA_VOL_NAME=kf-1-data-vol

KF_2_DATA_VOL_NAME=kf-2-data-vol

KF_3_DATA_VOL_NAME=kf-3-data-vol

KF_1_SECRETS_VOL_NAME=kf-1-secrets-vol

KF_2_SECRETS_VOL_NAME=kf-2-secrets-vol

KF_3_SECRETS_VOL_NAME=kf-3-secrets-vol



KF_DATA_VOL_SIZE=500

KF_DATA_VOL_EBS_TYPE=st1

deploy/docker-compose.kafka-1.dev.yml

+9 −18
Original line number Diff line number Diff line
@@ -3,28 +3,19 @@ version: '3.5' 
services:

  kafka-1:

    environment:

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://:${KF_PORT},EXT://${PUBLIC_HOSTNAME}:1${KF_PORT}

      - KAFKA_LISTENERS=PLAINTEXT://:${KF_PORT},EXT://0.0.0.0:${KF_PORT_EXT}

      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,EXT:PLAINTEXT

      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT

      - KAFKA_HEAP_OPTS=-Xmx1g -Xms1g

    ports:

      - target: ${KF_PORT_EXT}

        published: 1${KF_PORT}

        mode: host

      KAFKA_HEAP_OPTS: -Xmx1g -Xms1g

    deploy:

      mode: replicated

      replicas: 1

      restart_policy:

        delay: 30s

        window: 2m

      placement:

        constraints:

          - node.hostname == ${KF_1_PLACEMENT_CONSTRAINTS_NODE_HOSTNAME:-hostname}

      resources:

        limits:

          cpus: '1'

          memory: 2G

          cpus: '${RESOURCES_LIMITS_CPUS:-1}'

          memory: ${RESOURCES_LIMITS_MEMORY:-2G}

        reservations:

          memory: 1639M

          cpus: '${RESOURCES_RESERVATIONS_CPUS:-0.2}'

          memory: ${RESOURCES_RESERVATIONS_MEMORY:-1639M}



volumes:

  kf-data-vol:

    name: ${KF_DATA_VOL_NAME:-kf-1-data-vol}
 
    name: ${KF_1_DATA_VOL_NAME}