Commit 5e43dd83 authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Actualiza CI/CD y configs, limpia basura 

Aplica plantillas de CI/CD.
Pone al día las configuraciones de despliegue, reutilizando valores y
eliminando elementos antiguos (como el mapeo de puerto al host).
parent 72d7f649

.gitignore

deleted100644 → 0
+0 −0

Empty file deleted.

.gitlab-ci.yml

+125 −188
Original line number Diff line number Diff line 
include:

  - project: 'redmic-project/gitlab-ci-templates'

    ref: master

    file: '/_deployment.yml'



stages:

  - deploy



image: registry.gitlab.com/redmic-project/docker/docker-deploy:${DOCKER_DEPLOY_IMAGE_TAG}

variables:

  PROJECT_PARENT_NAME: confluent



.deploy-development:

  environment:

    name: dev/${SERVICE_NAME}



.deploy-production:

  environment:

    name: pro/${SERVICE_NAME}



.deploy-kf-1:

  variables: &deploy-kf-1-variables

    SERVICE_NAME: kf-1

    SERVICES_TO_CHECK: ${PROJECT_PARENT_NAME}_${CI_PROJECT_NAME}-1



.deploy-kf-2:

  variables: &deploy-kf-2-variables

    SERVICE_NAME: kf-2

    SERVICES_TO_CHECK: ${PROJECT_PARENT_NAME}_${CI_PROJECT_NAME}-2



services:

  - docker:dind

.deploy-kf-3:

  variables: &deploy-kf-3-variables

    SERVICE_NAME: kf-3

    SERVICES_TO_CHECK: ${PROJECT_PARENT_NAME}_${CI_PROJECT_NAME}-3



.deploy-kf-1-development:

  extends: .deploy-development

  variables:

  DOCKER_DRIVER: overlay2

  STACK: confluent

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-1.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-1.dev.yml

    <<: *deploy-kf-1-variables



deploy-kf-1-supporting-branch-develop:

  stage: deploy

.deploy-kf-2-development:

  extends: .deploy-development

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '1'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - branches

  except:

    - master

  when: manual



deploy-kf-2-supporting-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '2'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - branches

  except:

    - master

  when: manual



deploy-kf-3-supporting-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '3'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - branches

  except:

    - master

  when: manual



deploy-kf-1-stable-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '1'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - master

  when: manual



deploy-kf-2-stable-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '2'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - master

  when: manual



deploy-kf-3-stable-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    INSTANCE_ID: '3'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${DEV_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

  environment:

    name: dev/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - master

  when: manual



deploy-kf-1-supporting-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '1'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro/${CI_PROJECT_NAME}-${INSTANCE_ID}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-2.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-2.dev.yml

    <<: *deploy-kf-2-variables



.deploy-kf-3-development:

  extends: .deploy-development

  variables:

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-3.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-3.dev.yml

    <<: *deploy-kf-3-variables



.deploy-kf-1-production:

  extends: .deploy-production

  variables:

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-1.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-1.prod.yml

    <<: *deploy-kf-1-variables



.deploy-kf-2-production:

  extends: .deploy-production

  variables:

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-2.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-2.prod.yml

    <<: *deploy-kf-2-variables



.deploy-kf-3-production:

  extends: .deploy-production

  variables:

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-3.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-3.prod.yml

    <<: *deploy-kf-3-variables



.deploy-support-branch: &deploy-support-branch

  only:

    - branches

  except:

    - master

  when: manual



deploy-kf-2-supporting-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '2'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro/${CI_PROJECT_NAME}-${INSTANCE_ID}

    - schedules



deploy-kf-1-support-branch-development:

  extends: .deploy-kf-1-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-support-branch



deploy-kf-2-support-branch-development:

  extends: .deploy-kf-2-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-support-branch



deploy-kf-3-support-branch-development:

  extends: .deploy-kf-3-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-support-branch



.deploy-stable-branch: &deploy-stable-branch

  only:

    - branches

  except:

    - master

  when: manual



deploy-kf-3-supporting-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '3'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - branches

  except:

    - master

  when: manual

    - schedules



deploy-kf-1-stable-branch-development:

  extends: .deploy-kf-1-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



deploy-kf-2-stable-branch-development:

  extends: .deploy-kf-2-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



deploy-kf-3-stable-branch-development:

  extends: .deploy-kf-3-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



deploy-kf-1-stable-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '1'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - master

  when: manual

  extends: .deploy-kf-1-production

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



deploy-kf-2-stable-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '2'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro/${CI_PROJECT_NAME}-${INSTANCE_ID}

  only:

    - master

  when: manual

  extends: .deploy-kf-2-production

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



deploy-kf-3-stable-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    INSTANCE_ID: '3'

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-${INSTANCE_ID}

    IMAGE_TAG: ${PRO_CONFLUENT_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-${INSTANCE_ID}.prod.yml

    AWS_REGION: ${AWS_REGION}

  script:

    - deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} AWS_REGION=${AWS_REGION}

  environment:

    name: pro/${CI_PROJECT_NAME}-${INSTANCE_ID}

  extends: .deploy-kf-3-production

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



.deploy-tag: &deploy-tag

  only:

    - master

  when: manual
 
    - tags



deploy-kf-1-tag-development:

  extends: .deploy-kf-1-development

  <<: *deploy-tag



deploy-kf-2-tag-development:

  extends: .deploy-kf-2-development

  <<: *deploy-tag



deploy-kf-3-tag-development:

  extends: .deploy-kf-3-development

  <<: *deploy-tag



deploy-kf-1-tag-production:

  extends: .deploy-kf-1-production

  <<: *deploy-tag



deploy-kf-2-tag-production:

  extends: .deploy-kf-2-production

  <<: *deploy-tag



deploy-kf-3-tag-production:

  extends: .deploy-kf-3-production

  <<: *deploy-tag

deploy/.env

+11 −2
Original line number Diff line number Diff line 
IMAGE_NAME=confluentinc/cp-kafka

IMAGE_TAG=latest



ZK_PORT=2181

KF_PORT=9092

KF_PORT_EXT=9093

JMX_HTTP_PORT=9998

KAFKA_JMX_PORT=9999



JMX_PROMETHEUS_JAVAAGENT_PATH=/jmx_prometheus_javaagent.jar

JMX_PROMETHEUS_CONFIG_PATH=/config.yaml



KAFKA_LOG_DIRS=/var/lib/kafka/data/data

KAFKA_LOG_CLEANER_ENABLE=false

KAFKA_LOG_CLEANUP_POLICY=compact

KAFKA_LOG_CLEANER_MIN_CLEANABLE_RATIO=0.5

KAFKA_LOG_CLEANER_MIN_COMPACTION_LAG_MS=3600000

KAFKA_LOG_ROLL_HOURS=168



KAFKA_NUM_PARTITIONS=5

KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=2

KAFKA_DEFAULT_REPLICATION_FACTOR=2
@@ -16,4 +23,6 @@ KAFKA_MIN_INSYNC_REPLICAS=1 
KAFKA_OFFSETS_RETENTION_MINUTES=10080

KAFKA_SOCKET_RECEIVE_BUFFER_BYTES=-1

KAFKA_MESSAGE_MAX_BYTES=15728640

LOG_LEVEL=ERROR
 
KAFKA_LOG4J_ROOT_LOGLEVEL=ERROR

KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT

KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT

deploy/docker-compose.kafka-1.dev.yml

+8 −17
Original line number Diff line number Diff line
@@ -3,27 +3,18 @@ version: '3.5' 
services:

  kafka-1:

    environment:

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://:${KF_PORT},EXT://${PUBLIC_HOSTNAME}:1${KF_PORT}

      - KAFKA_LISTENERS=PLAINTEXT://:${KF_PORT},EXT://0.0.0.0:${KF_PORT_EXT}

      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,EXT:PLAINTEXT

      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT

      - KAFKA_HEAP_OPTS=-Xmx1g -Xms1g

    ports:

      - target: ${KF_PORT_EXT}

        published: 1${KF_PORT}

        mode: host

      KAFKA_HEAP_OPTS: -Xmx1g -Xms1g

    deploy:

      mode: replicated

      replicas: 1

      restart_policy:

        delay: 30s

        window: 2m

      placement:

        constraints:

          - node.hostname == ${KF_1_PLACEMENT_CONSTRAINTS_NODE_HOSTNAME:-hostname}

      resources:

        limits:

          cpus: '1'

          memory: 2G

          cpus: '${RESOURCES_LIMITS_CPUS:-1}'

          memory: ${RESOURCES_LIMITS_MEMORY:-2G}

        reservations:

          memory: 1639M

          cpus: '${RESOURCES_RESERVATIONS_CPUS:-0.2}'

          memory: ${RESOURCES_RESERVATIONS_MEMORY:-1639M}



volumes:

  kf-data-vol:

deploy/docker-compose.kafka-1.prod.yml

+5 −11
Original line number Diff line number Diff line
@@ -3,25 +3,19 @@ version: '3.5' 
services:

  kafka-1:

    environment:

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://:${KF_PORT}

      - KAFKA_LISTENERS=PLAINTEXT://:${KF_PORT}

      - KAFKA_HEAP_OPTS=-Xmx2g -Xms2g

      KAFKA_HEAP_OPTS: -Xmx2g -Xms2g

    deploy:

      mode: replicated

      replicas: 1

      placement:

        constraints:

          - node.role == worker

          - engine.labels.availability_zone == ${AWS_REGION}a

      restart_policy:

        delay: 30s

        window: 2m

      resources:

        limits:

          cpus: '1'

          memory: 3G

          cpus: '${RESOURCES_LIMITS_CPUS:-1}'

          memory: ${RESOURCES_LIMITS_MEMORY:-3G}

        reservations:

          memory: 2458M

          cpus: '${RESOURCES_RESERVATIONS_CPUS:-0.2}'

          memory: ${RESOURCES_RESERVATIONS_MEMORY:-2458M}



volumes:

  kf-data-vol: