Commit f78f5b21 authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Merge branch 'dev' into 'master' 

Habilita mantenimiento en prod, permite paso vars.

See merge request redmic-project/elastic/curator!9
parents 22c7e119 b01d9d65

.gitlab-ci.yml

+61 −49
Original line number Diff line number Diff line
@@ -2,13 +2,15 @@ stages: 
  - deploy

  - maintenance



image: registry.gitlab.com/redmic-project/docker/docker-deploy:latest

image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}



variables:

  DOCKER_DRIVER: overlay2

  STACK: elastic

  STATUS_CHECK_DELAY: 30

  IMAGE_TAG: ${CURATOR_IMAGE_TAG:-5.6.0}

  ELASTIC_NET_NAME: elastic6-net

  SERVICE: elastic_curator



services:

  - docker:dind
@@ -19,18 +21,13 @@ deploy-curator-supporting-branch-develop: 
    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: elastic_curator

    COMPOSE_FILE: docker-compose.curator.tmpl.yml:docker-compose.curator.dev.yml

    ELASTICSEARCH_URL: ${DEV_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${DEV_ELASTICSEARCH_USERNAME}:${DEV_ELASTICSEARCH_PASSWORD}

    ELASTICSEARCH_AUTH: ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}

    DELETE_OLD_LOGS_DISABLE: 'False'

    SNAPSHOT_DISABLE: 'False'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'False'

    LOG_INDEX_PREFIX: ${DEV_LOG_INDEX_PREFIX}

    DELETE_OLD_LOGS_DAYS_COUNT: ${DEV_DELETE_OLD_LOGS_DAYS_COUNT}

    SNAPSHOT_REPOSITORY: ${DEV_SNAPSHOT_REPOSITORY}

    DELETE_OLD_SNAPSHOTS_DAYS_COUNT: ${DEV_DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE} SNAPSHOT_DISABLE=${SNAPSHOT_DISABLE}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX}
@@ -52,18 +49,13 @@ deploy-curator-stable-branch-develop: 
    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: elastic_curator

    COMPOSE_FILE: docker-compose.curator.tmpl.yml:docker-compose.curator.dev.yml

    ELASTICSEARCH_URL: ${DEV_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${DEV_ELASTICSEARCH_USERNAME}:${DEV_ELASTICSEARCH_PASSWORD}

    ELASTICSEARCH_AUTH: ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}

    DELETE_OLD_LOGS_DISABLE: 'False'

    SNAPSHOT_DISABLE: 'False'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'False'

    LOG_INDEX_PREFIX: ${DEV_LOG_INDEX_PREFIX}

    DELETE_OLD_LOGS_DAYS_COUNT: ${DEV_DELETE_OLD_LOGS_DAYS_COUNT}

    SNAPSHOT_REPOSITORY: ${DEV_SNAPSHOT_REPOSITORY}

    DELETE_OLD_SNAPSHOTS_DAYS_COUNT: ${DEV_DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE} SNAPSHOT_DISABLE=${SNAPSHOT_DISABLE}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX}
@@ -84,18 +76,13 @@ deploy-curator-supporting-branch-production: 
    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: elastic_curator

    COMPOSE_FILE: docker-compose.curator.tmpl.yml:docker-compose.curator.prod.yml

    ELASTICSEARCH_URL: ${PRO_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${PRO_ELASTICSEARCH_USERNAME}:${PRO_ELASTICSEARCH_PASSWORD}

    ELASTICSEARCH_AUTH: ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}

    DELETE_OLD_LOGS_DISABLE: 'False'

    SNAPSHOT_DISABLE: 'False'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'False'

    LOG_INDEX_PREFIX: ${PRO_LOG_INDEX_PREFIX}

    DELETE_OLD_LOGS_DAYS_COUNT: ${PRO_DELETE_OLD_LOGS_DAYS_COUNT}

    SNAPSHOT_REPOSITORY: ${PRO_SNAPSHOT_REPOSITORY}

    DELETE_OLD_SNAPSHOTS_DAYS_COUNT: ${PRO_DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE} SNAPSHOT_DISABLE=${SNAPSHOT_DISABLE}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX}
@@ -117,18 +104,13 @@ deploy-curator-stable-branch-production: 
    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: elastic_curator

    COMPOSE_FILE: docker-compose.curator.tmpl.yml:docker-compose.curator.prod.yml

    ELASTICSEARCH_URL: ${PRO_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${PRO_ELASTICSEARCH_USERNAME}:${PRO_ELASTICSEARCH_PASSWORD}

    ELASTICSEARCH_AUTH: ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}

    DELETE_OLD_LOGS_DISABLE: 'False'

    SNAPSHOT_DISABLE: 'False'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'False'

    LOG_INDEX_PREFIX: ${PRO_LOG_INDEX_PREFIX}

    DELETE_OLD_LOGS_DAYS_COUNT: ${PRO_DELETE_OLD_LOGS_DAYS_COUNT}

    SNAPSHOT_REPOSITORY: ${PRO_SNAPSHOT_REPOSITORY}

    DELETE_OLD_SNAPSHOTS_DAYS_COUNT: ${PRO_DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE} SNAPSHOT_DISABLE=${SNAPSHOT_DISABLE}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX}
@@ -143,23 +125,31 @@ deploy-curator-stable-branch-production: 
    - web

  when: manual



scheduled-run-curator-stable-branch-develop:

scheduled-run-curator-develop:

  stage: maintenance

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICE: elastic_curator

  script: relaunch.sh

  only:

    - schedules



#scheduled-run-curator-stable-branch-production:

#  stage: maintenance

#  variables:

#    SSH_REMOTE: ${PRO_SSH_REMOTE}

#    SERVICE: elastic_curator

#  script: relaunch.sh

#  only:

#    - schedules

scheduled-run-curator-production:

  stage: maintenance

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

  script: relaunch.sh

  only:

    - schedules



# El siguiente job es temporal, hasta que se unifiquen elasticsearch v5 y v6

scheduled-run-old-curator-production:

  stage: maintenance

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICE: elastic_old-curator

  script: relaunch.sh

  only:

    - schedules



deploy-curator-restore-develop:

  stage: deploy
@@ -167,15 +157,13 @@ deploy-curator-restore-develop: 
    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: elastic_curator-restore

    COMPOSE_FILE: docker-compose.curator-restore.tmpl.yml:docker-compose.curator-restore.dev.yml

    ELASTICSEARCH_URL: ${DEV_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${DEV_ELASTICSEARCH_USERNAME}:${DEV_ELASTICSEARCH_PASSWORD}

    SNAPSHOT_REPOSITORY: ${DEV_SNAPSHOT_REPOSITORY}

    ELASTICSEARCH_AUTH: ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY} SNAPSHOT_RESTORE_NAME=${SNAPSHOT_RESTORE_NAME}

      SNAPSHOT_RESTORE_INDICES=${SNAPSHOT_RESTORE_INDICES}

      SNAPSHOT_RESTORE_INDICES=${SNAPSHOT_RESTORE_INDICES} ELASTIC_NET_NAME=${ELASTIC_NET_NAME}

  environment:

    name: dev/curator-restore

  only:
@@ -188,17 +176,41 @@ deploy-curator-restore-production: 
    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: elastic_curator-restore

    COMPOSE_FILE: docker-compose.curator-restore.tmpl.yml:docker-compose.curator-restore.prod.yml

    ELASTICSEARCH_URL: ${PRO_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${PRO_ELASTICSEARCH_USERNAME}:${PRO_ELASTICSEARCH_PASSWORD}

    SNAPSHOT_REPOSITORY: ${PRO_SNAPSHOT_REPOSITORY}

    ELASTICSEARCH_AUTH: ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY} SNAPSHOT_RESTORE_NAME=${SNAPSHOT_RESTORE_NAME}

      SNAPSHOT_RESTORE_INDICES=${SNAPSHOT_RESTORE_INDICES}

      SNAPSHOT_RESTORE_INDICES=${SNAPSHOT_RESTORE_INDICES} ELASTIC_NET_NAME=${ELASTIC_NET_NAME}

  environment:

    name: pro/curator-restore

  only:

    - web

  when: manual



# El siguiente job es temporal, hasta que se unifiquen elasticsearch v5 y v6

deploy-old-curator-supporting-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: elastic_old-curator

    COMPOSE_FILE: docker-compose.old-curator.tmpl.yml:docker-compose.old-curator.prod.yml

    ELASTICSEARCH_URL: http://es-1

    SNAPSHOT_DISABLE: 'False'

    DELETE_OLD_LOGS_DISABLE: 'True'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'True'

  script:

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} ELASTICSEARCH_URL=${ELASTICSEARCH_URL}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE} SNAPSHOT_DISABLE=${SNAPSHOT_DISABLE}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY}

  environment:

    name: pro/old-curator

  only:

    - branches

  except:

    - master

    - schedules

    - web

  when: manual

deploy/.env

+5 −0
Original line number Diff line number Diff line
@@ -8,3 +8,8 @@ SNAPSHOT_REPOSITORY=s3-backup 
DELETE_OLD_SNAPSHOTS_DAYS_COUNT=90

SNAPSHOT_RESTORE_NAME=

SNAPSHOT_RESTORE_INDICES=index.name

SNAPSHOT_RESTORE_INCLUDE_ALIASES=True

DELETE_OLD_LOGS_DISABLE=False

SNAPSHOT_DISABLE=False

DELETE_OLD_SNAPSHOTS_DISABLE=False

RESTORE_SNAPSHOT_DISABLE=True

deploy/config/actions.yml

+1 −0
Original line number Diff line number Diff line
@@ -69,6 +69,7 @@ actions: 
      name: ${SNAPSHOT_RESTORE_NAME}

      indices:

      - ${SNAPSHOT_RESTORE_INDICES}

      include_aliases: ${SNAPSHOT_RESTORE_INCLUDE_ALIASES:True}

      wait_interval: 30

      disable_action: ${RESTORE_SNAPSHOT_DISABLE:True}

    filters:

deploy/docker-compose.curator-restore.tmpl.yml

+3 −2
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ services: 
      - SNAPSHOT_REPOSITORY

      - SNAPSHOT_RESTORE_NAME

      - SNAPSHOT_RESTORE_INDICES

      - SNAPSHOT_RESTORE_INCLUDE_ALIASES

      - DELETE_OLD_LOGS_DISABLE=True

      - SNAPSHOT_DISABLE=True

      - DELETE_OLD_SNAPSHOTS_DISABLE=True
@@ -31,9 +32,9 @@ services: 
      resources:

        limits:

          cpus: '0.1'

          memory: 128M

          memory: 64M

        reservations:

          memory: 103M

          memory: 32M



networks:

  elastic-net:

deploy/docker-compose.curator.tmpl.yml

+6 −6
Original line number Diff line number Diff line
@@ -13,10 +13,10 @@ services: 
      - DELETE_OLD_LOGS_DAYS_COUNT

      - SNAPSHOT_REPOSITORY

      - DELETE_OLD_SNAPSHOTS_DAYS_COUNT

      - DELETE_OLD_LOGS_DISABLE=False

      - SNAPSHOT_DISABLE=False

      - DELETE_OLD_SNAPSHOTS_DISABLE=False

      - RESTORE_SNAPSHOT_DISABLE=True

      - DELETE_OLD_LOGS_DISABLE

      - SNAPSHOT_DISABLE

      - DELETE_OLD_SNAPSHOTS_DISABLE

      - RESTORE_SNAPSHOT_DISABLE

    networks:

      - elastic-net

    configs:
@@ -34,9 +34,9 @@ services: 
      resources:

        limits:

          cpus: '0.1'

          memory: 128M

          memory: 64M

        reservations:

          memory: 103M

          memory: 32M



networks:

  elastic-net: