Commit 62596f06 authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Merge branch 'dev' into 'master' 

Divide servicio en dos, separando restauración

See merge request redmic-project/elastic/curator!7
parents 14d84e46 e247facd

.gitlab-ci.yml

+52 −45
Original line number Diff line number Diff line
@@ -7,7 +7,6 @@ image: registry.gitlab.com/redmic-project/docker/docker-deploy:latest 
variables:

  DOCKER_DRIVER: overlay2

  STACK: elastic

  SERVICES_TO_CHECK: elastic_${CI_PROJECT_NAME}

  STATUS_CHECK_DELAY: 30

  IMAGE_TAG: ${CURATOR_IMAGE_TAG:-5.6.0}
@@ -18,13 +17,13 @@ deploy-curator-supporting-branch-develop: 
  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    COMPOSE_FILE: docker-compose.tmpl.yml:docker-compose.dev.yml

    SERVICES_TO_CHECK: elastic_curator

    COMPOSE_FILE: docker-compose.curator.tmpl.yml:docker-compose.curator.dev.yml

    ELASTICSEARCH_URL: ${DEV_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${DEV_ELASTICSEARCH_USERNAME}:${DEV_ELASTICSEARCH_PASSWORD}

    DELETE_OLD_LOGS_DISABLE: 'False'

    SNAPSHOT_DISABLE: 'False'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'False'

    RESTORE_SNAPSHOT_DISABLE: 'True'

    LOG_INDEX_PREFIX: ${DEV_LOG_INDEX_PREFIX}

    DELETE_OLD_LOGS_DAYS_COUNT: ${DEV_DELETE_OLD_LOGS_DAYS_COUNT}

    SNAPSHOT_REPOSITORY: ${DEV_SNAPSHOT_REPOSITORY}
@@ -34,11 +33,11 @@ deploy-curator-supporting-branch-develop: 
      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE} SNAPSHOT_DISABLE=${SNAPSHOT_DISABLE}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} RESTORE_SNAPSHOT_DISABLE=${RESTORE_SNAPSHOT_DISABLE}

      LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX} DELETE_OLD_LOGS_DAYS_COUNT=${DELETE_OLD_LOGS_DAYS_COUNT}

      SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY} DELETE_OLD_SNAPSHOTS_DAYS_COUNT=${DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX}

      DELETE_OLD_LOGS_DAYS_COUNT=${DELETE_OLD_LOGS_DAYS_COUNT} SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY}

      DELETE_OLD_SNAPSHOTS_DAYS_COUNT=${DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  environment:

    name: dev

    name: dev/curator

  only:

    - branches

  except:
@@ -50,13 +49,13 @@ deploy-curator-stable-branch-develop: 
  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    COMPOSE_FILE: docker-compose.tmpl.yml:docker-compose.dev.yml

    SERVICES_TO_CHECK: elastic_curator

    COMPOSE_FILE: docker-compose.curator.tmpl.yml:docker-compose.curator.dev.yml

    ELASTICSEARCH_URL: ${DEV_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${DEV_ELASTICSEARCH_USERNAME}:${DEV_ELASTICSEARCH_PASSWORD}

    DELETE_OLD_LOGS_DISABLE: 'False'

    SNAPSHOT_DISABLE: 'False'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'False'

    RESTORE_SNAPSHOT_DISABLE: 'True'

    LOG_INDEX_PREFIX: ${DEV_LOG_INDEX_PREFIX}

    DELETE_OLD_LOGS_DAYS_COUNT: ${DEV_DELETE_OLD_LOGS_DAYS_COUNT}

    SNAPSHOT_REPOSITORY: ${DEV_SNAPSHOT_REPOSITORY}
@@ -66,11 +65,11 @@ deploy-curator-stable-branch-develop: 
      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE} SNAPSHOT_DISABLE=${SNAPSHOT_DISABLE}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} RESTORE_SNAPSHOT_DISABLE=${RESTORE_SNAPSHOT_DISABLE}

      LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX} DELETE_OLD_LOGS_DAYS_COUNT=${DELETE_OLD_LOGS_DAYS_COUNT}

      SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY} DELETE_OLD_SNAPSHOTS_DAYS_COUNT=${DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX}

      DELETE_OLD_LOGS_DAYS_COUNT=${DELETE_OLD_LOGS_DAYS_COUNT} SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY}

      DELETE_OLD_SNAPSHOTS_DAYS_COUNT=${DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  environment:

    name: dev

    name: dev/curator

  only:

    - master

  except:
@@ -81,13 +80,13 @@ deploy-curator-supporting-branch-production: 
  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    COMPOSE_FILE: docker-compose.tmpl.yml:docker-compose.prod.yml

    SERVICES_TO_CHECK: elastic_curator

    COMPOSE_FILE: docker-compose.curator.tmpl.yml:docker-compose.curator.prod.yml

    ELASTICSEARCH_URL: ${PRO_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${PRO_ELASTICSEARCH_USERNAME}:${PRO_ELASTICSEARCH_PASSWORD}

    DELETE_OLD_LOGS_DISABLE: 'False'

    SNAPSHOT_DISABLE: 'False'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'False'

    RESTORE_SNAPSHOT_DISABLE: 'True'

    LOG_INDEX_PREFIX: ${PRO_LOG_INDEX_PREFIX}

    DELETE_OLD_LOGS_DAYS_COUNT: ${PRO_DELETE_OLD_LOGS_DAYS_COUNT}

    SNAPSHOT_REPOSITORY: ${PRO_SNAPSHOT_REPOSITORY}
@@ -97,11 +96,11 @@ deploy-curator-supporting-branch-production: 
      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE} SNAPSHOT_DISABLE=${SNAPSHOT_DISABLE}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} RESTORE_SNAPSHOT_DISABLE=${RESTORE_SNAPSHOT_DISABLE}

      LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX} DELETE_OLD_LOGS_DAYS_COUNT=${DELETE_OLD_LOGS_DAYS_COUNT}

      SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY} DELETE_OLD_SNAPSHOTS_DAYS_COUNT=${DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX}

      DELETE_OLD_LOGS_DAYS_COUNT=${DELETE_OLD_LOGS_DAYS_COUNT} SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY}

      DELETE_OLD_SNAPSHOTS_DAYS_COUNT=${DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  environment:

    name: pro

    name: pro/curator

  only:

    - branches

  except:
@@ -113,13 +112,13 @@ deploy-curator-stable-branch-production: 
  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    COMPOSE_FILE: docker-compose.tmpl.yml:docker-compose.prod.yml

    SERVICES_TO_CHECK: elastic_curator

    COMPOSE_FILE: docker-compose.curator.tmpl.yml:docker-compose.curator.prod.yml

    ELASTICSEARCH_URL: ${PRO_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${PRO_ELASTICSEARCH_USERNAME}:${PRO_ELASTICSEARCH_PASSWORD}

    DELETE_OLD_LOGS_DISABLE: 'False'

    SNAPSHOT_DISABLE: 'False'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'False'

    RESTORE_SNAPSHOT_DISABLE: 'True'

    LOG_INDEX_PREFIX: ${PRO_LOG_INDEX_PREFIX}

    DELETE_OLD_LOGS_DAYS_COUNT: ${PRO_DELETE_OLD_LOGS_DAYS_COUNT}

    SNAPSHOT_REPOSITORY: ${PRO_SNAPSHOT_REPOSITORY}
@@ -129,11 +128,11 @@ deploy-curator-stable-branch-production: 
      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE} SNAPSHOT_DISABLE=${SNAPSHOT_DISABLE}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} RESTORE_SNAPSHOT_DISABLE=${RESTORE_SNAPSHOT_DISABLE}

      LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX} DELETE_OLD_LOGS_DAYS_COUNT=${DELETE_OLD_LOGS_DAYS_COUNT}

      SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY} DELETE_OLD_SNAPSHOTS_DAYS_COUNT=${DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX}

      DELETE_OLD_LOGS_DAYS_COUNT=${DELETE_OLD_LOGS_DAYS_COUNT} SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY}

      DELETE_OLD_SNAPSHOTS_DAYS_COUNT=${DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  environment:

    name: pro

    name: pro/curator

  only:

    - master

  except:
@@ -144,7 +143,7 @@ scheduled-run-curator-stable-branch-develop: 
  stage: maintenance

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICE: elastic_${CI_PROJECT_NAME}

    SERVICE: elastic_curator

  script: relaunch.sh

  only:

    - schedules
@@ -153,41 +152,49 @@ scheduled-run-curator-stable-branch-develop: 
#  stage: maintenance

#  variables:

#    SSH_REMOTE: ${PRO_SSH_REMOTE}

#    SERVICE: elastic_${CI_PROJECT_NAME}

#    SERVICE: elastic_curator

#  script: relaunch.sh

#  only:

#    - schedules



run-pipeline-curator-restore-snapshot-stable-branch-develop:

  stage: maintenance

deploy-curator-restore-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICE: elastic_${CI_PROJECT_NAME}

    DELETE_OLD_LOGS_DISABLE: 'True'

    SNAPSHOT_DISABLE: 'True'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'True'

    RESTORE_SNAPSHOT_DISABLE: 'False'

    SERVICES_TO_CHECK: elastic_curator-restore

    COMPOSE_FILE: docker-compose.curator-restore.tmpl.yml:docker-compose.curator-restore.dev.yml

    ELASTICSEARCH_URL: ${DEV_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${DEV_ELASTICSEARCH_USERNAME}:${DEV_ELASTICSEARCH_PASSWORD}

    SNAPSHOT_REPOSITORY: ${DEV_SNAPSHOT_REPOSITORY}

  script:

    - >

      relaunch.sh DELETE_OLD_LOGS_DISABLE SNAPSHOT_DISABLE DELETE_OLD_SNAPSHOTS_DISABLE RESTORE_SNAPSHOT_DISABLE

      SNAPSHOT_RESTORE_NAME SNAPSHOT_RESTORE_INDICES SNAPSHOT_REPOSITORY

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY} SNAPSHOT_RESTORE_NAME=${SNAPSHOT_RESTORE_NAME}

      SNAPSHOT_RESTORE_INDICES=${SNAPSHOT_RESTORE_INDICES}

  environment:

    name: dev/curator-restore

  only:

    - web

  when: manual



run-pipeline-curator-restore-snapshot-stable-branch-production:

  stage: maintenance

deploy-curator-restore-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICE: elastic_${CI_PROJECT_NAME}

    DELETE_OLD_LOGS_DISABLE: 'True'

    SNAPSHOT_DISABLE: 'True'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'True'

    RESTORE_SNAPSHOT_DISABLE: 'False'

    SERVICES_TO_CHECK: elastic_curator-restore

    COMPOSE_FILE: docker-compose.curator-restore.tmpl.yml:docker-compose.curator-restore.prod.yml

    ELASTICSEARCH_URL: ${PRO_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${PRO_ELASTICSEARCH_USERNAME}:${PRO_ELASTICSEARCH_PASSWORD}

    SNAPSHOT_REPOSITORY: ${PRO_SNAPSHOT_REPOSITORY}

  script:

    - >

      relaunch.sh DELETE_OLD_LOGS_DISABLE SNAPSHOT_DISABLE DELETE_OLD_SNAPSHOTS_DISABLE RESTORE_SNAPSHOT_DISABLE

      SNAPSHOT_RESTORE_NAME SNAPSHOT_RESTORE_INDICES SNAPSHOT_REPOSITORY

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY} SNAPSHOT_RESTORE_NAME=${SNAPSHOT_RESTORE_NAME}

      SNAPSHOT_RESTORE_INDICES=${SNAPSHOT_RESTORE_INDICES}

  environment:

    name: pro/curator-restore

  only:

    - web

  when: manual

deploy/.env

+2 −4
Original line number Diff line number Diff line
@@ -2,11 +2,9 @@ ELASTICSEARCH_URL=http://elasticsearch 
ELASTICSEARCH_AUTH=user:pass

ELASTICSEARCH_PORT=9200

CURATOR_TIMEOUT=60

DELETE_OLD_LOGS_DISABLE=False

SNAPSHOT_DISABLE=False

DELETE_OLD_SNAPSHOTS_DISABLE=False

RESTORE_SNAPSHOT_DISABLE=True

LOG_INDEX_PREFIX=fluentd-

DELETE_OLD_LOGS_DAYS_COUNT=30

SNAPSHOT_REPOSITORY=s3-backup

DELETE_OLD_SNAPSHOTS_DAYS_COUNT=90

SNAPSHOT_RESTORE_NAME=

SNAPSHOT_RESTORE_INDICES=index.name

deploy/docker-compose.curator-restore.prod.yml

0 → 100644
+8 −0
Original line number Diff line number Diff line 
version: '3.5'



services:

  curator-restore:

    deploy:

      placement:

        constraints:

          - node.role == worker

deploy/docker-compose.curator-restore.tmpl.yml

0 → 100644
+50 −0
Original line number Diff line number Diff line 
version: '3.5'



services:

  curator-restore:

    image: bobrik/curator:${IMAGE_TAG:-latest}

    command: /.curator/actions.yml

    environment:

      - ELASTICSEARCH_URL

      - ELASTICSEARCH_AUTH

      - ELASTICSEARCH_PORT

      - CURATOR_TIMEOUT

      - SNAPSHOT_REPOSITORY

      - SNAPSHOT_RESTORE_NAME

      - SNAPSHOT_RESTORE_INDICES

      - DELETE_OLD_LOGS_DISABLE=True

      - SNAPSHOT_DISABLE=True

      - DELETE_OLD_SNAPSHOTS_DISABLE=True

      - RESTORE_SNAPSHOT_DISABLE=False

    networks:

      - elastic-net

    configs:

      - source: curator-config

        target: /.curator/curator.yml

      - source: curator-actions

        target: /.curator/actions.yml

    deploy:

      mode: replicated

      replicas: 1

      restart_policy:

        condition: none

      resources:

        limits:

          cpus: '0.1'

          memory: 128M

        reservations:

          memory: 103M



networks:

  elastic-net:

    name: ${ELASTIC_NET_NAME:-elastic6-net}

    external: true



configs:

  curator-config:

    name: ${CURATOR_CONFIG_NAME:-curator-config}

    file: ./config/curator.yml



  curator-actions:

    name: ${CURATOR_ACTIONS_NAME:-curator-actions}

    file: ./config/actions.yml