Commit 14d84e46 authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Merge branch 'dev' into 'master' 

Omite searchguard del backup y corrige restore

See merge request redmic-project/elastic/curator!6
parents 31c041df 25704d1b

.gitlab-ci.yml

+82 −8
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@ variables: 
services:

  - docker:dind



deploy-curator-delete-old-logs-supporting-branch-develop:

deploy-curator-supporting-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}
@@ -22,11 +22,21 @@ deploy-curator-delete-old-logs-supporting-branch-develop: 
    ELASTICSEARCH_URL: ${DEV_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${DEV_ELASTICSEARCH_USERNAME}:${DEV_ELASTICSEARCH_PASSWORD}

    DELETE_OLD_LOGS_DISABLE: 'False'

    SNAPSHOT_DISABLE: 'False'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'False'

    RESTORE_SNAPSHOT_DISABLE: 'True'

    LOG_INDEX_PREFIX: ${DEV_LOG_INDEX_PREFIX}

    DELETE_OLD_LOGS_DAYS_COUNT: ${DEV_DELETE_OLD_LOGS_DAYS_COUNT}

    SNAPSHOT_REPOSITORY: ${DEV_SNAPSHOT_REPOSITORY}

    DELETE_OLD_SNAPSHOTS_DAYS_COUNT: ${DEV_DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE} SNAPSHOT_DISABLE=${SNAPSHOT_DISABLE}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} RESTORE_SNAPSHOT_DISABLE=${RESTORE_SNAPSHOT_DISABLE}

      LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX} DELETE_OLD_LOGS_DAYS_COUNT=${DELETE_OLD_LOGS_DAYS_COUNT}

      SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY} DELETE_OLD_SNAPSHOTS_DAYS_COUNT=${DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  environment:

    name: dev

  only:
@@ -36,7 +46,7 @@ deploy-curator-delete-old-logs-supporting-branch-develop: 
    - schedules

  when: manual



deploy-curator-delete-old-logs-stable-branch-develop:

deploy-curator-stable-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}
@@ -44,11 +54,21 @@ deploy-curator-delete-old-logs-stable-branch-develop: 
    ELASTICSEARCH_URL: ${DEV_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${DEV_ELASTICSEARCH_USERNAME}:${DEV_ELASTICSEARCH_PASSWORD}

    DELETE_OLD_LOGS_DISABLE: 'False'

    SNAPSHOT_DISABLE: 'False'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'False'

    RESTORE_SNAPSHOT_DISABLE: 'True'

    LOG_INDEX_PREFIX: ${DEV_LOG_INDEX_PREFIX}

    DELETE_OLD_LOGS_DAYS_COUNT: ${DEV_DELETE_OLD_LOGS_DAYS_COUNT}

    SNAPSHOT_REPOSITORY: ${DEV_SNAPSHOT_REPOSITORY}

    DELETE_OLD_SNAPSHOTS_DAYS_COUNT: ${DEV_DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE} SNAPSHOT_DISABLE=${SNAPSHOT_DISABLE}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} RESTORE_SNAPSHOT_DISABLE=${RESTORE_SNAPSHOT_DISABLE}

      LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX} DELETE_OLD_LOGS_DAYS_COUNT=${DELETE_OLD_LOGS_DAYS_COUNT}

      SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY} DELETE_OLD_SNAPSHOTS_DAYS_COUNT=${DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  environment:

    name: dev

  only:
@@ -57,7 +77,7 @@ deploy-curator-delete-old-logs-stable-branch-develop: 
    - schedules

  when: manual



deploy-curator-delete-old-logs-supporting-branch-production:

deploy-curator-supporting-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}
@@ -65,11 +85,21 @@ deploy-curator-delete-old-logs-supporting-branch-production: 
    ELASTICSEARCH_URL: ${PRO_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${PRO_ELASTICSEARCH_USERNAME}:${PRO_ELASTICSEARCH_PASSWORD}

    DELETE_OLD_LOGS_DISABLE: 'False'

    SNAPSHOT_DISABLE: 'False'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'False'

    RESTORE_SNAPSHOT_DISABLE: 'True'

    LOG_INDEX_PREFIX: ${PRO_LOG_INDEX_PREFIX}

    DELETE_OLD_LOGS_DAYS_COUNT: ${PRO_DELETE_OLD_LOGS_DAYS_COUNT}

    SNAPSHOT_REPOSITORY: ${PRO_SNAPSHOT_REPOSITORY}

    DELETE_OLD_SNAPSHOTS_DAYS_COUNT: ${PRO_DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE} SNAPSHOT_DISABLE=${SNAPSHOT_DISABLE}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} RESTORE_SNAPSHOT_DISABLE=${RESTORE_SNAPSHOT_DISABLE}

      LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX} DELETE_OLD_LOGS_DAYS_COUNT=${DELETE_OLD_LOGS_DAYS_COUNT}

      SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY} DELETE_OLD_SNAPSHOTS_DAYS_COUNT=${DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  environment:

    name: pro

  only:
@@ -79,7 +109,7 @@ deploy-curator-delete-old-logs-supporting-branch-production: 
    - schedules

  when: manual



deploy-curator-delete-old-logs-stable-branch-production:

deploy-curator-stable-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}
@@ -87,11 +117,21 @@ deploy-curator-delete-old-logs-stable-branch-production: 
    ELASTICSEARCH_URL: ${PRO_ELASTICSEARCH_URL}

    ELASTICSEARCH_AUTH: ${PRO_ELASTICSEARCH_USERNAME}:${PRO_ELASTICSEARCH_PASSWORD}

    DELETE_OLD_LOGS_DISABLE: 'False'

    SNAPSHOT_DISABLE: 'False'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'False'

    RESTORE_SNAPSHOT_DISABLE: 'True'

    LOG_INDEX_PREFIX: ${PRO_LOG_INDEX_PREFIX}

    DELETE_OLD_LOGS_DAYS_COUNT: ${PRO_DELETE_OLD_LOGS_DAYS_COUNT}

    SNAPSHOT_REPOSITORY: ${PRO_SNAPSHOT_REPOSITORY}

    DELETE_OLD_SNAPSHOTS_DAYS_COUNT: ${PRO_DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      ELASTICSEARCH_URL=${ELASTICSEARCH_URL} ELASTICSEARCH_AUTH=${ELASTICSEARCH_AUTH}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE}

      DELETE_OLD_LOGS_DISABLE=${DELETE_OLD_LOGS_DISABLE} SNAPSHOT_DISABLE=${SNAPSHOT_DISABLE}

      DELETE_OLD_SNAPSHOTS_DISABLE=${DELETE_OLD_SNAPSHOTS_DISABLE} RESTORE_SNAPSHOT_DISABLE=${RESTORE_SNAPSHOT_DISABLE}

      LOG_INDEX_PREFIX=${LOG_INDEX_PREFIX} DELETE_OLD_LOGS_DAYS_COUNT=${DELETE_OLD_LOGS_DAYS_COUNT}

      SNAPSHOT_REPOSITORY=${SNAPSHOT_REPOSITORY} DELETE_OLD_SNAPSHOTS_DAYS_COUNT=${DELETE_OLD_SNAPSHOTS_DAYS_COUNT}

  environment:

    name: pro

  only:
@@ -117,3 +157,37 @@ scheduled-run-curator-stable-branch-develop: 
#  script: relaunch.sh

#  only:

#    - schedules



run-pipeline-curator-restore-snapshot-stable-branch-develop:

  stage: maintenance

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICE: elastic_${CI_PROJECT_NAME}

    DELETE_OLD_LOGS_DISABLE: 'True'

    SNAPSHOT_DISABLE: 'True'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'True'

    RESTORE_SNAPSHOT_DISABLE: 'False'

  script:

    - >

      relaunch.sh DELETE_OLD_LOGS_DISABLE SNAPSHOT_DISABLE DELETE_OLD_SNAPSHOTS_DISABLE RESTORE_SNAPSHOT_DISABLE

      SNAPSHOT_RESTORE_NAME SNAPSHOT_RESTORE_INDICES SNAPSHOT_REPOSITORY

  only:

    - web

  when: manual



run-pipeline-curator-restore-snapshot-stable-branch-production:

  stage: maintenance

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICE: elastic_${CI_PROJECT_NAME}

    DELETE_OLD_LOGS_DISABLE: 'True'

    SNAPSHOT_DISABLE: 'True'

    DELETE_OLD_SNAPSHOTS_DISABLE: 'True'

    RESTORE_SNAPSHOT_DISABLE: 'False'

  script:

    - >

      relaunch.sh DELETE_OLD_LOGS_DISABLE SNAPSHOT_DISABLE DELETE_OLD_SNAPSHOTS_DISABLE RESTORE_SNAPSHOT_DISABLE

      SNAPSHOT_RESTORE_NAME SNAPSHOT_RESTORE_INDICES SNAPSHOT_REPOSITORY

  only:

    - web

  when: manual

deploy/.env

+6 −0
Original line number Diff line number Diff line
@@ -3,4 +3,10 @@ ELASTICSEARCH_AUTH=user:pass 
ELASTICSEARCH_PORT=9200

CURATOR_TIMEOUT=60

DELETE_OLD_LOGS_DISABLE=False

SNAPSHOT_DISABLE=False

DELETE_OLD_SNAPSHOTS_DISABLE=False

RESTORE_SNAPSHOT_DISABLE=True

LOG_INDEX_PREFIX=fluentd-

DELETE_OLD_LOGS_DAYS_COUNT=30

SNAPSHOT_REPOSITORY=s3-backup

DELETE_OLD_SNAPSHOTS_DAYS_COUNT=90

deploy/config/actions.yml

+58 −7
Original line number Diff line number Diff line
@@ -3,23 +3,74 @@ actions: 
  1:

    action: delete_indices

    description: >-

      Delete indices older than ${DELETE_OLD_LOGS_DAYS_COUNT} (30 by default) days (based on index name), for 'fluentd-'

      prefixed indices. Ignore the error if the filter does not result in an actionable list of indices

      (ignore_empty_list) and exit cleanly.

      Delete indices older than 30 (by default) days (based on index name), for 'fluentd-' prefixed indices. Ignore

      the error if the filter does not result in an actionable list of indices (ignore_empty_list) and exit cleanly.

    options:

      ignore_empty_list: True

      timeout_override:

      continue_if_exception: False

      disable_action: ${DELETE_OLD_LOGS_DISABLE:False}

    filters:

    - filtertype: pattern

      kind: prefix

      value: fluentd-

      exclude:

      value: ${LOG_INDEX_PREFIX:fluentd-}

    - filtertype: age

      source: name

      direction: older

      timestring: '%Y%m%d'

      unit: days

      unit_count: ${DELETE_OLD_LOGS_DAYS_COUNT:30}

      exclude:
 


  2:

    action: snapshot

    description: >-

      Snapshot selected indices to repository ('s3-backup' by default) with automatic snapshot name.

      Exclude log indices.

    options:

      repository: ${SNAPSHOT_REPOSITORY:s3-backup}

      name: # default is 'curator-%Y%m%d%H%M%S'

      wait_for_completion: True

      max_wait: 3600

      wait_interval: 30

      disable_action: ${SNAPSHOT_DISABLE:False}

    filters:

    - filtertype: pattern

      kind: prefix

      value: ${LOG_INDEX_PREFIX:fluentd-}

      exclude: True

    - filtertype: pattern

      kind: prefix

      value: searchguard

      exclude: True



  3:

    action: delete_snapshots

    description: >-

      Delete old snapshots (90 by default) from repository ('s3-backup' by default).

    options:

      repository: ${SNAPSHOT_REPOSITORY:s3-backup}

      ignore_empty_list: True

      retry_interval: 120

      retry_count: 3

      disable_action: ${DELETE_OLD_SNAPSHOTS_DISABLE:False}

    filters:

    - filtertype: age

      source: name

      direction: older

      timestring: '%Y%m%d'

      unit: days

      unit_count: ${DELETE_OLD_SNAPSHOTS_DAYS_COUNT:90}



  4:

    action: restore

    description: >-

      Restore snapshot by name from repository ('s3-backup' by default). Disabled by default.

    options:

      repository: ${SNAPSHOT_REPOSITORY:s3-backup}

      name: ${SNAPSHOT_RESTORE_NAME}

      indices:

      - ${SNAPSHOT_RESTORE_INDICES}

      wait_interval: 30

      disable_action: ${RESTORE_SNAPSHOT_DISABLE:True}

    filters:

    - filtertype: state

      state: SUCCESS

deploy/docker-compose.tmpl.yml

+8 −0
Original line number Diff line number Diff line
@@ -10,7 +10,15 @@ services: 
      - ELASTICSEARCH_PORT

      - CURATOR_TIMEOUT

      - DELETE_OLD_LOGS_DISABLE

      - SNAPSHOT_DISABLE

      - DELETE_OLD_SNAPSHOTS_DISABLE

      - RESTORE_SNAPSHOT_DISABLE

      - LOG_INDEX_PREFIX

      - DELETE_OLD_LOGS_DAYS_COUNT

      - SNAPSHOT_REPOSITORY

      - DELETE_OLD_SNAPSHOTS_DAYS_COUNT

      - SNAPSHOT_RESTORE_NAME

      - SNAPSHOT_RESTORE_INDICES

    networks:

      - elastic-net

    configs: