Commit 3fdcab23 authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Merge branch 'dev' into 'master' 

dev

See merge request redmic-project/elastic/elasticsearch!47
parents bcd4983a fc66afd6

.dockerignore

+0 −2
Original line number Diff line number Diff line 
*
 
!config/**/*

!scripts/**/*

.gitignore

deleted100644 → 0
+0 −5
Original line number Diff line number Diff line 
.*

!.gitignore

!.gitlab-ci.yml

!.dockerignore

!.env

.gitlab-ci.yml

+241 −148
Original line number Diff line number Diff line 
image: docker:stable

include:

  - project: 'redmic-project/gitlab-ci-templates'

    ref: master

    file: '/packaging.yml'

  - project: 'redmic-project/gitlab-ci-templates'

    ref: master

    file: '/_deployment.yml'



stages:

  - pre-package

  - package

  - test-package

  - post-package

  - deploy



services:

  - docker:dind



variables:

  DOCKER_DRIVER: overlay2

  AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY}

  AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY}

  ES_PLUGINS: repository-s3



docker-build-commit-non-master-branches:

  stage: package

  variables:

    PARENT_IMAGE_NAME: registry.gitlab.com/redmic-project/docker/elasticsearch-xpack

    PARENT_IMAGE_TAG: latest

  only:

    - branches

  except:

    - master

  script:

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - >

      docker build --build-arg PARENT_IMAGE_NAME=${PARENT_IMAGE_NAME} --build-arg PARENT_IMAGE_TAG=${PARENT_IMAGE_TAG}

      -t ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} -t ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME}-latest .

    - docker push ${CI_REGISTRY_IMAGE}



docker-build-commit-master-branch:

  stage: package

  variables:

    PARENT_IMAGE_NAME: registry.gitlab.com/redmic-project/docker/elasticsearch-xpack

    PARENT_IMAGE_TAG: latest

  only:

    - master

  script:

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - >

      docker build --build-arg PARENT_IMAGE_NAME=${PARENT_IMAGE_NAME} --build-arg PARENT_IMAGE_TAG=${PARENT_IMAGE_TAG}

      -t ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} -t ${CI_REGISTRY_IMAGE}:latest .

    - docker push ${CI_REGISTRY_IMAGE}



docker-tag-already-built-image:

  stage: package

  only:

    - tags

  script:

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - docker tag ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} ${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}

    - docker push ${CI_REGISTRY_IMAGE}



container-scanning:

  stage: test-package

  allow_failure: true

  only:

    - branches

  script:

    - docker run -d --name db arminc/clair-db:latest

    - docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.1

    - apk add -U wget ca-certificates

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - docker pull ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}

    - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64

    - mv clair-scanner_linux_amd64 clair-scanner

    - chmod +x clair-scanner

    - touch clair-whitelist.yml

    - >

      ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-sast-container-report.json -l clair.log

      -w clair-whitelist.yml ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} || true

  artifacts:

    paths: [gl-sast-container-report.json]



deploy-supporting-branch:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    STACK: elastic

    SERVICES_TO_CHECK: elastic_${CI_PROJECT_NAME}-1

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-1.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-1.dev.yml

    OLD_ELASTIC_ADMIN_PASS: ${DEV_OLD_ELASTIC_ADMIN_PASS}

    ELASTIC_ADMIN_PASS: ${DEV_ELASTIC_ADMIN_PASS}

    ELASTIC_USER: ${DEV_ELASTIC_USER}

    ELASTIC_USER_PASS: ${DEV_ELASTIC_USER_PASS}

    ELASTIC_USER_ROLE: ${DEV_ELASTIC_USER_ROLE}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      OLD_ELASTIC_ADMIN_PASS=${OLD_ELASTIC_ADMIN_PASS} ELASTIC_ADMIN_PASS=${ELASTIC_ADMIN_PASS}

      ELASTIC_USER=${ELASTIC_USER} ELASTIC_USER_PASS=${ELASTIC_USER_PASS} ELASTIC_USER_ROLE=${ELASTIC_USER_ROLE}

      ES_PLUGINS=${ES_PLUGINS} AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}

      AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}

variables:

  PROJECT_PARENT_NAME: elastic



.docker-build:

  variables:

    COMPOSE_FILE_NAME: docker-compose.es6-1.tmpl.yml:docker-compose.es6-1.dev.yml



.deploy:

  variables:

    STACK: ${PROJECT_PARENT_NAME}

    DD_AWS_REGION: ${AWS_REGION}

  before_script:

    - mkdir -p deploy/certs deploy/config

    - echo "${CA_PEM}" > "deploy/certs/root-ca.pem"

    - echo "${CA_KEY}" > "deploy/certs/root-ca.key"

    - echo "${NODE_PEM}" > "deploy/certs/node.pem"

    - echo "${NODE_KEY}" > "deploy/certs/node.key"

    - echo "${NODE_CSR}" > "deploy/certs/node.csr"

    - echo "${ADMIN_PEM}" > "deploy/certs/admin.pem"

    - echo "${ADMIN_KEY}" > "deploy/certs/admin.key"

    - echo "${ADMIN_CSR}" > "deploy/certs/admin.csr"

    - echo "${SG_USERS}" > "deploy/config/sg_internal_users.yml"



.deploy-development:

  environment:

    name: dev

  only:

    - branches

  except:

    - master

  when: manual

    name: dev/${SERVICE_NAME}



deploy-pro:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    STACK: elastic

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    OLD_ELASTIC_ADMIN_PASS: ${PRO_OLD_ELASTIC_ADMIN_PASS}

    ELASTIC_ADMIN_PASS: ${PRO_ELASTIC_ADMIN_PASS}

    ELASTIC_USER: ${PRO_ELASTIC_USER}

    ELASTIC_USER_PASS: ${PRO_ELASTIC_USER_PASS}

    ELASTIC_USER_ROLE: ${PRO_ELASTIC_USER_ROLE}

  script:

    - create-nets.sh elastic-net

    - >

      export SERVICES_TO_CHECK=elastic_${CI_PROJECT_NAME}-1

        COMPOSE_FILE=docker-compose.${CI_PROJECT_NAME}-1.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-1.prod.yml

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      AWS_REGION=${AWS_REGION} OLD_ELASTIC_ADMIN_PASS=${OLD_ELASTIC_ADMIN_PASS}

      ELASTIC_ADMIN_PASS=${ELASTIC_ADMIN_PASS} ELASTIC_USER=${ELASTIC_USER}

      ELASTIC_USER_PASS=${ELASTIC_USER_PASS} ELASTIC_USER_ROLE=${ELASTIC_USER_ROLE}

      ES_PLUGINS=${ES_PLUGINS} AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}

      AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}

    - >

      export SERVICES_TO_CHECK=elastic_${CI_PROJECT_NAME}-2

        COMPOSE_FILE=docker-compose.${CI_PROJECT_NAME}-2.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-2.prod.yml

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      AWS_REGION=${AWS_REGION} OLD_ELASTIC_ADMIN_PASS=${OLD_ELASTIC_ADMIN_PASS}

      ELASTIC_ADMIN_PASS=${ELASTIC_ADMIN_PASS} ELASTIC_USER=${ELASTIC_USER}

      ELASTIC_USER_PASS=${ELASTIC_USER_PASS} ELASTIC_USER_ROLE=${ELASTIC_USER_ROLE}

      ES_PLUGINS=${ES_PLUGINS} AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}

      AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}

    - >

      export SERVICES_TO_CHECK=elastic_${CI_PROJECT_NAME}-3

        COMPOSE_FILE=docker-compose.${CI_PROJECT_NAME}-3.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-3.prod.yml

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      AWS_REGION=${AWS_REGION} OLD_ELASTIC_ADMIN_PASS=${OLD_ELASTIC_ADMIN_PASS}

      ELASTIC_ADMIN_PASS=${ELASTIC_ADMIN_PASS} ELASTIC_USER=${ELASTIC_USER}

      ELASTIC_USER_PASS=${ELASTIC_USER_PASS} ELASTIC_USER_ROLE=${ELASTIC_USER_ROLE}

      ES_PLUGINS=${ES_PLUGINS} AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}

      AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}

.deploy-production:

  environment:

    name: pro

  only:

    - master

    name: pro/${SERVICE_NAME}



.deploy-es6-1:

  variables: &deploy-es6-1-variables

    SERVICE_NAME: es6-1

    SERVICES_TO_CHECK: ${PROJECT_PARENT_NAME}_es6-1

    NODE_PEM: ${NODE_1_PEM}

    NODE_KEY: ${NODE_1_KEY}

    NODE_CSR: ${NODE_1_CSR}



.deploy-es6-2:

  variables: &deploy-es6-2-variables

    SERVICE_NAME: es6-2

    SERVICES_TO_CHECK: ${PROJECT_PARENT_NAME}_es6-2

    NODE_PEM: ${NODE_2_PEM}

    NODE_KEY: ${NODE_2_KEY}

    NODE_CSR: ${NODE_2_CSR}



.deploy-es6-3:

  variables: &deploy-es6-3-variables

    SERVICE_NAME: es6-3

    SERVICES_TO_CHECK: ${PROJECT_PARENT_NAME}_es6-3

    NODE_PEM: ${NODE_3_PEM}

    NODE_KEY: ${NODE_3_KEY}

    NODE_CSR: ${NODE_3_CSR}



.deploy-es6-1-development:

  extends: .deploy-development

  variables:

    COMPOSE_FILE: docker-compose.es6-1.tmpl.yml:docker-compose.es6-1.dev.yml

    <<: *deploy-es6-1-variables



.deploy-es6-2-development:

  extends: .deploy-development

  variables:

    COMPOSE_FILE: docker-compose.es6-2.tmpl.yml:docker-compose.es6-2.dev.yml

    <<: *deploy-es6-2-variables



.deploy-es6-3-development:

  extends: .deploy-development

  variables:

    COMPOSE_FILE: docker-compose.es6-3.tmpl.yml:docker-compose.es6-3.dev.yml

    <<: *deploy-es6-3-variables



.deploy-es6-1-production:

  extends: .deploy-production

  variables:

    COMPOSE_FILE: docker-compose.es6-1.tmpl.yml:docker-compose.es6-1.prod.yml

    <<: *deploy-es6-1-variables



.deploy-es6-2-production:

  extends: .deploy-production

  variables:

    COMPOSE_FILE: docker-compose.es6-2.tmpl.yml:docker-compose.es6-2.prod.yml

    <<: *deploy-es6-2-variables



.deploy-es6-3-production:

  extends: .deploy-production

  variables:

    COMPOSE_FILE: docker-compose.es6-3.tmpl.yml:docker-compose.es6-3.prod.yml

    <<: *deploy-es6-3-variables



.deploy-branch-base:

  variables: &deploy-branch-base-variables

    DD_IMAGE_NAME: ${CI_REGISTRY_IMAGE}/${CI_COMMIT_REF_SLUG}

    DD_IMAGE_TAG: ${CI_COMMIT_SHA}



.deploy-support-branch: &deploy-support-branch

  rules:

    - if: $CI_MERGE_REQUEST_ID ||

          $CI_COMMIT_TAG ||

          $CI_PIPELINE_SOURCE == "schedule" ||

          $CI_COMMIT_BRANCH == "master"

      when: never

    - if: $CI_COMMIT_BRANCH

      when: manual

      allow_failure: true



deploy-es6-1-support-branch-development:

  extends: .deploy-es6-1-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-support-branch



deploy-es6-2-support-branch-development:

  extends: .deploy-es6-2-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-support-branch



deploy-es6-3-support-branch-development:

  extends: .deploy-es6-3-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-support-branch



deploy-es6-1-support-branch-production:

  extends: .deploy-es6-1-production

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-support-branch



deploy-es6-2-support-branch-production:

  extends: .deploy-es6-2-production

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-support-branch



deploy-es6-3-support-branch-production:

  extends: .deploy-es6-3-production

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-support-branch



.deploy-stable-branch: &deploy-stable-branch

  rules:

    - if: $CI_MERGE_REQUEST_ID ||

          $CI_COMMIT_TAG ||

          $CI_PIPELINE_SOURCE == "schedule"

      when: never

    - if: $CI_COMMIT_BRANCH == "master"

      when: manual

      allow_failure: true



deploy-es6-1-stable-branch-development:

  extends: .deploy-es6-1-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



deploy-es6-2-stable-branch-development:

  extends: .deploy-es6-2-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



deploy-es6-3-stable-branch-development:

  extends: .deploy-es6-3-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



deploy-es6-1-stable-branch-production:

  extends: .deploy-es6-1-production

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



deploy-es6-2-stable-branch-production:

  extends: .deploy-es6-2-production

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



deploy-es6-3-stable-branch-production:

  extends: .deploy-es6-3-production

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



.deploy-tag-base:

  variables: &deploy-tag-base-variables

    DD_IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    DD_IMAGE_TAG: ${CI_COMMIT_TAG}



.deploy-tag: &deploy-tag

  rules:

    - if: $CI_COMMIT_TAG

      when: manual

      allow_failure: true



deploy-es6-1-tag-development:

  extends: .deploy-es6-1-development

  variables:

    <<: *deploy-tag-base-variables

  <<: *deploy-tag



deploy-es6-2-tag-development:

  extends: .deploy-es6-2-development

  variables:

    <<: *deploy-tag-base-variables

  <<: *deploy-tag



deploy-es6-3-tag-development:

  extends: .deploy-es6-3-development

  variables:

    <<: *deploy-tag-base-variables

  <<: *deploy-tag



deploy-es6-1-tag-production:

  extends: .deploy-es6-1-production

  variables:

    <<: *deploy-tag-base-variables

  <<: *deploy-tag



deploy-es6-2-tag-production:

  extends: .deploy-es6-2-production

  variables:

    <<: *deploy-tag-base-variables

  <<: *deploy-tag



deploy-es6-3-tag-production:

  extends: .deploy-es6-3-production

  variables:

    <<: *deploy-tag-base-variables

  <<: *deploy-tag

Dockerfile

+14 −27
Original line number Diff line number Diff line 
ARG PARENT_IMAGE_NAME

ARG PARENT_IMAGE_TAG

ARG PARENT_IMAGE_TAG="6.6.2"



FROM ${PARENT_IMAGE_NAME}:${PARENT_IMAGE_TAG}

FROM docker.elastic.co/elasticsearch/elasticsearch:${PARENT_IMAGE_TAG}



ENV ES_CLUSTER_NAME="clustername" \

	ES_NODE_NAME="nodename" \

	ES_NODE_MASTER="true" \

	ES_NODE_DATA="true" \

	ES_NODE_INGEST="true" \

	ES_BOOTSTRAP_MEMORY_LOCK="true" \

	ES_INDICES_QUERY_BOOL_MAX_CLAUSE_COUNT=30000 \

	ES_NETWORK_HOST="0.0.0.0" \

	ES_NETWORK_BIND_HOST="0.0.0.0" \

	ES_NETWORK_PUBLISH_HOST="_eth0_" \

	ES_DISCOVERY_ZEN_MINIMUM_MASTER_NODES=2 \

	ES_PATH="/usr/share/elasticsearch"

LABEL maintainer="info@redmic.es"



ENV ES_DATA_PATH="${ES_PATH}/data"

ARG ES_PATH="/usr/share/elasticsearch"



RUN apt-get update && \

	apt-get install -y --no-install-recommends \

		gettext-base \

		dnsutils && \

	ulimit -n 65536

ENV ES_PATH="${ES_PATH}" \

	cluster.name="clustername" \

	node.name="nodename" \

	path.data="${ES_PATH}/data" \

	bootstrap.memory_lock="true"



COPY config/ ${ES_PATH}/config/

COPY scripts/ /

ARG SEARCH_GUARD_VERSION="6.6.2-25.5"



VOLUME ["${ES_DATA_PATH}"]

RUN ulimit -n 65536 && \

	${ES_PATH}/bin/elasticsearch-plugin install --batch repository-s3 && \

	${ES_PATH}/bin/elasticsearch-plugin install --batch com.floragunn:search-guard-6:${SEARCH_GUARD_VERSION}



ENTRYPOINT ["/docker-entrypoint.sh"]



CMD ["elasticsearch"]
 
VOLUME [ "${ES_PATH}/data" ]

LICENSE

0 → 100644
+21 −0
Original line number Diff line number Diff line 
MIT License



Copyright (c) 2020 REDMIC Project / Elastic



Permission is hereby granted, free of charge, to any person obtaining a copy

of this software and associated documentation files (the "Software"), to deal

in the Software without restriction, including without limitation the rights

to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

copies of the Software, and to permit persons to whom the Software is

furnished to do so, subject to the following conditions:



The above copyright notice and this permission notice shall be included in all

copies or substantial portions of the Software.



THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

SOFTWARE.