Commit 7e36f819 authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Prepara la configuración y despliegue para v8 

Actualiza variables usadas para adaptarse a la versión actual de
elasticsearch.

Emplea imagen Docker oficial, en lugar de construir una propia. Ya no es
necesario porque no se necesitan los plugins que se usaban en versiones
anteriores.

Unifica en un mismo servicio replicado las 3 instancias.

Actualiza definición de ficheros compose y ci.
parent 2c5c003d

.dockerignore

deleted100644 → 0
+0 −1
Original line number Diff line number Diff line 
*

.gitlab-ci.yml

+2 −240
Original line number Diff line number Diff line 
include:

  - project: 'redmic-project/gitlab-ci-templates'

    ref: master

    file: '/packaging.yml'

  - project: 'redmic-project/gitlab-ci-templates'

    ref: master

    file: '/_deployment.yml'

    file: '/deployment-service/docker-deploy.yml'



stages:

  - pre-package

  - package

  - post-package

  - deploy



variables:

  PROJECT_PARENT_NAME: elastic



.docker-build:

  variables:

    COMPOSE_FILE_NAME: docker-compose.es6-1.tmpl.yml:docker-compose.es6-1.dev.yml



.deploy:

  variables:

    STACK: ${PROJECT_PARENT_NAME}

    DD_AWS_REGION: ${AWS_REGION}

  before_script:

    - mkdir -p deploy/certs deploy/config

    - echo "${CA_PEM}" > "deploy/certs/root-ca.pem"

    - echo "${CA_KEY}" > "deploy/certs/root-ca.key"

    - echo "${NODE_PEM}" > "deploy/certs/node.pem"

    - echo "${NODE_KEY}" > "deploy/certs/node.key"

    - echo "${NODE_CSR}" > "deploy/certs/node.csr"

    - echo "${ADMIN_PEM}" > "deploy/certs/admin.pem"

    - echo "${ADMIN_KEY}" > "deploy/certs/admin.key"

    - echo "${ADMIN_CSR}" > "deploy/certs/admin.csr"

    - echo "${SG_USERS}" > "deploy/config/sg_internal_users.yml"



.deploy-development:

  environment:

    name: dev/${SERVICE_NAME}



.deploy-production:

  environment:

    name: pro/${SERVICE_NAME}



.deploy-es6-1:

  variables: &deploy-es6-1-variables

    SERVICE_NAME: es6-1

    SERVICES_TO_CHECK: ${PROJECT_PARENT_NAME}_es6-1

    NODE_PEM: ${NODE_1_PEM}

    NODE_KEY: ${NODE_1_KEY}

    NODE_CSR: ${NODE_1_CSR}



.deploy-es6-2:

  variables: &deploy-es6-2-variables

    SERVICE_NAME: es6-2

    SERVICES_TO_CHECK: ${PROJECT_PARENT_NAME}_es6-2

    NODE_PEM: ${NODE_2_PEM}

    NODE_KEY: ${NODE_2_KEY}

    NODE_CSR: ${NODE_2_CSR}



.deploy-es6-3:

  variables: &deploy-es6-3-variables

    SERVICE_NAME: es6-3

    SERVICES_TO_CHECK: ${PROJECT_PARENT_NAME}_es6-3

    NODE_PEM: ${NODE_3_PEM}

    NODE_KEY: ${NODE_3_KEY}

    NODE_CSR: ${NODE_3_CSR}



.deploy-es6-1-development:

  extends: .deploy-development

  variables:

    COMPOSE_FILE: docker-compose.es6-1.tmpl.yml:docker-compose.es6-1.dev.yml

    <<: *deploy-es6-1-variables



.deploy-es6-2-development:

  extends: .deploy-development

  variables:

    COMPOSE_FILE: docker-compose.es6-2.tmpl.yml:docker-compose.es6-2.dev.yml

    <<: *deploy-es6-2-variables



.deploy-es6-3-development:

  extends: .deploy-development

  variables:

    COMPOSE_FILE: docker-compose.es6-3.tmpl.yml:docker-compose.es6-3.dev.yml

    <<: *deploy-es6-3-variables



.deploy-es6-1-production:

  extends: .deploy-production

  variables:

    COMPOSE_FILE: docker-compose.es6-1.tmpl.yml:docker-compose.es6-1.prod.yml

    <<: *deploy-es6-1-variables



.deploy-es6-2-production:

  extends: .deploy-production

  variables:

    COMPOSE_FILE: docker-compose.es6-2.tmpl.yml:docker-compose.es6-2.prod.yml

    <<: *deploy-es6-2-variables



.deploy-es6-3-production:

  extends: .deploy-production

  variables:

    COMPOSE_FILE: docker-compose.es6-3.tmpl.yml:docker-compose.es6-3.prod.yml

    <<: *deploy-es6-3-variables



.deploy-branch-base:

  variables: &deploy-branch-base-variables

    DD_IMAGE_NAME: ${CI_REGISTRY_IMAGE}/${CI_COMMIT_REF_SLUG}

    DD_IMAGE_TAG: ${CI_COMMIT_SHA}



.deploy-support-branch: &deploy-support-branch

  rules:

    - if: $CI_MERGE_REQUEST_ID ||

          $CI_COMMIT_TAG ||

          $CI_PIPELINE_SOURCE == "schedule" ||

          $CI_COMMIT_BRANCH == "master"

      when: never

    - if: $CI_COMMIT_BRANCH

      when: manual

      allow_failure: true



deploy-es6-1-support-branch-development:

  extends: .deploy-es6-1-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-support-branch



deploy-es6-2-support-branch-development:

  extends: .deploy-es6-2-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-support-branch



deploy-es6-3-support-branch-development:

  extends: .deploy-es6-3-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-support-branch



deploy-es6-1-support-branch-production:

  extends: .deploy-es6-1-production

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-support-branch



deploy-es6-2-support-branch-production:

  extends: .deploy-es6-2-production

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-support-branch



deploy-es6-3-support-branch-production:

  extends: .deploy-es6-3-production

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-support-branch



.deploy-stable-branch: &deploy-stable-branch

  rules:

    - if: $CI_MERGE_REQUEST_ID ||

          $CI_COMMIT_TAG ||

          $CI_PIPELINE_SOURCE == "schedule"

      when: never

    - if: $CI_COMMIT_BRANCH == "master"

      when: manual

      allow_failure: true



deploy-es6-1-stable-branch-development:

  extends: .deploy-es6-1-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



deploy-es6-2-stable-branch-development:

  extends: .deploy-es6-2-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



deploy-es6-3-stable-branch-development:

  extends: .deploy-es6-3-development

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



deploy-es6-1-stable-branch-production:

  extends: .deploy-es6-1-production

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



deploy-es6-2-stable-branch-production:

  extends: .deploy-es6-2-production

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



deploy-es6-3-stable-branch-production:

  extends: .deploy-es6-3-production

  variables:

    <<: *deploy-branch-base-variables

  <<: *deploy-stable-branch



.deploy-tag-base:

  variables: &deploy-tag-base-variables

    DD_IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    DD_IMAGE_TAG: ${CI_COMMIT_TAG}



.deploy-tag: &deploy-tag

  rules:

    - if: $CI_COMMIT_TAG

      when: manual

      allow_failure: true



deploy-es6-1-tag-development:

  extends: .deploy-es6-1-development

  variables:

    <<: *deploy-tag-base-variables

  <<: *deploy-tag



deploy-es6-2-tag-development:

  extends: .deploy-es6-2-development

  variables:

    <<: *deploy-tag-base-variables

  <<: *deploy-tag



deploy-es6-3-tag-development:

  extends: .deploy-es6-3-development

  variables:

    <<: *deploy-tag-base-variables

  <<: *deploy-tag



deploy-es6-1-tag-production:

  extends: .deploy-es6-1-production

  variables:

    <<: *deploy-tag-base-variables

  <<: *deploy-tag



deploy-es6-2-tag-production:

  extends: .deploy-es6-2-production

  variables:

    <<: *deploy-tag-base-variables

  <<: *deploy-tag



deploy-es6-3-tag-production:

  extends: .deploy-es6-3-production

  variables:

    <<: *deploy-tag-base-variables

  <<: *deploy-tag
 
    STACK: elastic

Dockerfile

deleted100644 → 0
+0 −21
Original line number Diff line number Diff line 
ARG PARENT_IMAGE_TAG="6.6.2"



FROM docker.elastic.co/elasticsearch/elasticsearch:${PARENT_IMAGE_TAG}



LABEL maintainer="info@redmic.es"



ARG ES_PATH="/usr/share/elasticsearch"



ENV ES_PATH="${ES_PATH}" \

	cluster.name="clustername" \

	node.name="nodename" \

	path.data="${ES_PATH}/data" \

	bootstrap.memory_lock="true"



ARG SEARCH_GUARD_VERSION="6.6.2-25.5"



RUN ulimit -n 65536 && \

	${ES_PATH}/bin/elasticsearch-plugin install --batch repository-s3 && \

	${ES_PATH}/bin/elasticsearch-plugin install --batch com.floragunn:search-guard-6:${SEARCH_GUARD_VERSION}



VOLUME [ "${ES_PATH}/data" ]

README.md

+1 −90
Original line number Diff line number Diff line 
# Elasticsearch



## Search Guard



### Certificates creation



Search Guard provides a tool, [Search Guard TLS Tool](https://search.maven.org/search?q=a:search-guard-tlstool). Download and extract it.



First, create a yaml file with certificates definition, at `config/example.yml` inside extracted content:



```yml

ca:

  root:

    dn: CN=root-ca.example.net,O=EXAMPLE

    keysize: 2048

    validityDays: 3650

    pkPassword: none

    file: root-ca.pem



defaults:

  validityDays: 3650

  pkPassword: none

  generatedPasswordLength: 12

  httpsEnabled: true

  reuseTransportCertificatesForHttp: true



nodes:

  - name: node1

    dn: CN=es1.example.net

    dns:

      - elasticsearch-1

      - es-1

  - name: node2

    dn: CN=es2.example.net

    dns:

      - elasticsearch-2

      - es-2

  - name: node3

    dn: CN=es3.example.net

    dns:

      - elasticsearch-3

      - es-3



clients:

  - name: admin

    dn: CN=admin.example.net

    admin: true

```



Then, use it with the script `tools/sgtlstool.sh` and generate the certificates:



```sh

./sgtlstool.sh -c ../config/example.yml -v -ca

./sgtlstool.sh -c ../config/example.yml -v -csr

./sgtlstool.sh -c ../config/example.yml -v -crt -f -o

```



Your certificates will be generated inside `tools/out` directory.



### Configuration



Before using Search Guard, you must update the content of `/usr/share/elasticsearch/plugins/search-guard-6/sgconfig/sg_internal_users.yml` file, which define the users to be created and its roles.



You can generate the password hashes with a [online tool](https://8gwifi.org/bccrypt.jsp), for example.



```yml

admin_elastic:

  readonly: true

  hash: $2a...

  roles:

    - admin



kibanaserver:

  readonly: true

  hash: $2a...

```



### Initialization



When using Search Guard at first time, is required to run a script as certified admin, to create the configuration index.



While running, get into container and run the following commands:



```sh

cd /usr/share/elasticsearch/plugins/search-guard-6/tools



bash sgadmin.sh -cd /usr/share/elasticsearch/plugins/search-guard-6/sgconfig -icl \

 -key /usr/share/elasticsearch/config/certs/admin.key \

 -cert /usr/share/elasticsearch/config/certs/admin.pem \

 -cacert /usr/share/elasticsearch/config/certs/root-ca.pem \

 -nhnv -h localhost

```

Elasticsearch is an open source distributed, RESTful search and analytics engine, scalable data store, and vector database



## Snapshots

deploy/.env

+10 −35
Original line number Diff line number Diff line 
PARENT_IMAGE_TAG=6.6.2

ES_PATH=/usr/share/elasticsearch

SEARCH_GUARD_VERSION=6.6.2-25.5



IMAGE_NAME=registry.gitlab.com/redmic-project/elastic/elasticsearch

IMAGE_TAG=latest



ELASTIC_PASSWORD=changeme

ES_CLUSTER_INITIAL_MASTER_NODES=es-node1,es-node2,es-node3

ES_DISCOVERY_SEED_HOSTS=elasticsearch

ES_PATH_DATA=/usr/share/elasticsearch/data

ES_XPACK_SECURITY_ENABLED=false

ES_SEARCHGUARD_SSL_TRANSPORT_PEMCERT_FILEPATH=certs/node.pem

ES_SEARCHGUARD_SSL_TRANSPORT_PEMKEY_FILEPATH=certs/node.key

ES_SEARCHGUARD_SSL_TRANSPORT_PEMTRUSTEDCAS_FILEPATH=certs/root-ca.pem

ES_SEARCHGUARD_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION=false

ES_SEARCHGUARD_SSL_TRANSPORT_RESOLVE_HOSTNAME=false

ES_SEARCHGUARD_SSL_HTTP_ENABLED=false

ES_SEARCHGUARD_SSL_HTTP_PEMCERT_FILEPATH=certs/node.pem

ES_SEARCHGUARD_SSL_HTTP_PEMKEY_FILEPATH=certs/node.key

ES_SEARCHGUARD_SSL_HTTP_PEMTRUSTEDCAS_FILEPATH=certs/root-ca.pem

ES_SEARCHGUARD_ENABLE_SNAPSHOT_RESTORE_PRIVILEGE=true

ES_SEARCHGUARD_ENTERPRISE_MODULES_ENABLED=false



PORT=9200

AWS_REGION=region



ELASTIC_NET_NAME=elastic6-net

ELASTIC_NET_DRIVER=overlay

METRIC_NET_NAME=metric-net

METRIC_NET_DRIVER=overlay



SG_FILE_UID=1000

SG_FILE_GID=1000

ES_CONFIG_PATH=/usr/share/elasticsearch/config

ES_BOOTSTRAP_MEMORY_LOCK=true

ES_XPACK_SECURITY_ENABLED=true



CA_PEM_NAME=ca-pem

CA_KEY_NAME=ca-key

DATA_VOL_NAME=elasticsearch-data-vol

CONF_VOL_NAME=elasticsearch-conf-vol



ES_CONFIG_PATH=/usr/share/elasticsearch/config

ES_CERT_PATH=/usr/share/elasticsearch/config/certs

SG_CONFIG_PATH=/usr/share/elasticsearch/plugins/search-guard-6/sgconfig
 
DEV_VOL_DRIVER=local

PRO_VOL_DRIVER=cloudstor:aws