Commit becc8a80 authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Usa vols NFS en dev, revisa vols en pro 

En desarrollo, intercambia restricción de nodo por volúmenes NFS.

El volumen state pasa de ser local a ser NFS/EFS en ambos entornos, para
conseguir persistencia.

En producción, actualiza los tamaños y tipos de discos EBS, e incluye
flowfile como EBS para ganar en rendimiento.
parent eda5161c

.gitlab-ci.yml

+1 −5
Original line number Diff line number Diff line
@@ -6,13 +6,9 @@ include: 
stages:

  - deploy



variables:

  PROJECT_PARENT_NAME: etl



.deploy:

  variables:

    STACK: ${PROJECT_PARENT_NAME}

    SERVICES_TO_CHECK: ${PROJECT_PARENT_NAME}_${CI_PROJECT_NAME}

    STACK: etl

    STATUS_CHECK_DELAY: 300

  environment:

    url: https://nifi.${PUBLIC_HOSTNAME}

deploy/.env

+6 −1
Original line number Diff line number Diff line
@@ -25,5 +25,10 @@ DATABASE_VOL_NAME=nifi-database-vol 
FLOWFILE_VOL_NAME=nifi-flowfile-vol

CONTENT_VOL_NAME=nifi-content-vol

PROVENANCE_VOL_NAME=nifi-provenance-vol

INGEST_VOL_NAME=ingest-data-vol

STATE_VOL_NAME=nifi-state-vol

SECURITY_VOL_NAME=nifi-security-vol

INGEST_VOL_NAME=ingest-data-vol



VOL_TYPE=nfs4

VOL_ADDR=127.0.0.1

VOL_OPTIONS=rw,nolock,noatime,tcp,rsize=8192,wsize=8192,timeo=14

deploy/docker-compose.dev.yml

+45 −9
Original line number Diff line number Diff line 
version: '3.5'



services:

  nifi:

    deploy:

      placement:

        constraints:

          - node.hostname == ${PLACEMENT_CONSTRAINTS_NODE_HOSTNAME:-hostname}



volumes:

  conf-vol:

    name: ${CONF_VOL_NAME}

    driver: local

    driver_opts:

      type: ${VOL_TYPE}

      o: addr=${VOL_ADDR},${VOL_OPTIONS}

      device: ${CONF_VOL_DEVICE:-:/nifi-conf-vol/}



  database-vol:

    name: ${DATABASE_VOL_NAME}

    driver: local

    driver_opts:

      type: ${VOL_TYPE}

      o: addr=${VOL_ADDR},${VOL_OPTIONS}

      device: ${DATABASE_VOL_DEVICE:-:/nifi-database-vol/}



  flowfile-vol:

    name: ${FLOWFILE_VOL_NAME}

    driver: local

    driver_opts:

      type: ${VOL_TYPE}

      o: addr=${VOL_ADDR},${VOL_OPTIONS}

      device: ${FLOWFILE_VOL_DEVICE:-:/nifi-flowfile-vol/}



  content-vol:

    name: ${CONTENT_VOL_NAME}

    driver: local

    driver_opts:

      type: ${VOL_TYPE}

      o: addr=${VOL_ADDR},${VOL_OPTIONS}

      device: ${CONTENT_VOL_DEVICE:-:/nifi-content-vol/}



  provenance-vol:

    name: ${PROVENANCE_VOL_NAME}

    driver: local

    driver_opts:

      type: ${VOL_TYPE}

      o: addr=${VOL_ADDR},${VOL_OPTIONS}

      device: ${PROVENANCE_VOL_DEVICE:-:/nifi-provenance-vol/}



  ingest-vol:

    name: ${INGEST_VOL_NAME}

  state-vol:

    name: ${STATE_VOL_NAME}

    driver: local

    driver_opts:

      type: ${VOL_TYPE}

      o: addr=${VOL_ADDR},${VOL_OPTIONS}

      device: ${STATE_VOL_DEVICE:-:/nifi-state-vol/}



  security-vol:

    name: ${SECURITY_VOL_NAME}

    driver: local

    driver_opts:

      type: ${VOL_TYPE}

      o: addr=${VOL_ADDR},${VOL_OPTIONS}

      device: ${SECURITY_VOL_DEVICE:-:/nifi-security-vol/}



  ingest-vol:

    name: ${INGEST_VOL_NAME}

    driver: local

    driver_opts:

      type: ${VOL_TYPE}

      o: addr=${VOL_ADDR},${VOL_OPTIONS}

      device: ${INGEST_VOL_DEVICE:-:/nifi-ingest-vol/}

deploy/docker-compose.prod.yml

+15 −7
Original line number Diff line number Diff line
@@ -24,26 +24,28 @@ volumes: 
    name: ${FLOWFILE_VOL_NAME}

    driver: 'cloudstor:aws'

    driver_opts:

      backing: shared

      backing: relocatable

      size: ${FLOWFILE_VOL_SIZE:-1}

      ebstype: gp3



  content-vol:

    name: ${CONTENT_VOL_NAME}

    driver: 'cloudstor:aws'

    driver_opts:

      backing: relocatable

      size: ${CONTENT_VOL_SIZE:-1}

      ebstype: gp2

      size: ${CONTENT_VOL_SIZE:-5}

      ebstype: gp3



  provenance-vol:

    name: ${PROVENANCE_VOL_NAME}

    driver: 'cloudstor:aws'

    driver_opts:

      backing: relocatable

      size: ${PROVENANCE_VOL_SIZE:-2}

      ebstype: gp2

      size: ${PROVENANCE_VOL_SIZE:-1}

      ebstype: gp3



  ingest-vol:

    name: ${INGEST_VOL_NAME}

  state-vol:

    name: ${STATE_VOL_NAME}

    driver: 'cloudstor:aws'

    driver_opts:

      backing: shared
@@ -53,3 +55,9 @@ volumes: 
    driver: 'cloudstor:aws'

    driver_opts:

      backing: shared



  ingest-vol:

    name: ${INGEST_VOL_NAME}

    driver: 'cloudstor:aws'

    driver_opts:

      backing: shared

deploy/docker-compose.tmpl.yml

+0 −3
Original line number Diff line number Diff line
@@ -106,9 +106,6 @@ volumes: 
  log-vol:

    name: ${LOG_VOL_NAME:-nifi-log-vol}



  state-vol:

    name: ${STATE_VOL_NAME:-nifi-state-vol}



configs:

  logback-xml:

    name: ${LOGBACK_XML_NAME:-nifi-logback-xml}