Commit 907d5a09 authored by Noel Alonso's avatar Noel Alonso
Browse files

Merge branch 'dev' of gitlab.com:redmic-project/etl/nifi into dev

parents 06b99d83 58a7a453

.gitlab-ci.yml

+44 −39
Original line number Diff line number Diff line
@@ -3,14 +3,15 @@ stages: 
  - test-package

  - deploy



image: registry.gitlab.com/redmic-project/docker/docker-deploy:${DOCKER_DEPLOY_IMAGE_TAG}



services:

  - docker:dind



variables:

  DOCKER_DRIVER: overlay2

  STACK: nifi

  STACK: etl

  NIFI_VERSION: latest

  NIFI_STATUS_CHECK_DELAY: 300

  NIFI_CONF_LOADER_STATUS_CHECK_DELAY: 30



docker-build-commit-non-master-branches:

  stage: package
@@ -71,14 +72,14 @@ container-scanning: 


deploy-nifi-supporting-branch-develop:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: 300

    IMAGE_TAG: ${DEV_NIFI_VERSION}

    COMPOSE_FILE: docker-compose.nifi.tmpl.yml:docker-compose.nifi.dev.yml

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: ${NIFI_STATUS_CHECK_DELAY}

    IMAGE_TAG: ${NIFI_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

    UI_AUTH: ${DEV_UI_AUTH}

  script:

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}
@@ -93,14 +94,14 @@ deploy-nifi-supporting-branch-develop: 


deploy-nifi-stable-branch-develop:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: 300

    IMAGE_TAG: ${DEV_NIFI_VERSION}

    COMPOSE_FILE: docker-compose.nifi.tmpl.yml:docker-compose.nifi.dev.yml

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: ${NIFI_STATUS_CHECK_DELAY}

    IMAGE_TAG: ${NIFI_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

    UI_AUTH: ${DEV_UI_AUTH}

  script:

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}
@@ -113,14 +114,14 @@ deploy-nifi-stable-branch-develop: 


deploy-nifi-supporting-branch-production:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: 300

    IMAGE_TAG: ${PRO_NIFI_VERSION}

    COMPOSE_FILE: docker-compose.nifi.tmpl.yml:docker-compose.nifi.prod.yml

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: ${NIFI_STATUS_CHECK_DELAY}

    IMAGE_TAG: ${NIFI_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}.prod.yml

    PUBLIC_HOSTNAME: ${PRO_PUBLIC_HOSTNAME}

    UI_AUTH: ${PRO_UI_AUTH}

  script:

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}
@@ -135,14 +136,14 @@ deploy-nifi-supporting-branch-production: 


deploy-nifi-stable-branch-production:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: 300

    IMAGE_TAG: ${PRO_NIFI_VERSION}

    COMPOSE_FILE: docker-compose.nifi.tmpl.yml:docker-compose.nifi.prod.yml

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: ${NIFI_STATUS_CHECK_DELAY}

    IMAGE_TAG: ${NIFI_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}.prod.yml

    PUBLIC_HOSTNAME: ${PRO_PUBLIC_HOSTNAME}

    UI_AUTH: ${PRO_UI_AUTH}

  script:

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}
@@ -155,17 +156,18 @@ deploy-nifi-stable-branch-production: 


deploy-nifi-conf-loader-supporting-branch-develop:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_nifi-conf-loader

    STATUS_CHECK_DELAY: 30

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-conf-loader

    STATUS_CHECK_DELAY: ${NIFI_CONF_LOADER_STATUS_CHECK_DELAY}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.nifi-conf-loader.tmpl.yml:docker-compose.nifi-conf-loader.dev.yml

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-conf-loader.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-conf-loader.dev.yml

  script:

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: dev/nifi-conf-loader

    name: dev/${CI_PROJECT_NAME}-conf-loader

  only:

    - branches

  except:
@@ -174,34 +176,36 @@ deploy-nifi-conf-loader-supporting-branch-develop: 


deploy-nifi-conf-loader-stable-branch-develop:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_nifi-conf-loader

    STATUS_CHECK_DELAY: 30

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-conf-loader

    STATUS_CHECK_DELAY: ${NIFI_CONF_LOADER_STATUS_CHECK_DELAY}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.nifi-conf-loader.tmpl.yml:docker-compose.nifi-conf-loader.dev.yml

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-conf-loader.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-conf-loader.dev.yml

  script:

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: dev/nifi-conf-loader

    name: dev/${CI_PROJECT_NAME}-conf-loader

  only:

    - master

  when: manual



deploy-nifi-conf-loader-supporting-branch-production:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_nifi-conf-loader

    STATUS_CHECK_DELAY: 30

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-conf-loader

    STATUS_CHECK_DELAY: ${NIFI_CONF_LOADER_STATUS_CHECK_DELAY}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.nifi-conf-loader.tmpl.yml:docker-compose.nifi-conf-loader.prod.yml

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-conf-loader.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-conf-loader.prod.yml

  script:

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: pro/nifi-conf-loader

    name: pro/${CI_PROJECT_NAME}-conf-loader

  only:

    - branches

  except:
@@ -210,17 +214,18 @@ deploy-nifi-conf-loader-supporting-branch-production: 


deploy-nifi-conf-loader-stable-branch-production:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_nifi-conf-loader

    STATUS_CHECK_DELAY: 30

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-conf-loader

    STATUS_CHECK_DELAY: ${NIFI_CONF_LOADER_STATUS_CHECK_DELAY}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.nifi-conf-loader.tmpl.yml:docker-compose.nifi-conf-loader.prod.yml

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-conf-loader.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-conf-loader.prod.yml

  script:

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: pro/nifi-conf-loader

    name: pro/${CI_PROJECT_NAME}-conf-loader

  only:

    - master

  when: manual

deploy/docker-compose.nifi.dev.yml

+5 −0
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ services: 
          cpus: '1'

          memory: 1G

        reservations:

          cpus: '0.2'

          memory: 820M



volumes:
@@ -25,3 +26,7 @@ volumes: 


  nifi-provenance-vol:

    name: ${NIFI_PROVENANCE_VOL_NAME:-nifi-provenance-vol}



  ingest-data-vol:

    name: ${INGEST_DATA_VOL_NAME:-ingest-data-vol}

deploy/docker-compose.nifi.prod.yml

+12 −6
Original line number Diff line number Diff line
@@ -11,31 +11,31 @@ services: 
          cpus: '1'

          memory: 2G

        reservations:

          cpus: '0.5'

          cpus: '0.2'

          memory: 1639M



volumes:

  nifi-conf-vol:

    name: ${NIFI_CONF_VOL_NAME:-nifi-conf-vol}

    driver: "cloudstor:aws"

    driver: 'cloudstor:aws'

    driver_opts:

      backing: shared



  nifi-database-vol:

    name: ${NIFI_DATABASE_VOL_NAME:-nifi-database-vol}

    driver: "cloudstor:aws"

    driver: 'cloudstor:aws'

    driver_opts:

      backing: shared



  nifi-flowfile-vol:

    name: ${NIFI_FLOWFILE_VOL_NAME:-nifi-flowfile-vol}

    driver: "cloudstor:aws"

    driver: 'cloudstor:aws'

    driver_opts:

      backing: shared



  nifi-content-vol:

    name: ${NIFI_CONTENT_VOL_NAME:-nifi-content-vol}

    driver: "cloudstor:aws"

    driver: 'cloudstor:aws'

    driver_opts:

      backing: relocatable

      size: 1
@@ -43,8 +43,14 @@ volumes: 


  nifi-provenance-vol:

    name: ${NIFI_PROVENANCE_VOL_NAME:-nifi-provenance-vol}

    driver: "cloudstor:aws"

    driver: 'cloudstor:aws'

    driver_opts:

      backing: relocatable

      size: 2

      ebstype: gp2



  ingest-data-vol:

    name: ${INGEST_DATA_VOL_NAME:-ingest-data-vol}

    driver: 'cloudstor:aws'

    driver_opts:

      backing: shared
 
 No newline at end of file

deploy/docker-compose.nifi.tmpl.yml

+10 −9
Original line number Diff line number Diff line
@@ -2,12 +2,12 @@ version: '3.5' 


services:

  nifi:

    image: apache/nifi:${IMAGE_TAG:-latest}

    image: ${IMAGE_NAME:-apache/nifi}:${IMAGE_TAG:-latest}

    hostname: apache-nifi

    environment:

      - NIFI_LOG_DIR

      - NIFI_HOME

      - NIFI_WEB_HTTP_HOST

      NIFI_LOG_DIR:

      NIFI_HOME:

      NIFI_WEB_HTTP_HOST:

    networks:

      - traefik-net

      - kafka-net
@@ -19,11 +19,12 @@ services: 
      - nifi-content-vol:${NIFI_HOME}/content_repository

      - nifi-provenance-vol:${NIFI_HOME}/provenance_repository

      - nifi-state-vol:${NIFI_HOME}/state

      - ingest-data-vol:/data

    configs:

      - source: logback-xml

        target: ${NIFI_HOME}/conf/logback.xml

    healthcheck:

      test: curl --silent http://apache-nifi:${PORT}

      test: curl --silent --output /dev/null http://apache-nifi:${PORT}/nifi

      interval: 30s

      timeout: 30s

      retries: 3
@@ -31,15 +32,15 @@ services: 
    deploy:

      mode: replicated

      replicas: 1

      restart_policy:

        delay: 10s

        window: 2m

      labels:

        traefik.frontend.auth.basic.users: ${UI_AUTH}

        traefik.frontend.rule: Host:nifi.${PUBLIC_HOSTNAME}

        traefik.frontend.headers.customRequestHeaders: X-ProxyScheme:https||X-ProxyHost:nifi.${PUBLIC_HOSTNAME}||X-ProxyPort:443

        traefik.backend: nifi

        traefik.port: "${PORT}"

      restart_policy:

        delay: 30s

        window: 2m

        traefik.port: '${PORT}'



networks:

  traefik-net: