Corrige cambio de stack, actualiza detalles de CI (50b02c1e) · Commits · REDMIC Project / ETL / NiFi Loader

.gitlab-ci.yml

+40 −35

Original line number	Diff line number	Diff line
		@@ -3,14 +3,15 @@ stages:
		- test-package
		- deploy

		image: registry.gitlab.com/redmic-project/docker/docker-deploy:${DOCKER_DEPLOY_IMAGE_TAG}

		services:
		- docker:dind

		variables:
		DOCKER_DRIVER: overlay2
		STACK: nifi
		STACK: etl
		NIFI_VERSION: latest
		NIFI_STATUS_CHECK_DELAY: 300
		NIFI_CONF_LOADER_STATUS_CHECK_DELAY: 30

		docker-build-commit-non-master-branches:
		stage: package
		@@ -71,14 +72,14 @@ container-scanning:

		deploy-nifi-supporting-branch-develop:
		stage: deploy
		image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}
		variables:
		SSH_REMOTE: ${DEV_SSH_REMOTE}
		SERVICES_TO_CHECK: etl_${CI_PROJECT_NAME}
		STATUS_CHECK_DELAY: 300
		IMAGE_TAG: ${DEV_NIFI_VERSION}
		COMPOSE_FILE: docker-compose.nifi.tmpl.yml:docker-compose.nifi.dev.yml
		SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}
		STATUS_CHECK_DELAY: ${NIFI_STATUS_CHECK_DELAY}
		IMAGE_TAG: ${NIFI_VERSION}
		COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}.dev.yml
		PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}
		UI_AUTH: ${DEV_UI_AUTH}
		script:
		- >
		deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}
		@@ -93,14 +94,14 @@ deploy-nifi-supporting-branch-develop:

		deploy-nifi-stable-branch-develop:
		stage: deploy
		image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}
		variables:
		SSH_REMOTE: ${DEV_SSH_REMOTE}
		SERVICES_TO_CHECK: etl_${CI_PROJECT_NAME}
		STATUS_CHECK_DELAY: 300
		IMAGE_TAG: ${DEV_NIFI_VERSION}
		COMPOSE_FILE: docker-compose.nifi.tmpl.yml:docker-compose.nifi.dev.yml
		SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}
		STATUS_CHECK_DELAY: ${NIFI_STATUS_CHECK_DELAY}
		IMAGE_TAG: ${NIFI_VERSION}
		COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}.dev.yml
		PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}
		UI_AUTH: ${DEV_UI_AUTH}
		script:
		- >
		deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}
		@@ -113,14 +114,14 @@ deploy-nifi-stable-branch-develop:

		deploy-nifi-supporting-branch-production:
		stage: deploy
		image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}
		variables:
		SSH_REMOTE: ${PRO_SSH_REMOTE}
		SERVICES_TO_CHECK: etl_${CI_PROJECT_NAME}
		STATUS_CHECK_DELAY: 300
		IMAGE_TAG: ${PRO_NIFI_VERSION}
		COMPOSE_FILE: docker-compose.nifi.tmpl.yml:docker-compose.nifi.prod.yml
		SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}
		STATUS_CHECK_DELAY: ${NIFI_STATUS_CHECK_DELAY}
		IMAGE_TAG: ${NIFI_VERSION}
		COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}.prod.yml
		PUBLIC_HOSTNAME: ${PRO_PUBLIC_HOSTNAME}
		UI_AUTH: ${PRO_UI_AUTH}
		script:
		- >
		deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}
		@@ -135,14 +136,14 @@ deploy-nifi-supporting-branch-production:

		deploy-nifi-stable-branch-production:
		stage: deploy
		image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}
		variables:
		SSH_REMOTE: ${PRO_SSH_REMOTE}
		SERVICES_TO_CHECK: etl_${CI_PROJECT_NAME}
		STATUS_CHECK_DELAY: 300
		IMAGE_TAG: ${PRO_NIFI_VERSION}
		COMPOSE_FILE: docker-compose.nifi.tmpl.yml:docker-compose.nifi.prod.yml
		SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}
		STATUS_CHECK_DELAY: ${NIFI_STATUS_CHECK_DELAY}
		IMAGE_TAG: ${NIFI_VERSION}
		COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}.prod.yml
		PUBLIC_HOSTNAME: ${PRO_PUBLIC_HOSTNAME}
		UI_AUTH: ${PRO_UI_AUTH}
		script:
		- >
		deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}
		@@ -155,13 +156,14 @@ deploy-nifi-stable-branch-production:

		deploy-nifi-conf-loader-supporting-branch-develop:
		stage: deploy
		image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}
		variables:
		SSH_REMOTE: ${DEV_SSH_REMOTE}
		SERVICES_TO_CHECK: etl_nifi-conf-loader
		STATUS_CHECK_DELAY: 30
		SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-conf-loader
		STATUS_CHECK_DELAY: ${NIFI_CONF_LOADER_STATUS_CHECK_DELAY}
		IMAGE_NAME: ${CI_REGISTRY_IMAGE}
		IMAGE_TAG: ${CI_COMMIT_SHA}
		COMPOSE_FILE: docker-compose.nifi-conf-loader.tmpl.yml:docker-compose.nifi-conf-loader.dev.yml
		COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-conf-loader.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-conf-loader.dev.yml
		script:
		- deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}
		environment:
		@@ -174,13 +176,14 @@ deploy-nifi-conf-loader-supporting-branch-develop:

		deploy-nifi-conf-loader-stable-branch-develop:
		stage: deploy
		image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}
		variables:
		SSH_REMOTE: ${DEV_SSH_REMOTE}
		SERVICES_TO_CHECK: etl_nifi-conf-loader
		STATUS_CHECK_DELAY: 30
		SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-conf-loader
		STATUS_CHECK_DELAY: ${NIFI_CONF_LOADER_STATUS_CHECK_DELAY}
		IMAGE_NAME: ${CI_REGISTRY_IMAGE}
		IMAGE_TAG: ${CI_COMMIT_SHA}
		COMPOSE_FILE: docker-compose.nifi-conf-loader.tmpl.yml:docker-compose.nifi-conf-loader.dev.yml
		COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-conf-loader.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-conf-loader.dev.yml
		script:
		- deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}
		environment:
		@@ -191,13 +194,14 @@ deploy-nifi-conf-loader-stable-branch-develop:

		deploy-nifi-conf-loader-supporting-branch-production:
		stage: deploy
		image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}
		variables:
		SSH_REMOTE: ${PRO_SSH_REMOTE}
		SERVICES_TO_CHECK: etl_nifi-conf-loader
		STATUS_CHECK_DELAY: 30
		SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-conf-loader
		STATUS_CHECK_DELAY: ${NIFI_CONF_LOADER_STATUS_CHECK_DELAY}
		IMAGE_NAME: ${CI_REGISTRY_IMAGE}
		IMAGE_TAG: ${CI_COMMIT_SHA}
		COMPOSE_FILE: docker-compose.nifi-conf-loader.tmpl.yml:docker-compose.nifi-conf-loader.prod.yml
		COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-conf-loader.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-conf-loader.prod.yml
		script:
		- deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}
		environment:
		@@ -210,13 +214,14 @@ deploy-nifi-conf-loader-supporting-branch-production:

		deploy-nifi-conf-loader-stable-branch-production:
		stage: deploy
		image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}
		variables:
		SSH_REMOTE: ${PRO_SSH_REMOTE}
		SERVICES_TO_CHECK: etl_nifi-conf-loader
		STATUS_CHECK_DELAY: 30
		SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-conf-loader
		STATUS_CHECK_DELAY: ${NIFI_CONF_LOADER_STATUS_CHECK_DELAY}
		IMAGE_NAME: ${CI_REGISTRY_IMAGE}
		IMAGE_TAG: ${CI_COMMIT_SHA}
		COMPOSE_FILE: docker-compose.nifi-conf-loader.tmpl.yml:docker-compose.nifi-conf-loader.prod.yml
		COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-conf-loader.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-conf-loader.prod.yml
		script:
		- deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}
		environment:

deploy/docker-compose.nifi.dev.yml

+1 −0

Original line number	Diff line number	Diff line
		@@ -8,6 +8,7 @@ services:
		cpus: '1'
		memory: 1G
		reservations:
		cpus: '0.2'
		memory: 820M

		volumes:

deploy/docker-compose.nifi.prod.yml

+7 −7

Original line number	Diff line number	Diff line
		@@ -11,31 +11,31 @@ services:
		cpus: '1'
		memory: 2G
		reservations:
		cpus: '0.5'
		cpus: '0.2'
		memory: 1639M

		volumes:
		nifi-conf-vol:
		name: ${NIFI_CONF_VOL_NAME:-nifi-conf-vol}
		driver: "cloudstor:aws"
		driver: 'cloudstor:aws'
		driver_opts:
		backing: shared

		nifi-database-vol:
		name: ${NIFI_DATABASE_VOL_NAME:-nifi-database-vol}
		driver: "cloudstor:aws"
		driver: 'cloudstor:aws'
		driver_opts:
		backing: shared

		nifi-flowfile-vol:
		name: ${NIFI_FLOWFILE_VOL_NAME:-nifi-flowfile-vol}
		driver: "cloudstor:aws"
		driver: 'cloudstor:aws'
		driver_opts:
		backing: shared

		nifi-content-vol:
		name: ${NIFI_CONTENT_VOL_NAME:-nifi-content-vol}
		driver: "cloudstor:aws"
		driver: 'cloudstor:aws'
		driver_opts:
		backing: relocatable
		size: 1
		@@ -43,7 +43,7 @@ volumes:

		nifi-provenance-vol:
		name: ${NIFI_PROVENANCE_VOL_NAME:-nifi-provenance-vol}
		driver: "cloudstor:aws"
		driver: 'cloudstor:aws'
		driver_opts:
		backing: relocatable
		size: 2
		@@ -51,6 +51,6 @@ volumes:

		ingest-data-vol:
		name: ${INGEST_DATA_VOL_NAME:-ingest-data-vol}
		driver: "cloudstor:aws"
		driver: 'cloudstor:aws'
		driver_opts:
		backing: shared
		No newline at end of file

deploy/docker-compose.nifi.tmpl.yml

+8 −8

Original line number	Diff line number	Diff line
		@@ -2,12 +2,12 @@ version: '3.5'

		services:
		nifi:
		image: apache/nifi:${IMAGE_TAG:-latest}
		image: ${IMAGE_NAME:-apache/nifi}:${IMAGE_TAG:-latest}
		hostname: apache-nifi
		environment:
		- NIFI_LOG_DIR
		- NIFI_HOME
		- NIFI_WEB_HTTP_HOST
		NIFI_LOG_DIR:
		NIFI_HOME:
		NIFI_WEB_HTTP_HOST:
		networks:
		- traefik-net
		- kafka-net
		@@ -32,15 +32,15 @@ services:
		deploy:
		mode: replicated
		replicas: 1
		restart_policy:
		delay: 10s
		window: 2m
		labels:
		traefik.frontend.auth.basic.users: ${UI_AUTH}
		traefik.frontend.rule: Host:nifi.${PUBLIC_HOSTNAME}
		traefik.frontend.headers.customRequestHeaders: X-ProxyScheme:https\|\|X-ProxyHost:nifi.${PUBLIC_HOSTNAME}\|\|X-ProxyPort:443
		traefik.backend: nifi
		traefik.port: "${PORT}"
		restart_policy:
		delay: 30s
		window: 2m
		traefik.port: '${PORT}'

		networks:
		traefik-net: