Commit 8ac32d85 authored by Ignacio's avatar Ignacio
Browse files

Elimina parte de despliegue de NIFI 

Se parte del repositorio de NIFI y se elimina
todo el despliegue de este.
parent 8ba0f50f

.gitlab-ci.yml

+13 −221
Original line number Diff line number Diff line 
include:

  - project: 'redmic-project/gitlab-ci-templates'

    ref: master

    file: '/packaging.yml'

  - project: 'redmic-project/gitlab-ci-templates'

    ref: master

    file: '/deployment.yml'



stages:

  - package

  - test-package

  - deploy



services:

  - docker:dind



variables:

  DOCKER_DRIVER: overlay2

  STACK: etl

  NIFI_VERSION: latest

  NIFI_STATUS_CHECK_DELAY: 300

  NIFI_CONF_LOADER_STATUS_CHECK_DELAY: 30



docker-build-commit-non-master-branches:

  stage: package

  image: docker:stable

  only:

    - branches

  except:

    - master

  script:

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - docker build -t ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} -t ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME}-latest .

    - docker push ${CI_REGISTRY_IMAGE}



docker-build-commit-master-branch:

  stage: package

  image: docker:stable

  only:

    - master

  script:

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - docker build -t ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} -t ${CI_REGISTRY_IMAGE}:latest .

    - docker push ${CI_REGISTRY_IMAGE}



docker-tag-already-built-image:

  stage: package

  image: docker:stable

  only:

    - tags

  script:

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - docker pull ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}

    - docker tag ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} ${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}

    - docker push ${CI_REGISTRY_IMAGE}



container-scanning:

  stage: test-package

  image: docker:stable

  allow_failure: true

  only:

    - branches

  except:

    - schedules

  script:

    - docker run -d --name db arminc/clair-db:latest

    - docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.1

    - apk add -U wget ca-certificates

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - docker pull ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}

    - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64

    - mv clair-scanner_linux_amd64 clair-scanner

    - chmod +x clair-scanner

    - touch clair-whitelist.yml

    - >

      ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-sast-container-report.json -l clair.log

      -w clair-whitelist.yml ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} || true

  artifacts:

    paths: [gl-sast-container-report.json]



deploy-nifi-supporting-branch-develop:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: ${NIFI_STATUS_CHECK_DELAY}

    IMAGE_TAG: ${NIFI_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

      UI_AUTH=${UI_AUTH}

  environment:

    name: dev/${CI_PROJECT_NAME}

  only:

    - branches

  except:

    - master

  when: manual



deploy-nifi-stable-branch-develop:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: ${NIFI_STATUS_CHECK_DELAY}

    IMAGE_TAG: ${NIFI_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

  script:

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

      UI_AUTH=${UI_AUTH}

  environment:

    name: dev/${CI_PROJECT_NAME}

  only:

    - master

  when: manual

  STATUS_CHECK_DELAY: 30



deploy-nifi-supporting-branch-production:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: ${NIFI_STATUS_CHECK_DELAY}

    IMAGE_TAG: ${NIFI_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}.prod.yml

    PUBLIC_HOSTNAME: ${PRO_PUBLIC_HOSTNAME}

.deploy:

  script:

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

      UI_AUTH=${UI_AUTH}

  environment:

    name: pro/${CI_PROJECT_NAME}

  only:

    - branches

  except:

    - master

  when: manual



deploy-nifi-stable-branch-production:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: ${NIFI_STATUS_CHECK_DELAY}

    IMAGE_TAG: ${NIFI_VERSION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}.prod.yml

    PUBLIC_HOSTNAME: ${PRO_PUBLIC_HOSTNAME}

  script:

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

      UI_AUTH=${UI_AUTH}

  environment:

    name: pro/${CI_PROJECT_NAME}

  only:

    - master

  when: manual



deploy-nifi-conf-loader-supporting-branch-develop:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-conf-loader

    STATUS_CHECK_DELAY: ${NIFI_CONF_LOADER_STATUS_CHECK_DELAY}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-conf-loader.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-conf-loader.dev.yml

  script:

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: dev/${CI_PROJECT_NAME}-conf-loader

  only:

    - branches

  except:

    - master

  when: manual



deploy-nifi-conf-loader-stable-branch-develop:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-conf-loader

    STATUS_CHECK_DELAY: ${NIFI_CONF_LOADER_STATUS_CHECK_DELAY}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-conf-loader.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-conf-loader.dev.yml

  script:

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: dev/${CI_PROJECT_NAME}-conf-loader

  only:

    - master

  when: manual



deploy-nifi-conf-loader-supporting-branch-production:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-conf-loader

    STATUS_CHECK_DELAY: ${NIFI_CONF_LOADER_STATUS_CHECK_DELAY}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-conf-loader.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-conf-loader.prod.yml

  script:

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: pro/${CI_PROJECT_NAME}-conf-loader

  only:

    - branches

  except:

    - master

  when: manual



deploy-nifi-conf-loader-stable-branch-production:

  stage: deploy

  image: ${DOCKER_DEPLOY_IMAGE_NAME}:${DOCKER_DEPLOY_IMAGE_TAG}

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: ${STACK}_${CI_PROJECT_NAME}-conf-loader

    STATUS_CHECK_DELAY: ${NIFI_CONF_LOADER_STATUS_CHECK_DELAY}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}-conf-loader.tmpl.yml:docker-compose.${CI_PROJECT_NAME}-conf-loader.prod.yml

  script:

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: pro/${CI_PROJECT_NAME}-conf-loader

  only:

    - master

  when: manual
 
      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG}

      NIFI_UID=${NNIFI_UID} NIFI_GID=${NIFI_GID}

      ADDITIONAL_LIBS_URLS="${ADDITIONAL_LIBS_URLS}"

Dockerfile

+1 −2
Original line number Diff line number Diff line
@@ -2,8 +2,7 @@ FROM alpine:3.9 


LABEL maintainer="info@redmic.es"



RUN apk add --no-cache --virtual \

		.build-deps \

RUN apk add --no-cache \

		curl=7.64.0-r2



COPY script /

deploy/.env

+2 −4
Original line number Diff line number Diff line 
PORT=8080

NIFI_LOG_DIR=/opt/nifi/nifi-current/logs

NIFI_HOME=/opt/nifi/nifi-current

NIFI_WEB_HTTP_HOST=0.0.0.0
 
NIFI_UID=1000

NIFI_GID=1000

deploy/config/logback.xml

deleted100644 → 0
+0 −170
Original line number Diff line number Diff line 
<?xml version="1.0" encoding="UTF-8"?>

<!--

  Licensed to the Apache Software Foundation (ASF) under one or more

  contributor license agreements.  See the NOTICE file distributed with

  this work for additional information regarding copyright ownership.

  The ASF licenses this file to You under the Apache License, Version 2.0

  (the "License"); you may not use this file except in compliance with

  the License.  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software

  distributed under the License is distributed on an "AS IS" BASIS,

  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  See the License for the specific language governing permissions and

  limitations under the License.

-->



<configuration scan="true" scanPeriod="30 seconds">

    <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">

        <resetJUL>true</resetJUL>

    </contextListener>



    <appender name="APP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">

        <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app.log</file>

        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">

            <!--

              For daily rollover, use 'app_%d.log'.

              For hourly rollover, use 'app_%d{yyyy-MM-dd_HH}.log'.

              To GZIP rolled files, replace '.log' with '.log.gz'.

              To ZIP rolled files, replace '.log' with '.log.zip'.

            -->

            <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.log</fileNamePattern>

            <!-- keep 2 log files worth of history -->

            <maxHistory>2</maxHistory>

        </rollingPolicy>

        <immediateFlush>true</immediateFlush>

        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">

            <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>

        </encoder>

    </appender>



    <appender name="USER_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">

        <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user.log</file>

        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">

            <!--

              For daily rollover, use 'user_%d.log'.

              For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.

              To GZIP rolled files, replace '.log' with '.log.gz'.

              To ZIP rolled files, replace '.log' with '.log.zip'.

            -->

            <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d{yyyy-MM-dd_HH}.log</fileNamePattern>

            <!-- keep 2 log files worth of history -->

            <maxHistory>2</maxHistory>

        </rollingPolicy>

        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">

            <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>

        </encoder>

    </appender>



    <appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">

        <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>

        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">

            <!--

              For daily rollover, use 'user_%d.log'.

              For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.

              To GZIP rolled files, replace '.log' with '.log.gz'.

              To ZIP rolled files, replace '.log' with '.log.zip'.

            -->

            <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d{yyyy-MM-dd_HH}.log</fileNamePattern>

            <!-- keep 2 log files worth of history -->

            <maxHistory>2</maxHistory>

        </rollingPolicy>

        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">

            <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>

        </encoder>

    </appender>



    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">

        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">

            <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>

        </encoder>

    </appender>



    <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->



    <logger name="org.apache.nifi" level="WARN"/>

    <logger name="org.apache.nifi.processors" level="WARN"/>

    <logger name="org.apache.nifi.processors.standard.LogAttribute" level="WARN"/>

    <logger name="org.apache.nifi.processors.standard.LogMessage" level="WARN"/>

    <logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" />





    <logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />

    <logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />

    <logger name="org.apache.zookeeper.server.NIOServerCnxnFactory" level="ERROR" />

    <logger name="org.apache.zookeeper.server.quorum" level="ERROR" />

    <logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" />

    <logger name="org.apache.zookeeper.server.PrepRequestProcessor" level="ERROR" />



    <logger name="org.apache.calcite.runtime.CalciteException" level="OFF" />



    <logger name="org.apache.curator.framework.recipes.leader.LeaderSelector" level="OFF" />

    <logger name="org.apache.curator.ConnectionState" level="OFF" />



    <!-- Logger for managing logging statements for nifi clusters. -->

    <logger name="org.apache.nifi.cluster" level="WARN"/>



    <!-- Logger for logging HTTP requests received by the web server. -->

    <logger name="org.apache.nifi.server.JettyServer" level="WARN"/>



    <!-- Logger for managing logging statements for jetty -->

    <logger name="org.eclipse.jetty" level="WARN"/>



    <!-- Suppress non-error messages due to excessive logging by class or library -->

    <logger name="org.springframework" level="ERROR"/>



    <!-- Suppress non-error messages due to known warning about redundant path annotation (NIFI-574) -->

    <logger name="org.glassfish.jersey.internal.Errors" level="ERROR"/>



    <!-- Suppress non-error messages due to Jetty AnnotationParser emitting a large amount of WARNS. Issue described in NIFI-5479. -->

    <logger name="org.eclipse.jetty.annotations.AnnotationParser" level="ERROR"/>



    <!--

        Logger for capturing user events. We do not want to propagate these

        log events to the root logger. These messages are only sent to the

        user-log appender.

    -->

    <logger name="org.apache.nifi.web.security" level="WARN" additivity="false">

        <appender-ref ref="USER_FILE"/>

    </logger>

    <logger name="org.apache.nifi.web.api.config" level="WARN" additivity="false">

        <appender-ref ref="USER_FILE"/>

    </logger>

    <logger name="org.apache.nifi.authorization" level="WARN" additivity="false">

        <appender-ref ref="USER_FILE"/>

    </logger>

    <logger name="org.apache.nifi.cluster.authorization" level="WARN" additivity="false">

        <appender-ref ref="USER_FILE"/>

    </logger>

    <logger name="org.apache.nifi.web.filter.RequestLogger" level="WARN" additivity="false">

        <appender-ref ref="USER_FILE"/>

    </logger>





    <!--

        Logger for capturing Bootstrap logs and NiFi's standard error and standard out.

    -->

    <logger name="org.apache.nifi.bootstrap" level="WARN" additivity="false">

        <appender-ref ref="BOOTSTRAP_FILE" />

    </logger>

    <logger name="org.apache.nifi.bootstrap.Command" level="WARN" additivity="false">

        <appender-ref ref="CONSOLE" />

        <appender-ref ref="BOOTSTRAP_FILE" />

    </logger>



    <!-- Everything written to NiFi's Standard Out will be logged with the logger org.apache.nifi.StdOut at INFO level -->

    <logger name="org.apache.nifi.StdOut" level="WARN" additivity="false">

        <appender-ref ref="BOOTSTRAP_FILE" />

    </logger>



    <!-- Everything written to NiFi's Standard Error will be logged with the logger org.apache.nifi.StdErr at ERROR level -->

    <logger name="org.apache.nifi.StdErr" level="ERROR" additivity="false">

        <appender-ref ref="BOOTSTRAP_FILE" />

    </logger>





    <root level="WARN">

        <appender-ref ref="APP_FILE"/>

    </root>



</configuration>
 
 No newline at end of file