Commit 08609974 authored by Ignacio's avatar Ignacio
Browse files

Merge branch 'dev' into 'master' 

Renombra script de procesado de playas

See merge request redmic-project/etl/nifi-loader!1
parents c9057ff7 04e9ed95

.gitlab-ci.yml

+14 −217
Original line number Diff line number Diff line 
include:

  - project: 'redmic-project/gitlab-ci-templates'

    ref: master

    file: '/packaging.yml'

  - project: 'redmic-project/gitlab-ci-templates'

    ref: master

    file: '/deployment.yml'



stages:

  - package

  - test-package

  - deploy



image: registry.gitlab.com/redmic-project/docker/docker-deploy:${DOCKER_DEPLOY_IMAGE_TAG}



services:

  - docker:dind



variables:

  DOCKER_DRIVER: overlay2

  STACK: nifi



docker-build-commit-non-master-branches:

  stage: package

  image: docker:stable

  only:

    - branches

  except:

    - master

  script:

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - docker build -t ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} -t ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME}-latest .

    - docker push ${CI_REGISTRY_IMAGE}



docker-build-commit-master-branch:

  stage: package

  image: docker:stable

  only:

    - master

  script:

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - docker build -t ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} -t ${CI_REGISTRY_IMAGE}:latest .

    - docker push ${CI_REGISTRY_IMAGE}



docker-tag-already-built-image:

  stage: package

  image: docker:stable

  only:

    - tags

  script:

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - docker pull ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}

    - docker tag ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} ${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}

    - docker push ${CI_REGISTRY_IMAGE}



container-scanning:

  stage: test-package

  image: docker:stable

  allow_failure: true

  only:

    - branches

  except:

    - schedules

  script:

    - docker run -d --name db arminc/clair-db:latest

    - docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.1

    - apk add -U wget ca-certificates

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - docker pull ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}

    - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64

    - mv clair-scanner_linux_amd64 clair-scanner

    - chmod +x clair-scanner

    - touch clair-whitelist.yml

    - >

      ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-sast-container-report.json -l clair.log

      -w clair-whitelist.yml ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} || true

  artifacts:

    paths: [gl-sast-container-report.json]



deploy-nifi-supporting-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: 300

    IMAGE_TAG: ${DEV_NIFI_VERSION}

    COMPOSE_FILE: docker-compose.nifi.tmpl.yml:docker-compose.nifi.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

    UI_AUTH: ${DEV_UI_AUTH}

  script:

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

      UI_AUTH=${UI_AUTH}

  environment:

    name: dev/${CI_PROJECT_NAME}

  only:

    - branches

  except:

    - master

  when: manual



deploy-nifi-stable-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: 300

    IMAGE_TAG: ${DEV_NIFI_VERSION}

    COMPOSE_FILE: docker-compose.nifi.tmpl.yml:docker-compose.nifi.dev.yml

    PUBLIC_HOSTNAME: ${DEV_PUBLIC_HOSTNAME}

    UI_AUTH: ${DEV_UI_AUTH}

  script:

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

      UI_AUTH=${UI_AUTH}

  environment:

    name: dev/${CI_PROJECT_NAME}

  only:

    - master

  when: manual



deploy-nifi-supporting-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: 300

    IMAGE_TAG: ${PRO_NIFI_VERSION}

    COMPOSE_FILE: docker-compose.nifi.tmpl.yml:docker-compose.nifi.prod.yml

    PUBLIC_HOSTNAME: ${PRO_PUBLIC_HOSTNAME}

    UI_AUTH: ${PRO_UI_AUTH}

  script:

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

      UI_AUTH=${UI_AUTH}

  environment:

    name: pro/${CI_PROJECT_NAME}

  only:

    - branches

  except:

    - master

  when: manual



deploy-nifi-stable-branch-production:

  stage: deploy

variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_${CI_PROJECT_NAME}

    STATUS_CHECK_DELAY: 300

    IMAGE_TAG: ${PRO_NIFI_VERSION}

    COMPOSE_FILE: docker-compose.nifi.tmpl.yml:docker-compose.nifi.prod.yml

    PUBLIC_HOSTNAME: ${PRO_PUBLIC_HOSTNAME}

    UI_AUTH: ${PRO_UI_AUTH}

  script:

    - >

      deploy.sh IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE} PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME}

      UI_AUTH=${UI_AUTH}

  environment:

    name: pro/${CI_PROJECT_NAME}

  only:

    - master

  when: manual



deploy-nifi-conf-loader-supporting-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_nifi-conf-loader

  STACK: etl

  STATUS_CHECK_DELAY: 30

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.nifi-conf-loader.tmpl.yml:docker-compose.nifi-conf-loader.dev.yml

  script:

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: dev/nifi-conf-loader

  only:

    - branches

  except:

    - master

  when: manual



deploy-nifi-conf-loader-stable-branch-develop:

  stage: deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_nifi-conf-loader

    STATUS_CHECK_DELAY: 30

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.nifi-conf-loader.tmpl.yml:docker-compose.nifi-conf-loader.dev.yml

  script:

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: dev/nifi-conf-loader

  only:

    - master

  when: manual



deploy-nifi-conf-loader-supporting-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_nifi-conf-loader

    STATUS_CHECK_DELAY: 30

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.nifi-conf-loader.tmpl.yml:docker-compose.nifi-conf-loader.prod.yml

.deploy:

  script:

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: pro/nifi-conf-loader

  only:

    - branches

  except:

    - master

  when: manual



deploy-nifi-conf-loader-stable-branch-production:

  stage: deploy

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICES_TO_CHECK: nifi_nifi-conf-loader

    STATUS_CHECK_DELAY: 30

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.nifi-conf-loader.tmpl.yml:docker-compose.nifi-conf-loader.prod.yml

  script:

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

  environment:

    name: pro/nifi-conf-loader

  only:

    - master

  when: manual
 
    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG}

      NIFI_UID=${NNIFI_UID} NIFI_GID=${NIFI_GID}

      ADDITIONAL_LIBS_URLS="${ADDITIONAL_LIBS_URLS}"

Dockerfile

+3 −0
Original line number Diff line number Diff line
@@ -2,6 +2,9 @@ FROM alpine:3.9 


LABEL maintainer="info@redmic.es"



RUN apk add --no-cache \

		curl=7.64.0-r2



COPY script /



COPY src /redmic-nifi-conf

deploy/.env

+2 −4
Original line number Diff line number Diff line 
PORT=8080

NIFI_LOG_DIR=/opt/nifi/nifi-current/logs

NIFI_HOME=/opt/nifi/nifi-current

NIFI_WEB_HTTP_HOST=0.0.0.0
 
NIFI_UID=1000

NIFI_GID=1000

deploy/config/logback.xml

deleted100644 → 0
+0 −170
Original line number Diff line number Diff line 
<?xml version="1.0" encoding="UTF-8"?>

<!--

  Licensed to the Apache Software Foundation (ASF) under one or more

  contributor license agreements.  See the NOTICE file distributed with

  this work for additional information regarding copyright ownership.

  The ASF licenses this file to You under the Apache License, Version 2.0

  (the "License"); you may not use this file except in compliance with

  the License.  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software

  distributed under the License is distributed on an "AS IS" BASIS,

  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  See the License for the specific language governing permissions and

  limitations under the License.

-->



<configuration scan="true" scanPeriod="30 seconds">

    <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">

        <resetJUL>true</resetJUL>

    </contextListener>



    <appender name="APP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">

        <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app.log</file>

        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">

            <!--

              For daily rollover, use 'app_%d.log'.

              For hourly rollover, use 'app_%d{yyyy-MM-dd_HH}.log'.

              To GZIP rolled files, replace '.log' with '.log.gz'.

              To ZIP rolled files, replace '.log' with '.log.zip'.

            -->

            <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.log</fileNamePattern>

            <!-- keep 2 log files worth of history -->

            <maxHistory>2</maxHistory>

        </rollingPolicy>

        <immediateFlush>true</immediateFlush>

        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">

            <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>

        </encoder>

    </appender>



    <appender name="USER_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">

        <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user.log</file>

        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">

            <!--

              For daily rollover, use 'user_%d.log'.

              For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.

              To GZIP rolled files, replace '.log' with '.log.gz'.

              To ZIP rolled files, replace '.log' with '.log.zip'.

            -->

            <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d{yyyy-MM-dd_HH}.log</fileNamePattern>

            <!-- keep 2 log files worth of history -->

            <maxHistory>2</maxHistory>

        </rollingPolicy>

        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">

            <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>

        </encoder>

    </appender>



    <appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">

        <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>

        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">

            <!--

              For daily rollover, use 'user_%d.log'.

              For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.

              To GZIP rolled files, replace '.log' with '.log.gz'.

              To ZIP rolled files, replace '.log' with '.log.zip'.

            -->

            <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d{yyyy-MM-dd_HH}.log</fileNamePattern>

            <!-- keep 2 log files worth of history -->

            <maxHistory>2</maxHistory>

        </rollingPolicy>

        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">

            <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>

        </encoder>

    </appender>



    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">

        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">

            <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>

        </encoder>

    </appender>



    <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->



    <logger name="org.apache.nifi" level="WARN"/>

    <logger name="org.apache.nifi.processors" level="WARN"/>

    <logger name="org.apache.nifi.processors.standard.LogAttribute" level="WARN"/>

    <logger name="org.apache.nifi.processors.standard.LogMessage" level="WARN"/>

    <logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" />





    <logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />

    <logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />

    <logger name="org.apache.zookeeper.server.NIOServerCnxnFactory" level="ERROR" />

    <logger name="org.apache.zookeeper.server.quorum" level="ERROR" />

    <logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" />

    <logger name="org.apache.zookeeper.server.PrepRequestProcessor" level="ERROR" />



    <logger name="org.apache.calcite.runtime.CalciteException" level="OFF" />



    <logger name="org.apache.curator.framework.recipes.leader.LeaderSelector" level="OFF" />

    <logger name="org.apache.curator.ConnectionState" level="OFF" />



    <!-- Logger for managing logging statements for nifi clusters. -->

    <logger name="org.apache.nifi.cluster" level="WARN"/>



    <!-- Logger for logging HTTP requests received by the web server. -->

    <logger name="org.apache.nifi.server.JettyServer" level="WARN"/>



    <!-- Logger for managing logging statements for jetty -->

    <logger name="org.eclipse.jetty" level="WARN"/>



    <!-- Suppress non-error messages due to excessive logging by class or library -->

    <logger name="org.springframework" level="ERROR"/>



    <!-- Suppress non-error messages due to known warning about redundant path annotation (NIFI-574) -->

    <logger name="org.glassfish.jersey.internal.Errors" level="ERROR"/>



    <!-- Suppress non-error messages due to Jetty AnnotationParser emitting a large amount of WARNS. Issue described in NIFI-5479. -->

    <logger name="org.eclipse.jetty.annotations.AnnotationParser" level="ERROR"/>



    <!--

        Logger for capturing user events. We do not want to propagate these

        log events to the root logger. These messages are only sent to the

        user-log appender.

    -->

    <logger name="org.apache.nifi.web.security" level="WARN" additivity="false">

        <appender-ref ref="USER_FILE"/>

    </logger>

    <logger name="org.apache.nifi.web.api.config" level="WARN" additivity="false">

        <appender-ref ref="USER_FILE"/>

    </logger>

    <logger name="org.apache.nifi.authorization" level="WARN" additivity="false">

        <appender-ref ref="USER_FILE"/>

    </logger>

    <logger name="org.apache.nifi.cluster.authorization" level="WARN" additivity="false">

        <appender-ref ref="USER_FILE"/>

    </logger>

    <logger name="org.apache.nifi.web.filter.RequestLogger" level="WARN" additivity="false">

        <appender-ref ref="USER_FILE"/>

    </logger>





    <!--

        Logger for capturing Bootstrap logs and NiFi's standard error and standard out.

    -->

    <logger name="org.apache.nifi.bootstrap" level="WARN" additivity="false">

        <appender-ref ref="BOOTSTRAP_FILE" />

    </logger>

    <logger name="org.apache.nifi.bootstrap.Command" level="WARN" additivity="false">

        <appender-ref ref="CONSOLE" />

        <appender-ref ref="BOOTSTRAP_FILE" />

    </logger>



    <!-- Everything written to NiFi's Standard Out will be logged with the logger org.apache.nifi.StdOut at INFO level -->

    <logger name="org.apache.nifi.StdOut" level="WARN" additivity="false">

        <appender-ref ref="BOOTSTRAP_FILE" />

    </logger>



    <!-- Everything written to NiFi's Standard Error will be logged with the logger org.apache.nifi.StdErr at ERROR level -->

    <logger name="org.apache.nifi.StdErr" level="ERROR" additivity="false">

        <appender-ref ref="BOOTSTRAP_FILE" />

    </logger>





    <root level="WARN">

        <appender-ref ref="APP_FILE"/>

    </root>



</configuration>
 
 No newline at end of file