Commit 5c7dba7e authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Merge branch 'dev' into 'master' 

Separa índices por servicio

See merge request redmic-project/log/fluentd!2
parents 642b0b88 4db0652a

.gitlab-ci.yml

+11 −42
Original line number Diff line number Diff line
@@ -5,6 +5,12 @@ stages: 


variables:

  DOCKER_DRIVER: overlay2

  FLUENTD_VERSION: '1.3'

  ES_PLUGIN_VERSION: '2.12.1'

  STACK: log

  SERVICES_TO_CHECK: log_${CI_PROJECT_NAME}

  IMAGE_NAME: ${CI_REGISTRY_IMAGE}

  IMAGE_TAG: ${CI_COMMIT_SHA}



services:

  - docker:dind
@@ -12,9 +18,6 @@ services: 
docker-build-commit-non-master-branches:

  stage: package

  image: docker:stable

  variables:

    FLUENTD_VERSION: '1.3'

    ES_PLUGIN_VERSION: '2.12.1'

  only:

    - branches

  except:
@@ -29,9 +32,6 @@ docker-build-commit-non-master-branches: 
docker-build-commit-master-branch:

  stage: package

  image: docker:stable

  variables:

    FLUENTD_VERSION: '1.3'

    ES_PLUGIN_VERSION: '2.12.1'

  only:

    - master

  script:
@@ -43,6 +43,7 @@ docker-build-commit-master-branch: 


docker-tag-already-built-image:

  stage: package

  image: docker:stable

  only:

    - tags

  script:
@@ -78,17 +79,9 @@ deploy-supporting-branch-develop: 
  image: registry.gitlab.com/redmic-project/docker/docker-deploy:latest

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    STACK: log

    SERVICES_TO_CHECK: log_${CI_PROJECT_NAME}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    ES_HOSTS: ${DEV_ES_HOSTS}

    ES_USER: ${DEV_ES_USER}

    ES_PASS: ${DEV_ES_PASS}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} ES_HOSTS=${ES_HOSTS} ES_USER=${ES_USER}

      ES_PASS=${ES_PASS}

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} ES_HOSTS=${ES_HOSTS}

  environment:

    name: dev

  only:
@@ -102,17 +95,9 @@ deploy-stable-branch-develop: 
  image: registry.gitlab.com/redmic-project/docker/docker-deploy:latest

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    STACK: log

    SERVICES_TO_CHECK: log_${CI_PROJECT_NAME}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    ES_HOSTS: ${DEV_ES_HOSTS}

    ES_USER: ${DEV_ES_USER}

    ES_PASS: ${DEV_ES_PASS}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} ES_HOSTS=${ES_HOSTS} ES_USER=${ES_USER}

      ES_PASS=${ES_PASS}

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} ES_HOSTS=${ES_HOSTS}

  environment:

    name: dev

  only:
@@ -124,17 +109,9 @@ deploy-supporting-branch-production: 
  image: registry.gitlab.com/redmic-project/docker/docker-deploy:latest

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    STACK: log

    SERVICES_TO_CHECK: log_${CI_PROJECT_NAME}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    ES_HOSTS: ${PRO_ES_HOSTS}

    ES_USER: ${PRO_ES_USER}

    ES_PASS: ${PRO_ES_PASS}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} ES_HOSTS=${ES_HOSTS} ES_USER=${ES_USER}

      ES_PASS=${ES_PASS}

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} ES_HOSTS=${ES_HOSTS}

  environment:

    name: pro

  only:
@@ -148,17 +125,9 @@ deploy-stable-branch-production: 
  image: registry.gitlab.com/redmic-project/docker/docker-deploy:latest

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    STACK: log

    SERVICES_TO_CHECK: log_${CI_PROJECT_NAME}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    ES_HOSTS: ${PRO_ES_HOSTS}

    ES_USER: ${PRO_ES_USER}

    ES_PASS: ${PRO_ES_PASS}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} ES_HOSTS=${ES_HOSTS} ES_USER=${ES_USER}

      ES_PASS=${ES_PASS}

    - deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} ES_HOSTS=${ES_HOSTS}

  environment:

    name: pro

  only:

Dockerfile

+7 −9
Original line number Diff line number Diff line
@@ -6,12 +6,10 @@ LABEL maintainer="info@redmic.es" 


ARG ES_PLUGIN_VERSION=2.12.1



RUN apk add --update --virtual .build-deps \

	sudo build-base ruby-dev && \

	sudo gem install \

		fluent-plugin-elasticsearch --no-rdoc --no-ri --version ${ES_PLUGIN_VERSION} && \

	sudo gem sources --clear-all && \

	apk del .build-deps && \

	rm -rf \

		/var/cache/apk/* \

		/home/fluent/.gem/ruby/2.5.0/cache/*.gem
 
RUN apk add --update --no-cache --virtual .build-deps \

		build-base \

		ruby-dev && \

	gem install --no-document \

		fluent-plugin-elasticsearch:${ES_PLUGIN_VERSION} && \

	gem sources --clear-all && \

	apk del .build-deps

deploy/.env

+0 −2
Original line number Diff line number Diff line 
ES_HOSTS=es-1:9200,es-2:9200
 
ES_USER=elastic_user

ES_PASS=changeme

deploy/conf/fluent.conf

+16 −11
Original line number Diff line number Diff line 
<source>

	@type forward

	port 24224

	bind 0.0.0.0

</source>

<match *.**>



<filter **>

	@type record_transformer

	<record>

		@service_name ${tag_parts[0]}

	</record>

</filter>



<match **>

	@type copy

	<store>

		@type elasticsearch

		@type elasticsearch_dynamic

		@id elasticsearch-output

		@log_level debug

		hosts "#{ENV['ES_HOSTS']}"

		user "#{ENV['ES_USER']}"

		password "#{ENV['ES_PASS']}"

		index_name fluentd

		type_name fluentd

		logstash_format true

		logstash_prefix fluentd

		logstash_prefix fluentd-${tag_parts[0]}

		logstash_dateformat %Y%m%d

		include_tag_key true

		tag_key @log_name

		flush_interval 1s

		flush_interval 5s

	</store>

	<store>

		@type stdout

		@id standard-output

		@log_level warn

	</store>

</match>

deploy/docker-compose.yml

+7 −7
Original line number Diff line number Diff line
@@ -5,14 +5,12 @@ services: 
    image: ${IMAGE_NAME}:${IMAGE_TAG:-latest}

    environment:

      - ES_HOSTS

      - ES_USER

      - ES_PASS

    networks:

      log-net:

        aliases:

          - fluentd

      elastic-net:

    secrets:

    configs:

      - source: fluent-conf

        target: /fluentd/etc/fluent.conf

    logging:
@@ -28,17 +26,19 @@ services: 
          cpus: '1'

          memory: 128M

        reservations:

          memory: 103M

          memory: 64M



networks:

  log-net:

    name: log-net

    name: ${LOG_NET_NAME:-log-net}

    driver: overlay

    attachable: true

    attachable: ${LOG_NET_ATTACHABLE:-true}



  elastic-net:

    name: ${ELASTIC_NET_NAME:-elastic6-net}

    external: true



secrets:

configs:

  fluent-conf:

    name: ${FLUENT_CONF_NAME:-fluent-conf}

    file: ./conf/fluent.conf