Commit 2fcbcd9d authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Actualiza config por defecto, revisa sintaxis 

Aplica cambios de configuración básica de Kafka: habilita cleaner de
compactación, aumenta réplicas y mínimo en sincronía y aumenta la
retención de offset de consumidores (ahora que se va a comenzar a
limpiar).

Limpia y unifica variables en gitlab-ci y docker-compose. Especifica
algunas a nivel de servicio, para permitir configurarlas desde fuera
realmente.

Modifica ligeramente chequeo de salud, para evitar warning en la salida
de logs.

Disminuye timeout de reinicio.
parent aac2c1c7

.gitlab-ci.yml

+1 −7
Original line number Diff line number Diff line
@@ -6,12 +6,9 @@ include: 
stages:

  - deploy



variables:

  PROJECT_PARENT_NAME: confluent



.deploy:

  variables:

    STACK: ${PROJECT_PARENT_NAME}

    STACK: confluent

    DD_AWS_REGION: ${AWS_REGION}



.deploy-development:
@@ -25,17 +22,14 @@ variables: 
.deploy-kf-1:

  variables: &deploy-kf-1-variables

    SERVICE_NAME: kf-1

    SERVICES_TO_CHECK: ${PROJECT_PARENT_NAME}_${CI_PROJECT_NAME}-1



.deploy-kf-2:

  variables: &deploy-kf-2-variables

    SERVICE_NAME: kf-2

    SERVICES_TO_CHECK: ${PROJECT_PARENT_NAME}_${CI_PROJECT_NAME}-2



.deploy-kf-3:

  variables: &deploy-kf-3-variables

    SERVICE_NAME: kf-3

    SERVICES_TO_CHECK: ${PROJECT_PARENT_NAME}_${CI_PROJECT_NAME}-3



.deploy-kf-1-development:

  extends: .deploy-development

deploy/.env

+18 −5
Original line number Diff line number Diff line
@@ -10,20 +10,33 @@ JMX_PROMETHEUS_JAVAAGENT_PATH=/jmx_prometheus_javaagent.jar 
JMX_PROMETHEUS_CONFIG_PATH=/config.yaml



KAFKA_LOG_DIRS=/var/lib/kafka/data/data

KAFKA_LOG_CLEANER_ENABLE=false

KAFKA_LOG_CLEANER_ENABLE=true

KAFKA_LOG_CLEANUP_POLICY=compact

KAFKA_LOG_CLEANER_MIN_CLEANABLE_RATIO=0.5

KAFKA_LOG_CLEANER_MIN_COMPACTION_LAG_MS=3600000

KAFKA_LOG_ROLL_HOURS=168



KAFKA_NUM_PARTITIONS=5

KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=2

KAFKA_DEFAULT_REPLICATION_FACTOR=2

KAFKA_MIN_INSYNC_REPLICAS=1

KAFKA_OFFSETS_RETENTION_MINUTES=10080

KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=3

KAFKA_DEFAULT_REPLICATION_FACTOR=3

KAFKA_MIN_INSYNC_REPLICAS=2

KAFKA_OFFSETS_RETENTION_MINUTES=43200

KAFKA_SOCKET_RECEIVE_BUFFER_BYTES=-1

KAFKA_MESSAGE_MAX_BYTES=15728640

KAFKA_LOG4J_ROOT_LOGLEVEL=ERROR

KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT

KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT

KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false



JMX_PROMETHEUS_JAVA_AGENT_NAME=jmx-prometheus-javaagent-0.11.0

JMX_PROMETHEUS_CONFIG_NAME=jmx-prometheus-config-0.11.0



KF_1_DATA_VOL_NAME=kf-1-data-vol

KF_2_DATA_VOL_NAME=kf-2-data-vol

KF_3_DATA_VOL_NAME=kf-3-data-vol

KF_1_SECRETS_VOL_NAME=kf-1-secrets-vol

KF_2_SECRETS_VOL_NAME=kf-2-secrets-vol

KF_3_SECRETS_VOL_NAME=kf-3-secrets-vol



KF_DATA_VOL_SIZE=500

KF_DATA_VOL_EBS_TYPE=st1

deploy/docker-compose.kafka-1.dev.yml

+1 −1
Original line number Diff line number Diff line
@@ -18,4 +18,4 @@ services: 


volumes:

  kf-data-vol:

    name: ${KF_DATA_VOL_NAME:-kf-1-data-vol}
 
    name: ${KF_1_DATA_VOL_NAME}

deploy/docker-compose.kafka-1.prod.yml

+3 −3
Original line number Diff line number Diff line
@@ -19,9 +19,9 @@ services: 


volumes:

  kf-data-vol:

    name: ${KF_DATA_VOL_NAME:-kf-1-data-vol}

    name: ${KF_1_DATA_VOL_NAME}

    driver: 'cloudstor:aws'

    driver_opts:

      backing: relocatable

      size: ${KF_DATA_VOL_SIZE:-500}

      ebstype: ${KF_DATA_VOL_EBS_TYPE:-st1}
 
      size: ${KF_DATA_VOL_SIZE}

      ebstype: ${KF_DATA_VOL_EBS_TYPE}

deploy/docker-compose.kafka-1.tmpl.yml

+5 −5
Original line number Diff line number Diff line
@@ -43,7 +43,7 @@ services: 
      - kf-data-vol:/var/lib/kafka/data

      - kf-secrets-vol:/etc/kafka/secrets

    healthcheck:

      test: wget --spider -nv -t 1 http://localhost:${JMX_HTTP_PORT}

      test: wget -q -O /dev/null -t 1 http://localhost:${JMX_HTTP_PORT}

      interval: ${HEALTHCHECK_INTERVAL:-30s}

      timeout: ${HEALTHCHECK_TIMEOUT:-15s}

      retries: ${HEALTHCHECK_RETRIES:-10}
@@ -52,7 +52,7 @@ services: 
      mode: replicated

      replicas: ${REPLICAS:-1}

      restart_policy:

        delay: ${RESTART_DELAY:-30s}

        delay: ${RESTART_DELAY:-10s}

      update_config:

        delay: ${UPDATE_DELAY:-5m}
@@ -69,13 +69,13 @@ networks: 


volumes:

  kf-secrets-vol:

    name: ${KF_SECRETS_VOL_NAME:-kf-1-secrets-vol}

    name: ${KF_1_SECRETS_VOL_NAME}



configs:

  jmx-prometheus-javaagent:

    name: ${JMX_PROMETHEUS_JAVA_AGENT_NAME:-jmx-prometheus-javaagent-0.11.0}

    name: ${JMX_PROMETHEUS_JAVA_AGENT_NAME}

    external: true



  jmx-prometheus-config:

    name: ${JMX_PROMETHEUS_CONFIG_NAME:-jmx-prometheus-config-0.11.0}

    name: ${JMX_PROMETHEUS_CONFIG_NAME}

    external: true