Commit 084c12fa authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Actualiza config de broker y schemas 

Schema registry ahora usa esquemas compatibles hacia atrás
transitivamente.

Se deshabilita por el momento la compactación de topics, aunque se deja
preparada la configuración para cuando se habilite.

Por defecto se trabaja con 5 particiones por topic, y se retocan otros
parámetros de los mismos.
parent 61280c92

kafka/.env

+9 −0
Original line number Diff line number Diff line 
ZK_PORT=2181

KF_PORT=9092

KAFKA_LOG_CLEANER_ENABLE=false

KAFKA_LOG_CLEANUP_POLICY=compact

KAFKA_LOG_CLEANER_MIN_CLEANABLE_RATIO=0.5

KAFKA_LOG_CLEANER_MIN_COMPACTION_LAG_MS=3600000

KAFKA_LOG_ROLL_HOURS=168

KAFKA_NUM_PARTITIONS=5

KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=3

KAFKA_DEFAULT_REPLICATION_FACTOR=3

KAFKA_MIN_INSYNC_REPLICAS=2

kafka/docker-compose.dev.yml

+12 −12
Original line number Diff line number Diff line
@@ -5,31 +5,31 @@ services: 
    container_name: kf-1

    restart: on-failure:3

    ports:

      - 1${KF_PORT}:${KF_PORT}

      - 9997:9997

      - '1${KF_PORT}:${KF_PORT}'

      - '9997:9997'

    environment:

      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${PUBLIC_HOSTNAME}:1${KF_PORT}

      KAFKA_HEAP_OPTS: '-Xmx256m -Xms256m'

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://${PUBLIC_HOSTNAME}:1${KF_PORT}

      - KAFKA_HEAP_OPTS=-Xmx256m -Xms256m



  kf-2:

    container_name: kf-2

    restart: on-failure:3

    ports:

      - 2${KF_PORT}:${KF_PORT}

      - 9998:9998

      - '2${KF_PORT}:${KF_PORT}'

      - '9998:9998'

    environment:

      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${PUBLIC_HOSTNAME}:2${KF_PORT}

      KAFKA_HEAP_OPTS: '-Xmx256m -Xms256m'

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://${PUBLIC_HOSTNAME}:2${KF_PORT}

      - KAFKA_HEAP_OPTS=-Xmx256m -Xms256m



  kf-3:

    container_name: kf-3

    restart: on-failure:3

    ports:

      - 3${KF_PORT}:${KF_PORT}

      - 9999:9999

      - '3${KF_PORT}:${KF_PORT}'

      - '9999:9999'

    environment:

      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${PUBLIC_HOSTNAME}:3${KF_PORT}

      KAFKA_HEAP_OPTS: '-Xmx256m -Xms256m'

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://${PUBLIC_HOSTNAME}:3${KF_PORT}

      - KAFKA_HEAP_OPTS=-Xmx256m -Xms256m



volumes:

  kf-1-data-vol:

kafka/docker-compose.prod.yml

+12 −12
Original line number Diff line number Diff line
@@ -3,11 +3,11 @@ version: '3.5' 
services:

  kf-1:

    ports:

      - ${KF_PORT}

      - 9997

      - '${KF_PORT}'

      - '9997'

    environment:

      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kf-1:${KF_PORT}

      KAFKA_HEAP_OPTS: '-Xmx4g -Xms4g'

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kf-1:${KF_PORT}

      - KAFKA_HEAP_OPTS=-Xmx4g -Xms4g

    deploy:

      mode: replicated

      replicas: 1
@@ -21,11 +21,11 @@ services: 


  kf-2:

    ports:

      - ${KF_PORT}

      - 9998

      - '${KF_PORT}'

      - '9998'

    environment:

      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kf-2:${KF_PORT}

      KAFKA_HEAP_OPTS: '-Xmx4g -Xms4g'

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kf-2:${KF_PORT}

      - KAFKA_HEAP_OPTS=-Xmx4g -Xms4g

    deploy:

      mode: replicated

      replicas: 1
@@ -39,11 +39,11 @@ services: 


  kf-3:

    ports:

      - ${KF_PORT}

      - 9999

      - '${KF_PORT}'

      - '9999'

    environment:

      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kf-3:${KF_PORT}

      KAFKA_HEAP_OPTS: '-Xmx4g -Xms4g'

      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kf-3:${KF_PORT}

      - KAFKA_HEAP_OPTS=-Xmx4g -Xms4g

    deploy:

      mode: replicated

      replicas: 1

kafka/docker-compose.tmpl.yml

+39 −30
Original line number Diff line number Diff line
@@ -9,16 +9,19 @@ services: 
    volumes:

      - kf-1-data-vol:/var/lib/kafka/data

    environment:

      KAFKA_BROKER_ID: 1

      KAFKA_ZOOKEEPER_CONNECT: zk-1:${ZK_PORT},zk-2:${ZK_PORT},zk-3:${ZK_PORT}

      KAFKA_LISTENERS: PLAINTEXT://kf-1:${KF_PORT}

      KAFKA_LOG_CLEANUP_POLICY: compact

      KAFKA_SEGMENT_MS: 60000

      KAFKA_NUM_PARTITIONS: 1

      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2

      KAFKA_DEFAULT_REPLICATION_FACTOR: 3

      KAFKA_MIN_INSYNC_REPLICAS: 2

      KAFKA_JMX_PORT: 9997

      - KAFKA_BROKER_ID=1

      - KAFKA_ZOOKEEPER_CONNECT=zk-1:${ZK_PORT},zk-2:${ZK_PORT},zk-3:${ZK_PORT}

      - KAFKA_LISTENERS=PLAINTEXT://kf-1:${KF_PORT}

      - KAFKA_JMX_PORT=9997

      - KAFKA_LOG_CLEANER_ENABLE

      - KAFKA_LOG_CLEANUP_POLICY

      - KAFKA_LOG_CLEANER_MIN_CLEANABLE_RATIO

      - KAFKA_LOG_CLEANER_MIN_COMPACTION_LAG_MS

      - KAFKA_LOG_ROLL_HOURS

      - KAFKA_NUM_PARTITIONS

      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR

      - KAFKA_DEFAULT_REPLICATION_FACTOR

      - KAFKA_MIN_INSYNC_REPLICAS

    healthcheck:

      test: kafka-topics --list -zookeeper $${KAFKA_ZOOKEEPER_CONNECT}

      interval: 30s
@@ -34,16 +37,19 @@ services: 
    volumes:

      - kf-2-data-vol:/var/lib/kafka/data

    environment:

      KAFKA_BROKER_ID: 2

      KAFKA_ZOOKEEPER_CONNECT: zk-1:${ZK_PORT},zk-2:${ZK_PORT},zk-3:${ZK_PORT}

      KAFKA_LISTENERS: PLAINTEXT://kf-2:${KF_PORT}

      KAFKA_LOG_CLEANUP_POLICY: compact

      KAFKA_SEGMENT_MS: 60000

      KAFKA_NUM_PARTITIONS: 1

      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2

      KAFKA_DEFAULT_REPLICATION_FACTOR: 3

      KAFKA_MIN_INSYNC_REPLICAS: 2

      KAFKA_JMX_PORT: 9998

      - KAFKA_BROKER_ID=2

      - KAFKA_ZOOKEEPER_CONNECT=zk-1:${ZK_PORT},zk-2:${ZK_PORT},zk-3:${ZK_PORT}

      - KAFKA_LISTENERS=PLAINTEXT://kf-2:${KF_PORT}

      - KAFKA_JMX_PORT=9998

      - KAFKA_LOG_CLEANER_ENABLE

      - KAFKA_LOG_CLEANUP_POLICY

      - KAFKA_LOG_CLEANER_MIN_CLEANABLE_RATIO

      - KAFKA_LOG_CLEANER_MIN_COMPACTION_LAG_MS

      - KAFKA_LOG_ROLL_HOURS

      - KAFKA_NUM_PARTITIONS

      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR

      - KAFKA_DEFAULT_REPLICATION_FACTOR

      - KAFKA_MIN_INSYNC_REPLICAS

    healthcheck:

      test: kafka-topics --list -zookeeper $${KAFKA_ZOOKEEPER_CONNECT}

      interval: 30s
@@ -59,16 +65,19 @@ services: 
    volumes:

      - kf-3-data-vol:/var/lib/kafka/data

    environment:

      KAFKA_BROKER_ID: 3

      KAFKA_ZOOKEEPER_CONNECT: zk-1:${ZK_PORT},zk-2:${ZK_PORT},zk-3:${ZK_PORT}

      KAFKA_LISTENERS: PLAINTEXT://kf-3:${KF_PORT}

      KAFKA_LOG_CLEANUP_POLICY: compact

      KAFKA_SEGMENT_MS: 60000

      KAFKA_NUM_PARTITIONS: 1

      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2

      KAFKA_DEFAULT_REPLICATION_FACTOR: 3

      KAFKA_MIN_INSYNC_REPLICAS: 2

      KAFKA_JMX_PORT: 9999

      - KAFKA_BROKER_ID=3

      - KAFKA_ZOOKEEPER_CONNECT=zk-1:${ZK_PORT},zk-2:${ZK_PORT},zk-3:${ZK_PORT}

      - KAFKA_LISTENERS=PLAINTEXT://kf-3:${KF_PORT}

      - KAFKA_JMX_PORT=9999

      - KAFKA_LOG_CLEANER_ENABLE

      - KAFKA_LOG_CLEANUP_POLICY

      - KAFKA_LOG_CLEANER_MIN_CLEANABLE_RATIO

      - KAFKA_LOG_CLEANER_MIN_COMPACTION_LAG_MS

      - KAFKA_LOG_ROLL_HOURS

      - KAFKA_NUM_PARTITIONS

      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR

      - KAFKA_DEFAULT_REPLICATION_FACTOR

      - KAFKA_MIN_INSYNC_REPLICAS

    healthcheck:

      test: kafka-topics --list -zookeeper $${KAFKA_ZOOKEEPER_CONNECT}

      interval: 30s

workers/docker-compose.schema-registry.dev.yml

+1 −1
Original line number Diff line number Diff line
@@ -7,4 +7,4 @@ services: 
    ports:

      - '1${SR_PORT}:${SR_PORT}'

    environment:

      SCHEMA_REGISTRY_HEAP_OPTS: '-Xmx256m -Xms256m'
 
      - SCHEMA_REGISTRY_HEAP_OPTS=-Xmx256m -Xms256m