Commit 92fc8d4c authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Aplica multitud de mejoras y amplía configuración 

Tras multitud de pruebas, se ha consolidado la presente configuración
de despliegue. A continuación, los cambios destacados para cada
servicio:

En general:

Permite definir desde variables de entorno más valores de los que se
recogen en el comando como parámetros. Esto facilita su actualización
editando las variables del servicio.

Se elimina el servicio etcd, debido a problemas de llenado de memoria
cuando los ficheros contenidos cambian con mucha frecuencia. Se usa
store local en filer como sustituto.

Permite deshabilitar la exposición mediante traefik desde variable de
entorno.

En servicio master:

Cambia esquema de replicación por defecto, distribuyendo entre
datacenters en lugar de entre racks. Esto hace que mejoren los tiempos
de lectura, ya que no paraleliza el proceso entre las diferentes
réplicas (que están siempre en diferentes nodos del cluster Swarm).
Mantiene la creación de 2 copias por volumen, para tener 3 en total
(con la original) y poder asumir la caída de uno de los nodos.

Amplía nombre de datacenter para incluir identificador del nodo,
necesario para la distribución a nivel de datacenter.

Permite definir 'garbageThreshold'.

En servicio volume:

Permite definir modo de lectura. Por defecto, cada volume hace de proxy
para los volúmenes que no tiene en local. Se define que por defecto,
redirija la petición al correcto.

Asigna tamaño de buffer de lectura (4MB por defecto) y flag de lectura
lenta (habilitado por defecto) para no bloquear con peticiones largas.

Asigna etiqueta de volumen 'hdd' por defecto.

En servicio filer:

Habilita store local filerdb3, que integra sincronización entre
diferentes replicas y sustituye a etcd.

Amplia recursos de memoria y CPU, ahora que hace también el trabajo de
store.
parent 106162eb

deploy/.env

+8 −14
Original line number Diff line number Diff line
@@ -4,15 +4,20 @@ IMAGE_TAG=latest 
MASTER_PORT=9333

MASTER_IP=swfs-master-{{.Task.Slot}}

MASTER_VOLUME_SIZE_LIMIT_MB=1024

DEFAULT_REPLICATION=020

MASTER_DEFAULT_REPLICATION=200

MASTER_GARBAGE_THRESHOLD=0.01



VOLUME_PORT=8080

VOLUME_IP=swfs-volume-{{.Node.ID}}

VOLUME_MAX=0

VOLUME_DISK=hdd

VOLUME_FILE_SIZE_LIMIT_MB=256

VOLUME_UPLOAD_LIMIT_MB=256

VOLUME_DOWNLOAD_LIMIT_MB=256

VOLUME_PUBLIC_HOST={{.Node.Hostname}}

VOLUME_READ_MODE=redirect

VOLUME_READ_BUFFER_SIZE=4

VOLUME_HAS_SLOW_READ=true

PUBLIC_VOLUME_PORT=8080

VOLUME_PORT_MODE=host
@@ -29,23 +34,16 @@ FILER_PORT_MODE=host 


MASTER_HOST=swfs-master

DATA_PATH=/data

DATACENTER=dc

DATACENTER=dc-{{.Node.ID}}

RACK=r-{{.Node.ID}}

METRICS_PORT=9090



ETCD_DATA_PATH=/etcd-data

ETCD_CLIENT_PORT=2379

ETCD_PEER_PORT=2380

ETCD_IP=swfs-etcd-{{.Task.Slot}}

ETCD_HOST=swfs-etcd

ETCD_CLUSTER_STATE=new

ETCD_CLUSTER_TOKEN=changeme



HEALTHCHECK_INTERVAL=15s

HEALTHCHECK_TIMEOUT=60s

HEALTHCHECK_RETRIES=10

HEALTHCHECK_START_PERIOD=2m



TRAEFIK_ENABLE=true

TRAEFIK_ENTRYPOINT=http

PUBLIC_HOSTNAME=change.me

MASTER_TRAEFIK_SUBDOMAIN=swfs-master
@@ -58,7 +56,3 @@ FILER_TRAEFIK_AUTH=user:$apr1$md5password 
MASTER_VOL_NAME=seaweedfs-master-{{.Task.Slot}}-vol

VOLUME_VOL_NAME=seaweedfs-volume-{{.Node.ID}}-vol

FILER_VOL_NAME=seaweedfs-filer-{{.Node.ID}}-vol
 
ETCD_VOL_NAME=seaweedfs-etcd-{{.Node.ID}}-vol



MASTER_NODE_LABEL_NAME=swfs-master

MASTER_NODE_LABEL_VALUE=1

deploy/docker-compose.tmpl.yml

+42 −70
Original line number Diff line number Diff line
@@ -11,8 +11,9 @@ services: 
          weed -logtostderr=true master

            -mdir=${DATA_PATH}

            -volumePreallocate

            -volumeSizeLimitMB=${MASTER_VOLUME_SIZE_LIMIT_MB}

            -defaultReplication=${DEFAULT_REPLICATION}

            -volumeSizeLimitMB=$${MASTER_VOLUME_SIZE_LIMIT_MB}

            -defaultReplication=$${MASTER_DEFAULT_REPLICATION}

            -garbageThreshold=$${MASTER_GARBAGE_THRESHOLD}

            -ip=$${MASTER_IP}

            -ip.bind=0.0.0.0

            -port=${MASTER_PORT}
@@ -22,6 +23,9 @@ services: 
    hostname: ${MASTER_IP}

    environment:

      MASTER_IP:

      MASTER_VOLUME_SIZE_LIMIT_MB:

      MASTER_DEFAULT_REPLICATION:

      MASTER_GARBAGE_THRESHOLD:

    networks:

      seaweedfs-net:

      traefik-net:
@@ -42,7 +46,7 @@ services: 
        delay: ${MASTER_UPDATE_DELAY:-1m}

      placement:

        constraints:

          - "node.labels.${MASTER_NODE_LABEL_NAME}==${MASTER_NODE_LABEL_VALUE}"

          - "node.labels.${MASTER_NODE_LABEL_NAME:-swfs-master}==${MASTER_NODE_LABEL_VALUE:-1}"

        max_replicas_per_node: 1

      resources:

        limits:
@@ -52,7 +56,7 @@ services: 
          cpus: '${MASTER_RESOURCES_RESERVATIONS_CPUS:-0.01}'

          memory: ${MASTER_RESOURCES_RESERVATIONS_MEMORY:-64M}

      labels:

        traefik.enable: 'true'

        traefik.enable: ${TRAEFIK_ENABLE}

        traefik.http.routers.swfs-master.entrypoints: ${TRAEFIK_ENTRYPOINT}

        traefik.http.routers.swfs-master.rule: Host(`${MASTER_TRAEFIK_SUBDOMAIN}.${PUBLIC_HOSTNAME}`)

        traefik.http.middlewares.swfs-master-auth.basicauth.users: ${MASTER_TRAEFIK_AUTH}
@@ -69,17 +73,20 @@ services: 
        $$(echo "

          weed -logtostderr=true volume

            -dir=${DATA_PATH}

            -max=${VOLUME_MAX}

            -max=$${VOLUME_MAX}

            -dataCenter=$${VOLUME_DATACENTER}

            -rack=$${VOLUME_RACK}

            -disk=${VOLUME_DISK}

            -fileSizeLimitMB=${VOLUME_FILE_SIZE_LIMIT_MB}

            -concurrentUploadLimitMB=${VOLUME_UPLOAD_LIMIT_MB}

            -concurrentDownloadLimitMB=${VOLUME_DOWNLOAD_LIMIT_MB}

            -disk=$${VOLUME_DISK}

            -fileSizeLimitMB=$${VOLUME_FILE_SIZE_LIMIT_MB}

            -concurrentUploadLimitMB=$${VOLUME_UPLOAD_LIMIT_MB}

            -concurrentDownloadLimitMB=$${VOLUME_DOWNLOAD_LIMIT_MB}

            -ip=$${VOLUME_IP}

            -ip.bind=0.0.0.0

            -port=${VOLUME_PORT}

            -publicUrl=$${VOLUME_PUBLIC_HOST}:${PUBLIC_VOLUME_PORT}

            -readMode=$${VOLUME_READ_MODE}

            -readBufferSizeMB=$${VOLUME_READ_BUFFER_SIZE}

            -hasSlowRead=$${VOLUME_HAS_SLOW_READ}

            -mserver=${MASTER_HOST}-1:${MASTER_PORT},${MASTER_HOST}-2:${MASTER_PORT},${MASTER_HOST}-3:${MASTER_PORT}

            -metricsPort=${METRICS_PORT}

        ")
@@ -87,8 +94,16 @@ services: 
    environment:

      VOLUME_IP:

      VOLUME_PUBLIC_HOST:

      VOLUME_MAX:

      VOLUME_DATACENTER: ${DATACENTER}

      VOLUME_RACK: ${RACK}

      VOLUME_DISK:

      VOLUME_FILE_SIZE_LIMIT_MB:

      VOLUME_UPLOAD_LIMIT_MB:

      VOLUME_DOWNLOAD_LIMIT_MB:

      VOLUME_READ_MODE:

      VOLUME_READ_BUFFER_SIZE:

      VOLUME_HAS_SLOW_READ:

    networks:

      seaweedfs-net:

      traefik-net:
@@ -123,7 +138,7 @@ services: 
          cpus: '${VOLUME_RESOURCES_RESERVATIONS_CPUS:-0.1}'

          memory: ${VOLUME_RESOURCES_RESERVATIONS_MEMORY:-320M}

      labels:

        traefik.enable: 'true'

        traefik.enable: ${TRAEFIK_ENABLE}

        traefik.http.routers.swfs-volume.entrypoints: ${TRAEFIK_ENTRYPOINT}

        traefik.http.routers.swfs-volume.rule: Host(`${VOLUME_TRAEFIK_SUBDOMAIN}.${PUBLIC_HOSTNAME}`)

        traefik.http.middlewares.swfs-volume-auth.basicauth.users: ${VOLUME_TRAEFIK_AUTH}
@@ -141,11 +156,11 @@ services: 
          weed -logtostderr=true filer

            -dataCenter=$${FILER_PREFERED_DATACENTER}

            -rack=$${FILER_PREFERED_RACK}

            -filerGroup=${FILER_GROUP}

            -collection=${FILER_COLLECTION}

            -maxMB=${FILER_MAX_MB}

            -concurrentUploadLimitMB=${FILER_UPLOAD_LIMIT_MB}

            -downloadMaxMBps=${FILER_DOWNLOAD_LIMIT_MB}

            -filerGroup=$${FILER_GROUP}

            -collection=$${FILER_COLLECTION}

            -maxMB=$${FILER_MAX_MB}

            -concurrentUploadLimitMB=$${FILER_UPLOAD_LIMIT_MB}

            -downloadMaxMBps=$${FILER_DOWNLOAD_LIMIT_MB}

            -ip=$${FILER_IP}

            -ip.bind=0.0.0.0

            -port=${FILER_PORT}
@@ -158,9 +173,14 @@ services: 
      FILER_IP:

      FILER_PREFERED_DATACENTER: ${DATACENTER}

      FILER_PREFERED_RACK: ${RACK}

      FILER_GROUP:

      FILER_COLLECTION:

      FILER_MAX_MB:

      FILER_UPLOAD_LIMIT_MB:

      FILER_DOWNLOAD_LIMIT_MB:

      WEED_LEVELDB2_ENABLED: 'false'

      WEED_ETCD_ENABLED: 'true'

      WEED_ETCD_SERVERS: http://${ETCD_HOST}-1:${ETCD_CLIENT_PORT},http://${ETCD_HOST}-2:${ETCD_CLIENT_PORT},http://${ETCD_HOST}-3:${ETCD_CLIENT_PORT}

      WEED_LEVELDB3_ENABLED: 'true'

      WEED_LEVELDB3_DIR: ${DATA_PATH}/filerldb3

    networks:

      seaweedfs-net:

      traefik-net:
@@ -187,13 +207,13 @@ services: 
        delay: ${FILER_UPDATE_DELAY:-15s}

      resources:

        limits:

          cpus: '${FILER_RESOURCES_LIMITS_CPUS:-1}'

          memory: ${FILER_RESOURCES_LIMITS_MEMORY:-128M}

          cpus: '${FILER_RESOURCES_LIMITS_CPUS:-2}'

          memory: ${FILER_RESOURCES_LIMITS_MEMORY:-256M}

        reservations:

          cpus: '${FILER_RESOURCES_RESERVATIONS_CPUS:-0.05}'

          memory: ${FILER_RESOURCES_RESERVATIONS_MEMORY:-64M}

          cpus: '${FILER_RESOURCES_RESERVATIONS_CPUS:-0.1}'

          memory: ${FILER_RESOURCES_RESERVATIONS_MEMORY:-128M}

      labels:

        traefik.enable: 'true'

        traefik.enable: ${TRAEFIK_ENABLE}

        traefik.http.routers.swfs-filer.entrypoints: ${TRAEFIK_ENTRYPOINT}

        traefik.http.routers.swfs-filer.rule: Host(`${FILER_TRAEFIK_SUBDOMAIN}.${PUBLIC_HOSTNAME}`)

        traefik.http.middlewares.swfs-filer-auth.basicauth.users: ${FILER_TRAEFIK_AUTH}
@@ -201,50 +221,6 @@ services: 
        traefik.http.services.swfs-filer.loadbalancer.server.port: ${FILER_PORT}

        traefik.http.services.swfs-filer.loadbalancer.sticky.cookie: 'true'



  swfs-etcd:

    image: ${ETCD_IMAGE_NAME:-quay.io/coreos/etcd}:${ETCD_IMAGE_TAG:-latest}

    entrypoint: /bin/sh

    command:

      - -c

      - |

        $$(echo "

          /usr/local/bin/etcd

            --data-dir=${ETCD_DATA_PATH}

            --name $${ETCD_IP}

            --initial-advertise-peer-urls http://$${ETCD_IP}:${ETCD_PEER_PORT}

            --listen-peer-urls http://0.0.0.0:${ETCD_PEER_PORT}

            --advertise-client-urls http://$${ETCD_IP}:${ETCD_CLIENT_PORT}

            --listen-client-urls http://0.0.0.0:${ETCD_CLIENT_PORT}

            --initial-cluster ${ETCD_HOST}-1=http://${ETCD_HOST}-1:${ETCD_PEER_PORT},${ETCD_HOST}-2=http://${ETCD_HOST}-2:${ETCD_PEER_PORT},${ETCD_HOST}-3=http://${ETCD_HOST}-3:${ETCD_PEER_PORT}

            --initial-cluster-state ${ETCD_CLUSTER_STATE}

            --initial-cluster-token ${ETCD_CLUSTER_TOKEN}

        ")

    hostname: ${ETCD_IP}

    environment:

      ETCD_IP:

    networks:

      seaweedfs-net:

    volumes:

      - etcd-vol:${ETCD_DATA_PATH}

    deploy:

      mode: replicated

      replicas: ${ETCD_REPLICAS:-3}

      restart_policy:

        delay: ${ETCD_RESTART_DELAY:-1s}

      update_config:

        delay: ${ETCD_UPDATE_DELAY:-10s}

      placement:

        constraints:

          - "node.labels.${MASTER_NODE_LABEL_NAME}==${MASTER_NODE_LABEL_VALUE}"

        max_replicas_per_node: 1

      resources:

        limits:

          cpus: '${ETCD_RESOURCES_LIMITS_CPUS:-1}'

          memory: ${ETCD_RESOURCES_LIMITS_MEMORY:-256M}

        reservations:

          cpus: '${ETCD_RESOURCES_RESERVATIONS_CPUS:-0.05}'

          memory: ${ETCD_RESOURCES_RESERVATIONS_MEMORY:-128M}



networks:

  seaweedfs-net:

    name: ${SEAWEEDFS_NET_NAME:-seaweedfs-net}
@@ -268,7 +244,3 @@ volumes: 
  filer-vol:

    name: ${FILER_VOL_NAME}

    driver: local
 


  etcd-vol:

    name: ${ETCD_VOL_NAME}

    driver: local