Commit 31afaa18 authored by Noel Alonso's avatar Noel Alonso
Browse files

Merge branch 'dev' into 'master' 

Dev

See merge request redmic-project/server/vessel/postgresql!16
parents aa0888e7 5e0eb656

.dockerignore

+1 −1
Original line number Diff line number Diff line 
*

!scripts
 
 No newline at end of file
 
!rootfs

.gitlab-ci.yml

+50 −215
Original line number Diff line number Diff line 
include:

  - project: 'redmic-project/gitlab-ci-templates'

    ref: master

    file: '/packaging.yml'

  - project: 'redmic-project/gitlab-ci-templates'

    ref: master

    file: '/deployment.yml'

  - project: 'redmic-project/gitlab-ci-templates'

    ref: master

    file: '/maintenance.yml'



stages:

  - package

  - test-package

  - deploy

  - backup



services:

  - docker:dind

  - maintenance



variables:

  DOCKER_DRIVER: overlay2



docker-build-commit-non-master-branches:

  stage: package

  image: docker:stable

  only:

    - branches

    - schedules

  except:

    - master

  script:

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - docker build -t ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} -t ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME}-latest .

    - docker push ${CI_REGISTRY_IMAGE}



docker-build-commit-master-branch:

  stage: package

  image: docker:stable

  only:

    - master

  except:

    - schedules

  script:

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - docker build -t ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} -t ${CI_REGISTRY_IMAGE}:latest .

    - docker push ${CI_REGISTRY_IMAGE}



docker-tag-already-built-image:

  stage: package

  image: docker:stable

  only:

    - tags

  except:

    - schedules

  script:

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - docker tag ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} ${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}

    - docker push ${CI_REGISTRY_IMAGE}



container-scanning:

  stage: test-package

  image: docker:stable

  allow_failure: true

  only:

    - branches

  except:

    - schedules

  script:

    - docker run -d --name db arminc/clair-db:latest

    - docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.1

    - apk add -U wget ca-certificates

    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}

    - docker pull ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}

    - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64

    - mv clair-scanner_linux_amd64 clair-scanner

    - chmod +x clair-scanner

    - touch clair-whitelist.yml

    - >

      ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-sast-container-report.json -l clair.log

      -w clair-whitelist.yml ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} || true

  artifacts:

    paths: [gl-sast-container-report.json]



deploy-dev:

  stage: deploy

  image: registry.gitlab.com/redmic-project/docker/docker-deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

  STACK: vessel

    SERVICES_TO_CHECK: vessel_${CI_PROJECT_NAME}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.postgresql.tmpl.yml:docker-compose.postgresql.dev.yml

    POSTGRES_USER: ${POSTGRES_USER}

    POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}

  script:

    - create-nets.sh postgres-net

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      POSTGRES_USER=${POSTGRES_USER} POSTGRES_PASSWORD=${POSTGRES_PASSWORD}

  environment:

    name: dev/postgresql

  only:

    - dev

  except:

    - schedules

  when: manual



deploy-supporting-branch:

  stage: deploy

  image: registry.gitlab.com/redmic-project/docker/docker-deploy

.deploy-development:

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    STACK: vessel

    SERVICES_TO_CHECK: vessel_${CI_PROJECT_NAME}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.postgresql.tmpl.yml:docker-compose.postgresql.dev.yml

    POSTGRES_USER: ${POSTGRES_USER}

    POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}.dev.yml

  script:

    - create-nets.sh postgres-net

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      POSTGRES_USER=${POSTGRES_USER} POSTGRES_PASSWORD=${POSTGRES_PASSWORD}

      POSTGRES_USER=${POSTGRES_USER} POSTGRES_PASSWORD=${POSTGRES_PASSWORD} INTERVAL=${INTERVAL}

  environment:

    name: dev/postgresql

  only:

    - branches

  except:

    - master

    - dev

    - schedules

  when: manual

    name: dev/${CI_PROJECT_NAME}



deploy-pro:

  stage: deploy

  image: registry.gitlab.com/redmic-project/docker/docker-deploy

.deploy-production:

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    STACK: vessel

    SERVICES_TO_CHECK: vessel_${CI_PROJECT_NAME}

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}

    IMAGE_TAG: ${CI_COMMIT_SHA}

    COMPOSE_FILE: docker-compose.postgresql.tmpl.yml:docker-compose.postgresql.prod.yml

    POSTGRES_USER: ${POSTGRES_USER}

    POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}

    AWS_REGION: ${AWS_REGION}

    COMPOSE_FILE: docker-compose.${CI_PROJECT_NAME}.tmpl.yml:docker-compose.${CI_PROJECT_NAME}.prod.yml

  script:

    - create-nets.sh postgres-net

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      POSTGRES_USER=${POSTGRES_USER} POSTGRES_PASSWORD=${POSTGRES_PASSWORD} AWS_REGION=${AWS_REGION}

      POSTGRES_USER=${POSTGRES_USER} POSTGRES_PASSWORD=${POSTGRES_PASSWORD} INTERVAL=${INTERVAL}

      AWS_REGION=${AWS_REGION}

  environment:

    name: pro/postgresql

  only:

    - master

  except:

    - schedules

  when: manual

    name: pro/${CI_PROJECT_NAME}



deploy-backup-dev:

  stage: deploy

  image: registry.gitlab.com/redmic-project/docker/docker-deploy:latest

.deploy-backup:

  extends: .deploy

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    STACK: vessel

    SERVICES_TO_CHECK: vessel_backup-db

    IMAGE_NAME: registry.gitlab.com/redmic-project/postgres/backup-db

    IMAGE_TAG: latest

    COMPOSE_FILE: docker-compose.backup-db.tmpl.yml:docker-compose.backup-db.dev.yml

    POSTGRES_USER: ${POSTGRES_USER}

    POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}

    POSTGRES_HOSTNAME: ${POSTGRES_HOSTNAME}

    BUCKET_BACKUP_DB: ${BUCKET_BACKUP_DB}

    AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}

    AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}

    PUSHGATEWAY_HOST: ${PUSHGATEWAY_HOST}

    PUSHGATEWAY_JOB: ${PUSHGATEWAY_JOB}

    SERVICES_TO_CHECK: ${STACK}_backup-db

    IMAGE_NAME: ${POSTGRES_BACKUP_IMAGE_NAME}

    IMAGE_TAG: ${POSTGRES_BACKUP_IMAGE_TAG}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      POSTGRES_USER=${POSTGRES_USER} POSTGRES_PASSWORD=${POSTGRES_PASSWORD}

      AWS_REGION=${AWS_REGION} POSTGRES_USER=${POSTGRES_USER} POSTGRES_PASSWORD=${POSTGRES_PASSWORD}

      POSTGRES_HOSTNAME=${POSTGRES_HOSTNAME} BUCKET_BACKUP_DB=${BUCKET_BACKUP_DB}

      AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}

      PUSHGATEWAY_HOST=${PUSHGATEWAY_HOST} PUSHGATEWAY_JOB=${PUSHGATEWAY_JOB}

  environment:

    name: dev/backup-db

  only:

    - dev

  except:

    - schedules

  when: manual



deploy-backup-supporting-branch:

  stage: deploy

  image: registry.gitlab.com/redmic-project/docker/docker-deploy:latest

.deploy-backup-development:

  extends: .deploy-backup

  variables:

    SSH_REMOTE: ${DEV_SSH_REMOTE}

    STACK: vessel

    SERVICES_TO_CHECK: vessel_backup-db

    IMAGE_NAME: registry.gitlab.com/redmic-project/postgres/backup-db

    IMAGE_TAG: latest

    COMPOSE_FILE: docker-compose.backup-db.tmpl.yml:docker-compose.backup-db.dev.yml

    POSTGRES_USER: ${POSTGRES_USER}

    POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}

    POSTGRES_HOSTNAME: ${POSTGRES_HOSTNAME}

    BUCKET_BACKUP_DB: ${BUCKET_BACKUP_DB}

    AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}

    AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      POSTGRES_USER=${POSTGRES_USER} POSTGRES_PASSWORD=${POSTGRES_PASSWORD} POSTGRES_HOSTNAME=${POSTGRES_HOSTNAME}

      BUCKET_BACKUP_DB=${BUCKET_BACKUP_DB} AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}

      AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}

  environment:

    name: dev/backup-db

  only:

    - branches

  except:

    - master

    - dev

    - schedules

  when: manual



deploy-backup-pro:

  stage: deploy

  image: registry.gitlab.com/redmic-project/docker/docker-deploy:latest

.deploy-backup-production:

  extends: .deploy-backup

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    STACK: vessel

    SERVICES_TO_CHECK: vessel_backup-db

    IMAGE_NAME: registry.gitlab.com/redmic-project/postgres/backup-db

    IMAGE_TAG: latest

    COMPOSE_FILE: docker-compose.backup-db.tmpl.yml:docker-compose.backup-db.prod.yml

    AWS_REGION: ${AWS_REGION}

    POSTGRES_USER: ${POSTGRES_USER}

    POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}

    POSTGRES_HOSTNAME: ${POSTGRES_HOSTNAME}

    BUCKET_BACKUP_DB: ${BUCKET_BACKUP_DB}

    AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}

    AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}

    PUSHGATEWAY_HOST: ${PUSHGATEWAY_HOST}

    PUSHGATEWAY_JOB: ${PUSHGATEWAY_JOB}

  script:

    - >

      deploy.sh IMAGE_NAME=${IMAGE_NAME} IMAGE_TAG=${IMAGE_TAG} COMPOSE_FILE=${COMPOSE_FILE}

      AWS_REGION=${AWS_REGION} POSTGRES_USER=${POSTGRES_USER} POSTGRES_PASSWORD=${POSTGRES_PASSWORD}

      POSTGRES_HOSTNAME=${POSTGRES_HOSTNAME} BUCKET_BACKUP_DB=${BUCKET_BACKUP_DB}

      AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}

      PUSHGATEWAY_HOST=${PUSHGATEWAY_HOST} PUSHGATEWAY_JOB=${PUSHGATEWAY_JOB}

  environment:

    name: pro/backup-db



deploy-backup-support-branch-development:

  extends: .deploy-backup-development

  only:

    - branches

  except:

    - master

    - schedules



deploy-backup-stable-branch-development:

  extends: .deploy-backup-development

  only:

    - master

  except:

    - schedules

  when: manual



scheduled-backup-pro:

  stage: backup

  image: registry.gitlab.com/redmic-project/docker/docker-deploy:latest

  variables:

    SSH_REMOTE: ${PRO_SSH_REMOTE}

    SERVICE: vessel_backup-db

  script:

    - relaunch.sh

deploy-backup-stable-branch-production:

  extends: .deploy-backup-production

  only:

    - master

  except:

    - schedules



scheduled-run-development:

  script: ":"

Dockerfile

+32 −2
Original line number Diff line number Diff line 
FROM mdillon/postgis:10-alpine

FROM mdillon/postgis:11-alpine



LABEL maintainer="info@redmic.es"



COPY /scripts/ /docker-entrypoint-initdb.d/
 
ENV PG_CRON_VERSION="1.1.4" \

	PG_PARTMAN_VERSION="4.0.0"



RUN apk add --no-cache --virtual \

		.build-deps \

		build-base \

		ca-certificates \

		openssl \

		tar && \

	wget -O /pg_cron.tgz https://github.com/citusdata/pg_cron/archive/v${PG_CRON_VERSION}.tar.gz && \

	tar xvzf /pg_cron.tgz && \

	cd pg_cron-${PG_CRON_VERSION} && \

	sed -i.bak -e 's/-Werror//g' Makefile && \

	sed -i.bak -e 's/-Wno-implicit-fallthrough//g' Makefile && \

	make && \

	make install && \

	cd .. && \

	rm -rf pg_cron.tgz pg_cron-* && \

	wget -O /pg_partman.tgz https://github.com/pgpartman/pg_partman/archive/v${PG_PARTMAN_VERSION}.tar.gz && \

	tar xvzf /pg_partman.tgz && \

	cd pg_partman-${PG_PARTMAN_VERSION} && \

	make && \

	make install && \

	cd .. && \

	rm -rf pg_partman.tgz pg_partman-* && \

	echo "shared_preload_libraries='pg_partman_bgw,pg_cron'" >> /usr/local/share/postgresql/postgresql.conf.sample && \

	echo "checkpoint_timeout = 30min" >> /usr/local/share/postgresql/postgresql.conf.sample && \

	echo "max_wal_size = 2GB" >> /usr/local/share/postgresql/postgresql.conf.sample && \

	mv /usr/local/bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint-origin.sh



COPY rootfs /

README.md

+4 −1
Original line number Diff line number Diff line 
# AIS Postgres

Base de datos PostgreSQL para almacenar datos AIS.



La tabla principal está particionada por tiempo, de esta forma sólo se accede agiliza las búsquedas que impliquen consultas por fechas.

deploy/.env

+2 −2
Original line number Diff line number Diff line 
POSTGRES_DB=ais

PGDATA=/var/lib/postgresql/data/db-files/

DESIRED_WORKER_AZ=b
 
INTERVAL=half-hour

POSTGRES_DB=ais