Multi deployment on docker Swarm

In this example, 3 stacks are deployed:

a traefik stack used as an http reverse proxy,
the elements of the keycloak stack are here in a dedicated stack,
the datachain stack, which will act as a client for the other two stacks.

All services are exposed via traefik. To facilitate deployment, each application has a dedicated domain, rather than working with uri. This avoids the risk of services sharing a URI. For example, here we have: traefik.demo.local, keycloak.demo.local, dc.demo.local for datachain, and spark.demo.local for the spark web GUI.

Traefik stack

This stack contains, with comments, the traefik configuration for requesting a certificate from let’s encrypt, to ensure https exchanges.

docker-compose.yml file

version: "3.7"

services:

  traefik:
    image: "traefik:${PRODUCT_VERSION}"
    command:
      - "--log.level=info"
      - "--api.insecure=true"
      - "--providers.docker=true"
      - "--providers.docker.swarmMode=true"
      - "--providers.docker.network=traefik_network"
      - "--providers.docker.exposedbydefault=false"
      - "--entrypoints.web.address=:80"
      ## Configuration in secure mode (https, whtih letsencrypt PKI (in this cas, URL must be public))
      # - "--entrypoints.web.http.redirections.entryPoint.to=websecure"
      # - "--entrypoints.web.http.redirections.entryPoint.scheme=https"
      # - "--entrypoints.web.http.redirections.entrypoint.permanent=true"
      # - "--entrypoints.websecure.address=:443"
      # - "--certificatesresolvers.resolver1.acme.httpchallenge=true"
      # - "--certificatesresolvers.resolver1.acme.httpchallenge.entrypoint=web"
      # - "--certificatesresolvers.resolver1.acme.email=postmaster@mysociety.acme.com"
      # - "--certificatesresolvers.resolver1.acme.storage=/letsencrypt/dc_certs.json"
    ports:
      - "80:80"
      ## Configuration in secure mode
      # - "443:443"
    volumes:
      - "/var/run/docker.sock:/var/run/docker.sock:ro"
      ## Configuration in secure mode
      ## Volume where letsencrypt certificates will be stored
      # - "traefik_certs:/letsencrypt"
    deploy:
      mode: global
      placement:
        constraints:
          # Always on manager for traefik
          - node.role == manager
      update_config:
        parallelism: 1
        delay: 10s
      restart_policy:
        condition: on-failure
      labels:
        ## CONFIGURATION DU DASHBOARD TRAEFIK
        ## Enable Dashboard visibility on traefik
        - "traefik.enable=true"
        - "traefik.docker.network=traefik_network"
        ## Dashboard config
        - "traefik.http.routers.dashboard.entrypoints=web"
        ## Configuration in secure mode : comment previous line and uncomment the three 
        # - "traefik.http.routers.dashboard.tls=true"
        # - "traefik.http.routers.dashboard.tls.certresolver=resolver1"
        # - "traefik.http.routers.dashboard.entrypoints=websecure"
        - "traefik.http.routers.dashboard.rule=Host(`${TRAEFIK_URL}`)"
        - "traefik.http.routers.dashboard.service=dashboard"
        - "traefik.http.services.dashboard.loadbalancer.server.port=8080"
        ## Basic authentication, generated with command : 
        ## echo $(htpasswd -nb admin password) | sed -e s/\\$/\\$\\$/g
        ## To authent, use admin as login and password as password
        - "traefik.http.routers.dashboard.middlewares=traefik-auth@docker"
        - "traefik.http.middlewares.traefik-auth.basicauth.users=admin:$$apr1$$mdAMA4f6$$2cfjZ7nxHcd.ABttomKqk1"
    networks:
      - traefik_network

## Configuration in secure mode : uncomment uncomment following lines 
#volumes:
#  traefik_certs:

networks:
   ## traefik_network is external : it must have been created before starting the stack
   traefik_network:
    external: true

Stack Keycloak

Note: traefik configuration, for application exposure.

file docker-compose.yml

version: "3.7"

services:
  dc_pg_keycloak:
    image: ${HARBOR_REGISTRY}/${HARBOR_PROJECT}/pg_keycloak:${PRODUCT_VERSION}
    networks:
      kc_network:
        aliases:
          - dc-pg-keycloak
    deploy:
      mode: replicated
      replicas: 1
    volumes:
      - keycloack_db:/var/lib/postgresql/data

  dc_keycloak:
    image: ${HARBOR_REGISTRY}/${HARBOR_PROJECT}/keycloak:${PRODUCT_VERSION}
    networks:
      traefik_network:
      kc_network:
        aliases:
          - dc-keycloak
    env_file: keycloak.env
    deploy:
      mode: replicated
      replicas: 1
      labels:
        - "traefik.enable=true"
        - "traefik.docker.network=traefik_network"
        - "traefik.http.routers.kc1_web_ui.rule=Host(`${KC_HOST}`) && PathPrefix(`/auth`)"
        - "traefik.http.routers.kc1_web_ui.entrypoints=web"
        ## Configuration in secure mode : comment previous line and uncomment the three 
        # - "traefik.http.routers.kc1_web_ui.tls=true"
        # - "traefik.http.routers.kc1_web_ui.tls.certresolver=resolver1"
        # - "traefik.http.routers.kc1_web_ui.entrypoints=websecure"       
        - "traefik.http.services.kc1_web_ui.loadbalancer.server.port=8080"
        ## Enable compression 
        - "traefik.http.routers.kc1_web_ui.middlewares=kc1_web_comp1@docker"
        - "traefik.http.middlewares.kc1_web_comp1.compress=true"

volumes:
  keycloack_db:

networks:
  ## traefik_network is external : it must have been created before starting the stack
  ## kc_network is external : it must have been created before starting the stack
  kc_network:
    external: true
  traefik_network:
    external: true

file keycloak.env

DATACHAIN_CLIENT_URL=${DC_URL_PUBLIC}
KEYCLOAK_FRONTEND_URL=${KC_URL_PUBLIC}/auth

Stack Datachain

Note: traefik configuration, for application exposure.

file docker-compose.yml

version: "3.7"
services:
  dc_redis_cache:
    image: redis:7.0-alpine
    command: redis-server --loglevel warning
    networks:
      dc_network:
        aliases:
          - dc-redis
    deploy:
      mode: replicated
      replicas: 1

  dc_pg:
    image: ${HARBOR_REGISTRY}/${HARBOR_PROJECT}/pg:${PRODUCT_VERSION}
    networks:
      dc_network:
        aliases:
          - dc-pg
    command: ["postgres", "-c", "log_min_duration_statement=2000"]
    deploy:
      mode: replicated
      replicas: 1
    volumes:
      - pg_data:/var/lib/postgresql/data

  dc_pg_migration:
    image: ${HARBOR_REGISTRY}/${HARBOR_PROJECT}/pg_migration:${PRODUCT_VERSION}
    networks:
      dc_network:
        aliases:
          - dc-pg-migration
      kc_network:
    deploy:
      restart_policy:
        condition: none
    volumes:
      - pg_dump:/data/pg_dump

  dc_pg_expose:
    image: ${HARBOR_REGISTRY}/${HARBOR_PROJECT}/pg_expose:${PRODUCT_VERSION}
    networks:
      dc_network:
        aliases:
          - dc-pg-expose
    deploy:
      mode: replicated
      replicas: 1
    volumes:
      - pg_expose:/var/lib/postgresql/data

  dc_backend:
    image: ${HARBOR_REGISTRY}/${HARBOR_PROJECT}/backend:${PRODUCT_VERSION}
    networks:
      traefik_network:
      dc_network:
        aliases:
          - dc-backend
    env_file: backend.env
    deploy:
      mode: replicated
      replicas: 1
      labels:
        - "traefik.enable=true"
        - "traefik.docker.network=traefik_network"
        - "traefik.http.routers.dc_backend_actuator.rule=Host(`${DC_HOST}`) && PathPrefix(`/actuator`)"
        - "traefik.http.routers.dc_backend_actuator.entrypoints=web"
        ## Configuration in secure mode : comment previous line and uncomment the three 
        # - "traefik.http.routers.dc_backend_actuator.tls=true"
        # - "traefik.http.routers.dc_backend_actuator.tls.certresolver=resolver1"
        # - "traefik.http.routers.dc_backend_actuator.entrypoints=websecure"       
        - "traefik.http.routers.dc_backend_actuator.service=dc_backend_actuator"
        - "traefik.http.services.dc_backend_actuator.loadbalancer.server.port=9090"
        ## Basic authentication, generated with command : 
        ## echo $(htpasswd -nb admin password) | sed -e s/\\$/\\$\\$/g
        ## To authent, use admin as login and password as password
        - "traefik.http.routers.dc_backend_actuator.middlewares=dc_backend_actuator-auth@docker"
        - "traefik.http.middlewares.dc_backend_actuator-auth.basicauth.users=admin:$$apr1$$mdAMA4f6$$2cfjZ7nxHcd.ABttomKqk1"
        - "traefik.http.middlewares.dc_backend_actuator-auth.basicauth.removeheader=true"

  dc_web_ui:
    image: ${HARBOR_REGISTRY}/${HARBOR_PROJECT}/web_ui:${PRODUCT_VERSION}
    networks:
      traefik_network:
      dc_network:
        aliases:
          - dc-web-ui
    env_file: web_ui.env
    deploy:
      mode: replicated
      replicas: 1
      labels:
        - "traefik.enable=true"
        - "traefik.docker.network=traefik_network"
        - "traefik.http.routers.dc_web_ui.rule=Host(`${DC_HOST}`) && PathPrefix(`/`)"
        - "traefik.http.routers.dc_web_ui.entrypoints=web"
        ## Configuration in secure mode : comment previous line and uncomment the three 
        # - "traefik.http.routers.dc_web_ui.entrypoints=websecure"
        # - "traefik.http.routers.dc_web_ui.tls=true"
        # - "traefik.http.routers.dc_web_ui.tls.certresolver=resolver1"
        - "traefik.http.routers.dc_web_ui.service=dc_web_ui"
        - "traefik.http.services.dc_web_ui.loadbalancer.server.port=80"
        ## Enable compression 
        - "traefik.http.routers.dc_web_ui1.middlewares=dc_web_comp1@docker"
        - "traefik.http.middlewares.dc_web_comp1.compress=true"

  dc_spark:
    image: ${HARBOR_REGISTRY}/${HARBOR_PROJECT}/spark:${PRODUCT_VERSION}
    networks:
      traefik_network:
      dc_network:
        aliases:
          - dc-spark1
    env_file: spark.env
    deploy:
      mode: replicated
      replicas: 1
      labels:
        - "traefik.enable=true"
        - "traefik.docker.network=traefik_network"
        - "traefik.http.routers.dc_spark11.rule=Host(`${SPARK_HOST}`) && PathPrefix(`/actuator`)"
        - "traefik.http.routers.dc_spark11.entrypoints=web"
        ## Configuration in secure mode : comment previous line and uncomment the three 
        # - "traefik.http.routers.dc_spark11.entrypoints=websecure"
        # - "traefik.http.routers.dc_spark11.tls=true"
        # - "traefik.http.routers.dc_spark11.tls.certresolver=resolver1"
        - "traefik.http.routers.dc_spark11.service=dc_spark11"
        - "traefik.http.services.dc_spark11.loadbalancer.server.port=9091"
        ## Basic authentication, generated with command : 
        ## echo $(htpasswd -nb admin password) | sed -e s/\\$/\\$\\$/g
        ## To authent, use admin as login and password as password
        - "traefik.http.routers.dc_spark11.middlewares=dc_spark11-auth@docker"
        - "traefik.http.middlewares.dc_spark11-auth.basicauth.users=admin:$$apr1$$mdAMA4f6$$2cfjZ7nxHcd.ABttomKqk1"
        - "traefik.http.middlewares.dc_spark11-auth.basicauth.removeheader=true"
        ## Expose spark UI : 
        - "traefik.http.routers.dc_spark11_web_ui.rule=Host(`${SPARK_HOST}`)"
        - "traefik.http.routers.dc_spark11_web_ui.entrypoints=web"
        ## Configuration in secure mode : comment previous line and uncomment the three 
        # - "traefik.http.routers.dc_spark11_web_ui.entrypoints=websecure"
        # - "traefik.http.routers.dc_spark11_web_ui.tls=true"
        # - "traefik.http.routers.dc_spark11_web_ui.tls.certresolver=resolver1"
        - "traefik.http.routers.dc_spark11_web_ui.service=dc_spark11_web_ui"
        - "traefik.http.services.dc_spark11_web_ui.loadbalancer.server.port=4040"
    volumes:
      - hdfs:/data
    healthcheck:
      test: ["CMD", "curl", "-u", "backend:70171933-e656-4183-9955-bfd0354d2250", "-f", "http://localhost:9091/actuator/health"]
      interval: 1m30s
      timeout: 10s
      retries: 6
      start_period: 40s

volumes:
  pg_data:
  pg_dump:
  hdfs:
  pg_expose:

networks:
  dc_network:
    external: true
  traefik_network:
    external: true
  kc_network:
    external: true

file backend.env

# URL public avec http ou https
dc.app.url.web-ui=${DC_URL_PUBLIC}

# Configuration Keycloak
keycloak.auth-server-url=http://dc-keycloak:8080/auth

# Configuration base de donnée
dc.app.db.host=dc-pg

# Configuration du datasource d'exposition de donnée
spring.expose-datasource.host=dc-pg-expose

# Configuration Redis
dc.app.notification.urls=redis://dc-redis:6379

# Monitoring
management.server.address=0.0.0.0
## * to see all exposed url by actuator and call /actuator 
# management.endpoints.web.exposure.include=*
management.endpoints.web.exposure.include=health,metrics,prometheus,info
management.metrics.export.prometheus.enabled=true

file spark.env

# Configuration du contexte Spark
dc.app.context.url=http://dc-spark1:8090

# Configuration spark
dc.app.spark.master=local[6]
dc.app.spark.cores.max=6
dc.app.spark.executor.memory=8g

# Configuration HDFS
dc.app.data-file.hdfs=file:///data

# Configuration notifications
dc.app.notification.urls=redis://dc-redis:6379

# Monitoring
management.server.address=0.0.0.0
## * to see all exposed url by actuator and call /actuator 
# management.endpoints.web.exposure.include=*
management.endpoints.web.exposure.include=health,metrics,prometheus,info
management.metrics.export.prometheus.enabled=true

file web_ui.env

KEYCLOAK_CONFIG={"url": "${KC_URL_PUBLIC}/auth","realm": "dc-realm","clientId": "dc_front", "backendClientId": "dc_backend", "userRole" : "dc_user", "projectRole" : "dc_project", "adminRole" : "dc_admin", "memberRole" : "dc_member"}

Download the example, along with sample scripts for starting up the solution.