docker-compose.test.yml

services:
  db:
    image: postgres:17
    restart: unless-stopped
    environment:
      TZ: UTC
      POSTGRES_DB: syncmaster
      POSTGRES_USER: syncmaster
      POSTGRES_PASSWORD: changeme
    ports:
      - 5432:5432
    volumes:
      - postgres_test_data:/var/lib/postgresql/data
    healthcheck:
      test: [CMD-SHELL, 'pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}']
      start_period: 5s
      interval: 30s
      timeout: 5s
      retries: 3

  rabbitmq:
    # not using 4.3 until https://github.com/celery/kombu/issues/2237
    image: rabbitmq:4.2
    restart: unless-stopped
    ports:
      - 5672:5672
    volumes:
      - rabbitmq_test_data:/var/lib/rabbitmq
    healthcheck:
      test: rabbitmq-diagnostics -q ping
      start_period: 5s
      interval: 30s
      timeout: 5s
      retries: 3

  # Using worker image here to skip building server image build in CI.
  # Worker test image already has all necessary dependencies
  migrations:
    image: mtsrus/syncmaster-worker:${WORKER_IMAGE_TAG:-test}
    restart: no
    build:
      dockerfile: docker/Dockerfile.worker
      context: .
      target: test
    volumes:
      - ./config.docker.yml:/app/config.yml
      - ./syncmaster:/app/syncmaster
      - ./tests:/app/tests
    entrypoint: [python, -m, syncmaster.db.migrations, upgrade, head]
    depends_on:
      db:
        condition: service_healthy

  server:
    image: mtsrus/syncmaster-server:${SERVER_IMAGE_TAG:-test}
    restart: unless-stopped
    build:
      dockerfile: docker/Dockerfile.server
      context: .
      target: test
    ports:
      - 8000:8000
    volumes:
      - ./config.docker.yml:/app/config.yml
      - ./syncmaster:/app/syncmaster
      - ./docs/_static:/app/mddocs/docs/_static
      - ./reports:/app/reports
      - ./tests:/app/tests
      - ./pyproject.toml:/app/pyproject.toml
    depends_on:
      db:
        condition: service_healthy
      migrations:
        condition: service_completed_successfully
      rabbitmq:
        condition: service_healthy
    profiles: [server, all]

  scheduler:
    image: mtsrus/syncmaster-scheduler:${SCHEDULER_IMAGE_TAG:-test}
    restart: unless-stopped
    build:
      dockerfile: docker/Dockerfile.scheduler
      context: .
      target: test
    volumes:
      - ./config.docker.yml:/app/config.yml
      - ./syncmaster:/app/syncmaster
      - ./tests:/app/tests
      - ./reports:/app/reports
      - ./pyproject.toml:/app/pyproject.toml
    depends_on:
      db:
        condition: service_healthy
      migrations:
        condition: service_completed_successfully
      rabbitmq:
        condition: service_healthy
    profiles: [scheduler, all]

  worker:
    image: mtsrus/syncmaster-worker:${WORKER_IMAGE_TAG:-test}
    restart: unless-stopped
    build:
      dockerfile: docker/Dockerfile.worker
      context: .
      target: test
    command: --queues 123-test_queue # Queue slug
    entrypoint: [coverage, run, -m, celery, -A, tests.test_integration.celery_test, worker, --max-tasks-per-child=1]
    env_file: .env.docker.test
    environment:
      # CI runs tests in the worker container, so we need to turn off interaction with static files for it
      - SYNCMASTER__SERVER__STATIC_FILES__ENABLED=false
    volumes:
      - ./config.docker.yml:/app/config.yml
      - ./syncmaster:/app/syncmaster
      - ./reports:/app/reports
      - ./tests:/app/tests
      - ./pyproject.toml:/app/pyproject.toml
    depends_on:
      db:
        condition: service_healthy
      migrations:
        condition: service_completed_successfully
      rabbitmq:
        condition: service_healthy
    profiles: [worker, scheduler, s3, hdfs, hive, iceberg, oracle, clickhouse, mysql, mssql, sftp, ftp, ftps, samba, webdav, all]

  horizon:
    image: mtsrus/horizon-backend:develop
    restart: unless-stopped
    environment:
      # list here usernames which should be assigned SUPERADMIN role on application start
      HORIZON__ENTRYPOINT__ADMIN_USERS: admin
      HORIZON__DATABASE__URL: postgresql+asyncpg://horizon:changeme@horizon-db:5432/horizon
      HORIZON__AUTH__ACCESS_TOKEN__SECRET_KEY: generate_another_random_string
      HORIZON__AUTH__PROVIDER: horizon.backend.providers.auth.dummy.DummyAuthProvider
      HORIZON__SERVER__LOGGING__PRESET: colored
      # PROMETHEUS_MULTIPROC_DIR is required for multiple workers, see:
      # https://prometheus.github.io/client_python/multiprocess/
      PROMETHEUS_MULTIPROC_DIR: /tmp/prometheus-metrics
    # tmpfs dir is cleaned up each container restart
    tmpfs:
      - /tmp/prometheus-metrics:mode=1777
    ports:
      - 8020:8000
    depends_on:
      horizon-db:
        condition: service_healthy
    profiles: [horizon, s3, hdfs, hive, iceberg, oracle, clickhouse, mysql, mssql, sftp, ftp, ftps, samba, webdav, all]

  horizon-db:
    image: postgres:17
    restart: unless-stopped
    environment:
      TZ: UTC
      POSTGRES_DB: horizon
      POSTGRES_USER: horizon
      POSTGRES_PASSWORD: changeme
    ports:
      - 5434:5432
    volumes:
      - horizon_test_data:/var/lib/postgresql/data
    healthcheck:
      test: [CMD-SHELL, 'pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}']
      start_period: 5s
      interval: 30s
      timeout: 5s
      retries: 3
    profiles: [horizon, s3, hdfs, hive, iceberg, oracle, clickhouse, mysql, mssql, sftp, ftp, ftps, samba, webdav, all]

  test-postgres:
    image: postgres:17
    restart: unless-stopped
    ports:
      - 5433:5432
    environment:
      TZ: UTC
      POSTGRES_DB: syncmaster
      POSTGRES_USER: syncmaster
      POSTGRES_PASSWORD: 123UsedForTestOnly@!
    healthcheck:
      test: [CMD-SHELL, 'pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}']
      start_period: 5s
      interval: 30s
      timeout: 5s
      retries: 3
    profiles: [s3, hdfs, hive, iceberg, oracle, clickhouse, mysql, mssql, sftp, ftp, ftps, samba, webdav, all]

  test-s3:
    image: bitnamilegacy/minio:latest
    container_name: test-s3
    restart: unless-stopped
    environment:
      TZ: UTC
      MINIO_ROOT_USER: syncmaster
      MINIO_ROOT_PASSWORD: 123UsedForTestOnly@!
    ports:
      - 9010:9000
      - 9011:9001
    healthcheck:
      test: curl -f http://localhost:9000/minio/health/live
      start_period: 5s
      interval: 30s
      timeout: 5s
      retries: 3
    profiles: [s3, iceberg, all]

  test-iceberg-rest:
    image: tabulario/iceberg-rest:latest
    container_name: test-iceberg-rest
    restart: unless-stopped
    environment:
      CATALOG_WAREHOUSE: s3a://syncmaster/data/
      CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO
      CATALOG_S3_ENDPOINT: http://test-s3:9000
      CATALOG_S3_PATH__STYLE__ACCESS: true
      AWS_ACCESS_KEY_ID: syncmaster
      AWS_SECRET_ACCESS_KEY: 123UsedForTestOnly@!
      AWS_REGION: us-east-1
    ports:
      - 8181:8181
    depends_on:
      test-s3:
        condition: service_healthy
    profiles: [iceberg, all]

  test-oracle:
    image: gvenzl/oracle-xe:slim-faststart
    restart: unless-stopped
    ports:
      - 1522:1521
    environment:
      TZ: UTC
      ORACLE_PASSWORD: 123AdminUserForTests@!
      APP_USER: syncmaster
      APP_USER_PASSWORD: 123UsedForTestOnly@!
    profiles: [oracle, all]

  test-clickhouse:
    # https://github.com/ClickHouse/clickhouse-java/issues/2636
    image: clickhouse/clickhouse-server:25.9
    restart: unless-stopped
    environment:
      TZ: UTC
      CLICKHOUSE_USER: syncmaster
      CLICKHOUSE_PASSWORD: 123UsedForTestOnly@!
      CLICKHOUSE_DB: syncmaster
    ports:
      - 8123:8123
      - 9001:9000
    profiles: [clickhouse, all]

  test-mssql:
    image: mcr.microsoft.com/mssql/server
    restart: unless-stopped
    environment:
      TZ: UTC
      ACCEPT_EULA: Y
      MSSQL_SA_PASSWORD: 123AdminUserForTests@!@!
    ports:
      - 1433:1433
    volumes:
      - ./docker/mssql/:/usr/config/
    entrypoint: [/usr/config/entrypoint.sh]
    platform: linux/amd64
    profiles: [mssql, all]

  test-mysql:
    image: mysql
    restart: unless-stopped
    environment:
      TZ: UTC
      MYSQL_ROOT_PASSWORD: 123RootUserForTestsOnly@!
      MYSQL_DATABASE: syncmaster
      MYSQL_USER: syncmaster
      MYSQL_PASSWORD: 123UsedForTestOnly@!
    ports:
      - 3306:3306
    platform: linux/amd64
    profiles: [mysql, all]

  metastore-hive:
    image: postgres:17
    restart: unless-stopped
    environment:
      TZ: UTC
      POSTGRES_DB: metastore
      POSTGRES_USER: test_hive
      POSTGRES_PASSWORD: test_hive
    ports:
      - 5440:5432
    healthcheck:
      test: [CMD-SHELL, 'pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}']
      start_period: 5s
      interval: 30s
      timeout: 5s
      retries: 3
    profiles: [hive, hdfs, s3, sftp, ftp, ftps, samba, webdav, all]

  keycloak:
    image: quay.io/keycloak/keycloak:latest
    command: start-dev
    restart: unless-stopped
    environment:
      TZ: UTC
      KEYCLOAK_ADMIN: admin
      KEYCLOAK_ADMIN_PASSWORD: admin
    ports:
      - 8080:8080
    volumes:
      - keycloak_data:/opt/keycloak/data
    profiles: [keycloak, all]

  test-hive:
    image: mtsrus/hadoop:hadoop2.7.3-hive2.3.10
    restart: unless-stopped
    hostname: test-hive
    depends_on:
      metastore-hive:
        condition: service_healthy
    ports:
      - 9820:9820 # HDFS IPC
      - 9870:9870 # HDFS WebHDFS
      - 8088:8088 # Yarn UI
      - 8042:8042 # NodeManager UI
      - 10000:10000 # Hive server
      - 10002:10002 # Hive server Admin UI
      - 19888:19888 # MapReduce JobServer History UI
      - 9083:9083 # Hive Metastore server
      - 9864:9864 # Datanode UI
    environment:
      TZ: UTC
      WITH_HIVE_SERVER: 'false' # We leave only the metastore server, we don’t need Hive itself, we don’t waste resources on it
      HIVE_METASTORE_DB_URL: jdbc:postgresql://metastore-hive:5432/metastore
      HIVE_METASTORE_DB_DRIVER: org.postgresql.Driver
      HIVE_METASTORE_DB_USER: test_hive
      HIVE_METASTORE_DB_PASSWORD: test_hive
    # writing spark dataframe to xml file fails if fs.defaultFS is not accessible
    profiles: [hive, hdfs, s3, ftp, all]

  test-sftp:
    image: ${SFTP_IMAGE:-linuxserver/openssh-server}
    restart: unless-stopped
    ports:
      - 2222:2222
    environment:
      TZ: UTC
      PUID: 1000
      PGID: 1000
      USER_NAME: syncmaster
      PASSWORD_ACCESS: 'true'
      SUDO_ACCESS: 'true'
      USER_PASSWORD: 123UsedForTestOnly@!
    profiles: [sftp, all]

  test-ftp:
    image: ${FTP_IMAGE:-chonjay21/ftps:latest}
    restart: unless-stopped
    environment:
      TZ: UTC
      USE_SSL: 'false'
      PASSV_MIN_PORT: 30000
      PASSV_MAX_PORT: 30010
      APP_USER_NAME: syncmaster
      APP_USER_PASSWD: 123UsedForTestOnly@!
      APP_UID: 1000
      APP_GID: 1000
    ports:
      - 2121:21
      - 30000-30010:30000-30010
    volumes:
      - ./docker/ftp/on_post_init.sh:/sources/ftp/eventscripts/on_post_init.sh
    profiles: [ftp, all]

  test-ftps:
    image: ${FTPS_IMAGE:-chonjay21/ftps:latest}
    restart: unless-stopped
    environment:
      TZ: UTC
      USE_SSL: 'true'
      PASSV_MIN_PORT: 30020
      PASSV_MAX_PORT: 30030
      APP_USER_NAME: syncmaster
      APP_USER_PASSWD: 123UsedForTestOnly@!
      APP_UID: 1000
      APP_GID: 1000
    ports:
      - 2122:21
      - 30020-30030:30020-30030
    volumes:
      - ./docker/ftp/on_post_init.sh:/sources/ftps/eventscripts/on_post_init.sh
    profiles: [ftps, all]

  test-samba:
    image: ${SAMBA_IMAGE:-elswork/samba}
    restart: unless-stopped
    ports:
      - 139:139
      - 1445:445
    environment:
      TZ: UTC
    volumes:
      - ./docker/samba/custom_entrypoint.sh:/custom_entrypoint.sh
    entrypoint: [/custom_entrypoint.sh]
    profiles: [samba, all]

  test-webdav:
    image: ${WEBDAV_IMAGE:-chonjay21/webdav:latest}
    restart: unless-stopped
    environment:
      TZ: UTC
      APP_USER_NAME: syncmaster
      APP_USER_PASSWD: 123UsedForTestOnly@!
      APP_UID: 1000
      APP_GID: 1000
    ports:
      - 8010:80
    volumes:
      # Remove after https://github.com/chonjay21/docker-webdav/pull/3
      - ./docker/webdav/on_post_init.sh:/sources/webdav/eventscripts/on_post_init.sh
    profiles: [webdav, all]

volumes:
  postgres_test_data:
  rabbitmq_test_data:
  horizon_test_data:
  keycloak_data: