From 6ff89f8fbcfbfa21c73b4f6bcf7fd8298266c4ac Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Fri, 20 Nov 2026 10:03:19 +1100 Subject: [PATCH] feat(docker): run the all-in-one image on DocumentDB instead of MongoDB 4.0.5 MongoDB 4.0.5 is EOL, SSPL licensed, and comes from Alpine 3.9 packages that carry OpenSSL 1.1.1k critical CVEs. The all-in-one image now runs DocumentDB v1.2-0 (MIT, Postgres 18) from the same Dockerfile stage the dev, integration and e2e stacks use. Metadata stays on the /data/db mount under /data/db/pgdata; the changeset documents moving old MongoDB data. --- .changeset/aio-documentdb.md | 23 ++++ .github/actions/e2e-setup/action.yml | 2 +- .github/workflows/docker-build.yml | 2 +- .vex/openssl-mongodb.vex.json | 150 ---------------------- agent_docs/development.md | 3 +- docker-compose.ci.yml | 5 +- docker-compose.dev.yml | 5 +- docker/documentdb/Dockerfile | 48 ------- docker/hyperdx/Dockerfile | 83 +++++++++--- docker/hyperdx/entry.local.base.sh | 15 ++- packages/app/tests/e2e/docker-compose.yml | 5 +- 11 files changed, 119 insertions(+), 222 deletions(-) create mode 100644 .changeset/aio-documentdb.md delete mode 100644 .vex/openssl-mongodb.vex.json delete mode 100644 docker/documentdb/Dockerfile diff --git a/.changeset/aio-documentdb.md b/.changeset/aio-documentdb.md new file mode 100644 index 0000000000..9dac214734 --- /dev/null +++ b/.changeset/aio-documentdb.md @@ -0,0 +1,23 @@ +--- +'@hyperdx/api': minor +--- + +feat: store all-in-one metadata in DocumentDB instead of MongoDB 4.0.5 + +DocumentDB is MIT licensed and runs on Postgres 18, which removes the SSPL +licence and the critical OpenSSL 1.1.1k CVEs that came with the bundled MongoDB. +Metadata still lives on the `/data/db` mount, now under `/data/db/pgdata`. The +image cannot read the MongoDB files already there and starts with an empty +database, so users, sources, dashboards and alerts saved by an older image need +to be moved across once: + +1. Stop the old container, then start its MongoDB on its own: + `docker run -d --name old-mongod -v :/data/db --entrypoint mongod --dbpath /data/db --bind_ip_all` +2. Dump it with the 4.0 tools, since newer `mongodump` refuses MongoDB 4.0: + `docker run --rm --network container:old-mongod mongo:4.0.5 mongodump --db hyperdx --archive > hyperdx.archive` +3. Remove `old-mongod`, start the new image with the same `/data/db` volume and + wait for the UI to load. +4. Restore into it, then restart the container: + `docker run --rm -i --network container: mongo:7.0 mongorestore --uri "mongodb://hyperdx:hyperdx@localhost:10260/?authMechanism=SCRAM-SHA-256" --ssl --tlsInsecure --drop --archive < hyperdx.archive` + +The old MongoDB files stay in `/data/db` until you delete them. diff --git a/.github/actions/e2e-setup/action.yml b/.github/actions/e2e-setup/action.yml index 138b1551cb..00cb17a7ec 100644 --- a/.github/actions/e2e-setup/action.yml +++ b/.github/actions/e2e-setup/action.yml @@ -77,7 +77,7 @@ runs: shell: bash run: | docker buildx create --use --driver=docker-container - docker buildx build docker/documentdb -t hyperdx/documentdb:local --cache-to "type=gha,mode=max,scope=documentdb" --cache-from "type=gha,scope=documentdb" --load + docker buildx build . -f docker/hyperdx/Dockerfile --target documentdb -t hyperdx/documentdb:local --cache-to "type=gha,mode=max,scope=documentdb" --cache-from "type=gha,scope=documentdb" --load - name: Start E2E Docker Compose shell: bash diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 513261b0d1..6d2ad339c7 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -168,7 +168,7 @@ jobs: docker/hyperdx/** docker/clickhouse/** docker/otel-collector/** - .vex/** + docker/documentdb/** - name: Setup Docker Buildx if: github.event_name != 'pull_request' || diff --git a/.vex/openssl-mongodb.vex.json b/.vex/openssl-mongodb.vex.json deleted file mode 100644 index 4a8aecdd8f..0000000000 --- a/.vex/openssl-mongodb.vex.json +++ /dev/null @@ -1,150 +0,0 @@ -{ - "@context": "https://openvex.dev/ns/v0.2.0", - "@id": "https://github.com/ClickHouse/ClickStack/blob/main/.vex/openssl-mongodb.vex.json", - "author": "HyperDX Team", - "role": "Supplier", - "timestamp": "2026-02-17T00:00:00Z", - "version": 1, - "statements": [ - { - "vulnerability": { "name": "CVE-2021-3711" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode and does not process external TLS certificates. OpenSSL certificate-parsing vulnerabilities are not exploitable in this deployment." - }, - { - "vulnerability": { "name": "CVE-2021-3712" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode and does not process external TLS certificates. OpenSSL certificate-parsing vulnerabilities are not exploitable in this deployment." - }, - { - "vulnerability": { "name": "CVE-2021-4044" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode and does not process external TLS certificates. OpenSSL vulnerabilities are not exploitable in this deployment." - }, - { - "vulnerability": { "name": "CVE-2022-0778" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode. Certificate processing from untrusted sources does not occur, making this infinite-loop vulnerability unexploitable." - }, - { - "vulnerability": { "name": "CVE-2022-1473" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode and does not process external TLS certificates. This memory leak vulnerability is not exploitable in this deployment." - }, - { - "vulnerability": { "name": "CVE-2022-3358" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode. Custom cipher usage from untrusted sources does not occur in this deployment." - }, - { - "vulnerability": { "name": "CVE-2022-3602" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode and does not process external TLS certificates. This buffer overflow vulnerability is not exploitable in this deployment." - }, - { - "vulnerability": { "name": "CVE-2022-3786" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode and does not process external TLS certificates. This buffer overflow vulnerability is not exploitable in this deployment." - }, - { - "vulnerability": { "name": "CVE-2022-3996" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode. The double-locking issue is not triggerable via external input in this deployment." - }, - { - "vulnerability": { "name": "CVE-2023-0286" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode. External GeneralName certificate parsing does not occur in this deployment." - }, - { - "vulnerability": { "name": "CVE-2023-0464" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode. Untrusted X.509 certificate chains with policy constraints are not processed in this deployment." - }, - { - "vulnerability": { "name": "CVE-2023-5363" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode. Untrusted key/IV inputs from external sources are not processed in this deployment." - }, - { - "vulnerability": { "name": "CVE-2024-4741" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode and does not process external TLS certificates. This use-after-free vulnerability is not exploitable in this deployment." - }, - { - "vulnerability": { "name": "CVE-2024-5535" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode. The SSL_select_next_proto vulnerability requires attacker-controlled input not present in this deployment." - }, - { - "vulnerability": { "name": "CVE-2024-6119" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode. OCSP responses from external sources are not processed in this deployment." - }, - { - "vulnerability": { "name": "CVE-2025-9230" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode and does not process external TLS certificates. This OpenSSL vulnerability is not exploitable in this deployment." - }, - { - "vulnerability": { "name": "CVE-2025-15467" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode and does not process external TLS certificates. This OpenSSL vulnerability is not exploitable in this deployment." - }, - { - "vulnerability": { "name": "CVE-2025-69419" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode and does not process external TLS certificates. This OpenSSL vulnerability is not exploitable in this deployment." - }, - { - "vulnerability": { "name": "CVE-2025-69420" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode and does not process external TLS certificates. This OpenSSL vulnerability is not exploitable in this deployment." - }, - { - "vulnerability": { "name": "CVE-2025-69421" }, - "products": [{ "@id": "pkg:oci/clickstack" }], - "status": "not_affected", - "justification": "inline_mitigations_already_exist", - "impact_statement": "MongoDB is deployed in localhost-only mode and does not process external TLS certificates. This OpenSSL vulnerability is not exploitable in this deployment." - } - ] -} diff --git a/agent_docs/development.md b/agent_docs/development.md index e7feee067f..fb3f8ee1dc 100644 --- a/agent_docs/development.md +++ b/agent_docs/development.md @@ -33,7 +33,8 @@ yarn dev # or equivalently: make dev - Docker Compose manages ClickHouse, DocumentDB, OTel Collector - The metadata store is [DocumentDB](https://github.com/documentdb/documentdb) (Postgres plus a Mongo wire-protocol gateway), built from - `docker/documentdb/Dockerfile`. The API still talks to it through Mongoose. + the `documentdb` stage of `docker/hyperdx/Dockerfile`, the same build the + all-in-one image ships. The API still talks to it through Mongoose. To browse it with MongoDB Compass or `mongosh`, connect to `mongodb://hyperdx:hyperdx@localhost:/?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256`. Everything works except Compass's Performance tab (the gateway doesn't diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index c47ecf97cd..e075a77ead 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -36,7 +36,10 @@ services: - internal db: image: hyperdx/documentdb:local - build: ./docker/documentdb + build: + context: . + dockerfile: docker/hyperdx/Dockerfile + target: documentdb environment: DOCDB_USER: hyperdx DOCDB_PASSWORD: hyperdx diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 619cd83a68..b94d5e2a57 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -10,7 +10,10 @@ services: hdx.dev.service: documentdb hdx.dev.port: '${HDX_DEV_MONGO_PORT:-27017}' image: hyperdx/documentdb:local - build: ./docker/documentdb + build: + context: . + dockerfile: docker/hyperdx/Dockerfile + target: documentdb environment: DOCDB_USER: hyperdx DOCDB_PASSWORD: hyperdx diff --git a/docker/documentdb/Dockerfile b/docker/documentdb/Dockerfile deleted file mode 100644 index 1da60fb441..0000000000 --- a/docker/documentdb/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -# DocumentDB (MIT): Postgres extensions plus a Mongo wire-protocol gateway, used -# in place of MongoDB (SSPL). Built from source because upstream ships glibc -# packages only; Alpine 3.24 matches the all-in-one base and its PostGIS targets PG18. -ARG DOCUMENTDB_VERSION=v1.2-0 -# Tags can move; the build fails unless the tag still resolves to this commit. -ARG DOCUMENTDB_COMMIT=965b6e7ad1c49aefb983a412a3fea0ff637cd7f1 - -FROM alpine:3.24 AS builder -ARG DOCUMENTDB_VERSION -ARG DOCUMENTDB_COMMIT -RUN apk add --no-cache build-base cmake git curl bash tar pkgconf perl linux-headers python3 \ - icu-dev krb5-dev openssl-dev rust cargo \ - postgresql18 postgresql18-dev postgresql18-contrib postgresql-pg_cron postgresql-pgvector postgis -ENV PG_CONFIG=/usr/libexec/postgresql18/pg_config INSTALL_DEPENDENCIES_ROOT=/deps -RUN git clone --depth 1 --branch ${DOCUMENTDB_VERSION} https://github.com/documentdb/documentdb.git /src \ - && test "$(git -C /src rev-parse HEAD)" = "${DOCUMENTDB_COMMIT}" \ - && mkdir -p /deps -# musl exports _init/_fini from every .so, which the duplicate-export check counts. -RUN sed -i "s#grep -vxE '_PG_init|Pg_magic_func'#grep -vxE '_PG_init|Pg_magic_func|_init|_fini'#g" \ - /src/pg_documentdb_extended_rum/validate_rum_core_exports.sh -RUN cd /src/scripts && MAKE_PROGRAM=cmake bash install_setup_libbson.sh \ - && bash install_setup_pcre2.sh && bash install_setup_intel_decimal_math_lib.sh -# Serial make: with -j the RUM export check can run before the library is linked. -RUN cd /src && make install-documentdb with_llvm=no PG_CONFIG=$PG_CONFIG -RUN cd /src/pg_documentdb_gw && cargo build --release -p documentdb_gateway \ - && install -s target/release/documentdb_gateway /usr/bin/documentdb-gateway -# Keep only what Postgres and DocumentDB link against: no GDAL, JIT or PostGIS raster/topology. -RUN set -e; PG=/usr/libexec/postgresql18; LIB=/usr/lib/postgresql18; SH=/usr/share/postgresql18; \ - rm -f $LIB/llvmjit*.so $LIB/postgis_raster-3.so $LIB/postgis_sfcgal-3.so $LIB/postgis_topology-3.so $LIB/address_standardizer*.so; rm -rf $LIB/bitcode; \ - rm -f $SH/extension/postgis_raster* $SH/extension/postgis_sfcgal* $SH/extension/postgis_topology* $SH/extension/postgis_tiger* $SH/extension/address_standardizer*; rm -rf $SH/contrib/postgis*; \ - for f in $PG/postgres $PG/pg_ctl $PG/initdb $PG/psql /usr/bin/documentdb-gateway $LIB/*.so; do echo $f; ldd $f 2>/dev/null | awk '/=> \//{print $3}'; done | sort -u > /tmp/files; \ - mkdir /out; tar -ch -T /tmp/files -f - | tar -x -C /out; \ - tar -ch $PG/postgres $PG/pg_ctl $PG/initdb $PG/psql $LIB $SH /usr/share/proj /usr/share/icu -f - | tar -x -C /out; \ - install -D -m 644 /src/pg_documentdb_gw/SetupConfiguration.json /out/etc/documentdb/gateway/SetupConfiguration.json - -FROM alpine:3.24 AS runtime -RUN apk add --no-cache bash openssl su-exec \ - && addgroup -S postgres && adduser -S -G postgres -h /var/lib/postgresql postgres \ - && install -d -o postgres -g postgres /data/docdb -COPY --from=builder /out/ / -COPY --chmod=755 entrypoint.sh /usr/local/bin/documentdb-entrypoint.sh -ENV PATH=/usr/libexec/postgresql18:$PATH PGDATA=/data/docdb/pgdata -EXPOSE 10260 -# The gateway only listens once Postgres is initialised and the user exists; the -# psql probe catches Postgres dying under a gateway that keeps its port open. -HEALTHCHECK --interval=2s --timeout=3s --start-period=60s --retries=30 \ - CMD nc -z 127.0.0.1 10260 && psql -h /tmp -U postgres -d postgres -qtAX -c 'SELECT 1' >/dev/null -ENTRYPOINT ["bash", "/usr/local/bin/documentdb-entrypoint.sh"] diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index a57caf91f7..f4419488c5 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -4,13 +4,16 @@ # - App (Frontend) # For all-in-one build: # - Clickhouse -# - Mongo +# - DocumentDB (Postgres with a Mongo wire-protocol gateway) # - Otel Collector (otelcol) ARG NODE_VERSION=22.23.4 ARG CLICKHOUSE_VERSION=26.10.3.7 ARG OTEL_COLLECTOR_VERSION=0.165.0 ARG OTEL_COLLECTOR_CORE_VERSION=1.71.0 +ARG DOCUMENTDB_VERSION=v1.2-0 +# Tags can move; the build fails unless the tag still resolves to this commit. +ARG DOCUMENTDB_COMMIT=965b6e7ad1c49aefb983a412a3fea0ff637cd7f1 # base ############################################################################################# # == Otel Collector Image == @@ -152,6 +155,57 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ ENTRYPOINT ["sh", "/etc/local/entry.sh"] +# == DocumentDB (MIT): Postgres extensions plus a Mongo wire-protocol gateway, used +# in place of MongoDB (SSPL). Built from source because upstream ships glibc +# packages only; Alpine 3.24 matches the all-in-one base and its PostGIS targets PG18. +FROM alpine:3.24 AS documentdb-builder +ARG DOCUMENTDB_VERSION +ARG DOCUMENTDB_COMMIT +RUN apk add --no-cache build-base cmake git curl bash tar pkgconf perl linux-headers python3 \ + icu-dev krb5-dev openssl-dev rust cargo \ + postgresql18 postgresql18-dev postgresql18-contrib postgresql-pg_cron postgresql-pgvector postgis +ENV PG_CONFIG=/usr/libexec/postgresql18/pg_config INSTALL_DEPENDENCIES_ROOT=/deps +RUN git clone --depth 1 --branch ${DOCUMENTDB_VERSION} https://github.com/documentdb/documentdb.git /src \ + && test "$(git -C /src rev-parse HEAD)" = "${DOCUMENTDB_COMMIT}" \ + && mkdir -p /deps +# musl exports _init/_fini from every .so, which the duplicate-export check counts. +RUN sed -i "s#grep -vxE '_PG_init|Pg_magic_func'#grep -vxE '_PG_init|Pg_magic_func|_init|_fini'#g" \ + /src/pg_documentdb_extended_rum/validate_rum_core_exports.sh +RUN cd /src/scripts && MAKE_PROGRAM=cmake bash install_setup_libbson.sh \ + && bash install_setup_pcre2.sh && bash install_setup_intel_decimal_math_lib.sh +# Serial make: with -j the RUM export check can run before the library is linked. +RUN cd /src && make install-documentdb with_llvm=no PG_CONFIG=$PG_CONFIG +RUN cd /src/pg_documentdb_gw && cargo build --release -p documentdb_gateway \ + && install -s target/release/documentdb_gateway /usr/bin/documentdb-gateway +# Keep only what Postgres and DocumentDB link against: no GDAL, JIT or PostGIS raster/topology. +# Shared libraries are listed as apk packages rather than copied, so the target installs +# its own point release instead of having its libssl, musl and libapk overwritten. +RUN set -e; PG=/usr/libexec/postgresql18; LIB=/usr/lib/postgresql18; SH=/usr/share/postgresql18; \ + rm -f $LIB/llvmjit*.so $LIB/postgis_raster-3.so $LIB/postgis_sfcgal-3.so $LIB/postgis_topology-3.so $LIB/address_standardizer*.so; rm -rf $LIB/bitcode; \ + rm -f $SH/extension/postgis_raster* $SH/extension/postgis_sfcgal* $SH/extension/postgis_topology* $SH/extension/postgis_tiger* $SH/extension/address_standardizer*; rm -rf $SH/contrib/postgis*; \ + for f in $PG/postgres $PG/pg_ctl $PG/initdb $PG/psql /usr/bin/documentdb-gateway $LIB/*.so; do ldd $f 2>/dev/null | awk '/=> \//{print $3}'; done | grep -E '^/(usr/)?lib/[^/]+$' | sort -u > /tmp/libs; \ + mkdir -p /out/etc/documentdb/gateway; \ + xargs apk info -W < /tmp/libs | sed -n 's/.* is owned by //p' > /tmp/owners; \ + test "$(wc -l < /tmp/owners)" = "$(wc -l < /tmp/libs)"; \ + sed 's/-[^-]*-r[0-9]*$//' /tmp/owners | sort -u > /out/etc/documentdb/apk-packages; \ + tar -ch $PG/postgres $PG/pg_ctl $PG/initdb $PG/psql /usr/bin/documentdb-gateway $LIB $SH -f - | tar -x -C /out; \ + install -m 644 /src/pg_documentdb_gw/SetupConfiguration.json /out/etc/documentdb/gateway/SetupConfiguration.json + +# Standalone DocumentDB used by the dev, integration and e2e compose files. +FROM alpine:3.24 AS documentdb +COPY --from=documentdb-builder /out/ / +RUN apk add --no-cache bash openssl su-exec $(cat /etc/documentdb/apk-packages) \ + && addgroup -S -g 70 postgres && adduser -S -u 70 -G postgres -h /var/lib/postgresql postgres \ + && install -d -o postgres -g postgres /data/docdb +COPY --chmod=755 docker/documentdb/entrypoint.sh /usr/local/bin/documentdb-entrypoint.sh +ENV PATH=/usr/libexec/postgresql18:$PATH PGDATA=/data/docdb/pgdata +EXPOSE 10260 +# The gateway only listens once Postgres is initialised and the user exists; the +# psql probe catches Postgres dying under a gateway that keeps its port open. +HEALTHCHECK --interval=2s --timeout=3s --start-period=60s --retries=30 \ + CMD nc -z 127.0.0.1 10260 && psql -h /tmp -U postgres -d postgres -qtAX -c 'SELECT 1' >/dev/null +ENTRYPOINT ["bash", "/usr/local/bin/documentdb-entrypoint.sh"] + # all-in-one base ############################################################################################ FROM hairyhenderson/gomplate:v5.3.1-alpine AS gomplate FROM clickhouse/clickhouse-server:${CLICKHOUSE_VERSION}-alpine AS all-in-one-base @@ -198,18 +252,20 @@ COPY --from=gomplate /bin/gomplate /usr/local/bin/gomplate # Copy migrate binary (Go-based goose migration tool with full TLS support) COPY --from=migrate-builder /migrate /usr/local/bin/migrate -# Install MongoDB and other dependencies (consolidated into a single RUN command). -# libstdc++/libgcc are the Node.js runtime deps, installed before the v3.9 -# repositories are appended so they resolve from this image's own Alpine release. -RUN apk add --no-cache libstdc++ libgcc && \ - echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/main' >> /etc/apk/repositories && \ - echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/community' >> /etc/apk/repositories && \ - apk update && \ - apk add --no-cache mongodb yaml-cpp=0.6.2-r2 ca-certificates && \ +COPY --from=documentdb-builder /out/ / + +# libstdc++/libgcc are the Node.js runtime deps; bash, openssl and su-exec run DocumentDB. +# ldd fails the build if the base's Alpine release lacks a library DocumentDB was built against. +# The gateway listens on localhost only, since its credentials are fixed. +RUN apk add --no-cache libstdc++ libgcc ca-certificates bash openssl su-exec $(cat /etc/documentdb/apk-packages) && \ apk add --no-cache --upgrade curl busybox && \ addgroup -S -g ${USER_GID} otel && \ adduser -S -u ${USER_UID} -G otel otel && \ + addgroup -S -g 70 postgres && adduser -S -u 70 -G postgres -h /var/lib/postgresql postgres && \ mkdir -p /data/db && \ + ! ldd /usr/libexec/postgresql18/postgres /usr/bin/documentdb-gateway /usr/lib/postgresql18/*.so 2>&1 | grep 'Error loading shared library' && \ + sed -i 's/"UseLocalHost": false/"UseLocalHost": true/' /etc/documentdb/gateway/SetupConfiguration.json && \ + grep -q '"UseLocalHost": true' /etc/documentdb/gateway/SetupConfiguration.json && \ install -d -m 0777 -o ${USER_UID} -g ${USER_GID} /etc/otel/supervisor-data && \ rm -rf /var/cache/apk/* && \ rm -rf /usr/local/lib/node_modules/npm /usr/local/lib/node_modules/corepack \ @@ -222,10 +278,7 @@ ENV NODE_ENV=production COPY --from=prod /app /app COPY --from=hyperdx ./refresh-env.js /etc/local/refresh-env.js COPY --from=hyperdx ./entry.local.base.sh /etc/local/entry.base.sh - -# Copy VEX exception files for Docker Scout vulnerability suppression -# Reference via: docker scout cves --vex-location /etc/vex/ -COPY .vex/*.vex.json /etc/vex/ +COPY --chmod=755 docker/documentdb/entrypoint.sh /usr/local/bin/documentdb-entrypoint.sh WORKDIR /app @@ -239,7 +292,7 @@ FROM all-in-one-base AS all-in-one-noauth LABEL org.opencontainers.image.vendor="HyperDX" \ org.opencontainers.image.title="HyperDX All-in-One (No Auth)" \ - org.opencontainers.image.description="HyperDX all-in-one image with ClickHouse, MongoDB, and OTel Collector (no authentication)" \ + org.opencontainers.image.description="HyperDX all-in-one image with ClickHouse, DocumentDB, and OTel Collector (no authentication)" \ org.opencontainers.image.source="https://github.com/ClickHouse/ClickStack" \ org.opencontainers.image.licenses="MIT" @@ -255,7 +308,7 @@ FROM all-in-one-base AS all-in-one-auth LABEL org.opencontainers.image.vendor="HyperDX" \ org.opencontainers.image.title="HyperDX All-in-One (With Auth)" \ - org.opencontainers.image.description="HyperDX all-in-one image with ClickHouse, MongoDB, and OTel Collector (with authentication)" \ + org.opencontainers.image.description="HyperDX all-in-one image with ClickHouse, DocumentDB, and OTel Collector (with authentication)" \ org.opencontainers.image.source="https://github.com/ClickHouse/ClickStack" \ org.opencontainers.image.licenses="MIT" diff --git a/docker/hyperdx/entry.local.base.sh b/docker/hyperdx/entry.local.base.sh index 82ceb0aebf..ec88695e71 100644 --- a/docker/hyperdx/entry.local.base.sh +++ b/docker/hyperdx/entry.local.base.sh @@ -14,7 +14,8 @@ export OPAMP_PORT=${HYPERDX_OPAMP_PORT:-4320} # Internal Services export HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE="${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE:-default}" export CLICKHOUSE_ENDPOINT="${CLICKHOUSE_ENDPOINT:-tcp://ch-server:9000?dial_timeout=10s}" -export MONGO_URI="mongodb://db:27017/hyperdx" +# DocumentDB gateway; the self-signed TLS cert never leaves the container. +export MONGO_URI="mongodb://hyperdx:hyperdx@db:10260/hyperdx?tls=true&tlsAllowInvalidCertificates=true&authMechanism=SCRAM-SHA-256" export OPAMP_SERVER_URL="http://127.0.0.1:${OPAMP_PORT}" export CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT="${CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT:-ch-server:9363}" @@ -59,8 +60,16 @@ echo "Exporting data to ClickHouse: # Start Clickhouse Server /entrypoint.sh > /var/log/clickhouse.log 2>&1 & -# Start Mongo Server -mongod --quiet --dbpath /data/db > /var/log/mongod.log 2>&1 & +# Images before DocumentDB kept MongoDB data directly in /data/db, which this image cannot read. +if [ -f /data/db/WiredTiger ] && [ ! -f /data/db/pgdata/.documentdb-initialised ]; then + echo "WARNING: found MongoDB data in /data/db. This image stores metadata in DocumentDB" + echo "WARNING: under /data/db/pgdata and starts empty; see the release notes to migrate it." +fi + +# Start DocumentDB (Postgres + Mongo wire-protocol gateway on :10260) +DOCDB_USER=hyperdx DOCDB_PASSWORD=hyperdx PGDATA=/data/db/pgdata \ + PATH="/usr/libexec/postgresql18:$PATH" \ + bash /usr/local/bin/documentdb-entrypoint.sh > /var/log/documentdb.log 2>&1 & # Wait for Clickhouse to be ready while ! curl -s "http://ch-server:8123" > /dev/null; do diff --git a/packages/app/tests/e2e/docker-compose.yml b/packages/app/tests/e2e/docker-compose.yml index d3c6653a14..eca2b2f0ff 100644 --- a/packages/app/tests/e2e/docker-compose.yml +++ b/packages/app/tests/e2e/docker-compose.yml @@ -2,7 +2,10 @@ name: hdx-e2e services: db: image: hyperdx/documentdb:local - build: ../../../../docker/documentdb + build: + context: ../../../.. + dockerfile: docker/hyperdx/Dockerfile + target: documentdb environment: DOCDB_USER: hyperdx DOCDB_PASSWORD: hyperdx