From f6d53359d77e3766ef4476ffb2f23427ce70fa97 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Wed, 23 Sep 2026 13:56:53 -0700 Subject: [PATCH] Add reproducible legacy container Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .dockerignore | 17 +++++ Dockerfile | 81 +++++++++++++++++++++ README.md | 52 ++++++++++++- docker/requirements.txt | 10 +++ docker/sitecustomize.py | 3 + docker/verify_environment.py | 77 ++++++++++++++++++++ graphormer/data/dgl_datasets/dgl_dataset.py | 10 ++- graphormer/data/pyg_datasets/pyg_dataset.py | 10 ++- 8 files changed, 255 insertions(+), 5 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker/requirements.txt create mode 100644 docker/sitecustomize.py create mode 100644 docker/verify_environment.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0129abc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +.git +.venv +**/__pycache__ +**/*.py[cod] +**/*.so +**/*.pt +**/*.csv +**/*.log +**/checkpoints +**/ckpts +**/sweeps +**/tensorboard +**/test_results +build +dist +exps +wandb diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..141f620 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,81 @@ +FROM python:3.9.19-slim-bookworm@sha256:69e712dbe4c4a166527cbf69374533125cfb6ee93a5e39031a0191c741d386d7 AS builder + +ENV DEBIAN_FRONTEND=noninteractive \ + DGLBACKEND=pytorch \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 \ + PYTHONPATH=/opt/graphormer \ + PYTHONUNBUFFERED=1 + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends \ + build-essential \ + ca-certificates \ + git \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/graphormer +COPY . . + +RUN test -f fairseq/setup.py \ + || (echo "Initialize submodules before building: git submodule update --init --recursive" >&2; exit 1) + +RUN python -m pip install \ + pip==23.3.2 \ + setuptools==69.5.1 \ + wheel==0.48.0 \ + && python -m pip install \ + torch==1.9.1+cu111 \ + torchaudio==0.9.1 \ + --find-links https://download.pytorch.org/whl/cu111/torch_stable.html \ + && python -m pip install --requirement docker/requirements.txt \ + && python -m pip install \ + dgl==0.7.2 \ + --find-links https://data.dgl.ai/wheels/repo.html + +COPY docker/sitecustomize.py /usr/local/lib/python3.9/site-packages/sitecustomize.py + +RUN cd fairseq \ + && CFLAGS="-include cstdint" CXXFLAGS="-include cstdint" \ + python -m pip install --no-build-isolation . + +RUN python - <<'PY' +import numpy +import pyximport + +pyximport.install( + setup_args={"include_dirs": numpy.get_include()}, + inplace=True, + language_level=3, +) +import graphormer.data.algos +PY + +RUN python docker/verify_environment.py + + +FROM python:3.9.19-slim-bookworm@sha256:69e712dbe4c4a166527cbf69374533125cfb6ee93a5e39031a0191c741d386d7 AS runtime + +LABEL org.opencontainers.image.title="Graphormer legacy runtime" \ + org.opencontainers.image.description="Pinned Python 3.9, PyTorch 1.9, and Fairseq environment for Graphormer" \ + org.opencontainers.image.source="https://github.com/microsoft/Graphormer" \ + org.opencontainers.image.version="legacy-py39-torch1.9" + +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ + DGLBACKEND=pytorch \ + PYTHONPATH=/opt/graphormer \ + PYTHONUNBUFFERED=1 + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends \ + ca-certificates \ + g++ \ + libgomp1 \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /usr/local /usr/local +COPY --from=builder /opt/graphormer /opt/graphormer + +WORKDIR /opt/graphormer + +CMD ["bash"] diff --git a/README.md b/README.md index 98c52bc..a150fca 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,57 @@ Next you may want to read: ## Requirements and Installation -#### Setup with Conda +### Containerized legacy environment + +Graphormer extends a pinned Fairseq revision and uses a legacy Python and +PyTorch stack. The container is the recommended way to run this environment +without changing packages in the host Python installation. + +Initialize the Fairseq submodule and build the image: + +```bash +git submodule update --init --recursive +docker build --tag graphormer:legacy . +``` + +Verify CPU imports and graph preprocessing: + +```bash +docker run --rm graphormer:legacy \ + python docker/verify_environment.py +``` + +GPU execution requires the NVIDIA Container Toolkit: + +```bash +docker run --rm --gpus all graphormer:legacy \ + python docker/verify_environment.py --require-cuda +``` + +Run an interactive shell and mount data or checkpoints separately from the +repository: + +```bash +docker run --rm --gpus all --interactive --tty \ + --volume /path/to/data:/data \ + graphormer:legacy +``` + +For single-host distributed training, provide an explicit process-group URL: + +```bash +docker run --rm --gpus all graphormer:legacy \ + fairseq-train ... \ + --distributed-world-size 2 \ + --distributed-init-method tcp://localhost:29500 +``` + +The image intentionally pins Python 3.9, PyTorch 1.9.1, CUDA 11.1 user-space +libraries, and pip 23.3.2. It is a reproducibility environment for the existing +Fairseq-based code, not a claim of compatibility with current Python or +PyTorch releases. Multi-node training is not covered by this setup. + +### Local installation ``` bash install.sh diff --git a/docker/requirements.txt b/docker/requirements.txt new file mode 100644 index 0000000..1fcdbfd --- /dev/null +++ b/docker/requirements.txt @@ -0,0 +1,10 @@ +Cython==0.29.36 +googledrivedownloader==0.4 +lmdb==2.3.0 +numpy==1.23.5 +ogb==1.3.2 +protobuf==3.20.3 +rdkit-pypi==2021.9.3 +tensorboard==2.11.2 +tensorboardX==2.4.1 +torch-geometric==2.3.1 diff --git a/docker/sitecustomize.py b/docker/sitecustomize.py new file mode 100644 index 0000000..043671b --- /dev/null +++ b/docker/sitecustomize.py @@ -0,0 +1,3 @@ +"""Compatibility imports for the pinned PyTorch 1.9 runtime.""" + +import distutils.version diff --git a/docker/verify_environment.py b/docker/verify_environment.py new file mode 100644 index 0000000..424fe65 --- /dev/null +++ b/docker/verify_environment.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import argparse +import json + +import numpy as np +import torch +from ogb.utils.mol import smiles2graph +from torch_geometric.data import Data + +import fairseq +import graphormer +import graphormer.models +import graphormer.tasks.graph_prediction +from graphormer.data import algos +from graphormer.data.wrapper import preprocess_item + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--require-cuda", action="store_true") + parser.add_argument("--expected-gpus", type=int) + return parser.parse_args() + + +def main(): + args = parse_args() + + adjacency = np.array( + [ + [False, True, False], + [True, False, True], + [False, True, False], + ] + ) + distances, _ = algos.floyd_warshall(adjacency) + if distances.tolist() != [[0, 1, 2], [1, 0, 1], [2, 1, 0]]: + raise RuntimeError("Graphormer shortest-path preprocessing returned bad data") + + graph = smiles2graph("CCO") + item = preprocess_item( + Data( + x=torch.from_numpy(graph["node_feat"]).long(), + edge_index=torch.from_numpy(graph["edge_index"]).long(), + edge_attr=torch.from_numpy(graph["edge_feat"]).long(), + y=torch.tensor([0.0]), + ) + ) + if tuple(item.spatial_pos.shape) != (3, 3): + raise RuntimeError("Graphormer molecular preprocessing returned a bad shape") + + cuda_available = torch.cuda.is_available() + gpu_count = torch.cuda.device_count() + if args.require_cuda and not cuda_available: + raise RuntimeError("CUDA was required but is not available") + if args.expected_gpus is not None and gpu_count != args.expected_gpus: + raise RuntimeError(f"Expected {args.expected_gpus} GPUs, found {gpu_count}") + + print( + json.dumps( + { + "cuda_available": cuda_available, + "fairseq": fairseq.__version__, + "gpu_count": gpu_count, + "graphormer": graphormer.__file__, + "preprocessed_nodes": item.spatial_pos.size(0), + "torch": torch.__version__, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/graphormer/data/dgl_datasets/dgl_dataset.py b/graphormer/data/dgl_datasets/dgl_dataset.py index 0bf0d25..c4f4f65 100644 --- a/graphormer/data/dgl_datasets/dgl_dataset.py +++ b/graphormer/data/dgl_datasets/dgl_dataset.py @@ -148,7 +148,7 @@ def __preprocess_dgl_graph( return pyg_graph - def __getitem__(self, idx): + def get(self, idx): if isinstance(idx, int): if self.__indices__ is not None: idx = self.__indices__[idx] @@ -157,5 +157,11 @@ def __getitem__(self, idx): else: raise TypeError("index to a GraphormerDGLDataset can only be an integer.") - def __len__(self) -> int: + def len(self) -> int: return len(self.dataset) if self.__indices__ is None else len(self.__indices__) + + def __getitem__(self, idx): + return self.get(idx) + + def __len__(self) -> int: + return self.len() diff --git a/graphormer/data/pyg_datasets/pyg_dataset.py b/graphormer/data/pyg_datasets/pyg_dataset.py index 9091878..e22a43b 100644 --- a/graphormer/data/pyg_datasets/pyg_dataset.py +++ b/graphormer/data/pyg_datasets/pyg_dataset.py @@ -93,7 +93,7 @@ def create_subset(self, subset): return dataset @lru_cache(maxsize=16) - def __getitem__(self, idx): + def get(self, idx): if isinstance(idx, int): item = self.dataset[idx] item.idx = idx @@ -102,5 +102,11 @@ def __getitem__(self, idx): else: raise TypeError("index to a GraphormerPYGDataset can only be an integer.") - def __len__(self): + def len(self): return self.num_data + + def __getitem__(self, idx): + return self.get(idx) + + def __len__(self): + return self.len()