diff --git a/packages/sie_sdk/README.md b/packages/sie_sdk/README.md index 8b11c182c..206b97cea 100644 --- a/packages/sie_sdk/README.md +++ b/packages/sie_sdk/README.md @@ -152,11 +152,13 @@ billed. The other items still succeed. On a CUDA server, the operator can load the DeBERTa-based GLiClass models with CUDA graphs, which cut the CPU time spent launching kernels. With `bucketed` graphs, sequence lengths are padded to buckets, which moves probabilities -slightly, as batching requests together does: by up to 0.006 on -`gliclass-large-v1.0` and 0.023 on `gliclass-multilang-mini` in our tests. -Send `options={"cuda_graphs": "off"}` to run a request eagerly; a request -cannot turn graphs on. See the server README for the supported models and -memory use. +slightly, as batching requests together does. `gliclass-base-v1.0`, +`gliclass-large-v1.0` and `opir-multitask-large-v1.0` load with `bucketed` +graphs by default: their probabilities differed from eager execution by up to +0.0144 in our tests, with no top label changed. Send +`options={"cuda_graphs": "off"}` to run a request eagerly; a request cannot +turn graphs on. See the server README for each model's measurements and the +memory graphs use. ## Generation prompts and guard verdicts diff --git a/packages/sie_server/README.md b/packages/sie_server/README.md index 61d4e1a3c..8e087f07a 100644 --- a/packages/sie_server/README.md +++ b/packages/sie_server/README.md @@ -101,20 +101,27 @@ documents twice), so that graphs were recorded and then replayed: 11,538 answers per model. A small change can still flip a near tie between the top two labels: -| Model | Largest probability change | Top label changed | -|--|--|--| -| `gliclass-small-v1.0` | 0.004 | 3 answers | -| `gliclass-base-v1.0` | 0.005 | none | -| `gliclass-large-v1.0` | 0.006 | none | -| `gliclass-base-v3.0` | 0.005 | 3 | -| `gliclass-large-v3.0` | 0.009 | 3 | -| `gliclass-instruct-base-v1.0` | 0.008 | 12 | -| `gliclass-instruct-large-v1.0` | 0.010 | 18 | -| `opir-multitask-large-v1.0` | 0.014 | none | -| `gliclass-multilang-mini` (100 descriptions, no joint groups: 2,016 answers) | 0.023 | 3 | +| Model | Largest probability change | Top label changed | Shipped profile | +|--|--|--|--| +| `gliclass-small-v1.0` | 0.0039 | 3 answers | `off` | +| `gliclass-base-v1.0` | 0.0049 | none | `bucketed` | +| `gliclass-large-v1.0` | 0.0056 | none | `bucketed` | +| `gliclass-base-v3.0` | 0.0054 | 3 | `off` | +| `gliclass-large-v3.0` | 0.0093 | 3 | `off` | +| `gliclass-instruct-base-v1.0` | 0.0076 | 12 | `off` | +| `gliclass-instruct-large-v1.0` | 0.0098 | 18 | `off` | +| `opir-multitask-large-v1.0` | 0.0144 | none | `bucketed` | +| `gliclass-multilang-mini` (100 descriptions, no joint groups: 2,016 answers) | 0.0227 | 3 | `off` | `exact` changed nothing. +The shipped profiles load with `bucketed` graphs only where no top label +changed: `gliclass-base-v1.0`, `gliclass-large-v1.0` and +`opir-multitask-large-v1.0`. Their probabilities can differ from eager +execution by up to the amounts above. To run one of them eagerly, set +`cuda_graphs: off` in its profile, or send `options={"cuda_graphs": "off"}` with +a request. The other models load with `off`. + Graphs apply on CUDA to the DeBERTa-based GLiClass models: the v1.0 models, `gliclass-base-v3.0` and `gliclass-large-v3.0`, the base and large instruct models, the Opir multitask models and `gliclass-multilang-mini`. The diff --git a/packages/sie_server/models/knowledgator__gliclass-base-v1.0.yaml b/packages/sie_server/models/knowledgator__gliclass-base-v1.0.yaml index 5099ea39e..3858fb038 100644 --- a/packages/sie_server/models/knowledgator__gliclass-base-v1.0.yaml +++ b/packages/sie_server/models/knowledgator__gliclass-base-v1.0.yaml @@ -19,5 +19,8 @@ profiles: adapter_options: loadtime: classification_type: single-label + # Replay forwards as CUDA graphs over bucketed lengths; "off" runs them eagerly. + # See "GLiClass CUDA graphs" in packages/sie_server/README.md. + cuda_graphs: bucketed runtime: threshold: 0.0 diff --git a/packages/sie_server/models/knowledgator__gliclass-large-v1.0.yaml b/packages/sie_server/models/knowledgator__gliclass-large-v1.0.yaml index bf4a13f99..5c1916c8c 100644 --- a/packages/sie_server/models/knowledgator__gliclass-large-v1.0.yaml +++ b/packages/sie_server/models/knowledgator__gliclass-large-v1.0.yaml @@ -19,5 +19,8 @@ profiles: adapter_options: loadtime: classification_type: single-label + # Replay forwards as CUDA graphs over bucketed lengths; "off" runs them eagerly. + # See "GLiClass CUDA graphs" in packages/sie_server/README.md. + cuda_graphs: bucketed runtime: threshold: 0.0 diff --git a/packages/sie_server/models/knowledgator__opir-multitask-large-v1.0.yaml b/packages/sie_server/models/knowledgator__opir-multitask-large-v1.0.yaml index 4cc3dcaa8..4a1ffa25e 100644 --- a/packages/sie_server/models/knowledgator__opir-multitask-large-v1.0.yaml +++ b/packages/sie_server/models/knowledgator__opir-multitask-large-v1.0.yaml @@ -19,6 +19,9 @@ profiles: adapter_options: loadtime: classification_type: single-label + # Replay forwards as CUDA graphs over bucketed lengths; "off" runs them eagerly. + # See "GLiClass CUDA graphs" in packages/sie_server/README.md. + cuda_graphs: bucketed runtime: threshold: 0.0 multi-label: diff --git a/packages/sie_server/tests/adapters/test_gliclass_cuda_graphs.py b/packages/sie_server/tests/adapters/test_gliclass_cuda_graphs.py index 43018d303..21b1b1fa1 100644 --- a/packages/sie_server/tests/adapters/test_gliclass_cuda_graphs.py +++ b/packages/sie_server/tests/adapters/test_gliclass_cuda_graphs.py @@ -7,6 +7,7 @@ from __future__ import annotations +from pathlib import Path from types import SimpleNamespace from typing import Any @@ -22,9 +23,18 @@ segment_ids, unsupported_reason, ) -from sie_server.core.loader import reject_unknown_loadtime_options +from sie_server.core.loader import _build_adapter_kwargs, load_model_configs, reject_unknown_loadtime_options from sie_server.types.inputs import InvalidInputError +_MODELS_DIR = Path(__file__).resolve().parents[2] / "models" +# The shipped profiles that load with graphs: DeBERTa-v3 models whose bucketed +# scores changed no top label against eager execution (see the server README). +_BUCKETED_BY_DEFAULT = { + "knowledgator/gliclass-base-v1.0", + "knowledgator/gliclass-large-v1.0", + "knowledgator/opir-multitask-large-v1.0", +} + class _Encoder: def __init__(self) -> None: @@ -542,3 +552,20 @@ def test_a_request_that_opts_out_runs_eagerly(self) -> None: _adapter_with(runner)._forward(_Pipe(), _inputs(1, 100), ["a", "b"], same_labels=True, graphs="off") assert runner.replayed == [] + + +def test_shipped_profiles_enable_bucketed_graphs_only_where_measured() -> None: + configs = load_model_configs(_MODELS_DIR) + # Named profiles (``model:profile``) inherit the default profile's load-time options. + modes = { + name.split(":")[0]: config.resolve_profile("default").loadtime.get("cuda_graphs", "off") + for name, config in configs.items() + if config.resolve_profile("default").adapter_path.endswith(":GLiClassAdapter") + } + + assert {name for name, mode in modes.items() if mode != "off"} == _BUCKETED_BY_DEFAULT + assert {modes[name] for name in _BUCKETED_BY_DEFAULT} == {"bucketed"} + for name in _BUCKETED_BY_DEFAULT: + adapter = GLiClassAdapter(**_build_adapter_kwargs(configs[name], "float16")) + assert adapter._cuda_graphs == "bucketed" + assert adapter._request_cuda_graphs({"cuda_graphs": "off"}) == "off"