From a571793c2ef57fa65e88a17a7f81e485ebb729ce Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Thu, 24 Sep 2026 13:42:14 +0200 Subject: [PATCH] perf(optimize): resolve each dependent's deadline only once To decide whether a task can be replaced, replace_tasks computes the latest deadline of its dependents. It resolved the deadline of every dependent for every task, although tasks such as docker images or toolchains share thousands of dependents. Now each task's deadline is resolved at most once, relative to a single `now`. Similarly, IndexSearch parsed the same deadline (and the expiration of tasks used as replacement for multiple tasks) over and over, so parsed timestamps are now cached. On a synthetic graph of 20,210 tasks where the 210 build and docker tasks are replaced, replace_tasks takes 0.17s instead of 0.24s. --- src/taskgraph/optimize/base.py | 21 ++++++++++++++------- src/taskgraph/optimize/strategies.py | 12 ++++++++++-- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/taskgraph/optimize/base.py b/src/taskgraph/optimize/base.py index ba07ca276..fc655f998 100644 --- a/src/taskgraph/optimize/base.py +++ b/src/taskgraph/optimize/base.py @@ -301,6 +301,18 @@ def replace_tasks( target_task_graph.graph.links_and_reverse_links_dict() ) + # Many tasks share dependents (e.g. docker images or toolchains), so + # resolve the deadline of each dependent only once. + now = datetime.datetime.now(datetime.timezone.utc) + deadlines = {} + + def get_deadline(label): + if label not in deadlines: + deadlines[label] = resolve_timestamps( + now, target_task_graph.tasks[label].task["deadline"] + ) + return deadlines[label] + for label in target_task_graph.graph.visit_postorder(): logger.debug(f"replace_tasks: {label}") # if we're not allowed to optimize, that's easy.. @@ -331,14 +343,9 @@ def replace_tasks( opt_by, opt, arg = optimizations(label) # compute latest deadline of dependents (if any) - dependents = [target_task_graph.tasks[l] for l in dependents_of[label]] deadline = None - if dependents: - now = datetime.datetime.now(datetime.timezone.utc) - deadline = max( - resolve_timestamps(now, task.task["deadline"]) - for task in dependents # type: ignore - ) + if dependents_of[label]: + deadline = max(get_deadline(l) for l in dependents_of[label]) if isinstance(opt, IndexSearch): arg = arg, index_to_taskid, taskid_to_status diff --git a/src/taskgraph/optimize/strategies.py b/src/taskgraph/optimize/strategies.py index 8fed9e54a..459412c8a 100644 --- a/src/taskgraph/optimize/strategies.py +++ b/src/taskgraph/optimize/strategies.py @@ -1,3 +1,4 @@ +import functools import logging from datetime import datetime @@ -10,6 +11,13 @@ logger = logging.getLogger("optimization") +@functools.cache +def _parse_time(timestamp, fmt): + # Many tasks share the same deadline or replacement task, so avoid parsing + # the same timestamps over and over. + return datetime.strptime(timestamp, fmt) + + @register_strategy("index-search") class IndexSearch(OptimizationStrategy): # A task with no dependencies remaining after optimization will be replaced @@ -56,10 +64,10 @@ def should_replace_task(self, task, params, deadline, arg): ) continue - if deadline and datetime.strptime( + if deadline and _parse_time( status["expires"], # type: ignore self.fmt, - ) < datetime.strptime(deadline, self.fmt): + ) < _parse_time(deadline, self.fmt): logger.debug( f"not replacing {task.label} with {task_id} because it expires before {deadline}" )