diff --git a/.gitignore b/.gitignore index 98a466e4..12ec352b 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,6 @@ bin/gpu_benchmark coverage.xml test/ + +# WfFormat schema (fetched from the WfFormat repository, not versioned here) +wfcommons-schema.json diff --git a/pyproject.toml b/pyproject.toml index ba2ced14..55b53333 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ ] license = {text = "GNU Lesser General Public License v3 (LGPLv3)"} dependencies = [ - "jsonschema", + "jsonschema>=4.18", "matplotlib", "networkx", "numpy", diff --git a/tests/recipes/test_recipes.py b/tests/recipes/test_recipes.py index 0fe05724..fff8c2bf 100644 --- a/tests/recipes/test_recipes.py +++ b/tests/recipes/test_recipes.py @@ -25,6 +25,7 @@ from wfcommons.wfchef.recipes import SrasearchRecipe from wfcommons import WorkflowGenerator +from wfcommons.wfinstances import SchemaValidator class TestRecipes: @@ -53,6 +54,10 @@ def test_recipes(self, recipe_class) -> None: workflows = WorkflowGenerator(recipe).build_workflows(1) assert len(workflows) == 1 + # generated instances (including their metrics) must be valid WfFormat instances + workflows[0].generate_json() + SchemaValidator().validate_instance(workflows[0].workflow_json) + @pytest.mark.unit diff --git a/tests/unit/common/test_task.py b/tests/unit/common/test_task.py index 9e33cac7..27c2cfdb 100644 --- a/tests/unit/common/test_task.py +++ b/tests/unit/common/test_task.py @@ -76,7 +76,7 @@ def test_task_execution(self, task: Task) -> None: "writtenBytes": 54321, "memoryInBytes": 10, "energyInKWh": 100, - "avgPowerInKWh": 1.0, + "avgPowerInW": 1.0, "priority": 100, "machines": ["machine_1"], "executedAt": "2024-09-15T08:59:33.699321-04:00", diff --git a/tests/unit/common/test_workflow.py b/tests/unit/common/test_workflow.py index bef2fdab..cfebfb2a 100644 --- a/tests/unit/common/test_workflow.py +++ b/tests/unit/common/test_workflow.py @@ -17,8 +17,8 @@ from typing import Any from datetime import datetime -from wfcommons.common import Task, Workflow -from wfcommons.wfinstances import Instance +from wfcommons.common import File, Task, Workflow +from wfcommons.wfinstances import Instance, SchemaValidator from wfcommons.version import __version__, __schema_version__ @@ -274,5 +274,98 @@ def test_workflow_dot_file(self): assert(num_dependencies == len(instance.workflow.edges)) +def _build_workflow(tasks: list[tuple[str, str]], dependencies: list[tuple[str, str]], **kwargs) -> Workflow: + """Build a workflow from (name, id) pairs and (parent id, child id) pairs.""" + workflow = Workflow(name="wfformat_test", makespan=10.0, **kwargs) + for name, task_id in tasks: + workflow.add_task(Task(name=name, task_id=task_id, runtime=1.0, + output_files=[File(f"{task_id}_out", 10)])) + for parent, child in dependencies: + workflow.add_dependency(parent, child) + workflow.generate_json() + return workflow +class TestWorkflowWfFormat: + + @pytest.mark.unit + def test_metrics_with_shared_task_names(self) -> None: + # four tasks sharing the same name fan into a fifth one + workflow = _build_workflow( + [("split", f"split_ID000000{i}") for i in range(1, 5)] + [("merge", "merge_ID0000005")], + [(f"split_ID000000{i}", "merge_ID0000005") for i in range(1, 5)]) + metrics = workflow.workflow_json["workflow"]["specification"]["metrics"] + assert metrics["numberOfLevels"] == 2 + assert metrics["minimumWidth"] == 1 + assert metrics["maximumWidth"] == 4 + SchemaValidator().validate_instance(workflow.workflow_json) + + @pytest.mark.unit + def test_metrics_of_a_chain_of_same_name_tasks(self) -> None: + ids = [f"step_ID000000{i}" for i in range(1, 5)] + workflow = _build_workflow([("step", task_id) for task_id in ids], list(zip(ids, ids[1:]))) + metrics = workflow.workflow_json["workflow"]["specification"]["metrics"] + assert (metrics["numberOfLevels"], metrics["minimumWidth"], metrics["maximumWidth"]) == (4, 1, 1) + SchemaValidator().validate_instance(workflow.workflow_json) + + @pytest.mark.unit + @pytest.mark.parametrize( + "task_ids", + [ + ["ID0000001", "ID0000002"], # as in the WfFormat documentation + ["ingest:raw", "proc/a"], # ':' and '/' are allowed from WfFormat 1.6 + ["main.cwl#step-1", "main.cwl#step-2"], + ["stage_in_local_local_0_0", "get_IDs_ID0000003"], + ] + ) + def test_task_ids_allowed_by_wfformat(self, task_ids: list[str]) -> None: + workflow = _build_workflow([("task", task_id) for task_id in task_ids], [(task_ids[0], task_ids[1])]) + SchemaValidator().validate_instance(workflow.workflow_json) + + @pytest.mark.unit + def test_byte_metrics_are_integers(self) -> None: + workflow = Workflow(name="bytes_test") + workflow.add_task(Task(name="a", task_id="a", runtime=0, bytes_read=2, bytes_written=3)) + workflow.add_task(Task(name="b", task_id="b", runtime=0, bytes_read=4, bytes_written=0.5)) + workflow.generate_json() + metrics = workflow.workflow_json["workflow"]["execution"]["metrics"] + # totals do not depend on runtimes, and fractional totals are omitted + assert metrics == {"totalNumBytesRead": 6} + assert isinstance(metrics["totalNumBytesRead"], int) + SchemaValidator().validate_instance(workflow.workflow_json) + + @pytest.mark.unit + def test_execution_fields(self) -> None: + task = Task(name="a", task_id="a", runtime=1.0, cores=0, args=["-x", "", 3], + executed_at="2025-03-01 10:00:01.000000") + execution = task.execution_as_dict() + assert "coreCount" not in execution + assert execution["command"] == {"arguments": ["-x", "3"]} + assert datetime.fromisoformat(execution["executedAt"]).tzinfo is not None + assert "command" not in Task(name="b", task_id="b", runtime=1.0).execution_as_dict() + + @pytest.mark.unit + def test_specification_only_workflow(self) -> None: + workflow = _build_workflow([("a", "a"), ("b", "b")], [("a", "b")], include_execution=False) + assert "execution" not in workflow.workflow_json["workflow"] + SchemaValidator().validate_instance(workflow.workflow_json) + + @pytest.mark.unit + def test_instance_properties(self) -> None: + workflow = _build_workflow([("a", "a")], [], author_name="Jane Doe", author_email="jane@example.org", + author_institution="Example University", author_country="US", + created_at="2026-08-04T15:19:28-08:00", executed_at="2026-08-04T15:20:00-08:00") + assert workflow.workflow_json["author"] == {"name": "Jane Doe", "email": "jane@example.org", + "institution": "Example University", "country": "US"} + assert workflow.workflow_json["createdAt"] == "2026-08-04T15:19:28-08:00" + assert workflow.workflow_json["workflow"]["execution"]["executedAt"] == "2026-08-04T15:20:00-08:00" + + @pytest.mark.unit + def test_duplicate_file_ids(self) -> None: + workflow = Workflow(name="files_test") + workflow.add_task(Task(name="a", task_id="a", runtime=1.0, output_files=[File("f.txt", 10)])) + workflow.add_task(Task(name="b", task_id="b", runtime=1.0, input_files=[File("f.txt", 20)])) + workflow.add_dependency("a", "b") + workflow.generate_json() + assert workflow.workflow_json["workflow"]["specification"]["files"] == [{"id": "f.txt", "sizeInBytes": 10}] + SchemaValidator().validate_instance(workflow.workflow_json) diff --git a/tests/unit/wfinstances/test_instance_analyzer.py b/tests/unit/wfinstances/test_instance_analyzer.py index 782a0edf..49779ace 100644 --- a/tests/unit/wfinstances/test_instance_analyzer.py +++ b/tests/unit/wfinstances/test_instance_analyzer.py @@ -34,7 +34,7 @@ def instance(self) -> pathlib.Path: { "name": "task_01", "id": "task_01", - "children": ["task_02", "task_03"], + "children": [], "inputFiles": ["input_01.txt", "input_02.txt"], "outputFiles": ["output_01.txt"], "parents": [] diff --git a/tests/unit/wfinstances/test_instances.py b/tests/unit/wfinstances/test_instances.py index ed9cfc61..d24e0e7a 100644 --- a/tests/unit/wfinstances/test_instances.py +++ b/tests/unit/wfinstances/test_instances.py @@ -12,7 +12,9 @@ import pathlib import pytest -from wfcommons.wfinstances import Instance +from wfcommons.version import __schema_version__ +from wfcommons.wfinstances import Instance, SchemaValidator +from wfcommons.wfinstances import schema as schema_module class TestInstance: @@ -34,7 +36,7 @@ def instance(self) -> pathlib.Path: { "name": "task_01", "id": "task_01", - "children": ["task_02", "task_03"], + "children": [], "inputFiles": ["input_01.txt", "input_02.txt"], "outputFiles": ["output_01.txt"], "parents": [] @@ -79,3 +81,158 @@ def instance(self) -> pathlib.Path: def test_instance_creation(self, instance : pathlib.Path) -> None: instance = Instance(instance) assert(instance.leaves() == instance.roots()) + + +def _wfformat_example() -> dict: + """The example instance of the WfFormat 1.6 documentation.""" + return { + "name": "my-workflow-instance", + "description": "A two-task workflow executed on a single node", + "createdAt": "2026-08-04T15:19:28-08:00", + "schemaVersion": "1.6", + "author": {"name": "Jane Doe", "email": "jane.doe@example.org", + "institution": "Example University", "country": "US"}, + "runtimeSystem": {"name": "pegasus", "version": "5.0.6", "url": "https://pegasus.isi.edu"}, + "workflow": { + "specification": { + "tasks": [ + {"name": "split", "id": "ID0000001", "parents": [], "children": ["ID0000002"], + "inputFiles": ["input.txt"], "outputFiles": ["chunk.txt"]}, + {"name": "analyze", "id": "ID0000002", "parents": ["ID0000001"], "children": [], + "inputFiles": ["chunk.txt"], "outputFiles": ["result.txt"]} + ], + "files": [ + {"id": "input.txt", "sizeInBytes": 1024}, {"id": "chunk.txt", "sizeInBytes": 512}, + {"id": "result.txt", "sizeInBytes": 128} + ], + "metrics": {"numberOfTasks": 2, "numberOfFiles": 3, "sumOfFileSizesInBytes": 1664, + "numberOfLevels": 2, "minimumWidth": 1, "maximumWidth": 1} + }, + "execution": { + "makespanInSeconds": 14.2, + "executedAt": "2026-08-04T15:20:00-08:00", + "tasks": [ + {"id": "ID0000001", "runtimeInSeconds": 4.2, "executedAt": "2026-08-04T15:20:00-08:00", + "command": {"program": "split", "arguments": ["-n", "2", "input.txt"]}, + "coreCount": 1, "avgCPU": 93.78, "readBytes": 1024, "writtenBytes": 512, + "memoryInBytes": 4194304, "energyInKWh": 0.25, "avgPowerInW": 120.0, + "machines": ["node01.example.org"]}, + {"id": "ID0000002", "runtimeInSeconds": 10.0, "executedAt": "2026-08-04T15:20:04-08:00", + "coreCount": 4, "readBytes": 512, "writtenBytes": 128, "machines": ["node01.example.org"]} + ], + "machines": [ + {"system": "linux", "architecture": "x86_64", "nodeName": "node01.example.org", + "release": "5.15.0", "memoryInBytes": 34359738368, + "cpu": {"coreCount": 8, "speedInMHz": 3600, "vendor": "AMD"}} + ], + "metrics": {"sumTaskRuntimesInSeconds": 14.2, "totalNumBytesRead": 1536, + "totalNumBytesWritten": 640} + } + } + } + + +def _load(instance: dict, tmp_path: pathlib.Path) -> Instance: + instance_path = tmp_path / "instance.json" + instance_path.write_text(json.dumps(instance)) + return Instance(instance_path) + + +class TestInstanceWfFormat: + + @pytest.mark.unit + def test_round_trip(self, tmp_path: pathlib.Path) -> None: + original = _wfformat_example() + output_path = tmp_path / "written.json" + _load(original, tmp_path).workflow.write_json(output_path) + written = json.loads(output_path.read_text()) + assert written == original + SchemaValidator().validate_instance(written) + + @pytest.mark.unit + def test_specification_only(self, tmp_path: pathlib.Path) -> None: + original = _wfformat_example() + del original["workflow"]["execution"] + instance = _load(original, tmp_path) + assert len(instance.workflow.tasks) == 2 + instance.workflow.generate_json() + assert "execution" not in instance.workflow.workflow_json["workflow"] + SchemaValidator().validate_instance(instance.workflow.workflow_json) + + @pytest.mark.unit + def test_minimal_instance(self, tmp_path: pathlib.Path) -> None: + # only name, schemaVersion, and workflow are required, and task IDs may hold ':' and '/' + minimal = { + "name": "minimal", + "schemaVersion": "1.6", + "workflow": { + "specification": {"tasks": [ + {"name": "t", "id": "t:1", "parents": [], "children": ["t/2"]}, + {"name": "t", "id": "t/2", "parents": ["t:1"], "children": []} + ]}, + "execution": {"makespanInSeconds": 1.0, "executedAt": "2026-08-04T15:20:00-08:00", + "tasks": [{"id": "t:1", "runtimeInSeconds": 0.5}]} + } + } + instance = _load(minimal, tmp_path) + assert instance.roots() == ["t:1"] and instance.leaves() == ["t/2"] + instance.workflow.generate_json() + written = instance.workflow.workflow_json + SchemaValidator().validate_instance(written) + # optional properties that the instance does not declare are not made up + assert not {"description", "createdAt", "author", "runtimeSystem"} & written.keys() + assert "executedAt" not in written["workflow"]["execution"]["tasks"][1] + + @pytest.mark.unit + def test_unused_machine(self, tmp_path: pathlib.Path) -> None: + original = _wfformat_example() + original["workflow"]["execution"]["machines"].append({"nodeName": "node02.example.org"}) + instance = _load(original, tmp_path) + instance.workflow.generate_json() + machines = instance.workflow.workflow_json["workflow"]["execution"]["machines"] + assert machines == original["workflow"]["execution"]["machines"] + + @pytest.mark.unit + def test_machine_without_cpu(self, tmp_path: pathlib.Path) -> None: + original = _wfformat_example() + del original["workflow"]["execution"]["machines"][0]["cpu"] + instance = _load(original, tmp_path) + assert instance.machines["node01.example.org"].cpu_cores is None + + @pytest.mark.unit + def test_integral_float_file_size(self, tmp_path: pathlib.Path) -> None: + # JSON Schema draft 2020-12 considers 1024.0 to be an integer + original = _wfformat_example() + original["workflow"]["specification"]["files"][0]["sizeInBytes"] = 1024.0 + _load(original, tmp_path) + + @pytest.mark.unit + def test_invalid_metrics(self, tmp_path: pathlib.Path) -> None: + original = _wfformat_example() + original["workflow"]["specification"]["metrics"]["maximumWidth"] = 2 + with pytest.raises(RuntimeError, match="semantic"): + _load(original, tmp_path) + + @pytest.mark.unit + def test_previous_schema_version(self, tmp_path: pathlib.Path) -> None: + original = _wfformat_example() + original["schemaVersion"] = "1.5" + with pytest.raises(RuntimeError, match="syntax"): + _load(original, tmp_path) + + @pytest.mark.unit + def test_schema_of_wfformat_release(self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: + # a schema file in the working directory (e.g., left by an older WfCommons) is ignored + (tmp_path / "wfcommons-schema.json").write_text(json.dumps({"type": "object", "required": ["bogus"]})) + monkeypatch.chdir(tmp_path) + # a cached schema that does not accept the current schema version is replaced + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + monkeypatch.setattr(schema_module, "_schema_cache", {}) + cache_file = tmp_path / "cache" / "wfcommons" / f"wfformat-v{__schema_version__}-wfcommons-schema.json" + cache_file.parent.mkdir(parents=True) + cache_file.write_text(json.dumps({"properties": {"schemaVersion": {"enum": ["1.5"]}}})) + + validator = SchemaValidator() + assert validator.schema["properties"]["schemaVersion"]["enum"] == [__schema_version__] + assert json.loads(cache_file.read_text()) == validator.schema + validator.validate_instance(_wfformat_example()) diff --git a/tests/unit/wfinstances/test_logs_common.py b/tests/unit/wfinstances/test_logs_common.py new file mode 100644 index 00000000..1752cc67 --- /dev/null +++ b/tests/unit/wfinstances/test_logs_common.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2025 The WfCommons Team. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +import logging +import pathlib +import sqlite3 + +import pytest + +from wfcommons.utils import WFFORMAT_ID_PATTERN +from wfcommons.wfinstances.logs import MakeflowLogsParser, NextflowLogsParser, PegasusLogsParser, \ + ROCrateLogsParser, SnakemakeLogsParser, StreamflowLogsParser, TaskVineLogsParser +from wfcommons.wfinstances.logs.abstract_logs_parser import LogsParser + + +class _DummyLogsParser(LogsParser): + def build_workflow(self, workflow_name=None): + return None + + +class TestLogsParsersCommon: + + @pytest.mark.unit + def test_exports(self) -> None: + import wfcommons.wfinstances as wfinstances + for name in ["MakeflowLogsParser", "NextflowLogsParser", "PegasusLogsParser", "ROCrateLogsParser", + "SnakemakeLogsParser", "StreamflowLogsParser", "TaskVineLogsParser"]: + assert hasattr(wfinstances, name) + + @pytest.mark.unit + def test_make_file_id(self) -> None: + parser = _DummyLogsParser("dummy") + + # invalid characters are replaced, valid IDs are kept + assert parser._make_file_id("data/f.txt") == "data/f.txt" + assert parser._make_file_id("sample A=1+x.fa") == "sample_A_1_x.fa" + assert parser._make_file_id("https://example.org/get?id=1&x=%20") == "https://example.org/get_id_1_x__20" + assert parser._make_file_id("data/d.txt->d_remote.txt") == "data/d.txt-_d_remote.txt" + for file_id in parser._file_keys: + assert WFFORMAT_ID_PATTERN.match(file_id) + + # the same file always gets the same ID + assert parser._make_file_id("sample A=1+x.fa") == "sample_A_1_x.fa" + + # distinct files whose names collide get distinct IDs + assert parser._make_file_id("out.txt", key="#f-d") == "out.txt" + assert parser._make_file_id("out.txt", key="#f-e") == "out.txt_2" + assert parser._make_file_id("out.txt", key="#f-d") == "out.txt" + assert parser._make_file_id("sample A+1+x.fa") == "sample_A_1_x.fa_2" + + @pytest.mark.unit + def test_constructors_keyword_wiring(self, tmp_path: pathlib.Path) -> None: + logger = logging.getLogger("test_logs_common") + + # Pegasus + (tmp_path / "pegasus").mkdir() + parser = PegasusLogsParser(tmp_path / "pegasus", description="pegasus run", logger=logger) + assert (parser.wms_name, parser.wms_version, parser.wms_url) == ("Pegasus", None, "https://pegasus.isi.edu") + assert parser.description == "pegasus run" and parser.logger is logger + + # Makeflow + mf = tmp_path / "makeflow" + (mf / "monitor").mkdir(parents=True) + (mf / "wf.makeflow").write_text("b.txt : a.txt\n\tcat a.txt > b.txt\n") + (mf / "wf.makeflow.makeflowlog").write_text("# NODE\t0\tcat a.txt > b.txt\n") + parser = MakeflowLogsParser(mf, mf / "monitor", description="makeflow run", logger=logger) + assert (parser.wms_name, parser.wms_version) == ("Makeflow", None) + assert parser.wms_url.startswith("https://ccl.cse.nd.edu/") + assert parser.description == "makeflow run" and parser.logger is logger + + # Snakemake + (tmp_path / "snakemake").mkdir() + sqlite3.connect(tmp_path / "snakemake" / "snkmt.sqlite").close() + parser = SnakemakeLogsParser(tmp_path / "snakemake", tmp_path / "snakemake" / "snkmt.sqlite", + description="snakemake run", logger=logger, snakemake_version="9.1.0") + assert (parser.wms_name, parser.wms_version) == ("Snakemake", "9.1.0") + assert parser.wms_url == "https://snakemake.readthedocs.io/" + assert parser.description == "snakemake run" and parser.logger is logger + + # TaskVine + tv = tmp_path / "vine-logs" + tv.mkdir() + for name in ["debug", "taskgraph", "transactions"]: + (tv / name).write_text("") + parser = TaskVineLogsParser(tv, ["wfbench"], "taskvine run", logger, taskvine_version="7.15.0") + assert (parser.wms_name, parser.wms_version) == ("TaskVine", "7.15.0") + assert parser.wms_url == "https://ccl.cse.nd.edu/software/taskvine/" + assert parser.description == "taskvine run" and parser.logger is logger + + # Nextflow, Streamflow, and RO-Crate + nf = tmp_path / "nextflow" + nf.mkdir() + (nf / "execution_trace.txt").write_text("") + parser = NextflowLogsParser(nf, "24.10.4", description="nextflow run", logger=logger) + assert (parser.wms_name, parser.wms_version, parser.wms_url) == ("Nextflow", "24.10.4", "https://www.nextflow.io") + assert parser.description == "nextflow run" and parser.logger is logger + parser = StreamflowLogsParser(nf, "0.2.0", description="streamflow run", logger=logger) + assert (parser.wms_name, parser.wms_version) == ("Streamflow", "0.2.0") + assert parser.description == "streamflow run" and parser.logger is logger + (nf / "ro-crate-metadata.json").write_text("{}") + parser = ROCrateLogsParser(nf, "Other", "1.0", "https://example.org", "crate run", logger) + assert (parser.wms_name, parser.wms_version, parser.wms_url) == ("Other", "1.0", "https://example.org") + assert parser.description == "crate run" and parser.logger is logger diff --git a/tests/unit/wfinstances/test_logs_makeflow.py b/tests/unit/wfinstances/test_logs_makeflow.py new file mode 100644 index 00000000..b0a9b615 --- /dev/null +++ b/tests/unit/wfinstances/test_logs_makeflow.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2025 The WfCommons Team. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +import json +import pathlib + +import pytest + +from wfcommons.wfinstances.logs import MakeflowLogsParser + + +def _write_makeflow_run(run_dir: pathlib.Path) -> None: + """Write a minimal Makeflow execution directory with resource monitor summaries.""" + (run_dir / "data").mkdir() + (run_dir / "monitor").mkdir() + (run_dir / "data/a.txt").write_bytes(b"x" * 100) + (run_dir / "workflow.makeflow").write_text( + "# Makeflow workflow specification\n" + "URL=http://example.org/data\n" + "OUT=data/b.txt\n" + "\n" + "$(OUT) : data/a.txt\n" + "\tg++ -o $(OUT) data/a.txt\n" + "\n" + "data/c.txt : ${OUT}->b_remote.txt\n" + "\twfbench --out data/c.txt\n") + commands = ["g++ -o data/b.txt data/a.txt", "wfbench --out data/c.txt"] + (run_dir / "workflow.makeflow.makeflowlog").write_text( + "# STARTED 1700000000000000\n" + + "".join(f"# NODE\t{k}\t{c}\n" for k, c in enumerate(commands)) + + "# FILE 1700000001000500 data/b.txt 1 2048\n" + "# FILE 1700000002000500 data/c.txt 1 4096\n" + "# COMPLETED 1700000007123456\n") + for k, command in enumerate(commands): + summary = {"command": command, "start": [1700000001000000 + k * 1000000, "us"], + "wall_time": [2.0, "s"] if k == 0 else [0, "s"], "cpu_time": [1.0, "s"], + "cores": [0, "cores"] if k == 1 else [2, "cores"], "memory": [12, "MB"], + "bytes_read": [0.75, "MB"], "bytes_written": [2.5, "MB"], "machine_cpus": [8, "cores"], + "host": "worker0", "monitor_version": "7.8.2 FINAL"} + (run_dir / f"monitor/resource-rule-{k}.summary").write_text(json.dumps(summary)) + + +class TestMakeflowLogsParser: + + @pytest.mark.unit + def test_build_workflow(self, tmp_path: pathlib.Path) -> None: + _write_makeflow_run(tmp_path) + workflow = MakeflowLogsParser(tmp_path, tmp_path / "monitor").build_workflow("makeflow-test") + workflow.generate_json() + wf_json = workflow.workflow_json + + # the version comes from the resource monitor (makeflow and resource monitor are part of cctools) + assert wf_json["runtimeSystem"]["version"] == "7.8.2" + + # task IDs are sanitized, variables are expanded, and renamed files use their workflow-level name + spec = {t["id"]: t for t in wf_json["workflow"]["specification"]["tasks"]} + assert list(spec) == ["g___ID0000001", "wfbench_ID0000002"] + assert spec["g___ID0000001"]["outputFiles"] == ["data/b.txt"] + assert spec["wfbench_ID0000002"]["inputFiles"] == ["data/b.txt"] + assert spec["wfbench_ID0000002"]["parents"] == ["g___ID0000001"] + files = {f["id"]: f["sizeInBytes"] for f in wf_json["workflow"]["specification"]["files"]} + assert files == {"data/a.txt": 100, "data/b.txt": 2048, "data/c.txt": 4096} + + # resource monitor values are converted from MB into bytes + tasks = {t["id"]: t for t in wf_json["workflow"]["execution"]["tasks"]} + assert tasks["g___ID0000001"]["memoryInBytes"] == 12 * 1024 ** 2 + assert tasks["g___ID0000001"]["readBytes"] == 786432 + assert tasks["g___ID0000001"]["writtenBytes"] == int(2.5 * 1024 ** 2) + assert tasks["g___ID0000001"]["coreCount"] == 2 + assert tasks["g___ID0000001"]["avgCPU"] == 50.0 + assert tasks["g___ID0000001"]["executedAt"] == "2023-11-14T22:13:21+00:00" + # a task runs on at least one core, and a zero runtime does not break the CPU utilization + assert tasks["wfbench_ID0000002"]["coreCount"] == 1 + assert "avgCPU" not in tasks["wfbench_ID0000002"] + + # machines are emitted (once) + assert tasks["g___ID0000001"]["machines"] == ["worker0"] + assert wf_json["workflow"]["execution"]["machines"] == [{"nodeName": "worker0", "cpu": {"coreCount": 8}}] diff --git a/tests/unit/wfinstances/test_logs_nextflow.py b/tests/unit/wfinstances/test_logs_nextflow.py new file mode 100644 index 00000000..b5146b7e --- /dev/null +++ b/tests/unit/wfinstances/test_logs_nextflow.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2025 The WfCommons Team. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +import json +import pathlib + +import pytest + +from wfcommons.wfinstances.logs import NextflowLogsParser + + +def _write_nextflow_run(run_dir: pathlib.Path) -> None: + """Write a minimal Nextflow execution directory (nf-prov RO-Crate and execution trace).""" + graph = [ + {"@id": "./", "@type": "Dataset", "mainEntity": {"@id": "main.nf"}}, + {"@id": "main.nf", "@type": ["File", "ComputationalWorkflow"], "name": "main.nf"}, + {"@id": "#run", "@type": "CreateAction", "name": "Nextflow workflow run 1234", + "startTime": "2025-03-01T10:00:00.000+01:00", "endTime": "2025-03-01T10:10:00.000+01:00", + "object": [], "result": []}, + {"@id": "data/in.fa", "@type": "File", "contentSize": "10"}, + {"@id": "#task/aa/out.txt", "@type": "File", "contentSize": "20"}, + {"@id": "#aa", "@type": "CreateAction", "name": "NFCORE:MAP_IDS (1)", + "instrument": {"@id": "#process-NFCORE:MAP_IDS"}, + "object": [{"@id": "data/in.fa"}], "result": [{"@id": "#task/aa/out.txt"}]}, + {"@id": "#bb", "@type": "CreateAction", "name": "NFCORE:CAT (2)", + "instrument": {"@id": "#process-NFCORE:CAT"}, + "object": {"@id": "#task/aa/out.txt"}, "result": []}, + ] + (run_dir / "ro-crate-metadata.json").write_text(json.dumps({"@graph": graph})) + (run_dir / "execution_trace.txt").write_text( + "task_id\tname\tsubmit\tduration\n" + "1\tNFCORE:MAP_IDS (1)\t2025-03-01 10:00:01.100\t519ms\n" + "2\tNFCORE:CAT (2)\t2025-03-01 10:00:02.000\t1d 2h 3m 4s\n") + + +class TestNextflowLogsParser: + + @pytest.mark.unit + @pytest.mark.parametrize("duration, seconds", [ + ("519ms", 0.519), + ("2m 40s", 160.0), + ("1h 3m 2s", 3782.0), + ("1d 2h 3m 4s", 93784.0), + ("4.5s", 4.5), + ("1m 1ms", 60.001), + ("-", 0.0), + ]) + def test_parse_duration(self, duration: str, seconds: float) -> None: + assert NextflowLogsParser._parse_duration(duration) == pytest.approx(seconds) + + @pytest.mark.unit + def test_build_workflow(self, tmp_path: pathlib.Path) -> None: + _write_nextflow_run(tmp_path) + workflow = NextflowLogsParser(tmp_path, nextflow_version="24.10.4").build_workflow("nextflow-test") + workflow.generate_json() + wf_json = workflow.workflow_json + + assert wf_json["runtimeSystem"]["version"] == "24.10.4" + tasks = {t["id"]: t for t in wf_json["workflow"]["execution"]["tasks"]} + # a process name containing "_ID" is fine + assert set(tasks) == {"NFCORE:MAP_IDS__1__ID0000001", "NFCORE:CAT__2__ID0000002"} + assert tasks["NFCORE:MAP_IDS__1__ID0000001"]["runtimeInSeconds"] == pytest.approx(0.519) + assert tasks["NFCORE:CAT__2__ID0000002"]["runtimeInSeconds"] == pytest.approx(93784.0) + # the trace timestamps are in the timezone of the workflow run + assert tasks["NFCORE:MAP_IDS__1__ID0000001"]["executedAt"] == "2025-03-01T10:00:01.100000+01:00" + assert wf_json["workflow"]["execution"]["executedAt"] == "2025-03-01T10:00:00+01:00" + spec = {t["id"]: t for t in wf_json["workflow"]["specification"]["tasks"]} + assert spec["NFCORE:CAT__2__ID0000002"]["parents"] == ["NFCORE:MAP_IDS__1__ID0000001"] diff --git a/tests/unit/wfinstances/test_logs_pegasus.py b/tests/unit/wfinstances/test_logs_pegasus.py new file mode 100644 index 00000000..37695524 --- /dev/null +++ b/tests/unit/wfinstances/test_logs_pegasus.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2025 The WfCommons Team. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +import json +import pathlib + +import pytest + +from wfcommons.wfinstances.logs import PegasusLogsParser + +_KICKSTART = """\ +- invocation: True + version: 3.0 + start: 2023-11-05T14:4{k}:30.123-08:00 + duration: 60.5 + transformation: "diamond::{name}:4.0" + mainjob: + start: 2023-11-05T14:4{k}:30.200-08:00 + duration: {duration} + usage: + utime: 59.5 + stime: 0.5 + maxrss: 2048 + procs: + - rchar: 1024 + wchar: 2048 + rbytes: 0 + wbytes: 4096 + machine: + uname_system: {system} + uname_nodename: {node}.example.org + uname_release: 5.15.0 + uname_machine: x86_64 + ram_total: 16384000 + cpu_count: 8 + cpu_speed: 2400 + cpu_vendor: GenuineIntel +""" + + +def _write_submit_dir(submit_dir: pathlib.Path) -> None: + """Write a minimal Pegasus 5 submit directory with two compute jobs and a stage-in job.""" + (submit_dir / "00/00").mkdir(parents=True) + (submit_dir / "braindump.yml").write_text( + "pegasus_wf_name: diamond-0\ntimestamp: 20231105T144500-0800\nplanner_version: 5.0.4\n") + (submit_dir / "workflow.yml").write_text("""\ +pegasus: '5.0' +name: diamond +x-pegasus: {apiLang: python, createdBy: user, createdOn: 11-05-23T14:44:58Z} +jobs: +- type: job + name: preprocess + id: ID0000001 + arguments: [-a, preprocess, -T, 60, '', -i, sample A=1+x.fa, -o, f.b] + uses: + - {lfn: sample A=1+x.fa, type: input} + - {lfn: f.b, type: output, stageOut: true} +- type: job + name: analyze + id: ID0000002 + profiles: + pegasus: {cores: '4'} + arguments: [-a, analyze, -i, f.b] + uses: + - {lfn: f.b, type: input} +""") + jobs = ["stage_in_local_local_0_0", "preprocess_ID0000001", "analyze_ID0000002"] + (submit_dir / "diamond-0.dag").write_text( + "".join(f"JOB {j} 00/00/{j}.sub\n" for j in jobs) + + "PARENT stage_in_local_local_0_0 CHILD preprocess_ID0000001\n" + "PARENT preprocess_ID0000001 CHILD analyze_ID0000002\n") + (submit_dir / "diamond-0.dag.dagman.out").write_text( + "11/05/23 14:45:01 ** condor_scheduniv_exec.1.0 (CONDOR_DAGMAN) STARTING UP\n" + "11/05/23 14:52:33 **** condor_scheduniv_exec.1.0 (condor_DAGMAN) EXITING WITH STATUS 0\n") + (submit_dir / "00/00/preprocess_ID0000001.meta").write_text(json.dumps( + [{"_id": "sample A=1+x.fa", "_attributes": {"size": "1024"}}, {"_id": "f.b", "_attributes": {"size": "2048"}}])) + (submit_dir / "00/00/preprocess_ID0000001.out.000").write_text( + _KICKSTART.format(k=6, name="preprocess", duration=60.0, system="linux", node="worker0")) + (submit_dir / "00/00/analyze_ID0000002.out.000").write_text( + _KICKSTART.format(k=7, name="analyze", duration=0.0, system="darwin", node="worker1")) + + +class TestPegasusLogsParser: + + @pytest.mark.unit + def test_build_workflow(self, tmp_path: pathlib.Path, monkeypatch) -> None: + _write_submit_dir(tmp_path / "submit") + monkeypatch.chdir(tmp_path) + workflow = PegasusLogsParser(tmp_path / "submit").build_workflow() + workflow.generate_json() + wf_json = workflow.workflow_json + + # workflow name, planner version, and DAGMan start time + assert wf_json["name"] == "diamond-0" + assert wf_json["runtimeSystem"]["version"] == "5.0.4" + assert wf_json["runtimeSystem"]["url"] == "https://pegasus.isi.edu" + execution = wf_json["workflow"]["execution"] + assert execution["executedAt"] == "2023-11-05T14:45:01-08:00" + assert execution["makespanInSeconds"] == 452.0 + + # file IDs are sanitized, and their sizes come from the meta files + files = {f["id"]: f["sizeInBytes"] for f in wf_json["workflow"]["specification"]["files"]} + assert files == {"sample_A_1_x.fa": 1024, "f.b": 2048} + + tasks = {t["id"]: t for t in execution["tasks"]} + assert set(tasks) == {"preprocess_ID0000001", "analyze_ID0000002"} + preprocess = tasks["preprocess_ID0000001"] + # at least one core, or the 'cores' profile + assert preprocess["coreCount"] == 1 + assert tasks["analyze_ID0000002"]["coreCount"] == 4 + # arguments are non-empty strings + assert preprocess["command"]["arguments"] == ["-a", "preprocess", "-T", "60", "-i", "sample A=1+x.fa", + "-o", "f.b"] + # kickstart start time and maxrss (in KB) + assert preprocess["executedAt"] == "2023-11-05T14:46:30.200000-08:00" + assert preprocess["memoryInBytes"] == 2048 * 1024 + assert preprocess["avgCPU"] == 100.0 + # a zero duration does not break the CPU utilization + assert tasks["analyze_ID0000002"]["runtimeInSeconds"] == 0.0 + assert "avgCPU" not in tasks["analyze_ID0000002"] + + # machines are emitted, and the total RAM (in KB) is converted into bytes + assert preprocess["machines"] == ["worker0.example.org"] + machines = {m["nodeName"]: m for m in execution["machines"]} + assert set(machines) == {"worker0.example.org", "worker1.example.org"} + assert machines["worker0.example.org"]["memoryInBytes"] == 16384000 * 1024 + assert machines["worker0.example.org"]["system"] == "linux" + assert machines["worker1.example.org"]["system"] == "macos" + + @pytest.mark.unit + def test_auxiliary_jobs(self, tmp_path: pathlib.Path, monkeypatch) -> None: + _write_submit_dir(tmp_path / "submit") + monkeypatch.chdir(tmp_path) + workflow = PegasusLogsParser(tmp_path / "submit", ignore_auxiliary=False).build_workflow("pegasus-test") + workflow.generate_json() + spec = {t["id"]: t for t in workflow.workflow_json["workflow"]["specification"]["tasks"]} + assert spec["preprocess_ID0000001"]["parents"] == ["stage_in_local_local_0_0"] + tasks = {t["id"]: t for t in workflow.workflow_json["workflow"]["execution"]["tasks"]} + assert tasks["stage_in_local_local_0_0"]["coreCount"] == 1 + + @pytest.mark.unit + def test_invalid_kickstart_record(self, tmp_path: pathlib.Path, monkeypatch) -> None: + _write_submit_dir(tmp_path / "submit") + (tmp_path / "submit/00/00/preprocess_ID0000001.out.000").write_text("- invocation: True\n mainjob: [x\n") + monkeypatch.chdir(tmp_path) + with pytest.raises(ValueError): + PegasusLogsParser(tmp_path / "submit").build_workflow("pegasus-test") + # the temporary file is removed + assert not list(tmp_path.glob(".pegasus-parser-tmp-*")) diff --git a/tests/unit/wfinstances/test_logs_ro_crate.py b/tests/unit/wfinstances/test_logs_ro_crate.py new file mode 100644 index 00000000..74df2281 --- /dev/null +++ b/tests/unit/wfinstances/test_logs_ro_crate.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2025 The WfCommons Team. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +import json +import pathlib + +import networkx as nx +import pytest + +from wfcommons.utils import WFFORMAT_ID_PATTERN +from wfcommons.wfinstances.logs import ROCrateLogsParser, StreamflowLogsParser + + +def _write_crate(crate_dir: pathlib.Path, instrument: str = "shell.cwl") -> None: + """Write a minimal StreamFlow-like RO-Crate: split -> (blast, blast) -> cat.""" + def step(k, name, inputs, outputs, start, end): + return {"@id": f"#step-{k}", "@type": "CreateAction", "name": f"Run of workflow/main.cwl#{name}", + "instrument": {"@id": instrument}, "object": inputs, "result": outputs, + "startTime": f"2025-03-01T10:{start}+00:00", "endTime": f"2025-03-01T10:{end}+00:00"} + + graph = [ + {"@id": "./", "@type": "Dataset", "mainEntity": {"@id": "main.cwl"}}, + {"@id": "main.cwl", "@type": ["File", "ComputationalWorkflow"], "name": "main.cwl"}, + {"@id": "#main", "@type": "CreateAction", "name": "Run of workflow/main.cwl", + "startTime": "2025-03-01T10:00:00+00:00", "endTime": "2025-03-01T10:05:00+00:00", + "object": [], "result": []}, + # a single (non-list) input object + step(0, "split (chr1, sample+1)", {"@id": "#f-a"}, [{"@id": "#f-b"}, {"@id": "#f-c"}], "00:01", "01:01"), + step(1, "blast", [{"@id": "#f-b"}], [{"@id": "#f-d"}], "01:02", "02:02"), + step(2, "blast", [{"@id": "#f-c"}], [{"@id": "#f-e"}], "01:02", "02:10"), + step(3, "cat", [{"@id": "#f-d"}, {"@id": "#f-e"}], [], "02:11", "03:20"), + {"@id": "#f-a", "@type": "File", "contentSize": "1000", "alternateName": "my input.txt"}, + {"@id": "#f-b", "@type": "File", "contentSize": "2000", "alternateName": "b.txt"}, + {"@id": "#f-c", "@type": "File", "contentSize": "3000", "alternateName": "c.txt"}, + # two distinct files with the same name + {"@id": "#f-d", "@type": "File", "contentSize": "4000", "alternateName": "out.txt"}, + {"@id": "#f-e", "@type": "File", "contentSize": "5000", "alternateName": "out.txt"}, + # a connection between the parameters of the same instrument + {"@id": "#conn", "@type": "ParameterConnection", "sourceParameter": {"@id": f"{instrument}#output"}, + "targetParameter": {"@id": f"{instrument}#input"}}, + ] + (crate_dir / "ro-crate-metadata.json").write_text(json.dumps({"@graph": graph})) + + +class TestROCrateLogsParser: + + @pytest.mark.unit + def test_build_workflow(self, tmp_path: pathlib.Path) -> None: + _write_crate(tmp_path) + workflow = StreamflowLogsParser(tmp_path, streamflow_version="0.2.0").build_workflow("crate-test") + workflow.generate_json() + wf_json = workflow.workflow_json + spec = {t["id"]: t for t in wf_json["workflow"]["specification"]["tasks"]} + + # the version is passed through + assert wf_json["runtimeSystem"]["version"] == "0.2.0" + + # task IDs are sanitized, and numbered with a counter incremented by one + assert list(spec) == ["main.cwl#split__chr1__sample_1__ID0000001", "main.cwl#blast_ID0000002", + "main.cwl#blast_ID0000003", "main.cwl#cat_ID0000004"] + + # distinct files with the same name have distinct (sanitized) IDs + files = {f["id"]: f["sizeInBytes"] for f in wf_json["workflow"]["specification"]["files"]} + assert files == {"my_input.txt": 1000, "b.txt": 2000, "c.txt": 3000, "out.txt": 4000, "out.txt_2": 5000} + assert spec["main.cwl#cat_ID0000004"]["inputFiles"] == ["out.txt", "out.txt_2"] + for identifier in list(spec) + list(files): + assert WFFORMAT_ID_PATTERN.match(identifier) + + # a single input object is read as a list + assert spec["main.cwl#split__chr1__sample_1__ID0000001"]["inputFiles"] == ["my_input.txt"] + + # the connection of an instrument to itself creates neither self-loops nor cycles + graph = nx.DiGraph() + for task_id, task in spec.items(): + assert task_id not in task["parents"] and task_id not in task["children"] + graph.add_edges_from((parent, task_id) for parent in task["parents"]) + assert nx.is_directed_acyclic_graph(graph) + assert sorted(spec["main.cwl#cat_ID0000004"]["parents"]) == ["main.cwl#blast_ID0000002", + "main.cwl#blast_ID0000003"] + assert spec["main.cwl#blast_ID0000002"]["parents"] == ["main.cwl#split__chr1__sample_1__ID0000001"] + + # task start times come from the crate + tasks = {t["id"]: t for t in wf_json["workflow"]["execution"]["tasks"]} + assert tasks["main.cwl#blast_ID0000003"]["executedAt"] == "2025-03-01T10:01:02+00:00" + assert tasks["main.cwl#blast_ID0000003"]["runtimeInSeconds"] == 68.0 + + @pytest.mark.unit + def test_instruments_to_ignore(self, tmp_path: pathlib.Path) -> None: + # instruments to ignore are matched by their exact name or by their base name + _write_crate(tmp_path, instrument="file:///wf/bin/shell.cwl") + for instruments_to_ignore in [["shell.cwl"], ["file:///wf/bin/shell.cwl"]]: + parser = ROCrateLogsParser(tmp_path, wms_name="Streamflow", instruments_to_ignore=instruments_to_ignore) + parser.build_workflow("crate-test") + assert parser._ignore_instrument("file:///wf/bin/shell.cwl") + assert not parser._self_connected_instruments + parser = ROCrateLogsParser(tmp_path, wms_name="Streamflow", instruments_to_ignore=["other.cwl"]) + parser.build_workflow("crate-test") + assert not parser._ignore_instrument("file:///wf/bin/shell.cwl") + assert parser._self_connected_instruments == {"file:///wf/bin/shell.cwl"} + + @pytest.mark.unit + def test_missing_timestamps(self, tmp_path: pathlib.Path) -> None: + _write_crate(tmp_path) + metadata = tmp_path / "ro-crate-metadata.json" + crate = json.loads(metadata.read_text()) + for item in crate["@graph"]: + item.pop("startTime", None) + item.pop("endTime", None) + metadata.write_text(json.dumps(crate)) + + workflow = ROCrateLogsParser(tmp_path, wms_name="Streamflow").build_workflow("crate-test") + workflow.generate_json() + assert workflow.makespan == 0.0 + assert all(t["runtimeInSeconds"] == 0.0 for t in workflow.workflow_json["workflow"]["execution"]["tasks"]) diff --git a/tests/unit/wfinstances/test_logs_snakemake.py b/tests/unit/wfinstances/test_logs_snakemake.py new file mode 100644 index 00000000..7f193662 --- /dev/null +++ b/tests/unit/wfinstances/test_logs_snakemake.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2025 The WfCommons Team. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +import pathlib +import sqlite3 + +import pytest + +from wfcommons.utils import sanitize_wfformat_id, to_rfc3339 +from wfcommons.wfinstances.logs import SnakemakeLogsParser + + +def _write_snkmt_db(run_dir: pathlib.Path) -> pathlib.Path: + """Write a minimal snkmt database (with the column positions read by the parser) and data files.""" + data = run_dir / "data" + (data / "sample~A,rep=1").mkdir(parents=True) + for name, size in [("a.txt", 100), ("sample~A,rep=1/b.txt", 200)]: + (data / name).write_bytes(b"x" * size) + db = run_dir / "snkmt.sqlite" + conn = sqlite3.connect(db) + conn.execute("CREATE TABLE workflows (id TEXT, snakefile TEXT, started_at TEXT, status TEXT, end_time TEXT)") + conn.execute("CREATE TABLE rules (id INTEGER, name TEXT, workflow_id TEXT)") + conn.execute("CREATE TABLE jobs (id INTEGER, workflow_id TEXT, snakemake_id INTEGER, rule_id INTEGER, " + "message TEXT, wildcards TEXT, reason TEXT, resources TEXT, shellcmd TEXT, threads INTEGER, " + "priority INTEGER, status TEXT, started_at TEXT, end_time TEXT)") + conn.execute("CREATE TABLE files (id INTEGER, path TEXT, file_type TEXT, job_id INTEGER)") + conn.execute("INSERT INTO workflows VALUES ('u1', 'Snakefile', '2025-03-01 10:00:00', 'RUNNING', NULL)") + conn.executemany("INSERT INTO rules VALUES (?, ?, 'u1')", [(1, "split"), (2, "get_IDs")]) + conn.executemany("INSERT INTO jobs VALUES (?, 'u1', ?, ?, NULL, NULL, NULL, NULL, ?, ?, 0, ?, ?, ?)", [ + (1, 1, 1, "wfbench --name split", 2, "SUCCESS", "2025-03-01 10:00:01.000000", "2025-03-01 10:01:01.500000"), + (2, 2, 2, None, 1, "FAILED", "2025-03-01 10:01:02.000000", None), + ]) + conn.executemany("INSERT INTO files VALUES (?, ?, ?, ?)", [ + (1, f"{data}/a.txt (access: read-only)", "INPUT", 1), + (2, f"{data}/sample~A,rep=1/b.txt", "OUTPUT", 1), + (3, f"{data}/sample~A,rep=1/b.txt", "INPUT", 2), + (4, f"{run_dir}/logs/1.log", "LOG", 1), + ]) + conn.commit() + conn.close() + return db + + +class TestSnakemakeLogsParser: + + @pytest.mark.unit + def test_build_workflow(self, tmp_path: pathlib.Path) -> None: + db = _write_snkmt_db(tmp_path) + workflow = SnakemakeLogsParser(tmp_path, snkmt_db=db, snakemake_version="9.1.0").build_workflow("smk-test") + workflow.generate_json() + wf_json = workflow.workflow_json + + assert wf_json["runtimeSystem"]["version"] == "9.1.0" + + # file IDs are sanitized + files = {f["id"]: f["sizeInBytes"] for f in wf_json["workflow"]["specification"]["files"]} + assert files == {sanitize_wfformat_id(f"{tmp_path}/data/a.txt"): 100, + sanitize_wfformat_id(f"{tmp_path}/data/sample~A,rep=1/b.txt"): 200} + assert any(file_id.endswith("/data/sample_A_rep_1/b.txt") for file_id in files) + + # a rule name containing "_ID" is fine + spec = {t["id"]: t for t in wf_json["workflow"]["specification"]["tasks"]} + assert spec["get_IDs_ID0000002"]["parents"] == ["split_ID0000001"] + + # the workflow start time comes from the database, and the makespan from the last job end time + # (the workflow has no end time) + execution = wf_json["workflow"]["execution"] + assert execution["executedAt"] == to_rfc3339("2025-03-01 10:00:00") + assert execution["makespanInSeconds"] == 61.5 + + # a job with no end time does not break the parser + tasks = {t["id"]: t for t in execution["tasks"]} + assert tasks["split_ID0000001"]["runtimeInSeconds"] == 60.5 + assert tasks["split_ID0000001"]["executedAt"] == to_rfc3339("2025-03-01 10:00:01") + assert tasks["split_ID0000001"]["coreCount"] == 2 + assert tasks["get_IDs_ID0000002"]["runtimeInSeconds"] == 0.0 + assert tasks["get_IDs_ID0000002"]["executedAt"] == to_rfc3339("2025-03-01 10:01:02") diff --git a/tests/unit/wfinstances/test_logs_taskvine.py b/tests/unit/wfinstances/test_logs_taskvine.py new file mode 100644 index 00000000..b257b281 --- /dev/null +++ b/tests/unit/wfinstances/test_logs_taskvine.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2025 The WfCommons Team. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +import pathlib + +import pytest + +from wfcommons.wfinstances.logs import TaskVineLogsParser + + +def _write_vine_logs(logs_dir: pathlib.Path) -> None: + """Write minimal TaskVine logs: a 'cat' task producing a file read by a 'wc' task.""" + (logs_dir / "debug").write_text( + "2026/01/31 12:00:01.00 vine_manager[245] vine: worker-1 (10.0.0.1:9123) busy on './cat a.txt b.txt'\n" + "2026/01/31 12:00:01.00 vine_manager[245] vine: Task 1 state change: READY (1) to RUNNING (2)\n" + "2026/01/31 12:00:02.00 vine_manager[245] vine: worker-1 (10.0.0.1:9123) busy on 'wc -l \"my file.txt\"'\n" + "2026/01/31 12:00:02.00 vine_manager[245] vine: Task 2 state change: READY (1) to RUNNING (2)\n") + (logs_dir / "taskgraph").write_text( + "# taskgraph\n" + 'FILE file-meta-x "bin/cat" 0\n' + 'FILE file-meta-w "wfbench" 0\n' + 'FILE file-meta-a "data/my input.txt" 0\n' + 'FILE file-rnd-c "concatenated.txt" 0\n' + 'FILE file-rnd-m "my file.txt" 0\n' + 'TASK T1 "./cat a.txt b.txt" INPUTS file-meta-x file-meta-w file-meta-a OUTPUTS file-rnd-c\n' + 'TASK T2 "wc -l "my file.txt" INPUTS" INPUTS file-rnd-c OUTPUTS file-rnd-m\n') + (logs_dir / "transactions").write_text( + "# time manager_pid MANAGER manager_pid START|END time_from_origin\n" + "1769907500000000 245 MANAGER 245 START 0\n" + "1769907502000000 245 TASK 1 RUNNING worker-1 FIRST_RESOURCES {}\n" + "1769907502000001 245 WORKER worker-1 TRANSFER INPUT file-meta-a 1007 72409 1769907492000000\n" + "1769907502000002 245 WORKER worker-1 TRANSFER OUTPUT file-rnd-c 2003 72409 1769907492000000\n" + "1769907506500000 245 TASK 1 DONE SUCCESS 0 {} {}\n" + "1769907512000000 245 TASK 2 RUNNING worker-1 FIRST_RESOURCES {}\n" + "1769907512000002 245 WORKER worker-1 TRANSFER OUTPUT file-rnd-m 11 72409 1769907492000000\n" + "1769907516500000 245 TASK 2 DONE SUCCESS 0 {} {}\n" + "1769907532000000 245 MANAGER 245 END 0\n") + + +class TestTaskVineLogsParser: + + @pytest.mark.unit + def test_default_filenames_to_ignore(self, tmp_path: pathlib.Path) -> None: + _write_vine_logs(tmp_path) + parser = TaskVineLogsParser(tmp_path) + assert parser.filenames_to_ignore == set() + + @pytest.mark.unit + def test_build_workflow(self, tmp_path: pathlib.Path, capsys) -> None: + _write_vine_logs(tmp_path) + parser = TaskVineLogsParser(tmp_path, filenames_to_ignore=["wfbench"], taskvine_version="7.15.0") + workflow = parser.build_workflow("taskvine-test") + workflow.generate_json() + wf_json = workflow.workflow_json + + # nothing is printed to stdout + assert capsys.readouterr().out == "" + + assert wf_json["runtimeSystem"]["version"] == "7.15.0" + files = {f["id"]: f["sizeInBytes"] for f in wf_json["workflow"]["specification"]["files"]} + # names are matched exactly (the 'cat' executable does not hide 'concatenated.txt'), quoted + # names may contain spaces, and file IDs are sanitized + assert files == {"data/my_input.txt": 1007, "concatenated.txt": 2003, "my_file.txt": 11} + + spec = {t["id"]: t for t in wf_json["workflow"]["specification"]["tasks"]} + assert spec["Task_ID0000001"]["inputFiles"] == ["data/my_input.txt"] + assert spec["Task_ID0000002"]["inputFiles"] == ["concatenated.txt"] + assert spec["Task_ID0000002"]["parents"] == ["Task_ID0000001"] + + # timestamps (in microseconds) come from the transactions log + execution = wf_json["workflow"]["execution"] + assert execution["executedAt"] == "2026-02-01T00:58:20+00:00" + assert execution["makespanInSeconds"] == 32.0 + tasks = {t["id"]: t for t in execution["tasks"]} + assert tasks["Task_ID0000001"]["executedAt"] == "2026-02-01T00:58:22+00:00" + assert tasks["Task_ID0000001"]["runtimeInSeconds"] == 4.5 diff --git a/tests/wfbench/test_translators_ids.py b/tests/wfbench/test_translators_ids.py new file mode 100644 index 00000000..b1104804 --- /dev/null +++ b/tests/wfbench/test_translators_ids.py @@ -0,0 +1,412 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2026 The WfCommons Team. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +""" +Regression tests (no Docker, no workflow systems) for creating workflow benchmarks and translating +them, with the task IDs, task names, and file IDs allowed by WfFormat 1.6. +""" + +import ast +import json +import pathlib +import re +import shutil +import subprocess +import sys +import types + +import pytest +import yaml + +from wfcommons import BlastRecipe +from wfcommons.common import File, Task, Workflow +from wfcommons.wfbench import WorkflowBenchmark +from wfcommons.wfbench import AirflowTranslator, BashTranslator, CWLTranslator, DaskTranslator, \ + MakeflowTranslator, NextflowTranslator, ParslTranslator, PegasusTranslator, PyCompssTranslator, \ + SnakemakeTranslator, StreamflowTranslator, SwiftTTranslator, TaskVineTranslator +from wfcommons.wfbench.translator.abstract_translator import Translator +from wfcommons.wfinstances import SchemaValidator + +ALL_TRANSLATORS = [AirflowTranslator, BashTranslator, CWLTranslator, DaskTranslator, MakeflowTranslator, + NextflowTranslator, ParslTranslator, PegasusTranslator, PyCompssTranslator, + SnakemakeTranslator, StreamflowTranslator, SwiftTTranslator, TaskVineTranslator] + +# task IDs that are valid in WfFormat 1.6 but not identifiers in the target languages +TRICKY_IDS = ["#1", "-x", "a:b", "a/b", "a.b", "a-b", "m#1", "class", "2"] +TRICKY_NAMES = ["split-fasta", "proc.step", "proc.step", "it's a name", "merge all", "proc.step", "m#1", "2", "x:y"] +# the first task is the parent, and the last task is the child, of all other tasks +TRICKY_EDGES = [(0, i) for i in range(1, len(TRICKY_IDS) - 1)] + \ + [(i, len(TRICKY_IDS) - 1) for i in range(1, len(TRICKY_IDS) - 1)] + + +@pytest.fixture(autouse=True) +def _no_wfbench_binary(monkeypatch): + # translations copy the wfbench executable, which may not be installed + monkeypatch.setattr(Translator, "_copy_binary_files", + lambda self, output_folder: output_folder.joinpath("bin").mkdir(exist_ok=True)) + + +def _build_workflow(ids, names, edges, with_files=True, file_ids=None, runtimes=None) -> Workflow: + """Build a workflow in which each task produces one file, used by all its children.""" + workflow = Workflow(name="tricky") + file_ids = file_ids or ["workflow_input"] + [f"f{i}" for i in range(len(ids))] + for i, (task_id, name) in enumerate(zip(ids, names)): + input_files, output_files = [], [] + if with_files: + parents = [parent for parent, child in edges if child == i] + input_files = [File(file_ids[parent + 1], 100 + parent) for parent in parents] or [File(file_ids[0], 1000)] + output_files = [File(file_ids[i + 1], 100 + i)] + workflow.add_task(Task(name=name, task_id=task_id, runtime=runtimes[i] if runtimes else 10.0 + i, + input_files=input_files, output_files=output_files, + bytes_read=100, bytes_written=200, avg_cpu=50.0, memory=1024)) + for parent, child in edges: + workflow.add_dependency(ids[parent], ids[child]) + return workflow + + +def _create_benchmark(tmp_path: pathlib.Path, workflow: Workflow, **kwargs) -> (WorkflowBenchmark, dict): + benchmark = WorkflowBenchmark(recipe=BlastRecipe, num_tasks=len(workflow.tasks)) + kwargs.setdefault("cpu_work", 100) + json_path = benchmark.create_benchmark_from_synthetic_workflow(tmp_path / "benchmark", workflow, **kwargs) + with open(json_path) as f: + data = json.load(f) + # the generated benchmark is a valid WfFormat instance + SchemaValidator().validate_instance(data) + return benchmark, data + + +def _arguments(data: dict) -> dict: + return {t["id"]: t["command"]["arguments"] for t in data["workflow"]["execution"]["tasks"]} + + +class TestWorkflowBenchmark: + + @pytest.mark.unit + def test_argument_order(self, tmp_path) -> None: + workflow = _build_workflow(["t1", "t2"], ["a", "b"], [(0, 1)]) + _, data = _create_benchmark(tmp_path, workflow, time=30, num_chunks=7) + for args in _arguments(data).values(): + assert "--num-chunks 7" in args + assert "--time 30" in args + + workflow = _build_workflow(["t1", "t2"], ["a", "b"], [(0, 1)]) + _, data = _create_benchmark(tmp_path / "no_time", workflow) + for args in _arguments(data).values(): + assert "--num-chunks 10" in args + assert not any(a.startswith("--time") or "None" in a for a in args) + + @pytest.mark.unit + def test_file_renaming(self, tmp_path) -> None: + # file IDs that are substrings of task IDs and of other file IDs + ids = ["s:1", "p/1", "p/2", "m#1"] + edges = [(0, 1), (0, 2), (1, 3), (2, 3)] + workflow = _build_workflow(ids, ["s", "p", "p", "m"], edges, file_ids=["a", "1", "11", "a1", "1a"]) + _, data = _create_benchmark(tmp_path, workflow) + arguments = _arguments(data) + tasks = {t["id"]: t for t in data["workflow"]["specification"]["tasks"]} + + for task_id, args in arguments.items(): + # the task name argument is not modified + assert args[0] == f"--name {task_id}" + # file arguments match the (renamed) files of the task + output_files = ast.literal_eval(next(a for a in args if a.startswith("--output-files"))[15:]) + input_files = ast.literal_eval(next(a for a in args if a.startswith("--input-files"))[14:]) + assert list(output_files) == tasks[task_id]["outputFiles"] + assert input_files == tasks[task_id]["inputFiles"] + # file names are safe to use as paths + for file_id in tasks[task_id]["outputFiles"] + tasks[task_id]["inputFiles"]: + assert re.fullmatch(r"[0-9A-Za-z_.-]+", file_id) + + # dependencies are still expressed by files + for parent, child in edges: + assert set(tasks[ids[parent]]["outputFiles"]) & set(tasks[ids[child]]["inputFiles"]) + + @pytest.mark.unit + def test_zero_runtimes(self, tmp_path) -> None: + workflow = _build_workflow(["t1", "t2", "t3"], ["a", "b", "b"], [(0, 1), (0, 2)], runtimes=[0, 0, 0]) + _, data = _create_benchmark(tmp_path, workflow, percent_cpu={"a": 0.5, "b": 0.4}) + for args in _arguments(data).values(): + assert "--cpu-work 100" in args + + @pytest.mark.unit + def test_execution_data_and_spec_only_workflow(self, tmp_path) -> None: + workflow = _build_workflow(["t1", "t2"], ["a", "b"], [(0, 1)]) + workflow.include_execution = False + _, data = _create_benchmark(tmp_path, workflow, mem=100) + # the benchmark commands are in the execution section + assert len(data["workflow"]["execution"]["tasks"]) == 2 + for task in data["workflow"]["execution"]["tasks"]: + assert task["command"]["program"] == "wfbench" + assert task["runtimeInSeconds"] == 0 + # measured data of the original workflow are not kept + for key in ("readBytes", "writtenBytes", "avgCPU"): + assert key not in task + # requested resources are set by the benchmark + assert task["coreCount"] >= 1 + assert task["memoryInBytes"] > 1024 + + @pytest.mark.unit + def test_values_per_task_name(self, tmp_path, monkeypatch) -> None: + # recipe tasks have no category: values are given per task name + benchmark = WorkflowBenchmark(recipe=BlastRecipe, num_tasks=45) + monkeypatch.setattr("builtins.input", lambda *args: "") + input_file = tmp_path / "input.json" + benchmark.generate_input_file(input_file) + params = json.loads(input_file.read_text()) + assert set(params["cpu_work"]) == {"split_fasta", "blastall", "cat_blast", "cat"} + + json_path = benchmark.create_benchmark(tmp_path / "benchmark", + percent_cpu=params["percent_cpu"], + cpu_work=params["cpu_work"], + gpu_work=params["gpu_work"]) + data = json.loads(json_path.read_text()) + SchemaValidator().validate_instance(data) + for args in _arguments(data).values(): + assert "--percent-cpu 0.6" in args and "--cpu-work 1000" in args and "--gpu-work 100" in args + + +class TestTranslators: + + @pytest.mark.unit + @pytest.mark.parametrize("translator_class", ALL_TRANSLATORS) + def test_missing_command(self, translator_class) -> None: + workflow = _build_workflow(["t1", "t2"], ["a", "b"], [(0, 1)]) + with pytest.raises(ValueError, match=r"'t1', 't2'.*WorkflowBenchmark"): + translator_class(workflow) + + @pytest.mark.unit + def test_unique_identifiers(self) -> None: + names = {"k1": "valid_name", "k2": "a.b", "k3": "a-b", "k4": "a_b", "k5": "1abc", "k6": "class", + "k7": "", "k8": "valid_name"} + identifiers = Translator._unique_identifiers(names, reserved={"class"}) + assert identifiers["k1"] == "valid_name" + assert identifiers["k4"] == "a_b" + assert identifiers["k2"] == "a_b_1" and identifiers["k3"] == "a_b_2" + assert identifiers["k5"] == "_1abc" + assert identifiers["k6"] == "class_1" + assert identifiers["k7"] == "_" + assert identifiers["k8"] == "valid_name_1" + assert len(set(identifiers.values())) == len(names) + # deterministic + assert identifiers == Translator._unique_identifiers(names, reserved={"class"}) + + @pytest.mark.unit + def test_graph(self) -> None: + workflow = BlastRecipe.from_num_tasks(45).build_workflow() + workflow.generate_json() + expected = workflow.workflow_json["workflow"]["specification"]["tasks"] + translator = BashTranslator(workflow) + assert list(translator.tasks) == [task["id"] for task in expected] + assert translator.root_task_names == [task["id"] for task in expected if not task["parents"]] + for task in expected: + assert translator.task_parents[task["id"]] == task["parents"] + assert translator.task_children[task["id"]] == task["children"] + + @pytest.mark.unit + def test_safe_ids_are_kept(self, tmp_path) -> None: + benchmark = WorkflowBenchmark(recipe=BlastRecipe, num_tasks=45) + benchmark.create_benchmark(tmp_path / "benchmark", cpu_work=10, data=1) + task_ids = set(benchmark.workflow.tasks) + assert all(task_id.isidentifier() for task_id in task_ids) + + ParslTranslator(benchmark.workflow).translate(tmp_path / "parsl") + code = (tmp_path / "parsl/parsl_workflow.py").read_text() + assert set(re.findall(r"^(\S+) = generic_shell_app", code, re.M)) == task_ids + + SnakemakeTranslator(benchmark.workflow).translate(tmp_path / "snakemake") + rules = re.findall(r"^rule (\S+):", (tmp_path / "snakemake/workflow.smk").read_text(), re.M) + assert set(rules) == task_ids | {"all_wfbench_tasks"} + assert ".wfbench_done" not in (tmp_path / "snakemake/workflow.smk").read_text() + + NextflowTranslator(benchmark.workflow).translate(tmp_path / "nextflow") + processes = re.findall(r"^process (\S+)\(\)", (tmp_path / "nextflow/modules/tasks_0.nf").read_text(), re.M) + assert set(processes) == task_ids + assert {p.name for p in (tmp_path / "nextflow/bin").glob("script_*.sh")} == \ + {f"script_{task_id}.sh" for task_id in task_ids} + + @pytest.mark.unit + def test_python_translators(self, tmp_path) -> None: + workflow = _build_workflow(TRICKY_IDS, TRICKY_NAMES, TRICKY_EDGES) + benchmark, _ = _create_benchmark(tmp_path, workflow) + + outputs = {AirflowTranslator: "workflow.py", DaskTranslator: "dask_workflow.py", + ParslTranslator: "parsl_workflow.py", PegasusTranslator: "pegasus_workflow.py", + PyCompssTranslator: "pycompss_workflow.py", TaskVineTranslator: "taskvine_workflow.py"} + for translator_class, file_name in outputs.items(): + output_folder = tmp_path / translator_class.__name__ + translator_class(benchmark.workflow).translate(output_folder) + code = (output_folder / file_name).read_text() + ast.parse(code) + # task IDs starting with '#' or '-' are attached to the '--name' flag + assert "--name=#1" in code and "--name=-x" in code + + airflow_ids = re.findall(r'task_id="([^"]+)"', (tmp_path / "AirflowTranslator/workflow.py").read_text()) + assert len(set(airflow_ids)) == len(TRICKY_IDS) + assert all(re.fullmatch(r"[\w.-]+", task_id) for task_id in airflow_ids) + + pegasus_ids = re.findall(r"_id='([^']+)'", (tmp_path / "PegasusTranslator/pegasus_workflow.py").read_text()) + assert len(set(pegasus_ids)) == len(TRICKY_IDS) + assert all(re.fullmatch(r"[-0-9a-zA-Z_]+", task_id) for task_id in pegasus_ids) + + @pytest.mark.unit + def test_shell_and_file_based_translators(self, tmp_path) -> None: + workflow = _build_workflow(TRICKY_IDS, TRICKY_NAMES, TRICKY_EDGES) + benchmark, _ = _create_benchmark(tmp_path, workflow) + + BashTranslator(benchmark.workflow).translate(tmp_path / "bash") + script = tmp_path / "bash/run_workflow.sh" + assert "--name=#1" in script.read_text() + if shutil.which("bash"): + assert subprocess.run(["bash", "-n", str(script)]).returncode == 0 + + SnakemakeTranslator(benchmark.workflow).translate(tmp_path / "snakemake") + rules = re.findall(r"^rule (\S+):", (tmp_path / "snakemake/workflow.smk").read_text(), re.M) + assert len(set(rules)) == len(TRICKY_IDS) + 1 + assert all(r.isidentifier() for r in rules) + + MakeflowTranslator(benchmark.workflow).translate(tmp_path / "makeflow") + lines = (tmp_path / "makeflow/workflow.makeflow").read_text().splitlines() + rules = [line for line in lines if line and not line.startswith(("#", "\t"))] + assert len(rules) == len(TRICKY_IDS) + assert all(line.count(":") == 1 and not line.startswith(" :") for line in rules) + # '#' starts a comment anywhere in a Makeflow command, unless quoted + commands = [line for line in lines if line.startswith("\t")] + assert any(" '--name=#1' " in command for command in commands) + assert any(" --name 'm#1' " in command for command in commands) + + NextflowTranslator(benchmark.workflow).translate(tmp_path / "nextflow") + module = (tmp_path / "nextflow/modules/tasks_0.nf").read_text() + processes = re.findall(r"^process (\S+)\(\)", module, re.M) + assert len(set(processes)) == len(TRICKY_IDS) + assert all(re.fullmatch(r"[A-Za-z_]\w*", p) for p in processes) and "class" not in processes + # map entries whose key is not an identifier use the subscript notation + assert all(re.fullmatch(r"[A-Za-z_]\w*", key) for key in re.findall(r"results\.(\S+) =", module)) + assert len(list((tmp_path / "nextflow/bin").glob("script_*.sh"))) == len(TRICKY_IDS) + + for translator_class in (CWLTranslator, StreamflowTranslator): + output_folder = tmp_path / translator_class.__name__ + translator_class(benchmark.workflow).translate(output_folder) + cwl = yaml.safe_load((output_folder / "main.cwl").read_text()) + assert all(re.fullmatch(r"[A-Za-z_][\w.-]*", str(step)) for step in cwl["steps"]) + sources = [s for step in cwl["steps"].values() if isinstance(step["in"], dict) + for s in (step["in"]["input_files"].get("source", []) + if isinstance(step["in"].get("input_files"), dict) else [])] + assert all(source.split("/")[0] in cwl["steps"] for source in sources) + inputs = yaml.safe_load((tmp_path / "StreamflowTranslator/inputs.yml").read_text()) + assert list(inputs) == list(yaml.safe_load((tmp_path / "StreamflowTranslator/main.cwl").read_text())["inputs"]) + + SwiftTTranslator(benchmark.workflow).translate(tmp_path / "swiftt") + code = (tmp_path / "swiftt/workflow.swift").read_text() + assert all(re.fullmatch(r"[A-Za-z_]\w*", v) for v in re.findall(r"^int (\S+)__out\[\];", code, re.M)) + + @pytest.mark.unit + def test_parsl_file_paths(self, tmp_path, monkeypatch) -> None: + # a workflow input file ID with '/', and short file IDs that are part of other names + ids = ["s:1", "p/1", "p/2", "m#1"] + workflow = _build_workflow(ids, ["s", "p", "p", "m"], [(0, 1), (0, 2), (1, 3), (2, 3)], + file_ids=["inputs/raw.dat", "1", "11", "a1", "1a"]) + benchmark = WorkflowBenchmark(recipe=BlastRecipe, num_tasks=4) + benchmark.workflow = workflow + for task in workflow.tasks.values(): + outputs = {f.file_id: f.size for f in task.output_files} + task.program = "wfbench" + task.args = [f"--name {task.task_id}", "--cpu-work 10", f"--output-files {outputs}", + f"--input-files {[f.file_id for f in task.input_files]}"] + ParslTranslator(workflow).translate(tmp_path / "parsl") + assert (tmp_path / "parsl/data/inputs/raw.dat").exists() + + commands = self._run_parsl_workflow(tmp_path / "parsl", monkeypatch) + assert len(commands) == 4 + folder = (tmp_path / "parsl").resolve() + for task_id, command in zip(ids, commands): + task = workflow.tasks[task_id] + assert f"--name {task_id} " in command + files = json.loads(re.search(r"--input-files '(\[.*?\])'", command).group(1)) + assert files == [str(folder / ("data" if task_id == "s:1" else "output") / f.file_id) + for f in task.input_files] + files = json.loads(re.search(r"--output-files '(\{.*?\})'", command).group(1)) + assert list(files) == [str(folder / "output" / f.file_id) for f in task.output_files] + + @staticmethod + def _run_parsl_workflow(folder: pathlib.Path, monkeypatch) -> list: + """Run a generated Parsl workflow with a fake Parsl module that records the commands.""" + commands = [] + + class FakeFile: + def __init__(self, path): + self.filepath = str(path) + + class FakeAppFuture: + def __init__(self, outputs): + self.outputs = outputs + self.stdout = self.stderr = None + + def result(self): + return 0 + + def bash_app(function): + def submit(cmd, inputs=None, outputs=None, stdout=None, stderr=None): + # futures are replaced by their results (app futures) or files (data futures) + inputs = [i.result() if isinstance(i, FakeAppFuture) else i for i in inputs or []] + commands.append(function(cmd, inputs=inputs, outputs=outputs or [])) + return FakeAppFuture(outputs or []) + return submit + + modules = {name: types.ModuleType(name) for name in + ["parsl", "parsl.app", "parsl.app.app", "parsl.config", "parsl.executors", + "parsl.executors.threads", "parsl.providers", "parsl.data_provider", + "parsl.data_provider.files"]} + modules["parsl"].clear = modules["parsl"].load = lambda *args: None + modules["parsl"].dfk = lambda: types.SimpleNamespace(cleanup=lambda: None) + modules["parsl.app.app"].bash_app = bash_app + modules["parsl.config"].Config = lambda **kwargs: None + modules["parsl.executors"].HighThroughputExecutor = lambda **kwargs: None + modules["parsl.executors.threads"].ThreadPoolExecutor = lambda **kwargs: None + modules["parsl.providers"].LocalProvider = lambda **kwargs: None + modules["parsl.data_provider.files"].File = FakeFile + for name, module in modules.items(): + monkeypatch.setitem(sys.modules, name, module) + + monkeypatch.chdir(folder) + code = (folder / "parsl_workflow.py").read_text() + exec(compile(code, str(folder / "parsl_workflow.py"), "exec"), {"__name__": "__main__"}) + return commands + + @pytest.mark.unit + def test_workflow_without_files(self, tmp_path) -> None: + workflow = _build_workflow(["t1", "t2", "t3", "t4"], ["a", "b", "b", "c"], + [(0, 1), (0, 2), (1, 3), (2, 3)], with_files=False) + benchmark, _ = _create_benchmark(tmp_path, workflow) + + for translator_class in ALL_TRANSLATORS: + translator_class(benchmark.workflow).translate(tmp_path / translator_class.__name__) + + # dependencies are expressed with marker files for systems that infer them from files + smk = (tmp_path / "SnakemakeTranslator/workflow.smk").read_text() + rule_t4 = smk[smk.index("rule t4:"):] + assert '"data/t2.wfbench_done"' in rule_t4 and '"data/t3.wfbench_done"' in rule_t4 + makeflow = (tmp_path / "MakeflowTranslator/workflow.makeflow").read_text() + assert "data/t4.wfbench_done : data/t2.wfbench_done data/t3.wfbench_done" in makeflow + script = (tmp_path / "NextflowTranslator/bin/script_t4.sh").read_text() + assert "t2.wfbench_done" in script and "t4.wfbench_done" in script + + # workflow without inputs + cwl = yaml.safe_load((tmp_path / "CWLTranslator/main.cwl").read_text()) + assert cwl["inputs"] == [] + assert "input_files" not in cwl["steps"]["t1"]["in"] + assert yaml.safe_load((tmp_path / "CWLTranslator/config.yml").read_text()) == {} + + @pytest.mark.unit + def test_swift_t_multiple_outputs(self, tmp_path) -> None: + workflow = _build_workflow(["t1", "t2", "t3"], ["a", "b", "b"], [(0, 1), (0, 2)]) + workflow.tasks["t1"].output_files.append(File("extra", 10)) + benchmark, _ = _create_benchmark(tmp_path, workflow) + with pytest.raises(ValueError, match="multiple outputs"): + SwiftTTranslator(benchmark.workflow) diff --git a/wfcommons/common/machine.py b/wfcommons/common/machine.py index 642f5930..04b44a65 100644 --- a/wfcommons/common/machine.py +++ b/wfcommons/common/machine.py @@ -57,7 +57,7 @@ class Machine: def __init__(self, name: str, - cpu: Dict[str, Union[int, str]], + cpu: Optional[Dict[str, Union[int, str]]] = None, system: Optional[MachineSystem] = None, architecture: Optional[str] = None, memory: Optional[int] = None, @@ -74,10 +74,12 @@ def __init__(self, self.release: str = release self.hashcode = hashcode - self.cpu_cores: int = cpu['coreCount'] - self.cpu_speed: int = cpu['speedInMHz'] if 'speedInMHz' in cpu else 0 - self.cpu_flops: int = cpu['coreCount'] * cpu['speedInMHz'] * 10 ^ 6 if 'speedInMHz' in cpu else 0 - self.cpu_vendor: str = cpu['vendor'] if 'vendor' in cpu else None + # the CPU information is optional in WfFormat + cpu = cpu if cpu else {} + self.cpu_cores: Optional[int] = cpu.get('coreCount', None) + self.cpu_speed: int = cpu.get('speedInMHz', 0) + self.cpu_flops: int = self.cpu_cores * self.cpu_speed * 10 ** 6 if self.cpu_cores and self.cpu_speed else 0 + self.cpu_vendor: Optional[str] = cpu.get('vendor', None) self.logger.debug(f"created machine: {self.name} with {self.cpu_cores} cores and {self.cpu_flops} FLOPS.") @@ -96,10 +98,13 @@ def as_dict(self) -> Dict[str, Union[int, str]]: machine['memoryInBytes'] = self.memory if self.release: machine['release'] = self.release + cpu = {} if self.cpu_cores: - machine['cpu'] = {'coreCount': self.cpu_cores} + cpu['coreCount'] = self.cpu_cores if self.cpu_speed: - machine['cpu']['speedInMHz'] = self.cpu_speed + cpu['speedInMHz'] = self.cpu_speed if self.cpu_vendor: - machine['cpu']['vendor'] = self.cpu_vendor + cpu['vendor'] = self.cpu_vendor + if cpu: + machine['cpu'] = cpu return machine diff --git a/wfcommons/common/task.py b/wfcommons/common/task.py index 4f50802c..72f7aead 100644 --- a/wfcommons/common/task.py +++ b/wfcommons/common/task.py @@ -10,13 +10,12 @@ import logging -from datetime import datetime from typing import Dict, List, Optional from logging import Logger from .machine import Machine from .file import File -from ..utils import NoValue +from ..utils import NoValue, to_rfc3339 class TaskType(NoValue): @@ -63,6 +62,8 @@ class Task: :type avg_power: Optional[float] :param priority: Task priority. :type priority: Optional[int] + :param executed_at: Task start timestamp in the ISO 8601 format. + :type executed_at: Optional[str] :param task_type: The type of the task. :type task_type: TaskType :param logger: The logger where to log information/warning or errors. @@ -114,7 +115,8 @@ def __init__(self, self.priority: Optional[int] = priority self.type: Optional[TaskType] = task_type self.launch_dir: Optional[str] = launch_dir - self.start_time: Optional[str] = str(datetime.now().astimezone().isoformat()) if not executed_at else executed_at + # the start time is optional in WfFormat, and is not made up when unknown + self.start_time: Optional[str] = executed_at if executed_at else None self.logger.debug( f"created task {self.task_id}: runtime => {self.runtime} seconds.") @@ -141,11 +143,14 @@ def execution_as_dict(self) -> Dict: """ task_obj = { 'id': self.task_id, - 'runtimeInSeconds': self.runtime, - 'command': {} + 'runtimeInSeconds': self.runtime } + command = {} if self.cores is not None: - task_obj['coreCount'] = self.cores + if self.cores >= 1: + task_obj['coreCount'] = self.cores + else: + self.logger.debug(f"Task {self.task_id}: coreCount {self.cores} is less than 1, omitting it.") if self.avg_cpu is not None: task_obj['avgCPU'] = self.avg_cpu if self.bytes_read is not None: @@ -157,17 +162,25 @@ def execution_as_dict(self) -> Dict: if self.energy is not None: task_obj['energyInKWh'] = self.energy if self.avg_power is not None: - task_obj['avgPowerInKWh'] = self.avg_power + task_obj['avgPowerInW'] = self.avg_power if self.priority is not None: task_obj['priority'] = self.priority if self.program is not None: - task_obj['command']['program'] = self.program - if self.args is not None: - task_obj['command']['arguments'] = self.args + command['program'] = self.program + if self.args: + # WfFormat task arguments are non-empty strings + command['arguments'] = [str(arg) for arg in self.args if arg is not None and str(arg) != ""] + if command: + task_obj['command'] = command if self.machines: task_obj['machines'] = [m.name for m in self.machines] if self.start_time: - task_obj['executedAt'] = self.start_time + try: + task_obj['executedAt'] = to_rfc3339(self.start_time) + except (ValueError, OverflowError, TypeError): + self.logger.warning(f"Task {self.task_id}: unable to convert timestamp '{self.start_time}' " + f"into the RFC 3339 format.") + task_obj['executedAt'] = self.start_time if self.category is not None: task_obj['category'] = self.category if self.launch_dir: diff --git a/wfcommons/common/workflow.py b/wfcommons/common/workflow.py index 4e1e20a1..1947d1b9 100644 --- a/wfcommons/common/workflow.py +++ b/wfcommons/common/workflow.py @@ -11,18 +11,34 @@ import getpass import importlib import json +import logging import networkx as nx import pathlib from datetime import datetime -from typing import Optional, List +from logging import Logger +from typing import Any, Dict, Optional, List, Tuple +from ..common.machine import Machine from ..common.task import Task, TaskType +from ..utils import to_rfc3339 from ..version import __version__, __schema_version__ -from ..wfchef.utils import create_graph, create_graph_from_json_object +from ..wfchef.utils import create_graph import tempfile +def _normalize_timestamp(timestamp: Any, logger: Logger) -> Any: + """ + Convert a timestamp into the RFC 3339 date-time format required by WfFormat, + leaving it untouched if it cannot be parsed. + """ + try: + return to_rfc3339(timestamp) + except (ValueError, OverflowError, TypeError): + logger.warning(f"Unable to convert timestamp '{timestamp}' into the RFC 3339 format.") + return timestamp + + class Workflow(nx.DiGraph): """ Representation of a workflow. The workflow representation is an extension of the @@ -50,6 +66,13 @@ class Workflow(nx.DiGraph): :type author_institution: Optional[str] :param author_country: Author country (preferably country code, ISO ALPHA-2 code). :type author_country: Optional[str] + :param created_at: Instance creation timestamp in the ISO 8601 format (defaults to now). + :type created_at: Optional[str] + :param include_execution: Whether the JSON instance includes the execution section + (a WfFormat instance with only a specification is valid). + :type include_execution: Optional[bool] + :param logger: The logger where to log information/warning or errors. + :type logger: Optional[Logger] """ def __init__(self, @@ -63,22 +86,30 @@ def __init__(self, author_name: Optional[str] = None, author_email: Optional[str] = None, author_institution: Optional[str] = None, - author_country: Optional[str] = None + author_country: Optional[str] = None, + created_at: Optional[str] = None, + include_execution: Optional[bool] = True, + logger: Optional[Logger] = None ) -> None: """Create an object of a workflow representation.""" + self.logger: Logger = logging.getLogger(__name__) if logger is None else logger + name = name if name else "workflow" self.description: Optional[ str] = description if description else "Instance generated with WfCommons - https://wfcommons.org" - self.created_at: str = str(datetime.now().astimezone().isoformat()) + self.created_at: str = created_at if created_at else str(datetime.now().astimezone().isoformat()) self.schema_version: str = f"{__schema_version__}" self.runtime_system_name: Optional[str] = "WfCommons" if not runtime_system_name else runtime_system_name self.runtime_system_version: Optional[str] = "unknown" if not runtime_system_version else runtime_system_version self.runtime_system_url: Optional[str] = f"https://docs.wfcommons.org/en/v{__version__}/" if not runtime_system_url else runtime_system_url self.executed_at: Optional[str] = str(datetime.now().astimezone().isoformat()) if not executed_at else executed_at - self.makespan: Optional[float] = makespan + self.makespan: Optional[float] = makespan if makespan is not None else 0.0 self.author_name: Optional[str] = author_name if author_name else str(getpass.getuser()) self.author_email: Optional[str] = author_email if author_email else "support@wfcommons.org" - self.author_institution: Optional[str] = None - self.author_country: Optional[str] = None + self.author_institution: Optional[str] = author_institution + self.author_country: Optional[str] = author_country + self.include_execution: bool = include_execution + # machines declared for the workflow, including the ones not used by any task + self.machines: Dict[str, Machine] = {} self.tasks: dict[str, Task] = {} self.tasks_parents = {} self.tasks_children = {} @@ -134,38 +165,51 @@ def generate_json(self) -> None: :return: A JSON representation of the workflow instance. :rtype: dict """ - workflow_machines = [] - machines_list = [] + workflow_machines = [machine.as_dict() for machine in self.machines.values()] + machines_list = list(self.machines.keys()) specification_tasks = [] execution_tasks = [] - files = set() - - workflow_json = { - "name": self.name, - "description": self.description, - "createdAt": self.created_at, - "schemaVersion": self.schema_version, - "author": { + files = {} + + # optional instance properties are omitted when not set + workflow_json = {"name": self.name} + if self.description: + workflow_json["description"] = self.description + if self.created_at: + workflow_json["createdAt"] = _normalize_timestamp(self.created_at, self.logger) + workflow_json["schemaVersion"] = self.schema_version + if self.author_name: + workflow_json["author"] = { "name": self.author_name, "email": self.author_email - }, - "workflow": { - "specification": { - "tasks": specification_tasks, - "files": [] - }, - "execution": { - "makespanInSeconds": self.makespan, - "executedAt": self.executed_at, - "tasks": execution_tasks - } - }, - "runtimeSystem": { - "name": self.runtime_system_name, - "version": self.runtime_system_version, - "url": self.runtime_system_url + } + if self.author_institution: + workflow_json["author"]["institution"] = self.author_institution + if self.author_country: + workflow_json["author"]["country"] = self.author_country + workflow_json["workflow"] = { + "specification": { + "tasks": specification_tasks, + "files": [] } } + if self.runtime_system_name: + workflow_json["runtimeSystem"] = { + "name": self.runtime_system_name, + "version": self.runtime_system_version + } + if self.runtime_system_url: + workflow_json["runtimeSystem"]["url"] = self.runtime_system_url + if self.include_execution: + workflow_json["workflow"]["execution"] = { + "makespanInSeconds": self.makespan if self.makespan is not None else 0.0, + "executedAt": _normalize_timestamp(self.executed_at, self.logger), + "tasks": execution_tasks + } + + if len(self.nodes) == 0: + self.logger.warning("The workflow has no tasks: WfFormat requires at least one task, " + "so the generated instance will not validate.") # generate tasks parents and children tasks_dependencies = {} @@ -196,17 +240,20 @@ def generate_json(self) -> None: machines_list.append(machine.name) workflow_machines.append(machine.as_dict()) - # add files to the workflow json object (input and output) - for file in task.input_files: - files.add(file) - for file in task.output_files: - files.add(file) + # add files to the workflow json object (input and output); a file ID must + # be declared only once, even if tasks report different sizes for it + for file in task.input_files + task.output_files: + if file.file_id not in files: + files[file.file_id] = file + elif files[file.file_id].size != file.size: + self.logger.warning(f"File '{file.file_id}' has different sizes ({files[file.file_id].size} " + f"and {file.size}): keeping the first one.") - if workflow_machines: + if workflow_machines and self.include_execution: workflow_json["workflow"]["execution"]["machines"] = workflow_machines - if files and len(files) > 0: - workflow_json["workflow"]["specification"]["files"] = list(file.as_dict() for file in files) + if files: + workflow_json["workflow"]["specification"]["files"] = [file.as_dict() for file in files.values()] self.workflow_json = workflow_json @@ -217,68 +264,68 @@ def generate_json(self) -> None: if execution_metrics != {}: self.workflow_json["workflow"]["execution"]["metrics"] = execution_metrics - - def compute_metrics(self, workflow_json: json) -> json: + def compute_metrics(self, workflow_json: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]: """ - Compute workflow specification and execution metrics. + Compute workflow specification and execution metrics, as defined by WfFormat. + :param workflow_json: Workflow instance as a JSON object. - :type workflow_json: json + :type workflow_json: Dict[str, Any] :return: specification and execution metrics. - :rtype: json + :rtype: Tuple[Dict[str, Any], Dict[str, Any]] """ - # Specification metrics - specification_metrics : json = {} - specification_metrics["numberOfTasks"] = len(workflow_json["workflow"]["specification"]["tasks"]) - if "files" in workflow_json["workflow"]["specification"]: - number_of_files = len(workflow_json["workflow"]["specification"]["files"]) - specification_metrics["numberOfFiles"] = number_of_files - if number_of_files > 0: - specification_metrics["sumOfFileSizesInBytes"] = sum( - file.get("sizeInBytes", 0) - for file in workflow_json["workflow"]["specification"]["files"] - ) - graph = create_graph_from_json_object(workflow_json) - levels = [ - tuple(generation) - for generation in nx.topological_generations(graph) - ][1:-1] # Remove the fictitious SRC and DST levels - - if len(levels) > 0: - widths = [len(level) for level in levels] # Remove the fictitious SRC and DST levels - + specification = workflow_json["workflow"]["specification"] + tasks = specification["tasks"] + specification_metrics = {"numberOfTasks": len(tasks)} + if "files" in specification: + files = specification["files"] + specification_metrics["numberOfFiles"] = len(files) + if len(files) > 0: + specification_metrics["sumOfFileSizesInBytes"] = sum(file.get("sizeInBytes", 0) for file in files) + + # A task with no parents is at level 0, and every other task is at one plus the + # maximum level of any of its parents. The graph is keyed by task ID, since task + # names are arbitrary and may be shared by many tasks. + graph = nx.DiGraph() + graph.add_nodes_from(task["id"] for task in tasks) + for task in tasks: + graph.add_edges_from((parent, task["id"]) for parent in task.get("parents", []) + if parent in graph and parent != task["id"]) + try: + widths = [len(level) for level in nx.topological_generations(graph)] + except nx.NetworkXUnfeasible: + self.logger.warning("The workflow graph contains a cycle: DAG level metrics are not computed.") + widths = [] + + if len(widths) > 0: specification_metrics["numberOfLevels"] = len(widths) specification_metrics["minimumWidth"] = min(widths) specification_metrics["maximumWidth"] = max(widths) # Execution metrics - execution_metrics : json = {} + execution_metrics = {} if "execution" in workflow_json["workflow"]: - total_runtime_in_seconds = sum( - task.get("runtimeInSeconds", 0) - for task in workflow_json["workflow"]["execution"]["tasks"] - ) - if total_runtime_in_seconds > 0: - execution_metrics["sumTaskRuntimesInSeconds"] = total_runtime_in_seconds + execution_tasks = workflow_json["workflow"]["execution"]["tasks"] - total_read_bytes = sum( - task.get("readBytes", 0) - for task in workflow_json["workflow"]["execution"]["tasks"] - ) + total_runtime_in_seconds = sum(task.get("runtimeInSeconds", 0) for task in execution_tasks) if total_runtime_in_seconds > 0: - execution_metrics["totalNumBytesRead"] = total_read_bytes + execution_metrics["sumTaskRuntimesInSeconds"] = total_runtime_in_seconds - total_written_bytes = sum( - task.get("writtenBytes", 0) - for task in workflow_json["workflow"]["execution"]["tasks"] - ) - if total_written_bytes > 0: - execution_metrics["totalNumBytesWritten"] = total_written_bytes + # byte totals are integers in WfFormat, while per-task byte counts may be fractional + for task_key, metric_key in (("readBytes", "totalNumBytesRead"), + ("writtenBytes", "totalNumBytesWritten")): + values = [task[task_key] for task in execution_tasks if task_key in task] + if values: + total = sum(values) + if float(total).is_integer() and total >= 0: + execution_metrics[metric_key] = int(total) + else: + self.logger.debug(f"Metric '{metric_key}' not computed: {total} is not a " + f"non-negative integer.") return specification_metrics, execution_metrics - def write_dot(self, dot_file_path: Optional[pathlib.Path] = None) -> None: """ Write a dot file of the workflow instance. diff --git a/wfcommons/utils.py b/wfcommons/utils.py index b65c784e..6485a7d6 100644 --- a/wfcommons/utils.py +++ b/wfcommons/utils.py @@ -10,19 +10,25 @@ import warnings warnings.filterwarnings('ignore') +import dateutil.parser import json import logging import math import pathlib +import re import scipy.stats import numpy as np import operator as op +from datetime import datetime from enum import Enum from functools import reduce from logging import Logger -from typing import Any, Dict, Optional, List, Tuple +from typing import Any, Dict, Optional, List, Tuple, Union +# Characters allowed in WfFormat task and file IDs (see "$defs/taskId" in the WfFormat schema) +WFFORMAT_ID_PATTERN = re.compile(r"^[0-9a-zA-Z_.#:/-]+$") +_WFFORMAT_ID_INVALID_CHARS = re.compile(r"[^0-9a-zA-Z_.#:/-]") class NoValue(Enum): @@ -30,6 +36,46 @@ def __repr__(self): return '<%s.%s>' % (self.__class__.__name__, self.name) +def sanitize_wfformat_id(identifier: Any, replacement: str = "_") -> str: + """ + Replace the characters that are not allowed in WfFormat task and file IDs. + + :param identifier: The raw identifier (e.g., a task name or a file name from a log). + :type identifier: Any + :param replacement: The string used in place of each invalid character. + :type replacement: str + + :return: An identifier that matches the WfFormat ID pattern. + :rtype: str + """ + sanitized = _WFFORMAT_ID_INVALID_CHARS.sub(replacement, str(identifier)) + return sanitized if sanitized else replacement + + +def to_rfc3339(timestamp: Union[None, str, int, float, datetime]) -> Optional[str]: + """ + Convert a timestamp into the RFC 3339 date-time format required by WfFormat. Naive + timestamps are assumed to be in the local timezone. + + :param timestamp: A datetime, a POSIX timestamp, or a date-time string. + :type timestamp: Union[None, str, int, float, datetime] + + :return: The timestamp in the RFC 3339 format, or None if no timestamp is provided. + :rtype: Optional[str] + """ + if timestamp is None or timestamp == "": + return None + if isinstance(timestamp, datetime): + dt = timestamp + elif isinstance(timestamp, (int, float)): + dt = datetime.fromtimestamp(timestamp) + else: + dt = dateutil.parser.parse(str(timestamp)) + if dt.tzinfo is None: + dt = dt.astimezone() + return dt.isoformat() + + def read_json(instance_filename: pathlib.Path) -> Dict[str, Any]: """ Read the JSON from the file path. diff --git a/wfcommons/wfbench/bench.py b/wfcommons/wfbench/bench.py index 7bee4eb0..c9f2b488 100644 --- a/wfcommons/wfbench/bench.py +++ b/wfcommons/wfbench/bench.py @@ -8,11 +8,13 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. +import ast import glob import json import logging import os import pathlib +import re import subprocess import time import shortuuid @@ -119,6 +121,9 @@ def create_benchmark_from_synthetic_workflow( """ self.workflow = workflow + # the benchmark tasks (i.e., their commands) are part of the execution section + self.workflow.include_execution = True + self.workflow.makespan = 0.0 save_dir = save_dir.resolve() save_dir.mkdir(exist_ok=True, parents=True) @@ -126,54 +131,57 @@ def create_benchmark_from_synthetic_workflow( json_path = save_dir.joinpath( f"{self.workflow.name.lower()}-{self.num_tasks}").with_suffix(".json") + # tasks are grouped by category (or by name, as tasks often do not have a category) + task_max_runtimes = {} + for task in self.workflow.tasks.values(): + task_type = self._task_type(task) + if task_type not in task_max_runtimes or task.runtime > task_max_runtimes[task_type]: + task_max_runtimes[task_type] = task.runtime + max_runtime = max(task_max_runtimes.values(), default=0) + # if no cpu_work is provided, use the maximum runtime of each task as a reference if cpu_work is None: - cpu_work = {} - for task in self.workflow.tasks.values(): - if task.category not in cpu_work or task.runtime > cpu_work[task.category]: - cpu_work[task.category] = task.runtime - for key in cpu_work.keys(): - cpu_work[key] *= 1000 + cpu_work = {task_type: runtime * 1000 for task_type, runtime in task_max_runtimes.items()} + if not max_runtime: + self.logger.warning("The workflow tasks have no runtimes (e.g., the workflow instance has no " + "execution data): the benchmark tasks will not perform any CPU work " + "unless cpu_work is provided.") cores, lock = self._creating_lock_files(lock_files_folder) - task_max_runtimes = {} - for task in self.workflow.tasks.values(): - if task.category not in task_max_runtimes or task.runtime > task_max_runtimes[task.category]: - task_max_runtimes[task.category] = task.runtime - max_runtime = max(runtime for runtime in task_max_runtimes.values()) - for task in self.workflow.tasks.values(): - runtime_factor = task.runtime / max_runtime - task_runtime_factor = task.runtime / task_max_runtimes[task.category] + # all runtimes are zero if the workflow has no execution data + runtime_factor = task.runtime / max_runtime if max_runtime else 1 + task_max_runtime = task_max_runtimes[self._task_type(task)] + task_runtime_factor = task.runtime / task_max_runtime if task_max_runtime else 1 # scale argument parameters to achieve a runtime distribution - task_percent_cpu = percent_cpu[task.category] * task_runtime_factor if isinstance(percent_cpu, dict) else percent_cpu * runtime_factor + task_percent_cpu = self._task_value(percent_cpu, task) * task_runtime_factor if isinstance(percent_cpu, dict) else percent_cpu * runtime_factor task_cores = int(10 * task_percent_cpu) # set number of cores to cpu threads in wfbench task_percent_cpu = max(0.1, task_percent_cpu) # set minimum to 0.1 which is equivalent to 1 thread in wfbench task_percent_cpu = round(task_percent_cpu, 2) if cpu_work is not None: - task_cpu_work = cpu_work[task.category] * task_runtime_factor if isinstance(cpu_work, dict) else cpu_work * runtime_factor + task_cpu_work = self._task_value(cpu_work, task) * task_runtime_factor if isinstance(cpu_work, dict) else cpu_work * runtime_factor task_cpu_work = int(task_cpu_work) else: task_cpu_work = None if gpu_work is not None: - task_gpu_work = gpu_work[task.category] * task_runtime_factor if isinstance(gpu_work, dict) else gpu_work * runtime_factor + task_gpu_work = self._task_value(gpu_work, task) * task_runtime_factor if isinstance(gpu_work, dict) else gpu_work * runtime_factor task_gpu_work = int(task_gpu_work) else: task_gpu_work = None task_memory = int(mem * runtime_factor) if mem else None self._set_argument_parameters( task, - task_percent_cpu, - task_cpu_work, - task_gpu_work, - num_chunks, - time, - task_memory, - lock_files_folder, - cores, - lock, - rundir + percent_cpu=task_percent_cpu, + cpu_work=task_cpu_work, + gpu_work=task_gpu_work, + time=time, + num_chunks=num_chunks, + mem=task_memory, + lock_files_folder=lock_files_folder, + cores=cores, + lock=lock, + rundir=rundir ) task.cores = task_cores + 1 if task_memory: @@ -213,10 +221,13 @@ def _rename_files_to_wfbench_format(self) -> List[File]: :rtype: List[File] """ new_file_names: Dict = {} + renamed_files: Set[int] = set() task_output_counter = 0 workflow_inputs: List[File] = [] for task in self.workflow.tasks.values(): + # task IDs may contain characters that are not safe in file names (e.g., '/' or ':') + file_prefix = re.sub(r"[^0-9A-Za-z_.-]", "_", task.task_id) output_files = sorted(task.output_files, key=lambda x: -len(x.file_id)) for file in output_files: if file.file_id in new_file_names: @@ -224,19 +235,20 @@ def _rename_files_to_wfbench_format(self) -> List[File]: task_output_counter += 1 # extension = ''.join(pathlib.Path(file.file_id).suffixes) - new_name = f"{task.task_id}_outfile_{task_output_counter:04d}" #{extension} + new_name = f"{file_prefix}_outfile_{task_output_counter:04d}" #{extension} new_file_names[file.file_id] = new_name - for i, item in enumerate(task.args): - if file.file_id in item: - task.args[i] = task.args[i].replace(file.file_id, new_name) file.file_id = new_name + renamed_files.add(id(file)) for task in self.workflow.tasks.values(): input_files = sorted(task.input_files, key=lambda x: -len(x.file_id)) for file in input_files: - org_name = file.file_id + if id(file) in renamed_files: + # file object shared with the task that produces it, already renamed + continue if file.file_id in new_file_names: - # file is an output file of another task and receives the corresponding name + # file is an output file of another task (or an input file for the workflow + # used by another task) and receives the corresponding name file.file_id = new_file_names[file.file_id] else: # file is an input file for the workflow and needs to be generated @@ -245,12 +257,73 @@ def _rename_files_to_wfbench_format(self) -> List[File]: new_name = f"workflow_infile_{len(workflow_inputs):04d}"#{extension} new_file_names[file.file_id] = new_name file.file_id = new_name - for i, item in enumerate(task.args): - if org_name in item: - task.args[i] = task.args[i].replace(org_name, file.file_id) + renamed_files.add(id(file)) + + # rename the files in the input/output files arguments of the tasks (only) + for task in self.workflow.tasks.values(): + task.args = [self._rename_files_in_argument(arg, new_file_names) for arg in task.args] return workflow_inputs + @staticmethod + def _rename_files_in_argument(argument: str, new_file_names: Dict[str, str]) -> str: + """ + Rename the files listed in an '--input-files' or '--output-files' task argument. + + :param argument: A task argument. + :type argument: str + :param new_file_names: The new names of the files, indexed by their current names. + :type new_file_names: Dict[str, str] + + :return: The argument with the renamed files (other arguments are returned unchanged). + :rtype: str + """ + if argument.startswith("--output-files "): + output_files = ast.literal_eval(argument[len("--output-files "):]) + output_files = {new_file_names.get(file_name, file_name): size for file_name, size in output_files.items()} + return f"--output-files {output_files}" + if argument.startswith("--input-files "): + input_files = ast.literal_eval(argument[len("--input-files "):]) + input_files = [new_file_names.get(file_name, file_name) for file_name in input_files] + return f"--input-files {input_files}" + return argument + + @staticmethod + def _task_type(task: Task) -> str: + """ + Get the type of a task, i.e., its category, or its name if the task has no category. + + :param task: The task. + :type task: Task + + :return: The task type. + :rtype: str + """ + return task.category if task.category is not None else task.name + + @staticmethod + def _task_value(values: Union[float, Dict[str, float]], task: Task) -> float: + """ + Get the value of a benchmark parameter for a task, if values are provided per task + category (or per task name, as tasks often do not have a category). + + :param values: A value, or values indexed by task category or task name. + :type values: Union[float, Dict[str, float]] + :param task: The task. + :type task: Task + + :return: The value of the parameter for the task. + :rtype: float + """ + if not isinstance(values, dict): + return values + if task.category is not None and task.category in values: + return values[task.category] + if task.name in values: + return values[task.name] + raise KeyError(f"No value provided for task '{task.task_id}' (category: '{task.category}', " + f"name: '{task.name}').") + def create_benchmark(self, save_dir: pathlib.Path, percent_cpu: Union[float, Dict[str, float]] = 0.6, @@ -301,6 +374,9 @@ def create_benchmark(self, self.recipe.from_num_tasks(self.num_tasks)) self.workflow = generator.build_workflow() self.workflow.name = f"{self.workflow.name.split('-')[0]}-Benchmark" + # the benchmark tasks (i.e., their commands) are part of the execution section + self.workflow.include_execution = True + self.workflow.makespan = 0.0 json_path = save_dir.joinpath( f"{self.workflow.name.lower()}-{self.num_tasks}").with_suffix(".json") @@ -311,16 +387,16 @@ def create_benchmark(self, for task in self.workflow.tasks.values(): self._set_argument_parameters( task, - percent_cpu, - cpu_work, - gpu_work, - time, - num_chunks, - mem, - lock_files_folder, - cores, - lock, - rundir, + percent_cpu=percent_cpu, + cpu_work=cpu_work, + gpu_work=gpu_work, + time=time, + num_chunks=num_chunks, + mem=mem, + lock_files_folder=lock_files_folder, + cores=cores, + lock=lock, + rundir=rundir, ) task.input_files = [] task.output_files = [] @@ -407,7 +483,15 @@ def _set_argument_parameters(self, if self.workflow.workflow_id: params.extend([f"--workflow_id {self.workflow.workflow_id}"]) + # the benchmark has not been executed yet: clear the measured execution data of the task task.runtime = 0 + task.avg_cpu = None + task.bytes_read = None + task.bytes_written = None + task.memory = None + task.energy = None + task.avg_power = None + task.machines = None task.program = "wfbench" task.args = [f"--name {task.task_id}"] @@ -426,10 +510,8 @@ def _generate_task_cpu_params(self, if not cpu_work: return [] - _percent_cpu = percent_cpu[task.category] if isinstance( - percent_cpu, dict) else percent_cpu - _cpu_work = cpu_work[task.category] if isinstance( - cpu_work, dict) else cpu_work + _percent_cpu = self._task_value(percent_cpu, task) + _cpu_work = self._task_value(cpu_work, task) params = [f"--percent-cpu {_percent_cpu}", f"--cpu-work {int(_cpu_work)}"] @@ -444,8 +526,7 @@ def _generate_task_gpu_params(self, task: Task, gpu_work: Union[int, Dict[str, i """ if not gpu_work: return [] - _gpu_work = gpu_work[task.category] if isinstance( - gpu_work, dict) else gpu_work + _gpu_work = self._task_value(gpu_work, task) return [f"--gpu-work {_gpu_work}"] @@ -591,7 +672,7 @@ def generate_input_file(self, path: pathlib.Path) -> None: } for node in workflow.nodes: task: Task = workflow.nodes[node]['task'] - task_type = task.task_id.split("_0")[0] + task_type = task.name for key in inputs.keys(): inputs[key].setdefault(task_type, defaults[key]) diff --git a/wfcommons/wfbench/translator/abstract_translator.py b/wfcommons/wfbench/translator/abstract_translator.py index 629b4b54..b70fdd1c 100644 --- a/wfcommons/wfbench/translator/abstract_translator.py +++ b/wfcommons/wfbench/translator/abstract_translator.py @@ -10,15 +10,16 @@ import logging import os +import re import sys import site import pathlib import shutil import textwrap from abc import ABC, abstractmethod -from typing import Optional, Union +from typing import Dict, Iterable, List, Optional, Tuple, Union -from ...common import Task, Workflow +from ...common import File, Task, Workflow from ...wfinstances.instance import Instance @@ -46,33 +47,21 @@ def __init__(self, instance = Instance(workflow, logger=logger) self.workflow: Workflow = instance.workflow - self.workflow.generate_json() - # find all tasks self.tasks = {} - for task in self.workflow.nodes.data(): - self.tasks[task[0]] = task[1]["task"] - - # find root, parents, and children tasks - self.root_task_names = [] - self.task_parents = {} - self.task_children = {} - for task in self.workflow.workflow_json["workflow"]["specification"]["tasks"]: - if len(task["parents"]) == 0: - if task["id"] not in self.root_task_names: - self.root_task_names.append(task["id"]) - self.task_parents.setdefault(task['id'], []) - else: - for parent in task["parents"]: - self.task_parents.setdefault(task['id'], []) - self.task_parents[task['id']].append(parent) - - if len(task["children"]) == 0: - self.task_children.setdefault(task['id'], []) - else: - for child in task["children"]: - self.task_children.setdefault(task['id'], []) - self.task_children[task['id']].append(child) + for task_id, task in self.workflow.nodes(data="task"): + self.tasks[task_id] = task + + # find root, parents, and children tasks (parents and children are listed in edge order) + self.task_parents = {task_id: [] for task_id in self.tasks} + self.task_children = {task_id: [] for task_id in self.tasks} + for parent, child in self.workflow.edges: + self.task_children[parent].append(child) + self.task_parents[child].append(parent) + self.root_task_names = [task_id for task_id in self.tasks if not self.task_parents[task_id]] + + self._check_benchmark_commands() + self._dependency_markers = None @abstractmethod def translate(self, output_folder: pathlib.Path) -> None: @@ -83,6 +72,206 @@ def translate(self, output_folder: pathlib.Path) -> None: :type output_folder: pathlib.Path """ + def _check_benchmark_commands(self) -> None: + """ + Check that every task has a command to run. The command is optional in WfFormat, but + translators generate applications that run the (WfBench) command of each task. + + :raises ValueError: If some tasks do not have a command. + """ + missing = [task_id for task_id, task in self.tasks.items() if not task.program] + if missing: + listed = ", ".join(f"'{task_id}'" for task_id in missing[:5]) + if len(missing) > 5: + listed += f", ... ({len(missing)} tasks in total)" + raise ValueError(f"Cannot translate workflow '{self.workflow.name}': task(s) {listed} have no command " + f"to run. Translators require a workflow benchmark, in which every task runs " + f"'wfbench'. Create one with WorkflowBenchmark (e.g., " + f"WorkflowBenchmark.create_benchmark_from_synthetic_workflow() for an existing " + f"workflow or instance), and translate the generated benchmark instead.") + + @staticmethod + def _unique_identifiers(names: Dict[str, str], + invalid_chars: str = r"[^0-9A-Za-z_]", + first_char: Optional[str] = r"[A-Za-z_]", + reserved: Iterable[str] = ()) -> Dict[str, str]: + """ + Map names (e.g., task IDs or task names, which may contain characters such as ':', '/', + '#', '.', or '-') to unique identifiers that are valid in the target language. A name that + is already a valid identifier is kept unchanged, so that the translation of a workflow with + well-behaved names does not change. Otherwise, each invalid character is replaced by '_', + the identifier is prefixed with '_' if it does not start with a valid character, and a + numeric suffix is appended if it collides with another identifier. + + :param names: The names to map, indexed by a unique key (e.g., the task ID). + :type names: Dict[str, str] + :param invalid_chars: Regular expression matching the characters not allowed in an identifier. + :type invalid_chars: str + :param first_char: Regular expression matching the allowed first characters (None for any). + :type first_char: Optional[str] + :param reserved: Identifiers that must not be used (e.g., keywords of the target language). + :type reserved: Iterable[str] + + :return: The identifier of each name, indexed by the same keys. + :rtype: Dict[str, str] + """ + invalid_re = re.compile(invalid_chars) + first_re = re.compile(first_char) if first_char else None + reserved = set(reserved) + identifiers = {} + used = set() + + def _is_valid(identifier: str) -> bool: + return (identifier != "" and not invalid_re.search(identifier) and identifier not in reserved + and (first_re is None or first_re.match(identifier) is not None)) + + # valid names keep their own identifier (unless already taken) + for key, name in names.items(): + name = str(name) + if _is_valid(name) and name not in used: + identifiers[key] = name + used.add(name) + + # other names are sanitized and disambiguated deterministically + for key, name in names.items(): + if key in identifiers: + continue + identifier = invalid_re.sub("_", str(name)) + if identifier == "" or (first_re is not None and first_re.match(identifier) is None): + identifier = f"_{identifier}" + candidate = identifier + suffix = 1 + while candidate in used or candidate in reserved: + candidate = f"{identifier}_{suffix}" + suffix += 1 + identifiers[key] = candidate + used.add(candidate) + + return identifiers + + @staticmethod + def _shell_safe_argument(argument: str) -> str: + """ + Make a task argument safe to be used as a word in a shell command. The '--name' value of a + WfBench task is its task ID, which may start with '#' (a shell comment) or '-' (an option + for the argument parser). Such a value is attached to the flag ('--name='), which + does not require any quoting and therefore works in every generated language. + + :param argument: A task argument. + :type argument: str + + :return: The argument to use in a shell command. + :rtype: str + """ + if argument.startswith("--name ") and argument[len("--name "):].startswith(("#", "-")): + return f"--name={argument[len('--name '):]}" + return argument + + def _dependencies_without_files(self) -> List[Tuple[str, str]]: + """ + Find the task dependencies that are not expressed by files, i.e., the (parent, child) + dependencies for which the child does not consume any file produced by the parent. + + :return: List of (parent, child) task IDs. + :rtype: List[Tuple[str, str]] + """ + dependencies = [] + for parent_id, children in self.task_children.items(): + parent_outputs = {file.file_id for file in self.tasks[parent_id].output_files} + for child_id in children: + if not parent_outputs.intersection(file.file_id for file in self.tasks[child_id].input_files): + dependencies.append((parent_id, child_id)) + return dependencies + + def _get_dependency_markers(self) -> Tuple[Dict[str, List[File]], Dict[str, List[File]]]: + """ + For workflow systems that infer task dependencies from files only, create empty marker + files that express the dependencies that are not expressed by files (e.g., in workflows + without files). A marker file is an additional (zero-size) output file of the parent task, + and an additional input file of its children. A task without output files also gets a + marker file, so that it is a target of the workflow. Workflows in which all dependencies + are expressed by files do not get any marker file. + + :return: The additional input files and the additional output files, indexed by task ID. + :rtype: Tuple[Dict[str, List[File]], Dict[str, List[File]]] + """ + if self._dependency_markers is None: + dependencies = self._dependencies_without_files() + marked_tasks = {parent_id for parent_id, _ in dependencies} + marked_tasks.update(task_id for task_id, task in self.tasks.items() if not task.output_files) + markers = self._create_marker_files(marked_tasks) if marked_tasks else {} + + extra_inputs = {task_id: [] for task_id in self.tasks} + for parent_id, child_id in dependencies: + extra_inputs[child_id].append(markers[parent_id]) + extra_outputs = {task_id: [markers[task_id]] if task_id in markers else [] for task_id in self.tasks} + if markers: + self.logger.info(f"Using {len(markers)} empty marker file(s) to express task dependencies " + f"that are not expressed by files.") + self._dependency_markers = (extra_inputs, extra_outputs) + + return self._dependency_markers + + def _create_marker_files(self, task_ids: Iterable[str]) -> Dict[str, File]: + """ + Create a (zero-size) dependency marker file for each task, with a unique file ID. + + :param task_ids: The IDs of the tasks. + :type task_ids: Iterable[str] + + :return: The marker file of each task, indexed by task ID. + :rtype: Dict[str, File] + """ + file_ids = {file.file_id for task in self.tasks.values() for file in task.input_files + task.output_files} + safe_names = self._unique_identifiers({task_id: task_id for task_id in self.tasks}, + invalid_chars=r"[^0-9A-Za-z_.-]", first_char=None) + markers = {} + for task_id in task_ids: + marker_id = f"{safe_names[task_id]}.wfbench_done" + while marker_id in file_ids: + marker_id = f"_{marker_id}" + markers[task_id] = File(marker_id, 0) + return markers + + def _task_input_files(self, task: Task) -> List[File]: + """ + Get the input files of a task, including dependency marker files (see _get_dependency_markers). + + :param task: The task. + :type task: Task + + :return: The input files of the task. + :rtype: List[File] + """ + return task.input_files + self._get_dependency_markers()[0][task.task_id] + + def _task_output_files(self, task: Task) -> List[File]: + """ + Get the output files of a task, including dependency marker files (see _get_dependency_markers). + + :param task: The task. + :type task: Task + + :return: The output files of the task. + :rtype: List[File] + """ + return task.output_files + self._get_dependency_markers()[1][task.task_id] + + def _warn_dependencies_without_files(self, system_name: str) -> None: + """ + Warn that the task dependencies not expressed by files cannot be enforced by a workflow + system that infers task dependencies from files. + + :param system_name: The name of the workflow system. + :type system_name: str + """ + dependencies = self._dependencies_without_files() + if dependencies: + examples = ", ".join(f"'{parent}' -> '{child}'" for parent, child in dependencies[:3]) + self.logger.warning(f"{system_name} infers task dependencies from files: {len(dependencies)} " + f"task dependencies that are not expressed by files (e.g., {examples}) " + f"will not be enforced in the generated workflow.") + def _copy_binary_files(self, output_folder: pathlib.Path) -> None: """ Copy binary files to workflow benchmark's bin folder. @@ -126,8 +315,15 @@ def _generate_input_files(self, output_folder: pathlib.Path) -> None: data_folder = output_folder.joinpath("data") data_folder.mkdir(exist_ok=True) for file in workflow_input_files: - with open(data_folder.joinpath(file.file_id), "wb") as fp: - fp.write(os.urandom(int(file.size))) + # file IDs may contain '/' + file_path = data_folder.joinpath(file.file_id) + file_path.parent.mkdir(parents=True, exist_ok=True) + with open(file_path, "wb") as fp: + fp.write(os.urandom(int(file.size))) + + # wfbench does not create the folders of the output files it writes + for file in output_files: + data_folder.joinpath(file.file_id).parent.mkdir(parents=True, exist_ok=True) def _write_output_file(self, contents: str, output_file_path: pathlib.Path) -> None: """ diff --git a/wfcommons/wfbench/translator/airflow.py b/wfcommons/wfbench/translator/airflow.py index 47570d8d..66056a1a 100644 --- a/wfcommons/wfbench/translator/airflow.py +++ b/wfcommons/wfbench/translator/airflow.py @@ -38,6 +38,9 @@ def __init__(self, self.sanitized_names = {} self.seq_num = 0 + # Airflow task IDs may only contain alphanumeric characters, '_', '.', and '-' + self.airflow_task_ids = self._unique_identifiers({task_id: task_id for task_id in self.tasks}, + invalid_chars=r"[^0-9A-Za-z_.-]", first_char=None) self.script = f""" from __future__ import annotations @@ -71,7 +74,7 @@ def translate(self, output_folder: pathlib.Path, name: Optional[str] = None) -> for task in self.tasks.values(): self.script += f""" {self._sanitize_varname(task.task_id)} = BashOperator( - task_id="{task.task_id}", + task_id="{self.airflow_task_ids[task.task_id]}", bash_command='{self.task_commands[task.task_id]}', ) """ @@ -133,7 +136,7 @@ def _prep_commands(self, output_folder: pathlib.Path) -> None: ast.literal_eval(input_files_arr)] a = f"{flag} {json.dumps(input_files_arr)}" else: - a = a.replace("'", "\"") + a = self._shell_safe_argument(a.replace("'", "\"")) args.append(a) command_str = " ".join([str(program)] + args) diff --git a/wfcommons/wfbench/translator/bash.py b/wfcommons/wfbench/translator/bash.py index 9a28b839..7d0fb615 100644 --- a/wfcommons/wfbench/translator/bash.py +++ b/wfcommons/wfbench/translator/bash.py @@ -109,7 +109,7 @@ def _bash_wftasks_codelines(self) -> list[str]: input_files_arr = json.dumps(input_files_arr).replace('"', '\\"') a = f"{flag} '{input_files_arr}'" - args.append(a) + args.append(self._shell_safe_argument(a)) code = f"bin/{task.program} {' '.join(args)}" diff --git a/wfcommons/wfbench/translator/cwl.py b/wfcommons/wfbench/translator/cwl.py index d0eb6991..08b640ca 100644 --- a/wfcommons/wfbench/translator/cwl.py +++ b/wfcommons/wfbench/translator/cwl.py @@ -80,6 +80,12 @@ def __init__(self, for task_name, level in levels.items(): self.task_level_map[level].append(task_name) + # task IDs are used as CWL step IDs (and to name workflow inputs and log files): they must + # not contain ':' (CURIE), '/' or '#' (URI separators), and must not be parsed as YAML numbers + self.step_ids = self._unique_identifiers({task_id: task_id for task_id in self.tasks}, + invalid_chars=r"[^0-9A-Za-z_.-]", + reserved=["compile_output_files", "compile_log_files"]) + def translate(self, output_folder: pathlib.Path) -> None: # Create output folder output_folder.mkdir(parents=True) @@ -128,7 +134,7 @@ def _parse_steps(self) -> None: input_files_arr = [f"{file}" for file in ast.literal_eval(input_files_arr)] input_files_arr = json.dumps(input_files_arr).replace('"', '\\"') a = f"{flag} '{input_files_arr}'" - args_array.append(a) + args_array.append(self._shell_safe_argument(a)) cmd = f"{task.program} {' '.join(args_array)}" @@ -136,16 +142,18 @@ def _parse_steps(self) -> None: output_files = [ f.file_id for f in task.output_files] + step_id = self.step_ids[task.task_id] if level == 0: - input_files = [f" input_files: {task.task_id}_input"] + # root tasks without input files have no workflow input (see _parse_inputs_outputs) + input_files = [f" input_files: {step_id}_input"] if task.input_files else [] else: input_files = [ " input_files:", " linkMerge: merge_flattened", " source:"] - input_files += [f" - {p}/output_files" for p in self.task_parents[task.task_id]] + input_files += [f" - {self.step_ids[p]}/output_files" for p in self.task_parents[task.task_id]] code = [ - f" {task.task_id}:", + f" {step_id}:", " run: clt/shell.cwl", " in:", ] @@ -155,7 +163,7 @@ def _parse_steps(self) -> None: code += [ f" command: {{default: \"{cmd}\"}}", " step_name:", - f" valueFrom: \"{task.task_id}\"", + f" valueFrom: \"{step_id}\"", f" output_filenames: {{default: {output_files}}}", " out: [" # Completed below ] @@ -169,11 +177,11 @@ def _parse_steps(self) -> None: code[-1] += "output_files]\n" self.cwl_script.extend(code) - output_files_sources.append(f" - {task.task_id}/output_files") + output_files_sources.append(f" - {step_id}/output_files") if self.generate_stdout_files: - log_files_sources.append(f" - {task.task_id}/out") + log_files_sources.append(f" - {step_id}/out") if self.generate_stderr_files: - log_files_sources.append(f" - {task.task_id}/err") + log_files_sources.append(f" - {step_id}/err") code = [ " compile_output_files:", @@ -222,16 +230,20 @@ def _parse_inputs_outputs(self) -> None: yml_written = False for f in task.input_files: if not cwl_written: - self.cwl_script.append(f" {task.task_id}_input:") + self.cwl_script.append(f" {self.step_ids[task.task_id]}_input:") self.cwl_script.append(" type: File[]") cwl_written = True if not yml_written: - self.yml_script.append(f"{task.task_id}_input:") + self.yml_script.append(f"{self.step_ids[task.task_id]}_input:") yml_written = True self.yml_script.append(" - class: File") self.yml_script.append(f" path: data/{f.file_id}") + # A workflow without input files has no inputs + if self.cwl_script[-1] == "\ninputs:": + self.cwl_script[-1] += " []" + # Appending the output to the cwl script code = ["\noutputs:", " data_folder:", @@ -268,14 +280,15 @@ def _write_cwl_files(self, output_folder: pathlib.Path) -> None: updates_shell_cwl += line.replace("#OPTIONAL_STDERR_FILE", "") else: updates_shell_cwl += line - with open(cwl_folder.joinpath(clt_folder / "shell.cwl"), "w", encoding="utf-8") as f: + with open(clt_folder.joinpath("shell.cwl"), "w", encoding="utf-8") as f: f.write(updates_shell_cwl) with open(cwl_folder.joinpath("main.cwl"), "w", encoding="utf-8") as f: f.write("\n".join(self.cwl_script)) with (open(cwl_folder.joinpath("config.yml"), "w", encoding="utf-8")) as f: - f.write("\n".join(self.yml_script)) + # a workflow without input files has an empty input object + f.write("\n".join(self.yml_script) if self.yml_script else "{}\n") def _write_readme_file(self, output_folder: pathlib.Path) -> None: readme_file_path = output_folder.joinpath("README") @@ -285,5 +298,5 @@ def _write_readme_file(self, output_folder: pathlib.Path) -> None: for task_id in self.workflow.roots(): task = self.workflow.tasks[task_id] for input_file in task.input_files: - out.write(f"--{task_id}_input ./data/{input_file} ") + out.write(f"--{self.step_ids[task_id]}_input ./data/{input_file} ") out.write("\n") \ No newline at end of file diff --git a/wfcommons/wfbench/translator/dask.py b/wfcommons/wfbench/translator/dask.py index 1b3da62d..a3a052b6 100644 --- a/wfcommons/wfbench/translator/dask.py +++ b/wfcommons/wfbench/translator/dask.py @@ -114,7 +114,7 @@ def _dask_wftasks_codelines(self, input_files_arr = [str(output_folder.joinpath(f"data/{file}")) for file in ast.literal_eval(input_files_arr)] a = f"{flag} '{json.dumps(input_files_arr)}'" else: - a = a.replace("'", "\"") + a = self._shell_safe_argument(a.replace("'", "\"")) args.append(a) code = [f"WorkflowTask(dag_id = '{task.task_id}',", diff --git a/wfcommons/wfbench/translator/makeflow.py b/wfcommons/wfbench/translator/makeflow.py index a76026f5..2a458a7a 100644 --- a/wfcommons/wfbench/translator/makeflow.py +++ b/wfcommons/wfbench/translator/makeflow.py @@ -15,7 +15,7 @@ from typing import Optional, Union from .abstract_translator import Translator -from ...common import Workflow +from ...common import File, Workflow this_dir = pathlib.Path(__file__).resolve().parent @@ -67,28 +67,33 @@ def _generate_code(self): """ self._script = "# Makeflow workflow specification\n\n" for task_name, task in self.workflow.tasks.items(): + # Makeflow infers task dependencies from files: the input/output files include + # dependency marker files when some dependencies are not expressed by files + input_files = self._task_input_files(task) + output_files = self._task_output_files(task) + make_clause = "" # output files - for output_file in task.output_files: - make_clause += f"data/{output_file.file_id} " + for output_file in output_files: + make_clause += f"{self._makeflow_file_name(output_file)} " make_clause += ": " # input files - for input_file in task.input_files: - make_clause += f"data/{input_file.file_id} " + for input_file in input_files: + make_clause += f"{self._makeflow_file_name(input_file)} " make_clause += "\n" # Command make_clause += "\t" make_clause += task.program + " " input_spec = "\"[" - for file in task.input_files: + for file in input_files: input_spec += f"\\\\\"data/{file.file_id}\\\\\"," - input_spec = input_spec[:-1] + "]\"" + input_spec = input_spec.rstrip(",") + "]\"" output_spec = "\"{" - for file in task.output_files: + for file in output_files: output_spec += f"\\\\\"data/{file.file_id}\\\\\":{str(file.size)}," - output_spec = output_spec[:-1] + "}\"" + output_spec = output_spec.rstrip(",") + "}\"" args = [] for a in task.args: @@ -97,13 +102,48 @@ def _generate_code(self): elif "--input-files" in a: args.append(f"--input-files {input_spec}") else: - args.append(a) + args.append(self._makeflow_argument(a)) + # output files (e.g., dependency marker files) are created by wfbench + if output_files and not any("--output-files" in a for a in task.args): + args.append(f"--output-files {output_spec}") args = " ".join(f"{a}" for a in args) make_clause += args + "\n" self._script += make_clause + "\n\n" return + @staticmethod + def _makeflow_file_name(file: File) -> str: + """ + Get the name of a file in a Makeflow rule. An unquoted '#' starts a comment anywhere in a + Makeflow file, and ':' separates the targets of a rule from its sources. + + :param file: The file. + :type file: File + + :return: The (quoted, if needed) file name. + :rtype: str + """ + file_name = f"data/{file.file_id}" + return f"\"{file_name}\"" if "#" in file_name or ":" in file_name else file_name + + def _makeflow_argument(self, argument: str) -> str: + """ + Get a task argument for a Makeflow command. An unquoted '#' starts a comment anywhere in a + Makeflow command (e.g., in a '--name' value), and quotes are passed to the shell. + + :param argument: A task argument. + :type argument: str + + :return: The (quoted, if needed) argument. + :rtype: str + """ + argument = self._shell_safe_argument(argument) + if "#" in argument: + flag, separator, value = argument.partition(" ") + argument = f"{flag} '{value}'" if separator else f"'{argument}'" + return argument + def _write_readme_file(self, output_folder: pathlib.Path) -> None: """ Write the README file. diff --git a/wfcommons/wfbench/translator/nextflow.py b/wfcommons/wfbench/translator/nextflow.py index 753bb0f5..044cf905 100644 --- a/wfcommons/wfbench/translator/nextflow.py +++ b/wfcommons/wfbench/translator/nextflow.py @@ -9,6 +9,7 @@ # (at your option) any later version. import pathlib +import re import shutil from logging import Logger @@ -22,6 +23,18 @@ this_dir = pathlib.Path(__file__).resolve().parent nextflow_config_file_name = "nextflow-wfcommons.config" +# Groovy keywords, and names used by the generated Nextflow code +GROOVY_RESERVED_NAMES = { + "abstract", "as", "assert", "boolean", "break", "byte", "case", "catch", "char", "class", "const", + "continue", "def", "default", "do", "double", "else", "enum", "extends", "false", "final", "finally", + "float", "for", "goto", "if", "implements", "import", "in", "instanceof", "int", "interface", "long", + "native", "new", "null", "package", "private", "protected", "public", "record", "return", "sealed", + "short", "static", "strictfp", "super", "switch", "synchronized", "this", "threadsafe", "throw", + "throws", "trait", "transient", "true", "try", "var", "void", "volatile", "while", "yield", + "Channel", "channel", "file", "files", "include", "params", "process", "pwd", "workflow", "results", + "inputs", "flowcept", "printUsage", "validateParams" +} + class NextflowTranslator(Translator): """ @@ -56,6 +69,12 @@ def __init__(self, self.out_files = set() self.subworkflows: List[List[Task]] = [] + # task IDs are used as process names (Groovy identifiers), and to name the task scripts + task_ids = {task_id: task_id for task_id in self.tasks} + self.process_names = self._unique_identifiers(task_ids, reserved=GROOVY_RESERVED_NAMES) + self.script_names = self._unique_identifiers(task_ids, invalid_chars=r"[^0-9A-Za-z_.-]", + first_char=None) + def translate(self, output_folder: pathlib.Path) -> None: """ Translate a workflow benchmark description (WfFormat) into a Nextflow workflow application. @@ -116,8 +135,9 @@ def _get_tasks_in_topological_order(self) -> List[Task]: break for potential_task in all_children: num_children = len(self.task_children[potential_task.task_id]) - if not num_children: - self.out_files.add(f"{self.output_folder.absolute()}/data/{potential_task.output_files[0]}") + output_files = self._task_output_files(potential_task) + if not num_children and output_files: + self.out_files.add(f"{self.output_folder.absolute()}/data/{output_files[0]}") if all(parent in sorted_tasks for parent in self._find_parents(potential_task.task_id)): tasks_in_current_level.append(potential_task) levels[current_level] = tasks_in_current_level @@ -136,10 +156,10 @@ def _create_task_script(self, task: Task) -> None: file_count_threshold = 10 input_files_list = [ - f"{self.output_folder.absolute()}/data/{f.file_id}" for f in task.input_files + f"{self.output_folder.absolute()}/data/{f.file_id}" for f in self._task_input_files(task) ] output_files_dict = { - f"{self.output_folder.absolute()}/data/{f.file_id}": f.size for f in task.output_files + f"{self.output_folder.absolute()}/data/{f.file_id}": f.size for f in self._task_output_files(task) } code = "#!/bin/bash\n" @@ -164,10 +184,13 @@ def _create_task_script(self, task: Task) -> None: elif "--input-files" in a: code += f"--input-files {input_arg} " else: - code += f"{a} " + code += f"{self._shell_safe_argument(a)} " + # output files (e.g., dependency marker files) are created by wfbench + if output_files_dict and not any("--output-files" in a for a in task.args): + code += f"--output-files {output_arg} " code += "\n" - script_file_path = self.output_folder.joinpath(f"bin/script_{task.task_id}.sh") + script_file_path = self.output_folder.joinpath(f"bin/script_{self.script_names[task.task_id]}.sh") with open(script_file_path, "w") as out: out.write(code) @@ -210,29 +233,34 @@ def _generate_task_process(self, task: Task) -> tuple[str, str]: code = f"process {function_name}()" + "{\n" code += "\tstoreDir \"${params.pwd_abs}/data\"\n" + # Nextflow infers task dependencies from files: the input/output files include + # dependency marker files when some dependencies are not expressed by files + input_files = self._task_input_files(task) + output_files = self._task_output_files(task) + # Input declaration # has_parents = self._find_parents(task.task_id) # if has_parents: - if len(task.input_files) > 1: + if len(input_files) > 1: # Fan-in: accept a collected list under a single path alias code += "input:\n" code += "\tpath \"input_files\"\n" - elif len(task.input_files) == 1: + elif len(input_files) == 1: # Single input: one path variable code += "input:\n" code += "\tpath input_file\n" # Output declaration - if len(task.output_files) > 0: + if len(output_files) > 0: code += "\toutput:\n" - for output_file in task.output_files: + for output_file in output_files: code += f"\t\tpath \"{output_file.file_id}\"\n" # code += "\t\tstdout\n" # Script code += "\tscript:\n" code += "\t\t\"\"\"\n" - code += f"\t\t${{params.simulate ? 'sleep 1' : \"bash ${{params.pwd_abs}}/bin/script_{task.task_id}.sh\"}}\n" + code += f"\t\t${{params.simulate ? 'sleep 1' : \"bash ${{params.pwd_abs}}/bin/script_{self.script_names[task.task_id]}.sh\"}}\n" # for f in task.output_files: # code += f"\t\techo -n \"${{params.pwd_abs}}/data/{f.file_id}\"\n" code += "\t\t\"\"\"\n" @@ -353,51 +381,69 @@ def _generate_task_call(self, code = "" has_parents = self._find_parents(task.task_id) has_children = self._find_children(task.task_id) + input_files = self._task_input_files(task) if include_comment: root_suffix = " (root)" if not has_parents else "" code += f"\t// Task: {task.task_id}{root_suffix}\n" - if has_parents: - if len(task.input_files) > 1: + if has_parents and input_files: + if len(input_files) > 1: # Fan-in: write dependency keys to a text file and load them at # runtime instead of unrolling one .mix() statement per parent. # This keeps run_module_N's compiled bytecode size independent of # fan-in width, avoiding Groovy's "method too long" (64KB) error. - deps_file = self._write_deps_file(function_name, task.input_files) + deps_file = self._write_deps_file(function_name, input_files) code += (f"\tdef {function_name}_deps = " f"file(\"${{params.pwd_abs}}/deps/{deps_file.name}\").readLines()\n") code += (f"\tdef {function_name}_input = Channel.empty()" f".mix(*{function_name}_deps.collect {{ {inputs_var}[it] }})\n") code += f"\tdef {function_name}_output = {function_name}({function_name}_input.collect())\n" else: - input_file_id = task.input_files[0].file_id - code += f"\tdef {function_name}_output = {function_name}({inputs_var}.{input_file_id})\n" - elif len(task.input_files) > 0: - if len(task.input_files) > 1: + input_file_id = input_files[0].file_id + code += f"\tdef {function_name}_output = {function_name}({self._map_entry(inputs_var, input_file_id)})\n" + elif len(input_files) > 0: + if len(input_files) > 1: code += f"\tdef {function_name}_input = Channel.of(\n" - for f in task.input_files: + for f in input_files: code += f"\t\tfile(params.pwd_abs + '/data/{f.file_id}'),\n" code += f"\t).collect()\n" code += f"\tdef {function_name}_output = {function_name}({function_name}_input)\n" else: - f = task.input_files[0] + f = input_files[0] code += f"\tdef {function_name}_input = Channel.of(file(params.pwd_abs + '/data/{f.file_id}'))\n" code += f"\tdef {function_name}_output = {function_name}({function_name}_input)\n" else: code += f"\tdef {function_name}_output = {function_name}()\n" if has_children: - output_files = task.output_files + output_files = self._task_output_files(task) if len(output_files) == 1: - code += f"\t{results_var}.{output_files[0].file_id} = {function_name}_output\n" + code += f"\t{self._map_entry(results_var, output_files[0].file_id)} = {function_name}_output\n" else: for i, f in enumerate(output_files): - code += f"\t{results_var}.{f.file_id} = {function_name}_output.map{{it[{i}]}}\n" + code += f"\t{self._map_entry(results_var, f.file_id)} = {function_name}_output.map{{it[{i}]}}\n" code += "\n" return code + @staticmethod + def _map_entry(map_var: str, key: str) -> str: + """ + Generate the Groovy code to access an entry of a map (e.g., the channel of a file). + + :param map_var: The name of the map variable. + :type map_var: str + :param key: The key of the entry (e.g., a file ID). + :type key: str + + :return: The code, i.e., a property access if the key is a valid identifier, or a subscript otherwise. + :rtype: str + """ + if re.fullmatch(r"[A-Za-z_][0-9A-Za-z_]*", key) and key not in GROOVY_RESERVED_NAMES: + return f"{map_var}.{key}" + return f"{map_var}['{key}']" + def _write_deps_file(self, function_name: str, input_files: List) -> pathlib.Path: """ Write the list of a fan-in task's dependency keys to a sidecar text file, @@ -591,7 +637,16 @@ def _generate_module_function(self, module_idx: int, tasks: List[Task]) -> str: return code def _sanitize_string(self, string: str) -> str: - return string.replace(".", "_").replace("-", "_") + """ + Get the (unique) process name of a task, i.e., its ID as a valid Groovy identifier. + + :param string: The task ID. + :type string: str + + :return: The process name. + :rtype: str + """ + return self.process_names[string] # ========================================================================= # Single-file mode methods diff --git a/wfcommons/wfbench/translator/parsl.py b/wfcommons/wfbench/translator/parsl.py index c6fd3ef1..17c170f9 100644 --- a/wfcommons/wfbench/translator/parsl.py +++ b/wfcommons/wfbench/translator/parsl.py @@ -8,6 +8,7 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. +import keyword import logging from typing import Union, Optional from collections import defaultdict, deque @@ -63,6 +64,14 @@ def __init__(self, for task_name, level in levels.items(): self.task_level_map[level].append(task_name) + # task IDs are used as Python variable names in the generated code, which must not + # shadow the names used by the generated code itself + reserved = set(keyword.kwlist) | {"parsl", "generic_shell_app", "get_parsl_files", "file_map", + "current_workdir", "File", "Path", "task_arr", "print", + "Exception"} + self.task_varnames = self._unique_identifiers({task_id: task_id for task_id in self.tasks}, + reserved=reserved) + def translate(self, output_folder: pathlib.Path) -> None: """ Translate a workflow benchmark description (WfFormat) into an actual workflow application. @@ -120,25 +129,28 @@ def _parsl_wftasks_codelines(self) -> list[str]: flag, input_files_arr = a.split(" ", 1) input_files_arr = json.dumps(ast.literal_eval(input_files_arr)).replace('"', '\\"') a = f"{flag} '{input_files_arr}'" - args.append(a) + args.append(self._shell_safe_argument(a)) args = " ".join(args) + varname = self.task_varnames[task.task_id] input_files = [f"{i.file_id}" for i in task.input_files] - dependency = [f"{p}.outputs" for p in self.task_parents[task.task_id]] + # a parent without output files is a dependency through its app future + dependency = [f"{self.task_varnames[p]}.outputs" if self.tasks[p].output_files + else f"[{self.task_varnames[p]}]" for p in self.task_parents[task.task_id]] if len(dependency) == 0: dependency.append(f"get_parsl_files({input_files})") dependency = " + ".join(dependency) output_files = [f"{o.file_id}" for o in task.output_files] code = [ - f"{task.task_id} = generic_shell_app(\"bin/{task.program} {args}\",", + f"{varname} = generic_shell_app(\"bin/{task.program} {args}\",", f" inputs={dependency},", f" outputs=get_parsl_files({output_files},", " True),", - f" stdout=\"logs/{task.task_id}_stdout.txt\",", - f" stderr=\"logs/{task.task_id}_stderr.txt\")", - f"task_arr.append({task.task_id})\n", + f" stdout=\"logs/{varname}_stdout.txt\",", + f" stderr=\"logs/{varname}_stderr.txt\")", + f"task_arr.append({varname})\n", ] codelines.extend(code) diff --git a/wfcommons/wfbench/translator/pegasus.py b/wfcommons/wfbench/translator/pegasus.py index 348fd5b5..2eb7d71c 100644 --- a/wfcommons/wfbench/translator/pegasus.py +++ b/wfcommons/wfbench/translator/pegasus.py @@ -39,6 +39,15 @@ def __init__(self, self.tasks_map = {} self.task_counter = 1 + # Pegasus job IDs may only contain alphanumeric characters, '_', and '-', and job names + # (i.e., transformation names) are also used by Pegasus to name the DAGMan jobs + pegasus_name_chars = r"[^0-9A-Za-z_-]" + self.job_ids = self._unique_identifiers({task_id: task_id for task_id in self.tasks}, + invalid_chars=pegasus_name_chars, first_char=None) + task_names = {task.name: task.name for task in self.tasks.values()} + self.transformation_names = self._unique_identifiers(task_names, invalid_chars=pegasus_name_chars, + first_char=None) + def translate(self, output_folder: pathlib.Path, tasks_priorities: Optional[Dict[str, int]] = None) -> None: """ Translate a workflow benchmark description (WfFormat) into a Pegasus workflow application. @@ -60,7 +69,7 @@ def translate(self, output_folder: pathlib.Path, tasks_priorities: Optional[Dict transformations.append(task.name) self.script += "if transformation_path is None:\n" \ f" raise RuntimeError('Unable to find {task.program}')\n" \ - f"transformation = Transformation('{task.name}', site='local',\n" \ + f"transformation = Transformation('{self.transformation_names[task.name]}', site='local',\n" \ f" pfn=transformation_path,\n" \ " is_stageable=True)\n" \ "transformation.add_env(PATH='/usr/bin:/bin:.')\n" \ @@ -111,7 +120,7 @@ def _add_task(self, task_name: str, parent_task: Optional[str] = None, tasks_pri if task_name not in self.parsed_tasks: task = self.tasks[task_name] job_name = f"job_{self.task_counter}" - self.script += f"{job_name} = Job('{task.name}', _id='{task_name}')\n" \ + self.script += f"{job_name} = Job('{self.transformation_names[task.name]}', _id='{self.job_ids[task_name]}')\n" \ f"task_output_files.setdefault('{job_name}', [])\n" # task priority @@ -125,7 +134,7 @@ def _add_task(self, task_name: str, parent_task: Optional[str] = None, tasks_pri input_spec = "\"[" for f in task.input_files: input_spec += f"\\\\\"{f.file_id}\\\\\"," - input_spec = input_spec[:-1] + "]\"" + input_spec = input_spec.rstrip(",") + "]\"" # output files output_spec = "\"{" @@ -136,7 +145,7 @@ def _add_task(self, task_name: str, parent_task: Optional[str] = None, tasks_pri f"task_output_files['{job_name}'].append(out_file_{self.task_counter})\n" \ f"{job_name}.add_outputs(out_file_{self.task_counter}, " \ f"stage_out={stage_out}, register_replica={stage_out})\n" - output_spec = output_spec[:-1] + "}\"" + output_spec = output_spec.rstrip(",") + "}\"" # arguments args = [] @@ -146,7 +155,7 @@ def _add_task(self, task_name: str, parent_task: Optional[str] = None, tasks_pri elif "--input-files" in a: args.append(f"--input-files {input_spec}") else: - args.append(a) + args.append(self._shell_safe_argument(a)) args = ", ".join(f"'{a}'" for a in args) self.script += f"{job_name}.add_args({args})\n" diff --git a/wfcommons/wfbench/translator/pycompss.py b/wfcommons/wfbench/translator/pycompss.py index d3056f88..6b619117 100644 --- a/wfcommons/wfbench/translator/pycompss.py +++ b/wfcommons/wfbench/translator/pycompss.py @@ -8,6 +8,7 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. +import keyword import pathlib import ast from logging import Logger @@ -38,6 +39,16 @@ def __init__(self, self.output_files_map = {} self.output_folder = pathlib.Path("") + # task names (followed by a task number) are used as Python function names + function_names = {task.task_id: f"{task.name}{task_number}" + for task_number, task in enumerate(self.tasks.values(), start=1)} + reserved = set(keyword.kwlist) | {"os", "task", "constraint", "binary", "mpi", "compss_open", + "compss_wait_on", "compss_barrier", "main_program"} + self.function_names = self._unique_identifiers(function_names, reserved=reserved) + + # task dependencies are inferred from files + self._warn_dependencies_without_files("PyCOMPSs") + def translate(self, output_folder: pathlib.Path) -> None: """ Translate a workflow benchmark description (WfFormat) into an actual workflow application. @@ -72,7 +83,7 @@ def _pycompss_code(self) -> None: all_pycompss_tasks_as_functions = {} task_number = 1 for task in self.tasks.values(): - function_name = f"{task.name}{task_number}" + function_name = self.function_names[task.task_id] task_number += 1 is_root_task = True if task.task_id in self.root_task_names else False all_input_files_name = [] @@ -157,7 +168,7 @@ def _pycompss_code(self) -> None: all_task_args += "--input-files " all_task_args += "\\\'[" i = 0 - for input_param in function_parameter_names_file_in.replace(' ', '').split(','): + for input_param in [param for param in function_parameter_names_file_in.replace(' ', '').split(',') if param]: if i == 0: all_task_args += "\\\\\\\"" + "{{"+ input_param +"}}" + "\\\\\\\"" else: @@ -165,7 +176,7 @@ def _pycompss_code(self) -> None: i += 1 all_task_args += "]\\\' " else: - all_task_args += f"{task_arg} " + all_task_args += f"{self._shell_safe_argument(task_arg)} " self.script += f", args='{all_task_args}'" self.script += f")\n" diff --git a/wfcommons/wfbench/translator/snakemake.py b/wfcommons/wfbench/translator/snakemake.py index 37280c96..c98f47d6 100644 --- a/wfcommons/wfbench/translator/snakemake.py +++ b/wfcommons/wfbench/translator/snakemake.py @@ -8,6 +8,7 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. +import keyword import pathlib import shutil @@ -69,16 +70,26 @@ def _generate_code(self): "rule all_wfbench_tasks:\n" "\tinput:\n") + # task IDs are used as rule names, which must be valid Python identifiers + rule_names = self._unique_identifiers({task_id: task_id for task_id in self.workflow.tasks}, + reserved=set(keyword.kwlist) | {"all_wfbench_tasks"}) + self._script = "\n# WfBench task rules\n" for task_name, task in self.workflow.tasks.items(): - rule = f"rule {task_name}:\n" + # Snakemake infers task dependencies from files: the input/output files include + # dependency marker files when some dependencies are not expressed by files + input_files = self._task_input_files(task) + output_files = self._task_output_files(task) + + rule = f"rule {rule_names[task_name]}:\n" # input files - rule += "\tinput:\n" - for input_file in task.input_files: + if input_files: + rule += "\tinput:\n" + for input_file in input_files: rule += f"\t\t\"data/{input_file.file_id}\",\n" # output files rule += "\toutput:\n" - for output_file in task.output_files: + for output_file in output_files: all_rule += f"\t\t\"data/{output_file.file_id}\",\n" rule += f"\t\t\"data/{output_file.file_id}\",\n" # shell @@ -86,14 +97,14 @@ def _generate_code(self): rule += "\t\t'" + task.program + " '\n" input_spec = "\\'[" - for file in task.input_files: + for file in input_files: input_spec += f"\"data/{file.file_id}\"," - input_spec = input_spec[:-1] + "]\\'" + input_spec = input_spec.rstrip(",") + "]\\'" output_spec = "\\'{{" - for file in task.output_files: + for file in output_files: output_spec += f"\"data/{file.file_id}\":{str(file.size)}," - output_spec = output_spec[:-1] + "}}\\'" + output_spec = output_spec.rstrip(",") + "}}\\'" args = [] for a in task.args: @@ -102,7 +113,10 @@ def _generate_code(self): elif "--input-files" in a: args.append(f"--input-files {input_spec}") else: - args.append(a) + args.append(self._shell_safe_argument(a)) + # output files (e.g., dependency marker files) are created by wfbench + if output_files and not any("--output-files" in a for a in task.args): + args.append(f"--output-files {output_spec}") for a in args: rule += "\t\t'" + a + " '\n" diff --git a/wfcommons/wfbench/translator/streamflow.py b/wfcommons/wfbench/translator/streamflow.py index c165ddf1..b3632624 100644 --- a/wfcommons/wfbench/translator/streamflow.py +++ b/wfcommons/wfbench/translator/streamflow.py @@ -13,7 +13,7 @@ import pathlib import ast import json -from typing import Union, Optional +from typing import Dict, Union, Optional from collections import defaultdict, deque @@ -62,8 +62,8 @@ def translate(self, output_folder: pathlib.Path) -> None: # Generate the streamflow.yml file self._generate_streamflow_file(output_folder) - # Generate the inputs.yml file - self._generate_inputs_file(output_folder) + # Generate the inputs.yml file (with the workflow input names of the CWL workflow) + self._generate_inputs_file(output_folder, cwl_translator.step_ids) # Generate the README file self._write_readme_file(output_folder) @@ -71,16 +71,24 @@ def translate(self, output_folder: pathlib.Path) -> None: def _generate_streamflow_file(self, output_folder: pathlib.Path) -> None: shutil.copy(this_dir.joinpath("templates/streamflow/streamflow.yml"), output_folder) - def _generate_inputs_file(self, output_folder: pathlib.Path) -> None: + def _generate_inputs_file(self, output_folder: pathlib.Path, step_ids: Dict[str, str]) -> None: file_path = output_folder.joinpath("inputs.yml") with open(file_path, "w") as out: + has_inputs = False for task_id in self.workflow.roots(): task = self.workflow.tasks[task_id] - out.write(f"{task.task_id}_input:\n") + # root tasks without input files have no workflow input + if not task.input_files: + continue + has_inputs = True + out.write(f"{step_ids[task.task_id]}_input:\n") for f in task.input_files: out.write(f" - class: File\n") out.write(f" path: ./data/{f}\n") + if not has_inputs: + # a workflow without input files has an empty input object + out.write("{}\n") diff --git a/wfcommons/wfbench/translator/swift_t.py b/wfcommons/wfbench/translator/swift_t.py index 407b4afa..ba63507f 100644 --- a/wfcommons/wfbench/translator/swift_t.py +++ b/wfcommons/wfbench/translator/swift_t.py @@ -41,10 +41,14 @@ def __init__(self, self.categories_input = {} self.parsed_tasks = set() self.files_map = {} + self.files_category = {} self.tasks_map = {} self.cmd_counter = 1 self.out_files = set() + # task names (i.e., categories) are used to name Swift/T variables and files + self.category_ids = self._unique_identifiers({task.name: task.name for task in self.tasks.values()}) + # find applications self.apps = set() for task in self.tasks.values(): @@ -53,13 +57,16 @@ def __init__(self, out_count = 0 for file in task.output_files: - self.files_map[file.file_id] = f"{task.name}__out" + self.files_map[file.file_id] = f"{self.category_ids[task.name]}__out" + self.files_category[file.file_id] = self.category_ids[task.name] out_count += 1 if out_count > 1: - self.logger.error( - "Swift/T does not allow an application to have multiple outputs.") - exit(1) + raise ValueError(f"Swift/T does not allow an application to have multiple outputs: task " + f"'{task.task_id}' has {out_count} output files.") + + # task dependencies are inferred from files + self._warn_dependencies_without_files("The Swift/T translator") def translate(self, output_folder: pathlib.Path) -> None: """ @@ -87,6 +94,7 @@ def translate(self, output_folder: pathlib.Path) -> None: self.script += f"root_in_files[{in_count}] = \"{in_file}\";\n" in_count += 1 self.files_map[file.file_id] = f"ins[{in_count}]" + self.files_category[file.file_id] = f"ins[{in_count}]" self.script += "\n" @@ -160,20 +168,23 @@ def _add_tasks(self, category: str) -> None: num_children = 0 input_files_cat = {} parsed_input_files = set() + category_name = category + category = self.category_ids[category_name] self.script += f"int {category}__out[];\n" for task in self.tasks.values(): - if task.name == category: + if task.name == category_name: # in/output files num_children = len(self.task_children[task.task_id]) input_files = set() prefix = "" + file_size = 0 for file in task.output_files: file_size = file.size for file in task.input_files: - cat_prefix = self.files_map[file.file_id].split("__out")[0] + cat_prefix = self.files_category[file.file_id] if file.file_id not in parsed_input_files: input_files_cat.setdefault(cat_prefix, 0) input_files_cat[cat_prefix] += 1 @@ -209,7 +220,7 @@ def _add_tasks(self, category: str) -> None: cats = " + ".join(f"{k}__out[{v - 1}]" for k, v in input_files_cat.items()) in_str = ", ".join(f"{k}_{v - 1}_output.txt" for k, v in input_files_cat.items()) - if "ins[" in cats: + if not cats or "ins[" in cats: cats = "0" in_str = "" self.script += f"int dep_{self.cmd_counter} = {cats};\n" diff --git a/wfcommons/wfbench/translator/taskvine.py b/wfcommons/wfbench/translator/taskvine.py index e1179eb0..c4280c78 100644 --- a/wfcommons/wfbench/translator/taskvine.py +++ b/wfcommons/wfbench/translator/taskvine.py @@ -128,7 +128,7 @@ def _add_task(self, task_name: str, parent_task: Optional[str] = None) -> list[s f"t_{self.task_counter}.add_input(in_{self.task_counter}_f_{f_counter}, '{file}')\n" f_counter += 1 input_spec += f"\\\\\"{file.file_id}\\\\\"," - input_spec = input_spec[:-1] + "]\"" + input_spec = input_spec.rstrip(",") + "]\"" # output files f_counter = 1 @@ -139,7 +139,7 @@ def _add_task(self, task_name: str, parent_task: Optional[str] = None) -> list[s f"t_{self.task_counter}.add_output(out_{self.task_counter}_f_{f_counter}, '{file}')\n" self.output_files_map[file.file_id] = f"out_{self.task_counter}_f_{f_counter}" f_counter += 1 - output_spec = output_spec[:-1] + "}\"" + output_spec = output_spec.rstrip(",") + "}\"" task_script += f"m.submit(t_{self.task_counter})\n" \ f"print(f'submitted task {{t_{self.task_counter}.id}}: {{t_{self.task_counter}.command}}')\n\n" @@ -152,7 +152,7 @@ def _add_task(self, task_name: str, parent_task: Optional[str] = None) -> list[s elif "--input-files" in a: args.append(f"--input-files {input_spec}") else: - args.append(a) + args.append(self._shell_safe_argument(a)) args = " ".join(f"{a}" for a in args) # write task diff --git a/wfcommons/wfbench/translator/templates/parsl_template.py b/wfcommons/wfbench/translator/templates/parsl_template.py index d5e1c74d..9477c55f 100644 --- a/wfcommons/wfbench/translator/templates/parsl_template.py +++ b/wfcommons/wfbench/translator/templates/parsl_template.py @@ -65,13 +65,24 @@ def generic_shell_app(cmd: str, inputs=None, outputs=None, if outputs is None: outputs = [] + def replace_file_name(cmd: str, file_path: str) -> str: + # File names appear in the command as JSON strings, and may contain '/': replace + # the (quoted) longest trailing part of the file path that appears in the command + parts = Path(file_path).parts + for i in range(1, len(parts)): + file_name = '"' + "/".join(parts[i:]) + '"' + if file_name in cmd: + return cmd.replace(file_name, '"' + file_path + '"') + return cmd + for i in inputs: - input_path = Path(i.filepath) - cmd = cmd.replace(input_path.name, i.filepath) + # inputs may also include app futures (dependencies without files) + if hasattr(i, "filepath"): + cmd = replace_file_name(cmd, i.filepath) for o in outputs: output_path = Path(o.filepath) output_path.parent.mkdir(parents=True, exist_ok=True) - cmd = cmd.replace(output_path.name, o.filepath) + cmd = replace_file_name(cmd, o.filepath) return cmd current_workdir = Path.cwd() diff --git a/wfcommons/wfchef/chef.py b/wfcommons/wfchef/chef.py index 38da0062..ea1a587b 100644 --- a/wfcommons/wfchef/chef.py +++ b/wfcommons/wfchef/chef.py @@ -9,6 +9,7 @@ # (at your option) any later version. import json +import logging import argparse import math import sys @@ -154,13 +155,20 @@ def analyzer_summary(path_to_instances: pathlib.Path) -> Dict: analyzer = InstanceAnalyzer() task_types = set() - for path in path_to_instances.glob("*.json"): + for path in pathlib.Path(path_to_instances).glob("*.json"): instance = Instance(input_instance=path) + # task statistics are built from execution data, which is optional in WfFormat + if "execution" not in instance.instance["workflow"]: + logging.getLogger(__name__).warning(f"Skipping instance without execution data: {path}") + continue analyzer.append_instance(instance) graph = create_graph(path) for node in graph.nodes: task_types.add(graph.nodes[node]["type"]) + if not analyzer.instances: + raise ValueError(f"No workflow instance with execution data found in: {path_to_instances}") + stats_dict = analyzer.build_summary(task_types - {"DST", "SRC"}, include_raw_data=False) return stats_dict @@ -336,6 +344,13 @@ def create_recipe(path_to_instances: Union[str, pathlib.Path], # Note: You'll need to define these paths in your actual code # skeleton_path = pathlib.Path(__file__).parent.joinpath("skeleton") + # Analyze the task statistics first, so that invalid instances are reported before + # any file of the recipe is written + if verbose: + print(f"Analyzing Workflow Statistics") + + stats = analyzer_summary(path_to_instances) + camelname = wf_name[:1].upper() + wf_name[1:] # Create a standalone package name @@ -473,10 +488,6 @@ def create_recipe(path_to_instances: Union[str, pathlib.Path], if verbose: print(f"Created pyproject.toml with entry point: {wf_name}_recipe") - if verbose: - print(f"Analyzing Workflow Statistics") - - stats = analyzer_summary(path_to_instances) recipe_dir.joinpath("task_type_stats.json").write_text(json.dumps(stats)) if verbose: diff --git a/wfcommons/wfchef/utils.py b/wfcommons/wfchef/utils.py index 0dc1ac67..cbfdb367 100644 --- a/wfcommons/wfchef/utils.py +++ b/wfcommons/wfchef/utils.py @@ -10,6 +10,7 @@ import json import pathlib +import re import matplotlib as mpl import matplotlib.patches as mpatches @@ -50,6 +51,25 @@ def create_graph(path: pathlib.Path) -> nx.DiGraph: return create_graph_from_json_object(content) +def _split_task_id(task_id: str) -> Optional[Tuple[str, str]]: + """ + Split a WfChef-friendly task ID (e.g., ``mProject_ID0000001`` or ``mProject_00000001``) + into its task type and numeric suffix. + + :param task_id: task ID. + :type task_id: str + + :return: the task type and the numeric suffix, or None if the ID does not follow + the WfChef naming convention. + :rtype: Optional[Tuple[str, str]] + """ + for separator in ("_ID", "_0"): + parts = task_id.split(separator) + if len(parts) == 2: + return parts[0], parts[1] + return None + + def create_graph_from_json_object(workflow_instance: dict) -> nx.DiGraph: """ Creates a networkX DiGraph from a JSON file in the WfFormat. @@ -66,24 +86,34 @@ def create_graph_from_json_object(workflow_instance: dict) -> nx.DiGraph: graph.add_node("SRC", label="SRC", type="SRC", id="SRC") graph.add_node("DST", label="DST", type="DST", id="DST") + tasks = workflow_instance["workflow"]["specification"]["tasks"] + task_ids = {task["id"] for task in tasks} id_count = 0 - for task in workflow_instance["workflow"]["specification"]["tasks"]: + for task in tasks: + # nodes are keyed by task ID, as task names are arbitrary and may be shared by many tasks + split_id = _split_task_id(task["id"]) # specific for epigenomics -- have to think about how to do it in general if "genome-dax" in workflow_instance["name"]: _type, *_ = task["name"].split("_") - graph.add_node(task["name"], label=_type, type=_type, id=str(id_count)) - id_count += 1 + _id = str(id_count) else: - try: - _type, _id = task["id"].split("_ID") - except ValueError: - _type, _id = task["id"].split("_0") - graph.add_node(task["name"], label=_type, type=_type, id=_id) - + if task["name"] != task["id"]: + # the task name holds the task type (possibly with a WfChef-friendly suffix) + _type = re.sub(r"_ID\d+$", "", task["name"]) + elif split_id: + _type = split_id[0] + else: + _type = task["name"] + _id = split_id[1] if split_id else str(id_count) + id_count += 1 + graph.add_node(task["id"], label=_type, type=_type, id=_id) + + for task in tasks: for parent in task["parents"]: - graph.add_edge(parent, task["name"]) + if parent in task_ids and parent != task["id"]: + graph.add_edge(parent, task["id"]) for node in graph.nodes: diff --git a/wfcommons/wfinstances/__init__.py b/wfcommons/wfinstances/__init__.py index e076efe5..460f5758 100644 --- a/wfcommons/wfinstances/__init__.py +++ b/wfcommons/wfinstances/__init__.py @@ -8,7 +8,8 @@ # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. -from .logs import MakeflowLogsParser, NextflowLogsParser, PegasusLogsParser, HierarchicalPegasusLogsParser, ROCrateLogsParser +from .logs import MakeflowLogsParser, NextflowLogsParser, PegasusLogsParser, HierarchicalPegasusLogsParser, \ + ROCrateLogsParser, SnakemakeLogsParser, StreamflowLogsParser, TaskVineLogsParser from .schema import SchemaValidator from .instance import Instance from .instance_analyzer import InstanceAnalyzer, InstanceElement diff --git a/wfcommons/wfinstances/instance.py b/wfcommons/wfinstances/instance.py index f267ba9e..547d4aff 100644 --- a/wfcommons/wfinstances/instance.py +++ b/wfcommons/wfinstances/instance.py @@ -39,11 +39,9 @@ class Instance: :param input_instance: The JSON instance. :type input_instance: pathlib.Path :param schema_file: The path to the JSON schema that defines the instance. - If no schema file is provided, it will look for a local - copy of the WfFormat, and if not available it will fetch - the latest schema from the - `WfFormat schema GitHub `_ - repository. + If no schema file is provided, it will use the schema of the + `WfFormat release `_ + that matches the schema version written by WfCommons. :type schema_file: Optional[str] :param logger: The logger where to log information/warning or errors. :type logger: Optional[Logger] @@ -67,124 +65,145 @@ def __init__(self, input_instance: pathlib.Path, schema_validator = SchemaValidator(schema_file, logger=logger) schema_validator.validate_instance(self.instance) - # Basic global properties + # Basic global properties (only name, schemaVersion, and workflow are required) self.name: str = self.instance["name"] - self.desc: str = self.instance["description"] - self.created_at: datetime = dateutil.parser.parse( - self.instance["createdAt"]) + self.desc: Optional[str] = self.instance.get("description", None) + self.created_at: Optional[datetime.datetime] = dateutil.parser.parse( + self.instance["createdAt"]) if "createdAt" in self.instance else None self.schema_version: str = self.instance["schemaVersion"] # Runtime system properties - self.runtime_system: Dict[str, str] = { - u: v for u, v in self.instance["runtimeSystem"].items() - } + self.runtime_system: Dict[str, str] = dict(self.instance.get("runtimeSystem", {})) # Author properties - self.author: Dict[str, str] = { - u: v for u, v in self.instance["author"].items() - } + self.author: Dict[str, str] = dict(self.instance.get("author", {})) # Workflow properties + specification: Dict[str, Any] = self.instance["workflow"]["specification"] + execution: Optional[Dict[str, Any]] = self.instance["workflow"].get("execution", None) + # Global properties - self.executed_at: datetime = dateutil.parser.parse( - self.instance["workflow"]["execution"]["executedAt"]) - self.makespan: float = self.instance["workflow"]["execution"]["makespanInSeconds"] + self.executed_at: Optional[datetime.datetime] = dateutil.parser.parse( + execution["executedAt"]) if execution else None + self.makespan: float = execution["makespanInSeconds"] if execution else 0.0 # Machines - if "machines" in self.instance["workflow"]["execution"].keys(): - self.machines: Dict[str, Machine] = { - machine['nodeName']: Machine( - name=machine['nodeName'], - cpu={k: v for k, v in machine['cpu'].items()}, - system=MachineSystem(machine.get('system', None)) if machine.get( - 'system', None) else None, - architecture=machine.get('architecture', None), - memory=machine.get('memoryInBytes', None), - release=machine.get('release', None), - hashcode=machine.get('machine_code', None), - logger=self.logger - ) for machine in self.instance["workflow"]["execution"]["machines"] - } + self.machines: Dict[str, Machine] = { + machine['nodeName']: Machine( + name=machine['nodeName'], + cpu=dict(machine.get('cpu', {})), + system=MachineSystem(machine['system']) if machine.get('system', None) else None, + architecture=machine.get('architecture', None), + memory=machine.get('memoryInBytes', None), + release=machine.get('release', None), + hashcode=machine.get('machine_code', None), + logger=self.logger + ) for machine in (execution.get("machines", []) if execution else []) + } # Files - files_map = {} - for file in self.instance["workflow"]["specification"]["files"]: - files_map[file["id"]] = file["sizeInBytes"] + files_map = {file["id"]: file["sizeInBytes"] for file in specification.get("files", [])} + if "files" not in specification and any( + "inputFiles" in task or "outputFiles" in task for task in specification["tasks"]): + self.logger.warning("The instance declares no list of files: file sizes are set to 0.") + + # Execution information, by task ID + execution_tasks: Dict[str, Dict[str, Any]] = { + t["id"]: t for t in (execution["tasks"] if execution else []) + } - # Tasks - tasks_map = {} - for task in self.instance["workflow"]["specification"]["tasks"]: - # Required arguments are defined in the JSON scheme - # Here name, type and runtime are required - # By default the value is set to None if we do not find the value + # Workflow + self.workflow: Workflow = Workflow( + name=self.name, + description=self.desc, + created_at=self.instance.get("createdAt", None), + executed_at=execution["executedAt"] if execution else None, + makespan=self.makespan, + runtime_system_name=self.runtime_system.get("name", None), + runtime_system_url=self.runtime_system.get("url", None), + runtime_system_version=self.runtime_system.get("version", None), + author_name=self.author.get("name", None), + author_email=self.author.get("email", None), + author_institution=self.author.get("institution", None), + author_country=self.author.get("country", None), + include_execution=execution is not None, + logger=self.logger + ) + # do not make up the optional properties the instance does not declare + if self.desc is None: + self.workflow.description = None + if self.created_at is None: + self.workflow.created_at = None + if not self.author: + self.workflow.author_name = None + if not self.runtime_system: + self.workflow.runtime_system_name = None + elif "url" not in self.runtime_system: + self.workflow.runtime_system_url = None + self.workflow.machines = self.machines + + # Tasks + for task in specification["tasks"]: # Create the list of files associated to this task input_files = [File( file_id=f, - size=files_map[f], + size=files_map.get(f, 0), logger=self.logger ) for f in task.get('inputFiles', [])] output_files = [File( file_id=f, - size=files_map[f], + size=files_map.get(f, 0), logger=self.logger ) for f in task.get('outputFiles', [])] - tasks_map[task['id']] = Task( + t = execution_tasks.get(task["id"], {}) + command = t.get("command", None) + + self.workflow.add_task(Task( name=task['name'], task_id=task['id'], - runtime=0, - category=task.get('category', None), + runtime=t.get('runtimeInSeconds', 0), + category=t.get('category', task.get('category', None)), input_files=input_files, output_files=output_files, + cores=t.get('coreCount', None), + avg_cpu=t.get('avgCPU', None), + bytes_read=t.get('readBytes', None), + bytes_written=t.get('writtenBytes', None), + memory=t.get('memoryInBytes', None), + energy=t.get('energyInKWh', None), + avg_power=t.get('avgPowerInW', None), + priority=t.get('priority', None), + machines=[self._get_machine(machine) for machine in t.get("machines", [])], + program=command.get('program', None) if command else None, + args=command.get('arguments', None) if command else None, + launch_dir=t.get('launchDir', None), + executed_at=t.get('executedAt', None), logger=self.logger - ) - - # Workflow - self.workflow: Workflow = Workflow( - name=self.name, - makespan=self.makespan, - runtime_system_name=self.runtime_system["name"], - runtime_system_url=self.runtime_system["url"], - runtime_system_version=self.runtime_system["version"], - author_name=self.author["name"], - author_email=self.author["email"] - ) - - for t in self.instance["workflow"]["execution"]["tasks"]: - task = tasks_map[t["id"]] - task.runtime=t['runtimeInSeconds'] if 'runtimeInSeconds' in t else 0 - task.cores=t.get('coreCount', None) - task.avg_cpu=t.get('avgCPU', None) - task.bytes_read=t.get('readBytes', None) - task.bytes_written=t.get('writtenBytes', None) - task.memory=t.get('memoryInBytes', None) - task.energy=t.get('energyInMhz', None) - task.avg_power=t.get('avgPowerInMhz', None) - task.priority=t.get('priority', None) - task.start_time=t.get('executedAt', None) - - # Fetch back the machine associated to this task - machines_list = t["machines"] if "machines" in t else [] - machines = [] - for machine in machines_list: - machines.append(self.machines[machine]) - task.machines = machines - - # Fetch the command associated to this task - command = t.get("command", None) - task.program=command.get('program', None) if command else None - task.args=command.get('arguments', None) if command else None - - self.workflow.add_task(task) + )) - for task in self.instance["workflow"]["specification"]["tasks"]: + for task in specification["tasks"]: for parent in task['parents']: self.workflow.add_dependency(parent, task["id"]) self.logger.info( f'Parsed an instance with {len(self.workflow.nodes)} tasks') + def _get_machine(self, machine_name: str) -> Machine: + """ + Get a machine by its node name, creating it if the instance does not declare it. + + :param machine_name: Machine node name. + :type machine_name: str + + :return: The machine. + :rtype: Machine + """ + if machine_name not in self.machines: + self.machines[machine_name] = Machine(name=machine_name, logger=self.logger) + return self.machines[machine_name] + def __iter__(self): """Produce an iterator based on a topological sort (e.g., scheduling order)""" self._n = 0 diff --git a/wfcommons/wfinstances/logs/abstract_logs_parser.py b/wfcommons/wfinstances/logs/abstract_logs_parser.py index a71fbd44..43069963 100644 --- a/wfcommons/wfinstances/logs/abstract_logs_parser.py +++ b/wfcommons/wfinstances/logs/abstract_logs_parser.py @@ -12,9 +12,10 @@ from abc import ABC, abstractmethod from logging import Logger -from typing import Optional +from typing import Any, Dict, Hashable, Optional from ...common.workflow import Workflow +from ...utils import sanitize_wfformat_id class LogsParser(ABC): @@ -49,6 +50,37 @@ def __init__(self, self.schema_version = None self.executed_at = None self.makespan = None + # WfFormat file IDs, by file key (and the reverse mapping, to detect collisions) + self._file_ids: Dict[Hashable, str] = {} + self._file_keys: Dict[str, Hashable] = {} + + def _make_file_id(self, file_name: Any, key: Optional[Hashable] = None) -> str: + """ + Build a WfFormat-compliant file ID from a file name found in the logs. The same file + (identified by its key) always gets the same ID, while distinct files whose sanitized + names collide get distinct IDs (suffixed with "_2", "_3", etc.). + + :param file_name: The file name found in the logs. + :type file_name: Any + :param key: A key that uniquely identifies the file in the logs (defaults to the file name). + :type key: Optional[Hashable] + + :return: A file ID that matches the WfFormat ID pattern. + :rtype: str + """ + key = str(file_name) if key is None else key + if key not in self._file_ids: + base_id = sanitize_wfformat_id(file_name) + file_id = base_id + suffix = 1 + while file_id in self._file_keys: + suffix += 1 + file_id = f"{base_id}_{suffix}" + if file_id != str(file_name): + self.logger.debug(f"File '{file_name}' is identified as '{file_id}'.") + self._file_ids[key] = file_id + self._file_keys[file_id] = key + return self._file_ids[key] @abstractmethod def build_workflow(self, workflow_name: Optional[str] = None) -> Workflow: diff --git a/wfcommons/wfinstances/logs/makeflow.py b/wfcommons/wfinstances/logs/makeflow.py index a5c9f188..83ffaaee 100644 --- a/wfcommons/wfinstances/logs/makeflow.py +++ b/wfcommons/wfinstances/logs/makeflow.py @@ -12,16 +12,27 @@ import itertools import math import pathlib +import re from datetime import datetime, timezone from logging import Logger -from typing import List, Optional +from typing import Any, Dict, List, Optional from .abstract_logs_parser import LogsParser from ...common.file import File from ...common.machine import Machine from ...common.task import Task, TaskType from ...common.workflow import Workflow +from ...utils import sanitize_wfformat_id + +# Makeflow variable assignment (e.g., "OUTPUT=data/out.txt") and reference (e.g., "$(OUTPUT)" or "${OUTPUT}") +_VARIABLE_ASSIGNMENT = re.compile(r'^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=(.*)$') +_VARIABLE_REFERENCE = re.compile(r'\$(?:\(([A-Za-z_][A-Za-z0-9_]*)\)|\{([A-Za-z_][A-Za-z0-9_]*)\})') + +# Resource Monitor units (cctools uses binary multiples) +_BYTES_UNITS = {'B': 1, 'kB': 1024, 'KB': 1024, 'KiB': 1024, 'MB': 1024 ** 2, 'MiB': 1024 ** 2, + 'GB': 1024 ** 3, 'GiB': 1024 ** 3, 'TB': 1024 ** 4, 'TiB': 1024 ** 4} +_SECONDS_UNITS = {'us': 1e-6, 'ms': 1e-3, 's': 1, 'm': 60, 'h': 3600} class MakeflowLogsParser(LogsParser): @@ -44,7 +55,8 @@ def __init__(self, description: Optional[str] = None, logger: Optional[Logger] = None) -> None: """Create an object of the makeflow log parser.""" - super().__init__('Makeflow', 'http://ccl.cse.nd.edu/software/makeflow/', description, logger) + super().__init__('Makeflow', wms_url='https://ccl.cse.nd.edu/software/makeflow/', description=description, + logger=logger) # Sanity checks if not execution_dir.is_dir(): @@ -78,6 +90,8 @@ def __init__(self, self._resource_monitor_logs_dir: pathlib.Path = resource_monitor_logs_dir self._files_map = {} self._args_map = {} + self._variables: Dict[str, str] = {} + self._machines: Dict[str, Machine] = {} def build_workflow(self, workflow_name: Optional[str] = None) -> Workflow: """ @@ -95,6 +109,7 @@ def build_workflow(self, workflow_name: Optional[str] = None) -> Workflow: self.workflow = Workflow(name=self.workflow_name, description=self.description, runtime_system_name=self.wms_name, + runtime_system_version=self.wms_version, runtime_system_url=self.wms_url) # parse workflow file @@ -119,9 +134,15 @@ def _parse_workflow_file(self) -> None: # print(f"Processing line: {line}") if line.lstrip().startswith('#'): continue + if '\t' not in line and _VARIABLE_ASSIGNMENT.match(line): + # variable assignment + match = _VARIABLE_ASSIGNMENT.match(line) + self._variables[match.group(1)] = self._expand_variables(match.group(2).strip().strip('"')) + continue if ':' in line and '\t' not in line: - outputs = line.split(':')[0].split() - inputs = line.split(':')[1].split() + line = self._expand_variables(line) + outputs = [self._get_file_name(file) for file in line.split(':')[0].split()] + inputs = [self._get_file_name(file) for file in line.split(':')[1].split()] for file in itertools.chain(outputs, inputs): if file not in self._files_map: @@ -129,8 +150,9 @@ def _parse_workflow_file(self) -> None: elif '\t' in line: # task execution command (likely olf here) + line = self._expand_variables(line) prefix = line.replace('./', '').strip().split()[1 if 'LOCAL' in line else 0] - task_name = "{}_ID{:07d}".format(prefix, task_id_counter) + task_name = "{}_ID{:07d}".format(sanitize_wfformat_id(prefix), task_id_counter) # create list of input and output files output_files = self._create_files(outputs, "output", task_name) @@ -156,8 +178,39 @@ def _parse_workflow_file(self) -> None: # adding edges for file in self._files_map: for child in self._files_map[file]['children']: - if self._files_map[file]['task_name']: - self.workflow.add_edge(self._files_map[file]['task_name'], child) + parent = self._files_map[file]['task_name'] + if parent and parent != child: + self.workflow.add_dependency(parent, child) + + def _expand_variables(self, line: str) -> str: + """ + Expand the references to the Makeflow variables defined so far (e.g., $(VAR) or ${VAR}). + + :param line: A line of the makeflow file. + :type line: str + + :return: The line with the known variables expanded. + :rtype: str + """ + def _replace(match: re.Match) -> str: + name = match.group(1) or match.group(2) + return self._variables.get(name, match.group(0)) + + return _VARIABLE_REFERENCE.sub(_replace, line) + + @staticmethod + def _get_file_name(file: str) -> str: + """ + Get the workflow-level name of a file from a makeflow rule, in which a file can be + renamed in the task sandbox (e.g., "local_name->remote_name"). + + :param file: The file as written in a makeflow rule. + :type file: str + + :return: The workflow-level file name. + :rtype: str + """ + return file.split('->')[0] if '->' in file and not file.startswith('->') else file def _create_files(self, files_list: List[str], input_or_output: str, task_name: str) -> List[File]: """ @@ -186,10 +239,10 @@ def _create_files(self, files_list: List[str], input_or_output: str, task_name: elif file_path.is_file(): size = int(file_path.stat().st_size) - file_obj_in = File(file_id=file, + file_obj_in = File(file_id=self._make_file_id(file), size=size, logger=self.logger) - file_obj_out = File(file_id=file, + file_obj_out = File(file_id=self._make_file_id(file), size=size, logger=self.logger) list_files.append(file_obj_in if input_or_output == "input" else file_obj_out) @@ -219,7 +272,7 @@ def _parse_makeflow_log_file(self): self.workflow.makespan = float('%.2f' % ((int(line.split()[2]) - start_time) / 1000000)) elif line.startswith('# FILE') and 'condorlog' not in line: - file_name = line.split()[3] + file_name = self._get_file_name(line.split()[3]) if file_name in self._files_map: size = int(line.split()[5]) for file_obj in self._files_map[file_name]['file']: @@ -232,16 +285,60 @@ def _parse_resource_monitor_logs(self): data = json.load(f) # task - task = self._args_map[data['command'].replace('perl', '').strip()] - task.runtime = float(data['wall_time'][0]) - task.cores = float(data['cores'][0]) - task.memory = int(data['memory'][0]) - task.bytes_read = int(data['bytes_read'][0]) - task.bytes_written = int(data['bytes_written'][0]) - task.avg_cpu = float('%.4f' % (float(data['cpu_time'][0]) / float(data['wall_time'][0]) * 100)) - task.machine = Machine(name=data['host'], - cpu={'coreCount': int(data['machine_cpus'][0]), 'speedInMHz': 0, 'vendor': ''}, - logger=self.logger) - - # workflow - self.workflow.wms_version = data['monitor_version'] + command = data['command'].replace('perl', '').strip() + if command not in self._args_map: + self.logger.warning(f'Unable to find the task of the resource monitor summary {file.name}: ' + f'{command}') + continue + task = self._args_map[command] + task.runtime = self._to_seconds(data['wall_time']) + if 'start' in data: + start_time = datetime.fromtimestamp(self._to_seconds(data['start']), tz=timezone.utc) + task.start_time = start_time.isoformat() + if 'cores' in data: + # a task runs on at least one core (the peak number of cores may be rounded down to 0) + task.cores = max(1, data['cores'][0] if isinstance(data['cores'], list) else data['cores']) + # resource monitor reports memory and I/O in MB + task.memory = self._to_bytes(data['memory']) + task.bytes_read = self._to_bytes(data['bytes_read']) + task.bytes_written = self._to_bytes(data['bytes_written']) + if task.runtime > 0: + task.avg_cpu = float('%.4f' % (self._to_seconds(data['cpu_time']) / task.runtime * 100)) + if data.get('host'): + if data['host'] not in self._machines: + cpu = {'coreCount': int(data['machine_cpus'][0])} if 'machine_cpus' in data else None + self._machines[data['host']] = Machine(name=data['host'], cpu=cpu, logger=self.logger) + task.machines = [self._machines[data['host']]] + + # workflow (resource monitor and makeflow are both part of cctools, e.g., "7.8.2 FINAL") + if data.get('monitor_version'): + self.wms_version = str(data['monitor_version']).split()[0] + self.workflow.runtime_system_version = self.wms_version + + @staticmethod + def _to_bytes(measurement: Any) -> int: + """ + Convert a resource monitor measurement (e.g., [12, "MB"]) into bytes. + + :param measurement: A [value, unit] measurement (a value with no unit is assumed in MB). + :type measurement: Any + + :return: The measurement in bytes. + :rtype: int + """ + value, unit = (measurement[0], measurement[1]) if isinstance(measurement, list) else (measurement, 'MB') + return int(round(float(value) * _BYTES_UNITS.get(unit, _BYTES_UNITS['MB']))) + + @staticmethod + def _to_seconds(measurement: Any) -> float: + """ + Convert a resource monitor measurement (e.g., [1.25, "s"] or [1700000001000000, "us"]) into seconds. + + :param measurement: A [value, unit] measurement (a value with no unit is assumed in seconds). + :type measurement: Any + + :return: The measurement in seconds. + :rtype: float + """ + value, unit = (measurement[0], measurement[1]) if isinstance(measurement, list) else (measurement, 's') + return float(value) * _SECONDS_UNITS.get(unit, 1) diff --git a/wfcommons/wfinstances/logs/nextflow.py b/wfcommons/wfinstances/logs/nextflow.py index fb5eb2f2..a5e8442d 100644 --- a/wfcommons/wfinstances/logs/nextflow.py +++ b/wfcommons/wfinstances/logs/nextflow.py @@ -11,7 +11,7 @@ import glob import json import pathlib -from datetime import datetime, timedelta +from datetime import datetime, timedelta, tzinfo import csv @@ -101,7 +101,8 @@ def build_workflow(self, workflow_name: Optional[str] = None) -> Workflow: # Parse the Nextflow (most recent) execution trace file to create a dict of task runtimes nextflow_execution_trace_file: pathlib.Path = max(self.nextflow_execution_trace_files, key=lambda p: p.stat().st_mtime) - nextflow_task_runtimes = self._load_nextflow_trace(nextflow_execution_trace_file) + nextflow_task_runtimes = self._load_nextflow_trace(nextflow_execution_trace_file, + self._get_run_timezone()) # Create an RO-Create parser @@ -119,9 +120,29 @@ def build_workflow(self, workflow_name: Optional[str] = None) -> Workflow: return ro_crate_parser.build_workflow(workflow_name) - def _load_nextflow_trace(self, trace_file: pathlib.Path) -> dict[str, tuple[str, str]]: + def _get_run_timezone(self) -> Optional[tzinfo]: """ - Parse a Nextflow execution trace file. + Get the timezone of the workflow run from the start time of the (nf-prov generated) RO-Crate + workflow run, since the execution trace timestamps are in the local time of the Nextflow host. + + :return: The timezone of the workflow run, or None if unknown. + :rtype: Optional[tzinfo] + """ + try: + with open(self.execution_dir / 'ro-crate-metadata.json') as f: + graph = json.load(f).get('@graph', []) + for item in graph: + if item.get('@type') == 'CreateAction' and 'Nextflow workflow run' in item.get('name', '') \ + and item.get('startTime'): + return datetime.fromisoformat(item['startTime']).tzinfo + except (OSError, ValueError, AttributeError, TypeError) as e: + self.logger.debug(f"Unable to determine the timezone of the workflow run: {e}") + return None + + def _load_nextflow_trace(self, trace_file: pathlib.Path, + timezone: Optional[tzinfo] = None) -> dict[str, tuple[str, str]]: + """ + Parse a Nextflow execution trace file (whose timestamps are in the given timezone, if any). Returns dict of task_name -> (start_iso, end_iso). """ times = {} @@ -134,7 +155,7 @@ def _load_nextflow_trace(self, trace_file: pathlib.Path) -> dict[str, tuple[str, if not submit_str or not duration_str: continue try: - start = datetime.strptime(submit_str, '%Y-%m-%d %H:%M:%S.%f') + start = datetime.strptime(submit_str, '%Y-%m-%d %H:%M:%S.%f').replace(tzinfo=timezone) duration_sec = self._parse_duration(duration_str) end = start + timedelta(seconds=duration_sec) # Store as ISO strings to match _time_diff expectations @@ -143,13 +164,17 @@ def _load_nextflow_trace(self, trace_file: pathlib.Path) -> dict[str, tuple[str, continue return times - def _parse_duration(self, duration_str: str) -> float: - """Parse Nextflow duration strings like '2m 40s', '519ms', '1h 3m 2s'.""" + @staticmethod + def _parse_duration(duration_str: str) -> float: + """Parse Nextflow duration strings like '2m 40s', '519ms', '1h 3m 2s', '1d 2h 3m'.""" import re total = 0.0 - for value, unit in re.findall(r'(\d+(?:\.\d+)?)\s*(h|m|s|ms)', duration_str): + # 'ms' must be matched before 'm' (otherwise '519ms' would be read as 519 minutes) + for value, unit in re.findall(r'(\d+(?:\.\d+)?)\s*(ms|d|h|m|s)', duration_str): value = float(value) - if unit == 'h': + if unit == 'd': + total += value * 86400 + elif unit == 'h': total += value * 3600 elif unit == 'm': total += value * 60 diff --git a/wfcommons/wfinstances/logs/pegasus.py b/wfcommons/wfinstances/logs/pegasus.py index 8847305c..2a775d33 100644 --- a/wfcommons/wfinstances/logs/pegasus.py +++ b/wfcommons/wfinstances/logs/pegasus.py @@ -18,13 +18,14 @@ from datetime import datetime from logging import Logger -from typing import List, Optional +from typing import Any, Dict, List, Optional from .abstract_logs_parser import LogsParser from ...common.file import File from ...common.machine import Machine, MachineSystem from ...common.task import Task, TaskType from ...common.workflow import Workflow +from ...utils import to_rfc3339 class PegasusLogsParser(LogsParser): @@ -47,7 +48,7 @@ def __init__(self, ignore_auxiliary: Optional[bool] = True, logger: Optional[Logger] = None) -> None: """Create an object of the pegasus log parser.""" - super().__init__('Pegasus', 'https://pegasus.isi.edu', description, logger) + super().__init__('Pegasus', wms_url='https://pegasus.isi.edu', description=description, logger=logger) # Sanity check if not submit_dir.is_dir(): @@ -59,6 +60,7 @@ def __init__(self, self.submit_dir: pathlib.Path = submit_dir self.ignore_auxiliary: Optional[bool] = ignore_auxiliary self.files_map = {} + self.machines: Dict[str, Machine] = {} self._tmp_file = f".pegasus-parser-tmp-{str(uuid.uuid4())}" def build_workflow(self, workflow_name: Optional[str] = None) -> Workflow: @@ -87,6 +89,9 @@ def _parse_braindump(self): """Parse the Pegasus braindump.txt file""" braindump_file = self.submit_dir / 'braindump.txt' braindump_file_yml = self.submit_dir / 'braindump.yml' + wms_version = None + executed_at = None + self.instance_name = None if braindump_file.exists(): with open(braindump_file) as f: @@ -100,15 +105,16 @@ def _parse_braindump(self): elif braindump_file_yml.exists(): with open(braindump_file_yml, "r") as f: data = yaml.safe_load(f) - wms_version = data['planner_version'] - self.instance_name = data['pegasus_wf_name'] - executed_at = data['timestamp'] + wms_version = data.get('planner_version') + self.instance_name = data.get('pegasus_wf_name') + executed_at = data.get('timestamp') else: raise OSError(f'Unable to find braindump files: {braindump_file} or {braindump_file_yml}') # sanity checks if not wms_version: raise ValueError(f"Unable to determine Pegasus version. Likely old logs generated by Pegasus 5.x. No longer supported") + wms_version = str(wms_version) # Remove anything that's not a number try: @@ -125,13 +131,17 @@ def _parse_braindump(self): if not executed_at: self.logger.warning('Unable to determine execution time from "timestamp".') + # the planner timestamp is used as the workflow execution start until the DAGMan log is parsed + self.wms_version = wms_version + self.executed_at = str(executed_at) if executed_at else None + # create base workflow instance object - self.workflow = Workflow(name=self.workflow_name, + self.workflow = Workflow(name=self.workflow_name if self.workflow_name else self.instance_name, description=self.description, runtime_system_name=self.wms_name, - runtime_system_version=wms_version, + runtime_system_version=self.wms_version, runtime_system_url=self.wms_url, - executed_at=executed_at) + executed_at=self.executed_at) def _parse_workflow(self): """Parse the Workflow file.""" @@ -146,31 +156,28 @@ def _parse_workflow(self): data = yaml.load(f, Loader=yaml.SafeLoader) self.logger.info(f'Processing Pegasus workflow file: {workflow_file.name}') - # create base workflow instance object - self.workflow = Workflow(name=self.workflow_name, - description=self.description, - runtime_system_name=self.wms_name, - runtime_system_version=data['pegasus'], - runtime_system_url=self.wms_url, - executed_at=data['x-pegasus']['createdOn']) - + # the base workflow instance object is created from the braindump file, since the + # 'pegasus' key is the version of the workflow YAML format (not the planner version), + # and 'x-pegasus.createdOn' is the workflow creation time (not the execution time) for j in data['jobs']: if j['type'] == "job": task_name = f"{j['name']}_{j['id']}" input_files = [File( - file_id=f['lfn'], + file_id=self._make_file_id(f['lfn']), size=0, logger=self.logger - ) for f in j['uses'] if f['type'] == "input"] + ) for f in j.get('uses', []) if f['type'] == "input"] output_files = [File( - file_id=f['lfn'], + file_id=self._make_file_id(f['lfn']), size=0, logger=self.logger - ) for f in j['uses'] if f['type'] == "output"] + ) for f in j.get('uses', []) if f['type'] == "output"] + # WfFormat task arguments are non-empty strings + args = [str(a) for a in j.get('arguments', []) if a is not None and str(a) != ""] self.workflow.add_task( Task( @@ -179,8 +186,8 @@ def _parse_workflow(self): category=j['name'], task_type=TaskType.COMPUTE, runtime=0, - args=j['arguments'], - cores=0, + args=args, + cores=self._get_job_cores(j), input_files=input_files, output_files=output_files, logger=self.logger @@ -191,6 +198,27 @@ def _parse_workflow(self): else: raise OSError('Unknown task type (not a job nor a sub-workflow') + def _get_job_cores(self, job: Dict[str, Any]) -> float: + """ + Get the number of cores requested by a job (Pegasus 'cores' profile), or 1 if not set. + + :param job: Job from the Pegasus workflow file. + :type job: Dict[str, Any] + + :return: The number of cores requested by the job. + :rtype: float + """ + profiles = job.get('profiles') or {} + cores = (profiles.get('pegasus') or {}).get('cores') + if cores is None: + return 1 + try: + cores = max(1.0, float(cores)) + return int(cores) if cores.is_integer() else cores + except (TypeError, ValueError): + self.logger.warning(f"Job {job.get('id')}: invalid 'cores' profile ({cores}), assuming 1 core.") + return 1 + # def _parse_dax(self): # """Parse the DAX file.""" # dax_list = self._fetch_all_files("dax", "*") @@ -313,7 +341,7 @@ def _parse_dag(self): task_type=TaskType.AUXILIARY, runtime=0, args=[], - cores=0, + cores=1, input_files=[], output_files=[], logger=self.logger @@ -352,19 +380,39 @@ def _parse_dag(self): # parse workflow makespan #TODO: can be replaced with .append_suffix('.dagman.out') in python 3.10 - dagman_file = pathlib.Path(str(dag_file) + '.dagman.out') + self._parse_dagman_output(pathlib.Path(str(dag_file) + '.dagman.out')) + + self.logger.debug(f'Found {num_tasks} jobs.') + + def _parse_dagman_output(self, dagman_file: pathlib.Path) -> None: + """ + Parse the DAGMan output file to obtain the workflow makespan and execution start time. + + :param dagman_file: DAGMan output file (e.g., .dag.dagman.out). + :type dagman_file: pathlib.Path + """ self.logger.debug('Processing Pegasus DAGMan output file.') with open(dagman_file) as f: lines = f.readlines() try: - s = datetime.strptime(' '.join(lines[0].split()[0:2]), '%m/%d %H:%M:%S') - e = datetime.strptime(' '.join(lines[-1].split()[0:2]), '%m/%d %H:%M:%S') - except ValueError: s = datetime.strptime(' '.join(lines[0].split()[0:2]), '%m/%d/%y %H:%M:%S') e = datetime.strptime(' '.join(lines[-1].split()[0:2]), '%m/%d/%y %H:%M:%S') + with_year = True + except ValueError: + s = datetime.strptime(' '.join(lines[0].split()[0:2]), '%m/%d %H:%M:%S') + e = datetime.strptime(' '.join(lines[-1].split()[0:2]), '%m/%d %H:%M:%S') + with_year = False self.workflow.makespan = (e - s).total_seconds() - self.logger.debug(f'Found {num_tasks} jobs.') + if with_year: + # the DAGMan start time is the workflow execution start (DAGMan logs the local + # time of the submit host, whose timezone is given by the planner timestamp) + tzinfo = None + try: + tzinfo = datetime.fromisoformat(to_rfc3339(self.executed_at)).tzinfo if self.executed_at else None + except (ValueError, OverflowError, TypeError): + self.logger.debug(f'Unable to determine the timezone from "timestamp": {self.executed_at}') + self.workflow.executed_at = (s.replace(tzinfo=tzinfo) if tzinfo else s.astimezone()).isoformat() def _parse_meta_file(self, task_name): """ @@ -381,8 +429,10 @@ def _parse_meta_file(self, task_name): with open(meta_list[0]) as metadata: m = json.load(metadata) for f in m: - if f['_id'] not in self.files_map: - self.files_map[f['_id']] = f['_attributes']['size'] + # files are keyed by their (sanitized) WfFormat ID + file_id = self._make_file_id(f['_id']) + if file_id not in self.files_map: + self.files_map[file_id] = f['_attributes']['size'] def _parse_job_output(self, task): """ @@ -443,56 +493,100 @@ def _parse_job_output_latest(self, task: Task, output_file_path: pathlib.Path) - if not (line.startswith('---------------') or "No such file or directory" in line): t.write(line) - with open(tmp_file, 'r') as f: - try: - data = yaml.load(f, Loader=yaml.FullLoader)[0] - except yaml.scanner.ScannerError as e: - print(f"File:{tmp_file.resolve()} => {e}") - exit(-1) - except TypeError as e: - print(f"[Error] File {output_file_path} is probably empty => {e}") - exit(-1) + try: + with open(tmp_file, 'r') as f: + try: + data = yaml.load(f, Loader=yaml.FullLoader)[0] + except yaml.YAMLError as e: + raise ValueError(f"Unable to parse the kickstart record {output_file_path}: {e}") from e + except (TypeError, KeyError, IndexError) as e: + raise ValueError(f"The kickstart record {output_file_path} is probably empty: {e}") from e + self._parse_kickstart_record(task, data) + finally: + tmp_file.unlink() + + def _parse_kickstart_record(self, task: Task, data: Dict[str, Any]) -> None: + """ + Update a task with the information from its kickstart record. + + :param task: Task object. + :type task: Task + :param data: Kickstart invocation record. + :type data: Dict[str, Any] + """ + task.program = data['transformation'] - task.program = data['transformation'] + if data['transformation'].startswith('pegasus:') or task.name.lower().startswith('chmod_'): + task.type = TaskType.AUXILIARY - if data['transformation'].startswith('pegasus:') or task.name.lower().startswith('chmod_'): - task.type = TaskType.AUXILIARY + mainjob = data['mainjob'] + task.runtime = float(mainjob['duration']) - mainjob = data['mainjob'] - task.runtime = float(mainjob['duration']) - task.memory = int(mainjob['usage']['maxrss']) - total_time = float(mainjob['usage']['utime']) + float(mainjob['usage']['stime']) - if total_time > 0: - task.avg_cpu = float('%.4f' % (100 * (total_time / task.runtime))) - - bytes_read = 0 - bytes_written = 0 - - # get job memory and I/O information - if mainjob['procs']: - for p in mainjob['procs']: - bytes_read += max(int(p['rbytes']), int(p['rchar'])) - bytes_written += max(int(p['wbytes']), int(p['wchar'])) - if bytes_read > 0: - task.bytes_read = bytes_read - if bytes_written > 0: - task.bytes_written = bytes_written - - # machine - task.machine = Machine( - name=data['machine']['uname_nodename'], - cpu={ - 'coreCount': data['machine']['cpu_count'], - 'speedInMHz': data['machine']['cpu_speed'], - 'vendor': data['machine']['cpu_vendor'] - }, - system=MachineSystem(data['machine']['uname_system']), - architecture=data['machine']['uname_machine'], - memory=data['machine']['ram_total'], - release=data['machine']['uname_release'] - ) - - tmp_file.unlink() + # task start time + start_time = mainjob.get('start', data.get('start')) + if start_time: + try: + task.start_time = to_rfc3339(start_time) + except (ValueError, OverflowError, TypeError): + self.logger.warning(f"Task {task.task_id}: unable to parse the start time '{start_time}'.") + + # kickstart reports the maximum resident set size in KB + task.memory = int(mainjob['usage']['maxrss']) * 1024 + total_time = float(mainjob['usage']['utime']) + float(mainjob['usage']['stime']) + if total_time > 0 and task.runtime > 0: + task.avg_cpu = float('%.4f' % (100 * (total_time / task.runtime))) + + bytes_read = 0 + bytes_written = 0 + + # get job memory and I/O information + if mainjob.get('procs'): + for p in mainjob['procs']: + bytes_read += max(int(p['rbytes']), int(p['rchar'])) + bytes_written += max(int(p['wbytes']), int(p['wchar'])) + if bytes_read > 0: + task.bytes_read = bytes_read + if bytes_written > 0: + task.bytes_written = bytes_written + + # machine (kickstart reports the total RAM in KB) + machine = data.get('machine') + if machine and machine.get('uname_nodename'): + name = machine['uname_nodename'] + if name not in self.machines: + self.machines[name] = Machine( + name=name, + cpu={ + 'coreCount': machine.get('cpu_count'), + 'speedInMHz': machine.get('cpu_speed'), + 'vendor': machine.get('cpu_vendor') + }, + system=self._get_machine_system(machine.get('uname_system')), + architecture=machine.get('uname_machine'), + memory=int(machine['ram_total']) * 1024 if machine.get('ram_total') else None, + release=machine.get('uname_release'), + logger=self.logger + ) + task.machines = [self.machines[name]] + + @staticmethod + def _get_machine_system(system: Optional[str]) -> Optional[MachineSystem]: + """ + Get the machine system from the kickstart 'uname_system' value. + + :param system: Kickstart 'uname_system' value (e.g., linux, darwin). + :type system: Optional[str] + + :return: The machine system, or None if unknown. + :rtype: Optional[MachineSystem] + """ + system = str(system).lower() if system else None + if system == 'darwin': + system = 'macos' + try: + return MachineSystem(system) + except ValueError: + return None # def _parse_job_output_legacy(self, task: Task, output_file_path: pathlib.Path) -> None: # """ diff --git a/wfcommons/wfinstances/logs/ro_crate.py b/wfcommons/wfinstances/logs/ro_crate.py index c6a84f73..1858b01a 100644 --- a/wfcommons/wfinstances/logs/ro_crate.py +++ b/wfcommons/wfinstances/logs/ro_crate.py @@ -25,6 +25,7 @@ from ...common.machine import Machine from ...common.task import Task, TaskType from ...common.workflow import Workflow +from ...utils import sanitize_wfformat_id class ROCrateLogsParser(LogsParser): @@ -92,6 +93,7 @@ def __init__(self, self.instruments_to_ignore : list[str] = instruments_to_ignore or [] self.task_runtimes : dict[str, tuple[str, str]] = task_runtimes or {} self.file_sizes : dict[str, int] = file_sizes or {} + self._self_connected_instruments : set[str] = set() def build_workflow(self, workflow_name: Optional[str] = None) -> Workflow: @@ -110,6 +112,7 @@ def build_workflow(self, workflow_name: Optional[str] = None) -> Workflow: self.workflow = Workflow(name=self.workflow_name, description=self.description, runtime_system_name=self.wms_name, + runtime_system_version=self.wms_version, runtime_system_url=self.wms_url) with open(self.metadata, 'r') as f: @@ -189,9 +192,9 @@ def _create_tasks(self, create_actions, main_workflow_id): if create_action["name"] in self.steps_to_ignore: continue - # Get all input & output for the create_action - input = [obj['@id'] if isinstance(obj, dict) else obj for obj in create_action['object']] - output = [obj['@id'] if isinstance(obj, dict) else obj for obj in create_action['result']] + # Get all input & output for the create_action (a single object may not be in a list) + input = [obj['@id'] if isinstance(obj, dict) else obj for obj in self._as_list(create_action.get('object'))] + output = [obj['@id'] if isinstance(obj, dict) else obj for obj in self._as_list(create_action.get('result'))] # Filter for actual files input_files = self._filter_file_ids(input) @@ -207,7 +210,7 @@ def _create_tasks(self, create_actions, main_workflow_id): # Using this name/id scheme below is WfChef-friendly task_name = self._sanitize_task_id(create_action['name'] + f"_ID{task_idx:07d}") task_id = task_name - task_idx += task_idx + task_idx += 1 task = Task(#name=create_action['name'], name=task_name, @@ -215,7 +218,7 @@ def _create_tasks(self, create_actions, main_workflow_id): task_id=task_id, task_type=TaskType.COMPUTE, runtime=self._time_diff(start_time, end_time), - executed_at=create_action.get('startTime',''), + executed_at=start_time, input_files=self._get_file_objects(input_files), output_files=self._get_file_objects(output_files), logger=self.logger) @@ -256,7 +259,9 @@ def _add_dependencies(self, files, instruments): for file in files.values(): for parent in file.get('out', []): for child in file.get('in', []): - self.workflow.add_dependency(self.task_id_name_map[parent], self.task_id_name_map[child]) + # a task that reads and writes the same file does not depend on itself + if parent != child: + self.workflow.add_dependency(self.task_id_name_map[parent], self.task_id_name_map[child]) parameter_connections = list(filter((lambda x: x.get('@type') == "ParameterConnection"), self.graph_data)) for parameter_connection in parameter_connections: @@ -272,18 +277,58 @@ def _add_dependencies(self, files, instruments): target = parameter_connection["targetParameter"]["@id"] target = target.rsplit("#", 1)[0] # Trim to get instrument - if source in self.instruments_to_ignore or target in self.instruments_to_ignore: + if self._ignore_instrument(source) or self._ignore_instrument(target): continue # print("source", source, "----> target", target) + # A connection between two parameters of the same instrument does not tell which of + # the tasks that use this instrument depend on each other, and linking all of them + # would create cycles + if source == target: + if source not in self._self_connected_instruments: + self._self_connected_instruments.add(source) + self.logger.warning(f"Ignoring the parameter connections of instrument '{source}' to " + f"itself (consider adding it to the instruments to ignore).") + continue + for parent in instruments.get(source, []): for child in instruments.get(target, []): - self.workflow.add_dependency(self.task_id_name_map[parent], self.task_id_name_map[child]) + parent_id = self.task_id_name_map[parent] + child_id = self.task_id_name_map[child] + # skip dependencies that would create a self-loop or a 2-cycle + if parent_id == child_id or self.workflow.has_edge(child_id, parent_id): + continue + self.workflow.add_dependency(parent_id, child_id) + + def _ignore_instrument(self, instrument: str) -> bool: + """ + Whether an instrument should be ignored, matching the instruments to ignore either by + their exact name or by their base name (e.g., "shell.cwl" matches "file:///wf/shell.cwl"). + + :param instrument: The instrument "@id". + :type instrument: str + + :return: True if the instrument should be ignored. + :rtype: bool + """ + return instrument in self.instruments_to_ignore or \ + instrument.rstrip('/').rsplit('/', 1)[-1] in self.instruments_to_ignore + + @staticmethod + def _as_list(value) -> list: + """Return a JSON-LD value as a list (a single value may not be in a list).""" + if value is None: + return [] + return value if isinstance(value, list) else [value] def _time_diff(self, start_time, end_time): if not start_time or not end_time: return 0.0 - diff = datetime.fromisoformat(end_time) - datetime.fromisoformat(start_time) + try: + diff = datetime.fromisoformat(end_time) - datetime.fromisoformat(start_time) + except (ValueError, TypeError) as e: + self.logger.warning(f"Unable to compute the time difference between {start_time} and {end_time}: {e}") + return 0.0 return diff.total_seconds() def _get_file_objects(self, files): @@ -313,8 +358,9 @@ def _get_file_objects(self, files): if not size: size = self._determine_nextflow_file_size(file) - # Create the file object - self.file_objects[file] = File(file_id=file_name, + # Create the file object (two distinct crate files may share the same name, so + # the file ID is unique per crate "@id") + self.file_objects[file] = File(file_id=self._make_file_id(file_name, key=file), size=size, logger=self.logger) output.append(self.file_objects[file]) @@ -371,7 +417,7 @@ def _filter_file_ids(self, ids): for file_id in file_ids: to_ignore = False for suffix in self.file_extensions_to_ignore: - if self.data_file_id_name_map[file_id].endswith(suffix): + if self.data_file_id_name_map.get(file_id, file_id).endswith(suffix): to_ignore = True break if not to_ignore: @@ -400,9 +446,10 @@ def _resolve_nextflow_task_file_path(self, file_id: str) -> Optional[pathlib.Pat @staticmethod def _sanitize_task_id(task_id: str) -> str: """Replace characters not allowed in WfCommons task IDs.""" - return task_id.replace(' ', '_').replace('(', '_').replace(')', '_') + return sanitize_wfformat_id(task_id) def _process_main_workflow(self, main_workflow): - self.workflow.makespan = self._time_diff(main_workflow['startTime'], main_workflow['endTime']) - self.workflow.executed_at = main_workflow['startTime'] + self.workflow.makespan = self._time_diff(main_workflow.get('startTime'), main_workflow.get('endTime')) + if main_workflow.get('startTime'): + self.workflow.executed_at = main_workflow['startTime'] diff --git a/wfcommons/wfinstances/logs/snakemake.py b/wfcommons/wfinstances/logs/snakemake.py index e3fbb73d..9e8d16af 100644 --- a/wfcommons/wfinstances/logs/snakemake.py +++ b/wfcommons/wfinstances/logs/snakemake.py @@ -25,6 +25,7 @@ from ...common.machine import Machine from ...common.task import Task, TaskType from ...common.workflow import Workflow +from ...utils import to_rfc3339 import sqlite3 class SnakemakeLogsParser(LogsParser): @@ -62,7 +63,8 @@ def __init__(self, ) -> None: """Create an object of the Snakemake parser.""" - super().__init__('Snakemake', 'https://snakemake.readthedocs.io/', description, logger) + super().__init__('Snakemake', wms_version=snakemake_version, wms_url='https://snakemake.readthedocs.io/', + description=description, logger=logger) # Sanity checks if not execution_dir.is_dir(): @@ -82,6 +84,7 @@ def __init__(self, self.task_input_files = {} self.task_output_files = {} self.file_input_output = {} + self.last_task_end_time: Optional[datetime] = None self.rules_to_ignore : list[str] = rules_to_ignore or [] self.path_prefix_rewrite : tuple[str, str] = path_prefix_rewrite or None @@ -103,7 +106,7 @@ def build_workflow(self, workflow_name: Optional[str] = None) -> Workflow: self.workflow = Workflow(name=self.workflow_name, description=self.description, runtime_system_name=self.wms_name, - runtime_system_version=self.snakemake_version, + runtime_system_version=self.wms_version, runtime_system_url=self.wms_url) # Parse the sqlite db for to identify rules @@ -115,19 +118,46 @@ def build_workflow(self, workflow_name: Optional[str] = None) -> Workflow: # Parse the sqlite db for to identify tasks self._create_tasks() - # Set the workflow's makespan + # Set the workflow's execution start time and makespan conn = sqlite3.connect(self.snkmt_db) cursor = conn.cursor() cursor.execute("SELECT * FROM workflows") rows = cursor.fetchall() if len(rows) != 1: - raise SystemError("The SQLite database has more than one entry in the workflows table") - workflow_start_time = datetime.fromisoformat(rows[0][2]).timestamp() - workflow_end_time = datetime.fromisoformat(rows[0][4]).timestamp() - self.workflow.makespan = workflow_end_time - workflow_start_time + raise SystemError(f"The SQLite database should have exactly one entry in the workflows table " + f"(found {len(rows)})") + workflow_start_time = self._parse_time(rows[0][2]) + workflow_end_time = self._parse_time(rows[0][4]) + if workflow_end_time is None: + # the workflow did not complete: use the end of its last task + workflow_end_time = self.last_task_end_time + if workflow_start_time is not None: + self.workflow.executed_at = to_rfc3339(workflow_start_time) + if workflow_end_time is not None: + self.workflow.makespan = (workflow_end_time - workflow_start_time).total_seconds() + else: + self.logger.warning("Unable to determine the workflow execution start time.") return self.workflow + def _parse_time(self, timestamp: Optional[str]) -> Optional[datetime]: + """ + Parse a timestamp from the snkmt database (e.g., "2025-03-01 10:00:00.000000"). + + :param timestamp: The timestamp. + :type timestamp: Optional[str] + + :return: The timestamp as a datetime object, or None if it is missing or invalid. + :rtype: Optional[datetime] + """ + if not timestamp: + return None + try: + return datetime.fromisoformat(str(timestamp)) + except ValueError: + self.logger.warning(f"Unable to parse timestamp '{timestamp}'.") + return None + def _build_task_map(self): conn = sqlite3.connect(self.snkmt_db) cursor = conn.cursor() @@ -184,7 +214,11 @@ def _build_file_map(self): full_path = full_path.split(" (access:")[0].split(" (cached)")[0].strip() if self.path_prefix_rewrite: full_path = full_path.replace(self.path_prefix_rewrite[0], self.path_prefix_rewrite[1]) - file_size = os.path.getsize(f"{full_path}") + try: + file_size = os.path.getsize(f"{full_path}") + except OSError: + self.logger.warning(f"Unable to determine the size of file {full_path}: assuming zero bytes.") + file_size = 0 input_or_output = row[2] if input_or_output == 'INPUT': self.task_input_files[task_idx].append(full_path) @@ -197,8 +231,9 @@ def _build_file_map(self): self.file_input_output[full_path] = [[],[]] self.file_input_output[full_path][1].append(task_idx) - if full_path not in self.file_map: - self.file_objects[full_path] = File(file_id=full_path, size=file_size) + if full_path not in self.file_objects: + self.file_objects[full_path] = File(file_id=self._make_file_id(full_path), size=file_size, + logger=self.logger) def _create_tasks(self): conn = sqlite3.connect(self.snkmt_db) @@ -210,12 +245,18 @@ def _create_tasks(self): if idx not in self.task_map: # One of the to-ignore ones continue cmd_line = row[8] - start_date = row[12] - end_date = row[13] - fmt = "%Y-%m-%d %H:%M:%S.%f" # adjust format to match your strings - t1 = datetime.strptime(start_date, fmt) - t2 = datetime.strptime(end_date, fmt) - elapsed = (t2 - t1).total_seconds() + t1 = self._parse_time(row[12]) + t2 = self._parse_time(row[13]) + start_date = to_rfc3339(t1) if t1 else None + if t1 is not None and t2 is not None: + elapsed = (t2 - t1).total_seconds() + if self.last_task_end_time is None or t2 > self.last_task_end_time: + self.last_task_end_time = t2 + else: + # e.g., a job that failed or did not complete + self.logger.warning(f"Job {idx} ({self.task_map[idx]}) has no start or end time: " + f"assuming a zero runtime.") + elapsed = 0.0 input_files = [self.file_objects[path] for path in self.task_input_files[idx]] output_files = [self.file_objects[path] for path in self.task_output_files[idx]] diff --git a/wfcommons/wfinstances/logs/taskvine.py b/wfcommons/wfinstances/logs/taskvine.py index bc960cb5..da775106 100644 --- a/wfcommons/wfinstances/logs/taskvine.py +++ b/wfcommons/wfinstances/logs/taskvine.py @@ -12,6 +12,7 @@ import itertools import math import pathlib +import re import sys from datetime import datetime, timezone @@ -30,6 +31,11 @@ ## https://github.com/cooperative-computing-lab/taskvine-example-logs?tab=readme-ov-file ### +# FILE file-meta-df1e8b0d0e056c4aedb917abe198a2ff "taskvine_poncho.tar.gz" 0 +# (the quoted file name may contain spaces) +_TASKGRAPH_FILE_LINE = re.compile(r'^FILE\s+(\S+)\s+"(.*)"\s+(\S+)\s*$') + + class TaskVineLogsParser(LogsParser): """ Parse TaskVine logs to generate workflow instance. This parser has some limitations in that @@ -45,20 +51,25 @@ class TaskVineLogsParser(LogsParser): are input to tasks. This argument is the list of names of files that should be ignored in the reconstructed instances, which typically do not include such files at task input. For instance, if reconstructing a workflow from an execution - of a WfBench-generated benchmark, one could pass ["wfbench", "stress-ng"] + of a WfBench-generated benchmark, one could pass ["wfbench", "stress-ng"]. A file + is ignored if its name, or its base name, is in this list. :type filenames_to_ignore: List[str] :param description: Workflow instance description. :type description: Optional[str] :param logger: The logger where to log information/warning or errors (optional). :type logger: Optional[Logger] + :param taskvine_version: The TaskVine (cctools) version used to execute the workflow (optional). + :type taskvine_version: Optional[str] """ def __init__(self, vine_logs_dir: pathlib.Path, filenames_to_ignore: Optional[List[str]] = None, description: Optional[str] = None, - logger: Optional[Logger] = None) -> None: - """Create an object of the makeflow log parser.""" - super().__init__('TaskVine', 'http://https://ccl.cse.nd.edu/software/taskvine/', description, logger) + logger: Optional[Logger] = None, + taskvine_version: Optional[str] = None) -> None: + """Create an object of the TaskVine log parser.""" + super().__init__('TaskVine', wms_version=taskvine_version, wms_url='https://ccl.cse.nd.edu/software/taskvine/', + description=description, logger=logger) # Sanity check if not vine_logs_dir.is_dir(): @@ -78,11 +89,12 @@ def __init__(self, self.taskgraph_file: pathlib.Path = taskgraph_file self.transactions_file: pathlib.Path = transactions_file - self.filenames_to_ignore: set[str] = set(filenames_to_ignore) or set({}) + self.filenames_to_ignore: set[str] = set(filenames_to_ignore or []) self.files_map = {} self.task_command_lines = {} self.task_runtimes = {} + self.task_start_times = {} self.task_input_files = {} self.task_output_files = {} self.known_task_ids = [] @@ -103,6 +115,7 @@ def build_workflow(self, workflow_name: Optional[str] = None) -> Workflow: self.workflow = Workflow(name=self.workflow_name, description=self.description, runtime_system_name=self.wms_name, + runtime_system_version=self.wms_version, runtime_system_url=self.wms_url) # Construct the task command-line array @@ -117,7 +130,7 @@ def build_workflow(self, workflow_name: Optional[str] = None) -> Workflow: # sys.stderr.write("FILEMAP: " + str(self.files_map) + "\n") for file_key in self.files_map.keys(): if not "size" in self.files_map[file_key]: - sys.stderr.write(f"Warning: Could not determine size for file with key {file_key}: assuming zero bytes.\n") + self.logger.warning(f"Could not determine size for file with key {file_key}: assuming zero bytes.") self.files_map[file_key]["size"] = 0 # sys.stderr.write(f"Identified {len(self.files_map)} valid files\n") @@ -129,7 +142,7 @@ def build_workflow(self, workflow_name: Optional[str] = None) -> Workflow: to_remove = [] for task_id in self.known_task_ids: if task_id not in self.task_runtimes.keys(): - sys.stderr.write(f"Warning: Ignoring task {task_id} because runtime could not be determined.\n") + self.logger.warning(f"Ignoring task {task_id} because runtime could not be determined.") to_remove.append(task_id) for victim in to_remove: self.known_task_ids.remove(victim) @@ -146,18 +159,35 @@ def build_workflow(self, workflow_name: Optional[str] = None) -> Workflow: def _construct_task_command_lines(self) -> None: + previous_line = "" with open(self.debug_file) as f: for line in f: if "state change: READY (1) to RUNNING (2)" in line: [task_index] = line[line.find("Task ") + len("Task "):].split()[0:1] command_line = previous_line[previous_line.find("busy on '") + len("busy on '"):-2] self.task_command_lines[int(task_index)] = command_line - executable = command_line.split()[0] - # May not be full-proof in case of commands like "export A=b; executable ..." but - # may help..... - self.filenames_to_ignore.add(executable) + if command_line.split(): + executable = command_line.split()[0] + # May not be full-proof in case of commands like "export A=b; executable ..." but + # may help..... (files are matched by their exact name or base name) + self.filenames_to_ignore.add(executable) + self.filenames_to_ignore.add(pathlib.PurePosixPath(executable).name) previous_line = line + def _is_ignored(self, filename: str) -> bool: + """ + Whether a file should be ignored, i.e., whether its name or base name is one of the + names of files to ignore (e.g., "wfbench" matches "bin/wfbench", but not "wfbench.txt"). + + :param filename: The file name. + :type filename: str + + :return: True if the file should be ignored. + :rtype: bool + """ + return filename in self.filenames_to_ignore or \ + pathlib.PurePosixPath(filename).name in self.filenames_to_ignore + def _construct_file_map(self) -> None: filename_to_key_map = {} @@ -169,12 +199,14 @@ def _construct_file_map(self) -> None: if line.startswith("FILE"): # FILE file-meta-df1e8b0d0e056c4aedb917abe198a2ff "taskvine_poncho.tar.gz" 0 - [ignore, file_key, filename, ignore] = line.split()[:4] - if len(filename) > 1 and filename[0] == "\"" and filename[-1] == "\"": - filename = filename[1:-1] - if self.filenames_to_ignore and any( - ignore_string in filename for ignore_string in self.filenames_to_ignore - ): + match = _TASKGRAPH_FILE_LINE.match(line.strip()) + if match: + file_key, filename = match.group(1), match.group(2) + else: + [ignore, file_key, filename] = line.split()[:3] + if len(filename) > 1 and filename[0] == "\"" and filename[-1] == "\"": + filename = filename[1:-1] + if self._is_ignored(filename): continue if file_key not in self.files_map: self.files_map[file_key] = {"filename": filename} @@ -199,19 +231,29 @@ def _construct_file_map(self) -> None: def _construct_task_runtimes(self) -> None: task_start_times = {} task_end_times = {} + manager_start_time = None + manager_end_time = None - # This method consists only of - + # Timestamps in the transactions log are in microseconds since the epoch with open(self.transactions_file) as f: for line in f: if line[0] == "#": continue + parts = line.split() + if len(parts) > 3 and parts[2] == "MANAGER": + if "START" in parts[3:] and manager_start_time is None: + manager_start_time = int(parts[0]) + elif "END" in parts[3:]: + manager_end_time = int(parts[0]) + continue + if len(parts) < 4 or parts[2] != "TASK": + continue if "RUNNING" in line: - [start_date, ignore, ignore, task_index] = line.split()[0:4] + [start_date, ignore, ignore, task_index] = parts[0:4] if int(task_index) in self.known_task_ids: task_start_times[int(task_index)] = int(start_date) elif "DONE" in line: - [end_date, ignore, ignore, task_index] = line.split()[0:4] + [end_date, ignore, ignore, task_index] = parts[0:4] if int(task_index) in self.known_task_ids: task_end_times[int(task_index)] = int(end_date) @@ -219,6 +261,30 @@ def _construct_task_runtimes(self) -> None: if task_index in task_end_times: self.task_runtimes[task_index] = ( float(task_end_times[task_index] - task_start_times[task_index]) / 1_000_000.0) + self.task_start_times[task_index] = self._to_datetime(task_start_times[task_index]) + + # Workflow execution start and makespan (from the manager start/end, or else from the tasks) + if manager_start_time is None and task_start_times: + manager_start_time = min(task_start_times.values()) + if manager_end_time is None and task_end_times: + manager_end_time = max(task_end_times.values()) + if manager_start_time is not None: + self.workflow.executed_at = self._to_datetime(manager_start_time) + if manager_end_time is not None and manager_end_time >= manager_start_time: + self.workflow.makespan = float(manager_end_time - manager_start_time) / 1_000_000.0 + + @staticmethod + def _to_datetime(timestamp: int) -> str: + """ + Convert a transactions log timestamp (in microseconds since the epoch) into the RFC 3339 format. + + :param timestamp: The timestamp in microseconds. + :type timestamp: int + + :return: The timestamp in the RFC 3339 format. + :rtype: str + """ + return datetime.fromtimestamp(timestamp / 1_000_000.0, tz=timezone.utc).isoformat() def _construct_task_input_output_files(self) -> None: # Initialize all entries @@ -230,8 +296,11 @@ def _construct_task_input_output_files(self) -> None: for line in f: if line.startswith("TASK"): # TASK T23 "__vine_env_task-rnd-twtxpejwzsyiebf/bin/run_in_env" INPUTS task-rnd-twtxpejwzsyiebf file-meta-d7504c061a7afd9401c612b4ac7d6be6 file-meta-baab4e4516c4d93a8fcdcbba1a680af7 file-meta-693cb61fedd032b4ddec444b8cce6c89 file-rnd-fnrudlxsaqmlpqq OUTPUTS file-rnd-pdtdqayfmmxyxyp + # (the quoted command line may contain spaces, so file keys are read after it) parts = line.split() task_key = parts[1] + if '"' in line: + parts = parts[:2] + line[line.rfind('"') + 1:].split() if not task_key.startswith("T"): continue try: @@ -269,7 +338,7 @@ def _construct_workflow(self) -> None: for file_key in self.files_map: filename = self.files_map[file_key]["filename"] file_size = self.files_map[file_key]["size"] - file_object_map[filename] = File(file_id=filename, + file_object_map[filename] = File(file_id=self._make_file_id(filename), size=file_size, logger=self.logger) @@ -277,11 +346,11 @@ def _construct_workflow(self) -> None: task_map = {} for task_id in self.known_task_ids: task_name = "Task_ID{:07d}".format(task_id) - print(f"DEBUGHERE: {task_name} ") task = Task(name=task_name, task_id=task_name, task_type=TaskType.COMPUTE, runtime=self.task_runtimes[task_id], + executed_at=self.task_start_times.get(task_id), program=self.task_command_lines[task_id].split()[0], args=self.task_command_lines[task_id].split()[1:], cores=1, diff --git a/wfcommons/wfinstances/schema.py b/wfcommons/wfinstances/schema.py index 59c28b3c..3aa2ec45 100644 --- a/wfcommons/wfinstances/schema.py +++ b/wfcommons/wfinstances/schema.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright (c) 2020-2024 The WfCommons Team. +# Copyright (c) 2020-2026 The WfCommons Team. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,20 +11,35 @@ import json import jsonschema import logging +import math +import os import pathlib import requests from logging import Logger -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional, Set + +from ..version import __schema_version__ + +SCHEMA_FILE_NAME = "wfcommons-schema.json" +# the schema is fetched from the WfFormat release that matches the version written by WfCommons +SCHEMA_URL = f"https://raw.githubusercontent.com/wfcommons/WfFormat/v{__schema_version__}/{SCHEMA_FILE_NAME}" +SCHEMA_FETCH_TIMEOUT_IN_SECONDS = 30 + +# schemas already loaded by this process, by URL +_schema_cache: Dict[str, Dict[str, Any]] = {} + +# tolerance used when comparing declared metrics against recomputed ones +METRICS_RELATIVE_TOLERANCE = 1e-9 class SchemaValidator: """ Validate JSON files against WfCommons schema (WfFormat). If schema file path - is not provided, it will look for a local copy of the WfFormat schema, and if - not available it will fetch the latest schema from the - `WfFormat schema GitHub `_ - repository. + is not provided, it will use the schema of the + `WfFormat release `_ that matches the + schema version written by WfCommons, which is fetched once and cached in the + user cache directory (e.g., ``~/.cache/wfcommons``). :param schema_file_path: JSON schema file path. :type schema_file_path: Optional[pathlib.Path] @@ -36,6 +51,8 @@ def __init__(self, schema_file_path: Optional[pathlib.Path] = None, logger: Opti """Create an object of the schema validator class.""" self.logger: Logger = logging.getLogger(__name__) if logger is None else logger self.schema = self._load_schema(schema_file_path) + self.errors: List[str] = [] + self.warnings: List[str] = [] def validate_instance(self, data: Dict[str, Any]) -> None: """ @@ -44,92 +61,319 @@ def validate_instance(self, data: Dict[str, Any]) -> None: :param data: Workflow instance in JSON format. :type data: Dict[str, Any] """ + self.errors = [] + self.warnings = [] self._syntax_validation(data) self._semantic_validation(data) - def _load_schema(self, schema_file_path: Optional[pathlib.Path] = None) -> json: + def _load_schema(self, schema_file_path: Optional[pathlib.Path] = None) -> Dict[str, Any]: """ - Load the schema file. If schema file path is not provided, it will look for - a local copy of the WfFormat schema, and if not available it will fetch - the latest schema from the GitHub repository. + Load the schema file. If schema file path is not provided, it will use the schema of the + WfFormat release that matches the schema version written by WfCommons, from the user cache + directory or, if not available, from the WfFormat GitHub repository. :param schema_file_path: JSON schema file path. :type schema_file_path: Optional[pathlib.Path] :return: The JSON schema. - :rtype: json + :rtype: Dict[str, Any] """ if schema_file_path: self.logger.info(f'Using schema file: {schema_file_path}') - return json.loads(open(schema_file_path).read()) - - # looking for local copy of schema file - schema_path = pathlib.Path(f"{pathlib.Path.cwd()}/wfcommons-schema.json") - if schema_path.exists(): - self.logger.info(f'Using schema file: {schema_path}') - return json.loads(open(schema_path).read()) - - # fetching latest schema file from GitHub repository - url = 'https://raw.githubusercontent.com/wfcommons/wfformat/master/wfcommons-schema.json' - response = requests.get(url) - schema = json.loads(response.content) - with open(schema_path, 'w') as outfile: - json.dump(schema, outfile) - self.logger.info(f"Using latest schema file from GitHub repository (saved local copy into {schema_path}).") + with open(schema_file_path, encoding="utf-8") as f: + return json.load(f) + + if SCHEMA_URL in _schema_cache: + return _schema_cache[SCHEMA_URL] + + # copy of the schema cached by a previous run + cache_path = _cache_dir().joinpath(f"wfformat-v{__schema_version__}-{SCHEMA_FILE_NAME}") + try: + with open(cache_path, encoding="utf-8") as f: + schema = json.load(f) + if _accepts_schema_version(schema): + self.logger.debug(f'Using schema file: {cache_path}') + _schema_cache[SCHEMA_URL] = schema + return schema + except (OSError, ValueError): + pass + + # fetching the schema file from the WfFormat GitHub repository + try: + response = requests.get(SCHEMA_URL, timeout=SCHEMA_FETCH_TIMEOUT_IN_SECONDS) + response.raise_for_status() + schema = response.json() + except (requests.RequestException, ValueError) as e: + raise RuntimeError(f"Unable to fetch the WfFormat {__schema_version__} schema from {SCHEMA_URL} ({e}). " + f"Provide a local copy of the schema file instead.") from e + if not _accepts_schema_version(schema): + raise RuntimeError(f"The schema fetched from {SCHEMA_URL} does not accept version {__schema_version__}.") + self.logger.info(f"Using WfFormat {__schema_version__} schema from {SCHEMA_URL}") + + try: + cache_path.parent.mkdir(parents=True, exist_ok=True) + with open(cache_path, "w", encoding="utf-8") as f: + json.dump(schema, f) + except OSError as e: + self.logger.debug(f"Unable to cache the schema file into {cache_path}: {e}") + + _schema_cache[SCHEMA_URL] = schema return schema - def _syntax_validation(self, data: Dict[str, Any]): + def _error(self, message: str) -> None: + self.errors.append(message) + self.logger.error(message) + + def _warning(self, message: str) -> None: + self.warnings.append(message) + self.logger.warning(message) + + def _syntax_validation(self, data: Dict[str, Any]) -> None: """ Validate the JSON workflow execution instance against the schema. :param data: Workflow instance in JSON format. :type data: Dict[str, Any] """ - v = jsonschema.Draft4Validator(self.schema) + # honor the draft the schema declares, rather than assuming one + validator_class = jsonschema.validators.validator_for(self.schema) + validator = validator_class(self.schema, format_checker=jsonschema.FormatChecker()) + has_error = False - for error in sorted(v.iter_errors(data), key=str): - msg = ' > '.join([str(e) for e in error.relative_path]) \ - + ': ' + error.message - self.logger.error(msg) - has_error = True + for error in sorted(validator.iter_errors(data), key=str): + location = ' > '.join([str(e) for e in error.absolute_path]) or "" + msg = f"{location}: {error.message}" + # "format" is an annotation rather than an assertion in JSON Schema + if error.validator == "format": + self._warning(msg) + else: + self._error(msg) + has_error = True if has_error: raise RuntimeError('The workflow instance has syntax errors.') - def _semantic_validation(self, data: Dict[str, Any]): + def _semantic_validation(self, data: Dict[str, Any]) -> None: """ Validate the semantics of the JSON workflow execution instance. :param data: Workflow instance in JSON format. :type data: Dict[str, Any] """ - has_error = False + errors_before = len(self.errors) + + workflow = data["workflow"] + specification = workflow["specification"] + execution = workflow.get("execution") - machine_ids = [] - if "machines" in data["workflow"]["execution"]: - for m in data["workflow"]["execution"]["machines"]: - machine_ids.append(m["nodeName"]) + tasks = specification["tasks"] + task_ids = self._collect_ids(tasks, "id", "task") + file_ids = self._collect_ids(specification.get("files", []), "id", "file") + + self._validate_dependencies(tasks, task_ids) + self._validate_file_references(tasks, specification, file_ids) + level_widths = self._compute_level_widths(tasks) + self._validate_specification_metrics(specification, level_widths) + + if execution is None: + self.logger.debug("Skipping execution processing.") else: - self.logger.debug("Skipping machines processing.") - - tasks_ids = [] - for j in data["workflow"]["execution"]["tasks"]: - tasks_ids.append(j["id"]) - if "machines" in j: - for m in j["machines"]: - if m not in machine_ids: - self.logger.error(f"Machine \"{j['machine']}\" is not declared in the list of machines.") - has_error = True - - # since tasks may be declared out of order, their dependencies are only verified here - for j in data["workflow"]["specification"]["tasks"]: - for p in j["parents"]: - if p not in tasks_ids: - self.logger.error(f"Parent task \"{p}\" is not declared in the list of workflow tasks.") - has_error = True - - self.logger.debug(f'The workflow has {len(tasks_ids)} tasks.') - self.logger.debug(f'The workflow has {len(machine_ids)} machines.') + self._validate_execution(execution, task_ids) + self._validate_execution_metrics(execution) - if has_error: + self.logger.debug(f'The workflow has {len(task_ids)} tasks.') + self.logger.debug(f'The workflow has {len(file_ids)} files.') + + if len(self.errors) > errors_before: raise RuntimeError('The workflow instance has semantic errors.') + + def _collect_ids(self, items: List[Dict[str, Any]], key: str, kind: str) -> Set[str]: + identifiers = set() + for item in items: + identifier = item.get(key) + if identifier is None: + continue + if identifier in identifiers: + self._error(f"The {kind} \"{identifier}\" is declared more than once.") + identifiers.add(identifier) + return identifiers + + def _validate_dependencies(self, tasks: List[Dict[str, Any]], task_ids: Set[str]) -> None: + parents_of = {} + children_of = {} + for task in tasks: + task_id = task["id"] + for kind, key in (("parent", "parents"), ("child", "children")): + references = task.get(key, []) + for reference in references: + if reference == task_id: + self._error(f"Task \"{task_id}\" declares itself as its own {kind}.") + elif reference not in task_ids: + self._error(f"{kind.capitalize()} task \"{reference}\" is not declared in the list " + f"of workflow tasks.") + if len(references) != len(set(references)): + self._warning(f"Task \"{task_id}\" declares the same {kind} task more than once.") + parents_of[task_id] = set(task.get("parents", [])) + children_of[task_id] = set(task.get("children", [])) + + # both ends of every dependency have to agree + for relation, reverse, kind, reverse_kind in ((parents_of, children_of, "parent", "child"), + (children_of, parents_of, "child", "parent")): + for task_id, references in relation.items(): + for reference in references: + if reference != task_id and reference in reverse and task_id not in reverse[reference]: + self._warning(f"Task \"{task_id}\" declares \"{reference}\" as a {kind}, but task " + f"\"{reference}\" does not declare \"{task_id}\" as a {reverse_kind}.") + + def _validate_file_references(self, + tasks: List[Dict[str, Any]], + specification: Dict[str, Any], + file_ids: Set[str]) -> None: + files_declared = "files" in specification + referenced_file_ids = set() + for task in tasks: + for key in ("inputFiles", "outputFiles"): + for file_id in task.get(key, []): + referenced_file_ids.add(file_id) + if files_declared and file_id not in file_ids: + self._error(f"File \"{file_id}\", used by task \"{task['id']}\" as one of its {key}, " + f"is not declared in the list of workflow files.") + + if not files_declared and referenced_file_ids: + self._warning(f"{len(referenced_file_ids)} file(s) are used by tasks while the workflow " + f"declares no list of files.") + + unused_file_ids = file_ids - referenced_file_ids + if unused_file_ids: + self._warning(f"{len(unused_file_ids)} file(s) are declared but not used by any task.") + + def _compute_level_widths(self, tasks: List[Dict[str, Any]]) -> Optional[List[int]]: + """ + Compute the width of each level of the workflow DAG, where a task with no parents + is at level 0 and every other task is at one plus the maximum level of any of its + parents. Reports cycles as errors. + """ + task_ids = [task["id"] for task in tasks] + index_of = {task_id: index for index, task_id in enumerate(task_ids)} + parent_counts = [0] * len(task_ids) + children = [[] for _ in task_ids] + + for task in tasks: + for parent in set(task.get("parents", [])): + if parent in index_of and parent != task["id"]: + parent_counts[index_of[task["id"]]] += 1 + children[index_of[parent]].append(index_of[task["id"]]) + + ready = [index for index, count in enumerate(parent_counts) if count == 0] + level_widths = [] + number_processed = 0 + while ready: + level_widths.append(len(ready)) + number_processed += len(ready) + next_ready = [] + for index in ready: + for child_index in children[index]: + parent_counts[child_index] -= 1 + if parent_counts[child_index] == 0: + next_ready.append(child_index) + ready = next_ready + + if number_processed != len(task_ids): + unresolved = sorted(task_ids[index] for index, count in enumerate(parent_counts) if count > 0) + preview = ", ".join(f"\"{task_id}\"" for task_id in unresolved[:5]) + self._error(f"The workflow graph contains a cycle: {len(unresolved)} task(s) are part of, " + f"or depend on, a cycle ({preview}).") + return None + + return level_widths + + def _validate_execution(self, execution: Dict[str, Any], task_ids: Set[str]) -> None: + machines_declared = "machines" in execution + machine_names = self._collect_ids(execution.get("machines", []), "nodeName", "machine") + executed_task_ids = set() + + for task in execution["tasks"]: + task_id = task["id"] + if task_id in executed_task_ids: + self._error(f"The task \"{task_id}\" is declared more than once in the execution section.") + executed_task_ids.add(task_id) + + if task_id not in task_ids: + self._error(f"Task \"{task_id}\" of the execution section is not declared in the list " + f"of workflow tasks.") + + for machine_name in task.get("machines", []): + if machines_declared and machine_name not in machine_names: + self._error(f"Machine \"{machine_name}\", used by task \"{task_id}\", is not declared " + f"in the list of machines.") + elif not machines_declared: + self._warning(f"Task \"{task_id}\" ran on machine \"{machine_name}\" while the workflow " + f"declares no list of machines.") + + missing_task_ids = task_ids - executed_task_ids + if missing_task_ids: + self._warning(f"{len(missing_task_ids)} task(s) have no entry in the execution section.") + + self.logger.debug(f'The workflow has {len(machine_names)} machines.') + + def _validate_specification_metrics(self, + specification: Dict[str, Any], + level_widths: Optional[List[int]]) -> None: + metrics = specification.get("metrics") + if not metrics: + return + + files = specification.get("files", []) + expected = { + "numberOfTasks": len(specification["tasks"]), + "numberOfFiles": len(files), + "sumOfFileSizesInBytes": sum(f.get("sizeInBytes", 0) for f in files) + } + if level_widths: + expected["numberOfLevels"] = len(level_widths) + expected["minimumWidth"] = min(level_widths) + expected["maximumWidth"] = max(level_widths) + + self._compare_metrics(metrics, expected, "specification") + + def _validate_execution_metrics(self, execution: Dict[str, Any]) -> None: + metrics = execution.get("metrics") + if not metrics: + return + + tasks = execution["tasks"] + expected = { + "sumTaskRuntimesInSeconds": sum(t["runtimeInSeconds"] for t in tasks if "runtimeInSeconds" in t), + "totalNumBytesRead": sum(t["readBytes"] for t in tasks if "readBytes" in t), + "totalNumBytesWritten": sum(t["writtenBytes"] for t in tasks if "writtenBytes" in t) + } + + self._compare_metrics(metrics, expected, "execution") + + def _compare_metrics(self, metrics: Dict[str, Any], expected: Dict[str, Any], section: str) -> None: + for key, expected_value in expected.items(): + if key in metrics and not math.isclose(metrics[key], expected_value, + rel_tol=METRICS_RELATIVE_TOLERANCE): + self._error(f"The {section} metric \"{key}\" is declared as {metrics[key]}, but the " + f"instance data gives {expected_value}.") + + +def _cache_dir() -> pathlib.Path: + """ + :return: The WfCommons directory in the user cache directory. + :rtype: pathlib.Path + """ + cache_home = os.environ.get("XDG_CACHE_HOME") + base = pathlib.Path(cache_home) if cache_home else pathlib.Path.home().joinpath(".cache") + return base.joinpath("wfcommons") + + +def _accepts_schema_version(schema: Dict[str, Any]) -> bool: + """ + :param schema: A WfFormat JSON schema. + :type schema: Dict[str, Any] + + :return: Whether the schema accepts the schema version written by WfCommons. + :rtype: bool + """ + return __schema_version__ in schema.get("properties", {}).get("schemaVersion", {}).get("enum", [])