diff --git a/CHANGELOG.md b/CHANGELOG.md index def5edc0f..67bf297a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## Version 2.1.0 + +ARTIST 2.1.0 makes loading and saving scenarios with thousands of heliostats fast and introduces the columnar heliostat layout for scenario files. + +### What's Changed + +- Sped up loading HDF5 scenarios with many heliostats: all heliostat data is now read onto the CPU first, moved to the device once per heliostat group, and the surfaces of each group are computed in batched NURBS evaluations. The new optional `surface_evaluation_chunk_size` argument of `Scenario.load_scenario_from_hdf5()` (default 1024) limits how many heliostats are evaluated at once, trading speed for device memory. +- Added the columnar heliostat layout for scenario files, which stores each heliostat parameter of all heliostats in one dataset. `H5ScenarioGenerator` now writes this layout. Scenarios in the legacy layout still load, with a warning, and can be converted with `artist.io.convert_scenario_to_columnar()`. Older `ARTIST` versions cannot load scenarios in the columnar layout. +- Warnings about missing kinematics or actuator parameters and the use of prototypes are now summarized over all heliostats instead of being logged per heliostat. + + ## Version 2.0.0 ARTIST 2.0.0 reorganizes the package, adds cylindrical-receiver ray tracing and heliostat blocking, and corrects inverse kinematics and its reconstruction workflow. diff --git a/CITATION.cff b/CITATION.cff index 2411b6065..1aee02a71 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -80,4 +80,4 @@ keywords: - Surface Reconstruction - Python license: MIT -version: 2.0.0 +version: 2.1.0 diff --git a/artist/field/heliostat_field.py b/artist/field/heliostat_field.py index 321a05439..acafd9fd4 100644 --- a/artist/field/heliostat_field.py +++ b/artist/field/heliostat_field.py @@ -2,15 +2,15 @@ import math from collections import defaultdict from collections.abc import Sequence -from typing import cast +from typing import Any, cast import h5py +import numpy as np import torch from typing_extensions import Self from artist.field.heliostat_group import HeliostatGroup -from artist.field.surface import Surface -from artist.io import h5_scenario_parser +from artist.io import h5_columnar_heliostats from artist.nurbs.surfaces import NURBSSurfaces from artist.nurbs.utils import ( create_nurbs_evaluation_grid, @@ -87,10 +87,15 @@ def from_hdf5( number_of_surface_points_per_facet: torch.Tensor, change_number_of_control_points_per_facet: torch.Tensor | None = None, device: torch.device | None = None, + surface_evaluation_chunk_size: int = 1024, ) -> Self: """ Load a heliostat field from an HDF5 file. + The heliostats are read onto the CPU as columns, i.e., one tensor per parameter for all heliostats, whichever + layout the file uses. They are grouped by kinematics and actuator type, moved to the device once per + group, and the surface points and normals of each group are computed in batched NURBS evaluations. + Parameters ---------- config_file : h5py.File @@ -114,19 +119,26 @@ def from_hdf5( The device on which to perform computations or load tensors and models (default is None). If None, ``ARTIST`` will automatically select the most appropriate device (CUDA or CPU) based on availability and OS. + surface_evaluation_chunk_size : int + The maximum number of heliostats whose surfaces are evaluated in one batched NURBS evaluation + (default is 1024). Larger values are faster but need more memory on the device. Raises ------ ValueError - If neither prototypes nor individual heliostat parameters are provided. + If neither prototypes nor individual heliostat parameters are provided, if a kinematics or actuator + type is unknown, or if ``surface_evaluation_chunk_size`` is not positive. Returns ------- HeliostatField The heliostat field loaded from the HDF5 file. """ + if surface_evaluation_chunk_size < 1: + raise ValueError( + f"The surface evaluation chunk size must be positive, but is {surface_evaluation_chunk_size}." + ) device = get_device(device=device) - rank = ( torch.distributed.get_rank() if torch.distributed.is_available() and torch.distributed.is_initialized() @@ -135,304 +147,502 @@ def from_hdf5( if rank == 0: log.info("Loading a heliostat field from an HDF5 file.") - grouped_field_data: defaultdict[ - str, defaultdict[str, list[str | torch.Tensor]] - ] = defaultdict(lambda: defaultdict(list)) + heliostats_config = config_file[constants.heliostat_key] + if rank == 0 and not h5_columnar_heliostats.is_columnar(heliostats_config): + log.warning( + "This scenario stores its heliostats in the legacy layout with one HDF5 group per heliostat, " + "which loads slower for large heliostat fields. Regenerate the scenario or convert it with " + "artist.io.convert_scenario_to_columnar() to load it faster." + ) - for heliostat_name in config_file[constants.heliostat_key].keys(): - single_heliostat_config = config_file[constants.heliostat_key][ - heliostat_name - ] + # Usually all heliostats are read as one set of columns. Groups of heliostats that had to be read + # separately, because their parameters differ in shape, are concatenated. + groups: dict[str, dict[str, Any]] = {} + for columns in h5_columnar_heliostats.read_heliostat_columns(heliostats_config): + for group_key, group in cls._group_heliostats( + columns=columns, + prototype_surface=prototype_surface, + prototype_kinematics=prototype_kinematics, + prototype_actuators=prototype_actuators, + change_number_of_control_points_per_facet=change_number_of_control_points_per_facet, + rank=rank, + ).items(): + if group_key not in groups: + groups[group_key] = group + else: + for key, value in group.items(): + if key == constants.names: + groups[group_key][key] += value + elif key != constants.heliostat_group_type: + groups[group_key][key] = torch.cat( + (groups[group_key][key], value) + ) - if constants.heliostat_surface_key in single_heliostat_config.keys(): - surface_config = h5_scenario_parser.surface_config( - prototype=False, - scenario_file=single_heliostat_config, - device=device, - ) - else: - if prototype_surface is None: - raise ValueError( - "If the heliostat does not have individual surface parameters, a surface prototype must be provided!" - ) - if rank == 0: - log.info( - "Individual surface parameters not provided - loading a heliostat with the surface prototype." - ) - surface_config = prototype_surface - - if constants.heliostat_kinematics_key in single_heliostat_config.keys(): - initial_orientation = torch.tensor( - single_heliostat_config[constants.heliostat_kinematics_key][ - constants.kinematics_initial_orientation - ][()], - dtype=torch.float, + heliostat_groups = [] + for group_key, group in groups.items(): + number_of_heliostats = len(group[constants.names]) + surface_points, surface_normals = cls._calculate_surface_points_and_normals( + control_points=group[constants.evaluation_control_points], + degrees=group[constants.facet_degrees], + canting=group[constants.facets_canting].to(device), + facet_translations=group[constants.facet_translations].to(device), + number_of_surface_points_per_facet=number_of_surface_points_per_facet, + surface_evaluation_chunk_size=surface_evaluation_chunk_size, + device=device, + ) + heliostat_groups.append( + type_registry.heliostat_group_type_mapping[group_key]( + names=group[constants.names], + positions=group[constants.positions].to(device), + surface_points=surface_points.reshape(number_of_heliostats, -1, 4), + surface_normals=surface_normals.reshape( + number_of_heliostats, -1, 4 + ), + canting=group[constants.facets_canting].to(device), + facet_translations=group[constants.facet_translations].to(device), + nurbs_control_points=group[constants.facet_control_points].to( + device + ), + nurbs_degrees=group[constants.facet_degrees][-1, -1].to(device), + initial_orientations=group[constants.initial_orientations].to( + device + ), + kinematics_translation_deviation_parameters=group[ + constants.translation_deviations + ].to(device), + kinematics_rotation_deviation_parameters=group[ + constants.rotation_deviations + ].to(device), + actuator_parameters_non_optimizable=group[ + constants.actuator_parameters_non_optimizable + ].to(device), + actuator_parameters_optimizable=group[ + constants.actuator_parameters_optimizable + ].to(device), device=device, ) - kinematics_type = single_heliostat_config[ - constants.heliostat_kinematics_key - ][constants.kinematics_type][()].decode("utf-8") - - translation_deviations, rotation_deviations, number_of_actuators = ( - h5_scenario_parser.kinematics_deviations( - prototype=False, - kinematics_type=kinematics_type, - scenario_file=single_heliostat_config, - log=log, - heliostat_name=heliostat_name, - device=device, - ) + ) + if rank == 0: + kinematics_type, actuator_type = group[constants.heliostat_group_type] + log.info( + f"Added a heliostat group with kinematics type: {kinematics_type}, and actuator type: {actuator_type}, to the heliostat field." ) - else: - if prototype_kinematics is None: - raise ValueError( - "If the heliostat does not have an individual kinematics, a kinematics prototype must be provided!" - ) - if rank == 0: - log.info( - "Individual kinematics configuration not provided - loading a heliostat with the kinematics prototype." - ) - kinematics_type = prototype_kinematics[constants.kinematics_type] - initial_orientation = prototype_kinematics[ - constants.kinematics_initial_orientation - ] - translation_deviations = prototype_kinematics[ - constants.translation_deviations - ] - rotation_deviations = prototype_kinematics[ - constants.rotation_deviations - ] + return cls(heliostat_groups=heliostat_groups, device=device) - if constants.heliostat_actuator_key in single_heliostat_config.keys(): - actuator_keys = list( - single_heliostat_config[constants.heliostat_actuator_key].keys() - ) + @staticmethod + def _group_heliostats( + columns: dict[str, Any], + prototype_surface: "SurfaceConfig", + prototype_kinematics: dict[str, str | torch.Tensor], + prototype_actuators: dict[str, str | torch.Tensor], + change_number_of_control_points_per_facet: torch.Tensor | None, + rank: int, + ) -> dict[str, dict[str, Any]]: + """ + Group heliostats read as columns by kinematics and actuator type and gather the CPU tensors of each group. - actuator_type_list = [] - for key in actuator_keys: - actuator_type_list.append( - single_heliostat_config[constants.heliostat_actuator_key][key][ - constants.actuator_type_key - ][()].decode("utf-8") - ) + Heliostats without individual parameters use the prototypes. Which prototypes and default parameters + were used is logged on rank 0, summarized over all heliostats. - unique_actuators = {a for a in actuator_type_list} + Parameters + ---------- + columns : dict[str, Any] + The heliostat columns, see ``h5_columnar_heliostats.heliostat_columns()``. + prototype_surface : SurfaceConfig + The prototype for the surface configuration to be used if a heliostat has no individual surface. + prototype_kinematics : dict[str, str | torch.Tensor] + The prototype for the kinematics, including type, initial orientation and deviations. + prototype_actuators : dict[str, str | torch.Tensor] + The prototype for the actuators, including type and parameters. + change_number_of_control_points_per_facet : torch.Tensor | None + The updated number of control points along each direction of each 2D facet. + Shape is ``[2]``. + rank : int + The rank of this process, logging only happens on rank 0. - if kinematics_type == constants.rigid_body_key: - if len(unique_actuators) > 1: - raise ValueError( - "When using the rigid body kinematics, all actuators for a given heliostat must have the same type." - ) - else: - actuator_type: str = actuator_type_list[indices.actuator_type] - - actuator_parameters_non_optimizable, actuator_parameters_optimizable = ( - h5_scenario_parser.actuator_parameters( - prototype=False, - scenario_file=single_heliostat_config, - actuator_type=actuator_type, - number_of_actuators=number_of_actuators, - log=log, - heliostat_name=heliostat_name, - device=device, - ) - ) - else: - if prototype_actuators is None: - raise ValueError( - "If the heliostat does not have individual actuators, an actuator prototype must be provided!" - ) - if rank == 0: - log.info( - "Individual actuator configurations not provided - loading a heliostat with the actuator prototype." - ) - actuator_type = str(prototype_actuators[constants.actuator_type_key]) - actuator_parameters_non_optimizable = prototype_actuators[ - constants.actuator_parameters_non_optimizable - ] - actuator_parameters_optimizable = prototype_actuators[ - constants.actuator_parameters_optimizable - ] + Raises + ------ + ValueError + If neither prototypes nor individual heliostat parameters are provided, or if a kinematics or actuator + type is unknown or does not fit the heliostat. - surface = Surface(surface_config, device=device) + Returns + ------- + dict[str, dict[str, Any]] + Per heliostat group, the heliostat names, the group type and the stacked CPU tensors of all heliostats. + """ + names = list(columns[constants.names]) - number_of_facets = len(surface_config.facet_list) - degrees = torch.empty( - indices.nurbs_degrees, - dtype=torch.int32, - device=device, + # The row of each heliostat in the columns of a component, or -1 if the heliostat uses the prototype. + rows = {} + for component, prototype, error in ( + ( + constants.heliostat_surface_key, + prototype_surface, + "If the heliostat does not have individual surface parameters, a surface prototype must be provided!", + ), + ( + constants.heliostat_kinematics_key, + prototype_kinematics, + "If the heliostat does not have an individual kinematics, a kinematics prototype must be provided!", + ), + ( + constants.heliostat_actuator_key, + prototype_actuators, + "If the heliostat does not have individual actuators, an actuator prototype must be provided!", + ), + ): + rows[component] = torch.full((len(names),), -1) + heliostat_indices = columns.get( + f"{component}/{constants.heliostat_indices_key}" ) - # Each facet automatically has the same control points dimensions. This is required in ARTIST. - # control_points: Shape is - # [number_of_surfaces, number_of_facets_per_surface, number_of_control_points_u_direction, number_of_control_points_v_direction, 3]. - control_points = torch.empty( - ( - number_of_facets, - surface_config.facet_list[indices.first_facet].control_points.shape[ - indices.h5_control_points_u - ], - surface_config.facet_list[indices.first_facet].control_points.shape[ - indices.h5_control_points_v - ], - indices.control_point_dimension, - ), - device=device, + if heliostat_indices is not None: + rows[component][torch.as_tensor(heliostat_indices)] = torch.arange( + len(heliostat_indices) + ) + if prototype is None and (rows[component] < 0).any(): + raise ValueError(error) + + surface = f"{constants.heliostat_surface_key}/" + kinematics = f"{constants.heliostat_kinematics_key}/" + actuator = f"{constants.heliostat_actuator_key}/" + kinematics_types = columns.get(f"{kinematics}{constants.kinematics_type}") + actuator_types = columns.get(f"{actuator}{constants.actuator_type_key}") + for kinematics_type in set( + kinematics_types if kinematics_types is not None else [] + ): + if kinematics_type != constants.rigid_body_key: + raise ValueError( + f"The kinematics type: {kinematics_type} is not yet implemented!" + ) + if actuator_types is not None: + if any(len(set(types)) > 1 for types in actuator_types): + raise ValueError( + "When using the rigid body kinematics, all actuators for a given heliostat must have the same type." + ) + if actuator_types.shape[1] != constants.rigid_body_number_of_actuators: + raise ValueError( + f"This scenario file contains the wrong amount of actuators for this heliostat and its kinematics type." + f" Expected {constants.rigid_body_number_of_actuators} actuators, found {actuator_types.shape[1]} actuator(s)." + ) + + cpu = torch.device("cpu") + individual: dict[str, torch.Tensor | None] = dict.fromkeys( + ( + constants.facet_control_points, + constants.facet_degrees, + constants.facet_translations, + constants.facets_canting, + constants.initial_orientations, + constants.translation_deviations, + constants.rotation_deviations, + ) + ) + prototypes: dict[str, torch.Tensor] = {} + if f"{surface}{constants.facet_control_points}" in columns: + for key, file_key in ( + (constants.facet_control_points, constants.facet_control_points), + (constants.facet_translations, constants.facets_translation_vector), + (constants.facets_canting, constants.facets_canting), + ): + individual[key] = torch.as_tensor( + columns[f"{surface}{file_key}"], dtype=torch.float + ) + individual[constants.facet_degrees] = torch.as_tensor( + columns[f"{surface}{constants.facet_degrees}"] + )[..., [indices.nurbs_u, indices.nurbs_v]].to(torch.int32) + if (rows[constants.heliostat_surface_key] < 0).any(): + for key, attribute in ( + (constants.facet_control_points, constants.facet_control_points), + (constants.facet_degrees, constants.facet_degrees), + (constants.facet_translations, constants.facet_translations), + (constants.facets_canting, constants.facets_canting), + ): + prototypes[key] = torch.stack( + [ + getattr(facet, attribute) + for facet in prototype_surface.facet_list + ] + ).to(cpu) + + missing_deviations: dict[str, torch.Tensor] = {} + if kinematics_types is not None: + individual[constants.initial_orientations] = torch.as_tensor( + columns[f"{kinematics}{constants.kinematics_initial_orientation}"], + dtype=torch.float, ) - canting = torch.empty( + ( + individual[constants.translation_deviations], + individual[constants.rotation_deviations], + missing_deviations, + ) = h5_columnar_heliostats.read_kinematics_deviations(columns) + if (rows[constants.heliostat_kinematics_key] < 0).any(): + for key, prototype_key in ( ( - number_of_facets, - indices.canting_direction_dimension, - 4, + constants.initial_orientations, + constants.kinematics_initial_orientation, ), - device=device, + (constants.translation_deviations, constants.translation_deviations), + (constants.rotation_deviations, constants.rotation_deviations), + ): + prototypes[key] = cast( + torch.Tensor, prototype_kinematics[prototype_key] + ).to(cpu) + + # The actuator parameters are read for all heliostats with individual actuators per actuator type. + actuator_parameters = { + actuator_type: h5_columnar_heliostats.read_actuator_parameters( + columns, actuator_type + ) + for actuator_type in set( + actuator_types[:, indices.actuator_type] + if actuator_types is not None + else [] + ) + } + + group_keys = [] + for kinematics_row, actuator_row in zip( + rows[constants.heliostat_kinematics_key].tolist(), + rows[constants.heliostat_actuator_key].tolist(), + ): + kinematics_type = ( + cast(np.ndarray, kinematics_types)[kinematics_row] + if kinematics_row >= 0 + else prototype_kinematics[constants.kinematics_type] ) - facet_translation_vectors = torch.empty( - (number_of_facets, 4), device=device + actuator_type = ( + cast(np.ndarray, actuator_types)[actuator_row, indices.actuator_type] + if actuator_row >= 0 + else prototype_actuators[constants.actuator_type_key] ) - for facet_index in range(number_of_facets): - degrees = surface_config.facet_list[facet_index].degrees - control_points[facet_index] = surface_config.facet_list[ - facet_index - ].control_points - canting[facet_index] = surface_config.facet_list[facet_index].canting - facet_translation_vectors[facet_index] = surface_config.facet_list[ - facet_index - ].translation_vector + group_keys.append((str(kinematics_type), str(actuator_type))) + positions = torch.as_tensor( + columns[constants.heliostat_position], dtype=torch.float + ) + groups = {} + for kinematics_type, actuator_type in dict.fromkeys(group_keys): + heliostat_indices = torch.tensor( + [ + heliostat_index + for heliostat_index, group_key in enumerate(group_keys) + if group_key == (kinematics_type, actuator_type) + ] + ) + surface_rows, kinematics_rows, actuator_rows = ( + rows[component][heliostat_indices] + for component in ( + constants.heliostat_surface_key, + constants.heliostat_kinematics_key, + constants.heliostat_actuator_key, + ) + ) + group: dict[str, Any] = { + constants.names: [names[index] for index in heliostat_indices.tolist()], + constants.heliostat_group_type: [kinematics_type, actuator_type], + constants.positions: positions[heliostat_indices], + } + for key in ( + constants.facet_control_points, + constants.facet_degrees, + constants.facet_translations, + constants.facets_canting, + ): + group[key] = _select(surface_rows, individual[key], prototypes.get(key)) + for key in ( + constants.initial_orientations, + constants.translation_deviations, + constants.rotation_deviations, + ): + group[key] = _select( + kinematics_rows, individual[key], prototypes.get(key) + ) + non_optimizable, optimizable, _ = actuator_parameters.get( + actuator_type, (None, None, None) + ) + for key, individual_parameters in ( + (constants.actuator_parameters_non_optimizable, non_optimizable), + (constants.actuator_parameters_optimizable, optimizable), + ): + group[key] = _select( + actuator_rows, + individual_parameters, + None + if (actuator_rows >= 0).all() + else cast(torch.Tensor, prototype_actuators[key]).to(cpu), + ) + + # As before, the surface points are evaluated from the control points in the file, the changed + # control points only replace the stored control points. + group[constants.evaluation_control_points] = group[ + constants.facet_control_points + ] if change_number_of_control_points_per_facet is not None: - control_points = create_planar_nurbs_control_points( - number_of_control_points=change_number_of_control_points_per_facet, - canting=canting, - device=device, + group[constants.facet_control_points] = torch.stack( + [ + create_planar_nurbs_control_points( + number_of_control_points=change_number_of_control_points_per_facet, + canting=canting, + device=cpu, + ) + for canting in group[constants.facets_canting] + ] ) + groups[f"{kinematics_type}_{actuator_type}"] = group + + if rank == 0: - heliostat_group_key = f"{kinematics_type}_{actuator_type}" + def summary(heliostat_names: list[str]) -> str: + more = ", ..." if len(heliostat_names) > 5 else "" + return f"{len(heliostat_names)} heliostat(s) ({', '.join(heliostat_names[:5])}{more})" + + for component, configuration in ( + (constants.heliostat_surface_key, "surface parameters"), + (constants.heliostat_kinematics_key, "kinematics configuration"), + (constants.heliostat_actuator_key, "actuator configurations"), + ): + number_of_prototypes = int((rows[component] < 0).sum()) + if number_of_prototypes > 0: + log.info( + f"Individual {configuration} not provided for {number_of_prototypes} heliostat(s) - " + f"loading these heliostats with the {component} prototype." + ) - grouped_field_data[heliostat_group_key][constants.names].append( - heliostat_name - ) - grouped_field_data[heliostat_group_key][constants.heliostat_group_type] = [ - kinematics_type, - actuator_type, - ] + for parameter, missing in missing_deviations.items(): + if missing.any(): + heliostat_indices = columns[ + f"{kinematics}{constants.heliostat_indices_key}" + ][missing.numpy()] + log.warning( + f"No individual kinematics {parameter} set for " + f"{summary([names[index] for index in heliostat_indices])}. Using default values!" + ) - grouped_field_data[heliostat_group_key][constants.positions].append( - torch.tensor( - single_heliostat_config[constants.heliostat_position][()], - dtype=torch.float, - device=device, + if actuator_types is not None: + heliostat_indices = columns[ + f"{actuator}{constants.heliostat_indices_key}" + ] + actuator_names = columns[f"{actuator}{constants.actuator_names_key}"] + is_linear = torch.as_tensor( + actuator_types[:, indices.actuator_type] + == constants.linear_actuator_key ) - ) - grouped_field_data[heliostat_group_key][constants.surface_points].append( - surface.get_surface_points_and_normals( - number_of_points_per_facet=number_of_surface_points_per_facet, - canting=canting, - facet_translations=facet_translation_vectors, - device=device, - )[indices.surface_points_from_tuple] - ) - grouped_field_data[heliostat_group_key][constants.surface_normals].append( - surface.get_surface_points_and_normals( - number_of_points_per_facet=number_of_surface_points_per_facet, - canting=canting, - facet_translations=facet_translation_vectors, - device=device, - )[indices.surface_normals_from_tuple] - ) - grouped_field_data[heliostat_group_key][constants.facets_canting].append( - canting - ) - grouped_field_data[heliostat_group_key][ - constants.facet_translations - ].append(facet_translation_vectors) - grouped_field_data[heliostat_group_key][ - constants.facet_control_points - ].append(control_points) - grouped_field_data[heliostat_group_key][ - constants.initial_orientations - ].append(initial_orientation) - grouped_field_data[heliostat_group_key][ - constants.translation_deviations - ].append(translation_deviations) - grouped_field_data[heliostat_group_key][ - constants.rotation_deviations - ].append(rotation_deviations) - grouped_field_data[heliostat_group_key][ - constants.actuator_parameters_non_optimizable - ].append(actuator_parameters_non_optimizable) - grouped_field_data[heliostat_group_key][ - constants.actuator_parameters_optimizable - ].append(actuator_parameters_optimizable) - - for group in grouped_field_data: - for key in grouped_field_data[group]: - if key not in [ - constants.names, - constants.kinematics_type, - constants.actuator_type_key, - ]: - grouped_field_data[group][key] = torch.stack( - grouped_field_data[group][key] + missing_parameters: defaultdict[tuple[str, str], list[str]] = ( + defaultdict(list) + ) + for parameter, missing in actuator_parameters.get( + constants.linear_actuator_key, (None, None, {}) + )[2].items(): + for row, actuator_index in torch.nonzero( + missing & is_linear[:, None] + ).tolist(): + missing_parameters[ + (actuator_names[row][actuator_index], parameter) + ].append(names[heliostat_indices[row]]) + for ( + actuator_name, + parameter, + ), heliostat_names in missing_parameters.items(): + log.warning( + f"No individual {parameter} set for {actuator_name} on {summary(heliostat_names)}. " + f"Using default values!" ) - heliostat_groups = [] - for heliostat_group_name in grouped_field_data.keys(): - number_of_heliostats_in_group = len( - grouped_field_data[heliostat_group_name][constants.names] - ) - heliostat_groups.append( - type_registry.heliostat_group_type_mapping[heliostat_group_name]( - names=grouped_field_data[heliostat_group_name][constants.names], - positions=grouped_field_data[heliostat_group_name][ - constants.positions - ], - surface_points=cast( - torch.Tensor, - grouped_field_data[heliostat_group_name][ - constants.surface_points - ], - ).reshape(number_of_heliostats_in_group, -1, 4), - surface_normals=cast( - torch.Tensor, - grouped_field_data[heliostat_group_name][ - constants.surface_normals - ], - ).reshape(number_of_heliostats_in_group, -1, 4), - canting=grouped_field_data[heliostat_group_name][ - constants.facets_canting - ], - facet_translations=grouped_field_data[heliostat_group_name][ - constants.facet_translations - ], - nurbs_control_points=grouped_field_data[heliostat_group_name][ - constants.facet_control_points - ], - nurbs_degrees=degrees, - initial_orientations=grouped_field_data[heliostat_group_name][ - constants.initial_orientations - ], - kinematics_translation_deviation_parameters=grouped_field_data[ - heliostat_group_name - ][constants.translation_deviations], - kinematics_rotation_deviation_parameters=grouped_field_data[ - heliostat_group_name - ][constants.rotation_deviations], - actuator_parameters_non_optimizable=grouped_field_data[ - heliostat_group_name - ][constants.actuator_parameters_non_optimizable], - actuator_parameters_optimizable=grouped_field_data[ - heliostat_group_name - ][constants.actuator_parameters_optimizable], + return groups + + @staticmethod + def _calculate_surface_points_and_normals( + control_points: torch.Tensor, + degrees: torch.Tensor, + canting: torch.Tensor, + facet_translations: torch.Tensor, + number_of_surface_points_per_facet: torch.Tensor, + surface_evaluation_chunk_size: int, + device: torch.device, + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + Calculate the surface points and normals of many heliostats in batched NURBS evaluations. + + Heliostats are evaluated together if they share the NURBS degrees of their first facet and whether their + control points are flat. As for a single ``Surface``, canting and facet translations are only applied to + heliostats with flat control points, all others already contain them. + + Parameters + ---------- + control_points : torch.Tensor + The control points of all heliostats on the CPU. + Shape is ``[number_of_heliostats, number_of_facets, number_of_control_points_u_direction, number_of_control_points_v_direction, 3]``. + degrees : torch.Tensor + The NURBS degrees of all facets of all heliostats on the CPU. + Shape is ``[number_of_heliostats, number_of_facets, 2]``. + canting : torch.Tensor + The canting vectors of all facets of all heliostats. + Shape is ``[number_of_heliostats, number_of_facets, 2, 4]``. + facet_translations : torch.Tensor + The translation vectors of all facets of all heliostats. + Shape is ``[number_of_heliostats, number_of_facets, 4]``. + number_of_surface_points_per_facet : torch.Tensor + The number of sampling points along each direction of each 2D facet. + Shape is ``[2]``. + surface_evaluation_chunk_size : int + The maximum number of heliostats whose surfaces are evaluated in one batched NURBS evaluation. + device : torch.device + The device on which to perform computations or load tensors and models. + + Returns + ------- + torch.Tensor + The surface points of all heliostats. + Shape is ``[number_of_heliostats, number_of_facets, number_of_surface_points_per_facet, 4]``. + torch.Tensor + The surface normals of all heliostats. + Shape is ``[number_of_heliostats, number_of_facets, number_of_surface_points_per_facet, 4]``. + """ + number_of_heliostats, number_of_facets = control_points.shape[:2] + evaluation_points = create_nurbs_evaluation_grid( + number_of_evaluation_points=number_of_surface_points_per_facet, + device=device, + ) + surface_points = torch.empty( + (number_of_heliostats, number_of_facets, evaluation_points.shape[0], 4), + device=device, + ) + surface_normals = torch.empty_like(surface_points) + + is_flat = (control_points[..., 2] == 0).flatten(start_dim=1).all(dim=1) + evaluation_keys = torch.cat( + (degrees[:, indices.first_facet], is_flat.unsqueeze(1).int()), dim=1 + ) + unique_keys, key_per_heliostat = torch.unique( + evaluation_keys, dim=0, return_inverse=True + ) + + for key_index, key in enumerate(unique_keys): + apply_canting = bool(key[-1]) + heliostat_indices = torch.nonzero(key_per_heliostat == key_index).squeeze(1) + for chunk in heliostat_indices.split(surface_evaluation_chunk_size): + chunk_on_device = chunk.to(device) + nurbs_surfaces = NURBSSurfaces( + degrees=key[:2].to(device), + control_points=control_points[chunk].to(device), device=device, ) - ) - if rank == 0: - log.info( - f"Added a heliostat group with kinematics type: {grouped_field_data[heliostat_group_name][constants.heliostat_group_type][indices.kinematics_type_index]}, and actuator type: {grouped_field_data[heliostat_group_name][constants.heliostat_group_type][indices.actuator_type_index]}, to the heliostat field." + points, normals = nurbs_surfaces.calculate_surface_points_and_normals( + evaluation_points=evaluation_points.expand( + len(chunk), number_of_facets, -1, -1 + ), + canting=canting[chunk_on_device] if apply_canting else None, + facet_translations=facet_translations[chunk_on_device] + if apply_canting + else None, + device=device, ) + surface_points[chunk_on_device] = points + surface_normals[chunk_on_device] = normals - return cls(heliostat_groups=heliostat_groups, device=device) + return surface_points, surface_normals def update_surfaces( self, @@ -501,3 +711,34 @@ def update_surfaces( -1, 4, ).detach() + + +def _select( + rows: torch.Tensor, individual: torch.Tensor | None, prototype: torch.Tensor | None +) -> torch.Tensor: + """ + Select the individual parameters of heliostats by their rows, and the prototype for heliostats without. + + Parameters + ---------- + rows : torch.Tensor + The row of each heliostat in the individual parameters, or -1 if the heliostat uses the prototype. + Shape is ``[number_of_heliostats]``. + individual : torch.Tensor | None + The individual parameters, or None if no heliostat has them. + prototype : torch.Tensor | None + The prototype parameters, or None if every heliostat has individual parameters. + + Returns + ------- + torch.Tensor + The parameters of each heliostat. + """ + has_individual = rows >= 0 + if has_individual.all(): + return cast(torch.Tensor, individual)[rows] + prototype = cast(torch.Tensor, prototype) + selected = prototype.expand(len(rows), *prototype.shape).clone() + if has_individual.any(): + selected[has_individual] = cast(torch.Tensor, individual)[rows[has_individual]] + return selected diff --git a/artist/io/__init__.py b/artist/io/__init__.py index 8362f9ebb..4e64b5e78 100644 --- a/artist/io/__init__.py +++ b/artist/io/__init__.py @@ -1,7 +1,9 @@ from .calibration_parser import CalibrationDataParser +from .h5_columnar_heliostats import convert_scenario_to_columnar from .paint_calibration_parser import PaintCalibrationDataParser __all__ = [ "CalibrationDataParser", "PaintCalibrationDataParser", + "convert_scenario_to_columnar", ] diff --git a/artist/io/h5_columnar_heliostats.py b/artist/io/h5_columnar_heliostats.py new file mode 100644 index 000000000..d2eaa8ea1 --- /dev/null +++ b/artist/io/h5_columnar_heliostats.py @@ -0,0 +1,599 @@ +import pathlib +from collections import defaultdict +from collections.abc import Sequence +from typing import Any + +import h5py +import numpy as np +import torch + +from artist.geometry import rotations +from artist.util import constants, indices + +required_fields = { + "": (constants.heliostat_id, constants.heliostat_position), + constants.heliostat_surface_key: ( + constants.facet_control_points, + constants.facet_degrees, + constants.facets_translation_vector, + constants.facets_canting, + ), + constants.heliostat_kinematics_key: ( + constants.kinematics_type, + constants.kinematics_initial_orientation, + ), + constants.heliostat_actuator_key: ( + constants.actuator_type_key, + constants.actuator_clockwise_axis_movement, + constants.actuator_min_max_motor_positions, + ), +} +"""The parameters every heliostat, facet, kinematics, and actuator must have, per component.""" + +optional_fields = { + constants.heliostat_kinematics_key: constants.kinematics_deviations, + constants.heliostat_actuator_key: constants.actuator_parameters_key, +} +"""The group of scalar parameters per component that may be missing, which are stored as NaN.""" + +string_fields = (constants.kinematics_type, constants.actuator_type_key) +"""The parameters that are stored as strings.""" + +rigid_body_deviations = ( + ( + (indices.first_joint_translation_e, constants.first_joint_translation_e), + (indices.first_joint_translation_n, constants.first_joint_translation_n), + (indices.first_joint_translation_u, constants.first_joint_translation_u), + (indices.second_joint_translation_e, constants.second_joint_translation_e), + (indices.second_joint_translation_n, constants.second_joint_translation_n), + (indices.second_joint_translation_u, constants.second_joint_translation_u), + (indices.concentrator_translation_e, constants.concentrator_translation_e), + (indices.concentrator_translation_n, constants.concentrator_translation_n), + (indices.concentrator_translation_u, constants.concentrator_translation_u), + ), + ( + (indices.first_joint_tilt_n, constants.first_joint_tilt_n), + (indices.first_joint_tilt_u, constants.first_joint_tilt_u), + (indices.second_joint_tilt_e, constants.second_joint_tilt_e), + (indices.second_joint_tilt_n, constants.second_joint_tilt_n), + ), +) +"""Index and name of each translation and each rotation deviation parameter of the rigid body kinematics.""" + +linear_actuator_parameters = ( + ( + (indices.actuator_increment, constants.actuator_increment), + (indices.actuator_offset, constants.actuator_offset), + (indices.actuator_pivot_radius, constants.actuator_pivot_radius), + ), + ( + (indices.actuator_initial_angle, constants.actuator_initial_angle), + ( + indices.actuator_initial_stroke_length, + constants.actuator_initial_stroke_length, + ), + ), +) +"""Index and name of each non-optimizable and each optimizable linear actuator parameter.""" + + +class ColumnarLayoutError(ValueError): + """Raised if heliostats cannot be stored in the columnar layout, e.g., because their parameters differ in shape.""" + + +def is_columnar(heliostats_config: h5py.Group) -> bool: + """ + Check whether the heliostats of a scenario file are stored in the columnar layout. + + Parameters + ---------- + heliostats_config : h5py.Group + The heliostat group of the scenario file. + + Returns + ------- + bool + Whether the heliostats are stored in the columnar layout. + """ + return ( + heliostats_config.attrs.get(constants.layout_attribute) + == constants.columnar_layout + ) + + +def heliostat_columns( + heliostats: Sequence[tuple[str, dict[str, Any]]], +) -> dict[str, np.ndarray]: + """ + Arrange the parameters of all heliostats in the columnar layout. + + Every heliostat parameter is stored in one dataset for all heliostats, with the heliostats as the first + dimension, sorted by name. Facets and actuators are sorted by name as well, as HDF5 sorts the members of a + group. Individual surfaces, kinematics, and actuators are only stored for the heliostats that have them, + together with the indices of these heliostats. Missing kinematics deviations and actuator parameters are + stored as NaN. + + Parameters + ---------- + heliostats : Sequence[tuple[str, dict[str, Any]]] + The name and the flattened parameters of each heliostat, as stored in the legacy layout, e.g., + ``{"position": ..., "surface/facets/facet_1/control_points": ...}``. + + Raises + ------ + ColumnarLayoutError + If the heliostats cannot be stored in the columnar layout. Nothing is silently left out: parameters + that differ in shape between heliostats, or that the layout does not know, raise this error. + + Returns + ------- + dict[str, np.ndarray] + The path of each dataset relative to the heliostat group and its data. + """ + heliostats = sorted(heliostats, key=lambda heliostat: heliostat[0]) + names = [name for name, _ in heliostats] + if not names: + raise ColumnarLayoutError("The scenario contains no heliostats.") + if len(set(names)) != len(names): + raise ColumnarLayoutError("Heliostat names must be unique.") + + # Per component, the index, name, and the parameters of each item (facet or actuator) of every heliostat + # that has the component. Heliostat-level parameters and the kinematics are a single unnamed item. + rows: dict[str, list[tuple[int, str, dict[str, dict[str, Any]]]]] = { + component: [] for component in required_fields + } + for heliostat_index, (name, parameters) in enumerate(heliostats): + items_per_component: dict[str, defaultdict[str, dict[str, Any]]] = { + component: defaultdict(dict) for component in required_fields + } + for key, value in parameters.items(): + component, _, rest = key.partition("/") if "/" in key else ("", "", key) + if component == constants.heliostat_surface_key: + facets, _, rest = rest.partition("/") + item, _, field = ( + rest.partition("/") + if facets == constants.facets_key + else ("", "", "") + ) + elif component == constants.heliostat_actuator_key: + item, _, field = rest.partition("/") + else: + item, field = "", rest + if component not in required_fields or not field: + raise ColumnarLayoutError( + f"Heliostat {name} contains the parameter {key}, which the columnar layout cannot store." + ) + items_per_component[component][item][field] = value + for component, items in items_per_component.items(): + if items or not component: + rows[component].append( + (heliostat_index, name, dict(sorted(items.items())) or {"": {}}) + ) + + columns = {constants.names: np.array(names, dtype=h5py.string_dtype())} + for component, component_rows in rows.items(): + if component_rows: + columns.update(_component_columns(component, component_rows)) + return columns + + +def _component_columns( + component: str, rows: list[tuple[int, str, dict[str, dict[str, Any]]]] +) -> dict[str, np.ndarray]: + """ + Arrange the parameters of one component of all heliostats that have it in the columnar layout. + + Parameters + ---------- + component : str + The component, i.e., ``""`` for heliostat-level parameters, surface, kinematics, or actuators. + rows : list[tuple[int, str, dict[str, dict[str, Any]]]] + The index, name, and the parameters of each item (facet or actuator) of every heliostat with the + component. + + Raises + ------ + ColumnarLayoutError + If a parameter is missing, unknown, not scalar where it must be, or differs in shape between heliostats. + + Returns + ------- + dict[str, np.ndarray] + The path of each dataset relative to the heliostat group and its data. + """ + prefix = f"{component}/" if component else "" + has_items = component in ( + constants.heliostat_surface_key, + constants.heliostat_actuator_key, + ) + columns = {} + if component: + columns[f"{prefix}{constants.heliostat_indices_key}"] = np.array( + [heliostat_index for heliostat_index, _, _ in rows], dtype=np.int64 + ) + if component == constants.heliostat_actuator_key: + if len({len(items) for _, _, items in rows}) > 1: + raise ColumnarLayoutError( + "Heliostats have different numbers of actuators, so the columnar layout cannot store them." + ) + columns[f"{prefix}{constants.actuator_names_key}"] = np.array( + [list(items) for _, _, items in rows], dtype=h5py.string_dtype() + ) + + for field in required_fields[component]: + values = [] + for _, name, items in rows: + if any(field not in parameters for parameters in items.values()): + raise ColumnarLayoutError( + f"Heliostat {name} has no {prefix}{field}, which the columnar layout requires." + ) + item_values = [ + _to_str(parameters[field]) + if field in string_fields + else _to_numpy(parameters[field]) + for parameters in items.values() + ] + values.append(item_values if has_items else item_values[0]) + try: + columns[f"{prefix}{field}"] = ( + np.array(values, dtype=h5py.string_dtype()) + if field in string_fields + else np.stack( + [np.stack(value) if has_items else value for value in values] + ) + ) + except ValueError as error: + raise ColumnarLayoutError( + f"The {prefix}{field} differs in shape between heliostats or their facets or actuators, " + f"so the columnar layout cannot store it." + ) from error + + optional = f"{optional_fields.get(component)}/" + optional_keys = set() + for _, name, items in rows: + for parameters in items.values(): + for field in parameters: + if field.startswith(optional): + optional_keys.add(field) + elif field not in required_fields[component]: + raise ColumnarLayoutError( + f"Heliostat {name} contains the parameter {prefix}{field}, which the columnar layout " + f"cannot store." + ) + for key in sorted(optional_keys): + scalars = [] + for _, name, items in rows: + arrays = [ + _to_numpy(parameters.get(key, np.nan)) for parameters in items.values() + ] + if any(array.size != 1 for array in arrays): + raise ColumnarLayoutError( + f"Heliostat {name} has a non-scalar {prefix}{key}, which the columnar layout cannot store." + ) + scalars.append([float(array.reshape(())) for array in arrays]) + column = np.array(scalars, dtype=np.float64) + columns[f"{prefix}{key}"] = column if has_items else column[:, 0] + return columns + + +def _to_numpy(value: Any) -> np.ndarray: + """ + Convert a parameter value, i.e., a tensor, NumPy array, or Python scalar, to a NumPy array. + + Parameters + ---------- + value : Any + The value. + + Returns + ------- + np.ndarray + The value as a NumPy array. + """ + return value.detach().cpu().numpy() if torch.is_tensor(value) else np.asarray(value) + + +def _to_str(value: Any) -> str: + """ + Convert a string parameter value, which may be stored as bytes or a NumPy scalar, to a string. + + Parameters + ---------- + value : Any + The value. + + Returns + ------- + str + The value as a string. + """ + value = value.item() if isinstance(value, np.ndarray | np.generic) else value + return value.decode("utf-8") if isinstance(value, bytes) else str(value) + + +def write_columnar_heliostats( + heliostats_config: h5py.Group, columns: dict[str, np.ndarray] +) -> None: + """ + Write heliostats arranged by ``heliostat_columns()`` into an empty HDF5 group and mark its layout. + + Parameters + ---------- + heliostats_config : h5py.Group + The empty heliostat group of the scenario file. + columns : dict[str, np.ndarray] + The path of each dataset relative to the heliostat group and its data. + """ + for path, data in columns.items(): + heliostats_config.create_dataset(path, data=data) + heliostats_config.attrs[constants.layout_attribute] = constants.columnar_layout + + +def read_legacy_heliostats( + heliostats_config: h5py.Group, +) -> list[tuple[str, dict[str, Any]]]: + """ + Read all heliostats stored in the legacy layout, with one HDF5 group per heliostat, as flattened parameters. + + Parameters + ---------- + heliostats_config : h5py.Group + The heliostat group of the scenario file. + + Returns + ------- + list[tuple[str, dict[str, Any]]] + The name and the flattened parameters of each heliostat. + """ + heliostats = [] + for name, heliostat_config in heliostats_config.items(): + parameters: dict[str, Any] = {} + heliostat_config.visititems( + lambda key, item, parameters=parameters: ( + parameters.__setitem__(key, item[()]) + if isinstance(item, h5py.Dataset) + else None + ) + ) + heliostats.append((name, parameters)) + return heliostats + + +def read_heliostat_columns( + heliostats_config: h5py.Group, +) -> list[dict[str, np.ndarray]]: + """ + Read the heliostats of a scenario file in the columnar layout, whichever layout the file uses. + + Heliostats in the legacy layout are arranged in columns while reading. If their parameters differ in shape, + e.g., because their surfaces have different numbers of control points, each heliostat is arranged on its own. + + Parameters + ---------- + heliostats_config : h5py.Group + The heliostat group of the scenario file. + + Raises + ------ + ColumnarLayoutError + If a heliostat in the legacy layout contains parameters that are missing or unknown. + + Returns + ------- + list[dict[str, np.ndarray]] + One or more sets of columns, each mapping the path of a dataset relative to the heliostat group to its + data. Together they contain all heliostats, sorted by name. + """ + if is_columnar(heliostats_config): + columns: dict[str, np.ndarray] = {} + heliostats_config.visititems( + lambda path, item: ( + columns.__setitem__( + path, + item.asstr()[()] + if h5py.check_string_dtype(item.dtype) + else item[()], + ) + if isinstance(item, h5py.Dataset) + else None + ) + ) + return [columns] + + heliostats = read_legacy_heliostats(heliostats_config) + try: + return [heliostat_columns(heliostats)] + except ColumnarLayoutError: + return [ + heliostat_columns([heliostat]) + for heliostat in sorted(heliostats, key=lambda heliostat: heliostat[0]) + ] + + +def read_kinematics_deviations( + columns: dict[str, np.ndarray], +) -> tuple[torch.Tensor, torch.Tensor, dict[str, torch.Tensor]]: + """ + Read the rigid body deviations of all heliostats with individual kinematics onto the CPU. + + Missing deviations default to zero. + + Parameters + ---------- + columns : dict[str, np.ndarray] + The heliostat columns. + + Returns + ------- + torch.Tensor + The translation deviations. + Shape is ``[number_of_heliostats_with_kinematics, 9]``. + torch.Tensor + The rotation deviations. + Shape is ``[number_of_heliostats_with_kinematics, 4]``. + dict[str, torch.Tensor] + For each deviation, which heliostats with individual kinematics miss it. + Shape is ``[number_of_heliostats_with_kinematics]``. + """ + prefix = f"{constants.heliostat_kinematics_key}/{constants.kinematics_deviations}/" + number_of_rows = len( + columns[ + f"{constants.heliostat_kinematics_key}/{constants.heliostat_indices_key}" + ] + ) + deviations, missing = [], {} + for parameters in rigid_body_deviations: + values = torch.zeros((number_of_rows, len(parameters))) + for index, key in parameters: + column = torch.as_tensor( + columns.get(f"{prefix}{key}", np.full(number_of_rows, np.nan)) + ) + missing[key] = column.isnan() + values[:, index] = column.nan_to_num(0.0).float() + deviations.append(values) + return deviations[0], deviations[1], missing + + +def read_actuator_parameters( + columns: dict[str, np.ndarray], actuator_type: str +) -> tuple[torch.Tensor, torch.Tensor, dict[str, torch.Tensor]]: + """ + Read the parameters of all heliostats with individual actuators onto the CPU, as actuators of one type. + + Missing parameters default to zero. For linear actuators, the initial angle of the first actuator is + corrected for the orientation of the surface, as in ``h5_scenario_parser.linear_actuators()``. + + Parameters + ---------- + columns : dict[str, np.ndarray] + The heliostat columns. + actuator_type : str + The actuator type. + + Raises + ------ + ValueError + If the actuator type is unknown. + + Returns + ------- + torch.Tensor + The non-optimizable parameters. + Shape is ``[number_of_heliostats_with_actuators, number_of_parameters, number_of_actuators]``. + torch.Tensor + The optimizable parameters. + Shape is ``[number_of_heliostats_with_actuators, 2, number_of_actuators]`` for linear actuators. + dict[str, torch.Tensor] + For each linear actuator parameter, which actuators of the heliostats miss it. + Shape is ``[number_of_heliostats_with_actuators, number_of_actuators]``. + """ + prefix = f"{constants.heliostat_actuator_key}/" + clockwise_axis_movement = torch.as_tensor( + columns[f"{prefix}{constants.actuator_clockwise_axis_movement}"] + ) + min_max_motor_positions = torch.as_tensor( + columns[f"{prefix}{constants.actuator_min_max_motor_positions}"] + ).double() + number_of_rows, number_of_actuators = clockwise_axis_movement.shape + + if actuator_type == constants.ideal_actuator_key: + actuator_type_int, number_of_non_optimizable_parameters = ( + constants.ideal_actuator_int, + 4, + ) + elif actuator_type == constants.linear_actuator_key: + actuator_type_int, number_of_non_optimizable_parameters = ( + constants.linear_actuator_int, + 7, + ) + else: + raise ValueError(f"The actuator type: {actuator_type} is not yet implemented!") + + non_optimizable = torch.zeros( + (number_of_rows, number_of_non_optimizable_parameters, number_of_actuators) + ) + non_optimizable[:, indices.actuator_type] = actuator_type_int + non_optimizable[:, indices.actuator_clockwise_movement] = ( + clockwise_axis_movement.float() + ) + non_optimizable[:, indices.actuator_min_motor_position] = min_max_motor_positions[ + ..., indices.data_actuator_min_motor_position + ].float() + non_optimizable[:, indices.actuator_max_motor_position] = min_max_motor_positions[ + ..., indices.data_actuator_max_motor_position + ].float() + if actuator_type == constants.ideal_actuator_key: + return non_optimizable, torch.empty((number_of_rows, 0)), {} + + optimizable = torch.zeros((number_of_rows, 2, number_of_actuators)) + missing = {} + for values, parameters in zip( + (non_optimizable, optimizable), linear_actuator_parameters + ): + for index, key in parameters: + column = torch.as_tensor( + columns.get( + f"{prefix}{constants.actuator_parameters_key}/{key}", + np.full((number_of_rows, number_of_actuators), np.nan), + ) + ) + missing[key] = column.isnan() + values[:, index] = column.nan_to_num(0.0).float() + + # As in h5_scenario_parser.linear_actuators(): the first actuator rotates along the east axis, and its initial + # angle is relative to the surface, which is stored pointing up, while ARTIST expects heliostats to initially + # face south. The rotation from south to up, projected onto the east axis, is added to the initial angle. + cpu = torch.device("cpu") + axis, angle = rotations.rotation_angle_and_axis( + from_orientation=torch.tensor([0.0, -1.0, 0.0, 0.0]), + to_orientation=torch.tensor([0.0, 0.0, 1.0, 0.0]), + device=cpu, + ) + optimizable[:, indices.actuator_initial_angle, indices.actuator_one_index] += ( + axis[indices.e] * angle + ) + return non_optimizable, optimizable, missing + + +def convert_scenario_to_columnar( + input_path: pathlib.Path | str, output_path: pathlib.Path | str +) -> None: + """ + Convert a scenario file with heliostats in the legacy layout to the columnar layout. + + Everything apart from the heliostats is copied unchanged. Scenarios in the columnar layout load faster, + especially for large heliostat fields, and yield exactly the same scenario. + + Parameters + ---------- + input_path : pathlib.Path | str + The scenario file with heliostats in the legacy layout. + output_path : pathlib.Path | str + The file to write the converted scenario to. It must differ from the input file. + + Raises + ------ + ValueError + If the heliostats of the input file are already stored in the columnar layout, or input and output are + the same file. + ColumnarLayoutError + If the heliostats cannot be stored in the columnar layout. + """ + if pathlib.Path(input_path).resolve() == pathlib.Path(output_path).resolve(): + raise ValueError("The converted scenario must be written to a different file.") + with h5py.File(input_path, "r") as input_file: + heliostats_config = input_file[constants.heliostat_key] + if is_columnar(heliostats_config): + raise ValueError( + f"The heliostats in {input_path} are already stored in the columnar layout." + ) + columns = heliostat_columns(read_legacy_heliostats(heliostats_config)) + with h5py.File(output_path, "w") as output_file: + for key, value in input_file.attrs.items(): + output_file.attrs[key] = value + for key in input_file.keys(): + if key != constants.heliostat_key: + input_file.copy(input_file[key], output_file, key) + write_columnar_heliostats( + output_file.create_group(constants.heliostat_key), columns + ) diff --git a/artist/io/h5_scenario_parser.py b/artist/io/h5_scenario_parser.py index e92200d91..2e7a778b4 100644 --- a/artist/io/h5_scenario_parser.py +++ b/artist/io/h5_scenario_parser.py @@ -60,7 +60,7 @@ def surface_config( dtype=torch.int32, device=device, ), - translation_vector=torch.tensor( + facet_translations=torch.tensor( facet_config[facet][constants.facets_translation_vector][()], dtype=torch.float, device=device, diff --git a/artist/scenario/h5_scenario_generator.py b/artist/scenario/h5_scenario_generator.py index fc02de21d..68c6752a6 100644 --- a/artist/scenario/h5_scenario_generator.py +++ b/artist/scenario/h5_scenario_generator.py @@ -6,6 +6,7 @@ import h5py import torch +from artist.io import h5_columnar_heliostats from artist.util import constants from artist.util.config import ( ActuatorConfig, @@ -218,7 +219,14 @@ def _include_parameters( file[f"{prefix}/{key}"] = value def generate_scenario(self) -> None: - """Generate the scenario and save it as an HDF5 file.""" + """ + Generate the scenario and save it as an HDF5 file. + + The heliostats are stored in the columnar layout, which stores each heliostat parameter of all heliostats + in one dataset and loads fast even for large heliostat fields. If the heliostats cannot be stored in this + layout, e.g., because their surfaces have different numbers of control points, they are stored in the + legacy layout with one HDF5 group per heliostat instead. + """ log.info(f"Generating a scenario saved to: {self.file_path}.") if self.file_path.suffix == ".h5": save_name = self.file_path @@ -292,10 +300,27 @@ def generate_scenario(self) -> None: # Include heliostat parameters. log.info("Including parameters for the heliostats.") - self._include_parameters( - file=f, - prefix=constants.heliostat_key, - parameters=self._flatten_dict( - self.heliostat_list_config.create_heliostat_list_dict() - ), + heliostat_parameters = ( + self.heliostat_list_config.create_heliostat_list_dict() ) + try: + columns = h5_columnar_heliostats.heliostat_columns( + [ + (name, self._flatten_dict(parameters)) + for name, parameters in heliostat_parameters.items() + ] + ) + except h5_columnar_heliostats.ColumnarLayoutError as error: + log.warning( + f"Storing the heliostats in the legacy layout, which loads slower for large heliostat fields: {error}" + ) + self._include_parameters( + file=f, + prefix=constants.heliostat_key, + parameters=self._flatten_dict(heliostat_parameters), + ) + else: + h5_columnar_heliostats.write_columnar_heliostats( + heliostats_config=f.create_group(constants.heliostat_key), + columns=columns, + ) diff --git a/artist/scenario/scenario.py b/artist/scenario/scenario.py index d493f60dd..a8ebd25f6 100644 --- a/artist/scenario/scenario.py +++ b/artist/scenario/scenario.py @@ -108,6 +108,7 @@ def load_scenario_from_hdf5( number_of_surface_points_per_facet: torch.Tensor = torch.tensor([50, 50]), change_number_of_control_points_per_facet: torch.Tensor | None = None, device: torch.device | None = None, + surface_evaluation_chunk_size: int = 1024, ) -> Self: """ Class method to load the scenario from an HDF5 file. @@ -129,6 +130,9 @@ def load_scenario_from_hdf5( The device on which to perform computations or load tensors and models (default is None). If None, ``ARTIST`` will automatically select the most appropriate device (CUDA or CPU) based on availability and OS. + surface_evaluation_chunk_size : int + The maximum number of heliostats whose surfaces are evaluated in one batched NURBS evaluation + (default is 1024). Larger values are faster but need more memory on the device. Returns ------- @@ -249,6 +253,7 @@ def load_scenario_from_hdf5( ), change_number_of_control_points_per_facet=change_number_of_control_points_per_facet, device=device, + surface_evaluation_chunk_size=surface_evaluation_chunk_size, ) return cls( diff --git a/artist/scenario/surface_generator.py b/artist/scenario/surface_generator.py index 335132897..5df7e260d 100644 --- a/artist/scenario/surface_generator.py +++ b/artist/scenario/surface_generator.py @@ -366,7 +366,7 @@ def generate_fitted_surface_config( facet_key=f"facet_{i + 1}", control_points=translated_control_points.detach(), degrees=nurbs.degrees, - translation_vector=facet_translation_vectors[i], + facet_translations=facet_translation_vectors[i], canting=canting[i], ) ) @@ -424,7 +424,7 @@ def generate_ideal_surface_config( facet_key=f"facet_{facet_index + 1}", control_points=control_points[facet_index], degrees=self.degrees, - translation_vector=facet_translation_vectors[facet_index], + facet_translations=facet_translation_vectors[facet_index], canting=canting[facet_index], ) facet_config_list.append(facet_config) diff --git a/artist/util/config.py b/artist/util/config.py index fee99623c..072a9ede1 100644 --- a/artist/util/config.py +++ b/artist/util/config.py @@ -433,9 +433,9 @@ class FacetConfig: The NURBS control points. degrees : torch.Tensor The NURBS degree in the east and north direction. - translation_vector : torch.Tensor - The translation_vector of the facet. - canting: torch.Tensor + facet_translations : torch.Tensor + The translation vector of the facet. + canting : torch.Tensor The canting vectors in the east and north direction. Methods @@ -449,7 +449,7 @@ def __init__( facet_key: str, control_points: torch.Tensor, degrees: torch.Tensor, - translation_vector: torch.Tensor, + facet_translations: torch.Tensor, canting: torch.Tensor, ) -> None: """ @@ -463,15 +463,15 @@ def __init__( The NURBS control points. degrees : torch.Tensor The NURBS degree in the east and north direction. - translation_vector : torch.Tensor - The translation_vector of the facet. - canting: torch.Tensor + facet_translations : torch.Tensor + The translation vector of the facet. + canting : torch.Tensor The canting vectors in the east and north direction. """ self.facet_key = facet_key self.control_points = control_points self.degrees = degrees - self.translation_vector = translation_vector + self.facet_translations = facet_translations self.canting = canting def create_facet_dict(self) -> dict[str, Any]: @@ -486,7 +486,7 @@ def create_facet_dict(self) -> dict[str, Any]: return { constants.facet_control_points: self.control_points, constants.facet_degrees: self.degrees, - constants.facets_translation_vector: self.translation_vector, + constants.facets_translation_vector: self.facet_translations, constants.facets_canting: self.canting, } diff --git a/artist/util/constants.py b/artist/util/constants.py index 8c49a11e7..d4c69f229 100644 --- a/artist/util/constants.py +++ b/artist/util/constants.py @@ -53,6 +53,8 @@ """Key to access the facet canting in the east and north direction.""" facet_translations = "facet_translations" """Key to access the facet translation vectors.""" +evaluation_control_points = "evaluation_control_points" +"""Key to access the control points of the surfaces of a heliostat group.""" first_joint_translation_e = "first_joint_translation_e" """Key to access the kinematics first joint translation in the east direction.""" @@ -134,6 +136,15 @@ actuators_prototype_key = "actuator" """Key to access the actuators prototype.""" +layout_attribute = "layout" +"""Attribute of the heliostat group in a scenario file that names the layout of the heliostats.""" +columnar_layout = "columnar" +"""Layout storing each heliostat parameter of all heliostats in one dataset, attribute in hdf5 file.""" +heliostat_indices_key = "heliostat_indices" +"""Key for the indices of the heliostats that have individual parameters for a component.""" +actuator_names_key = "names" +"""Key for the actuator names of all heliostats with individual actuators.""" + heliostat_id = "id" """Key to access the heliostat ID.""" heliostat_position = "position" diff --git a/docs/conf.py b/docs/conf.py index 96b4ff03f..4f0e3b950 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,7 +10,7 @@ project = "ARTIST" copyright = f"{datetime.now().year}, ARTIST consortium" author = "ARTIST Consortium" -release = "2.0.0" +release = "2.1.0" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/docs/scenario.rst b/docs/scenario.rst index b660ebb25..614a62526 100644 --- a/docs/scenario.rst +++ b/docs/scenario.rst @@ -108,43 +108,36 @@ These five elements result in an ``ARTIST`` scenario HDF5 file with the followin │ ├── lightsource2 │ │ └── ... │ └── ... - ├── heliostats [1,*] - │ ├── heliostat1 - │ │ ├── id [1,1] - │ │ ├── position [1,1] - │ │ ├── surface [0,1] - │ │ │ └── facets [1,*] - │ │ │ ├── facet_1 - │ │ │ │ ├── control_points [1,1] - │ │ │ │ ├── degrees [1,1] - │ │ │ │ ├── position [1,1] - │ │ │ │ └── canting [1,1] - │ │ │ ├── facet_2 - │ │ │ │ └── ... - │ │ │ └── ... - │ │ ├── kinematics [0,1] - │ │ │ ├── type [1,1] - │ │ │ ├── initial_orientation [1,1] - │ │ │ └── deviations [0,1] - │ │ │ ├── first_joint_translation_e [0,1] - │ │ │ ├── first_joint_tilt_n [0,1] - │ │ │ └── ... - │ │ └── actuator [0,*] - │ │ ├── actuator_1 - │ │ │ ├── type [1,1] - │ │ │ ├── clockwise_axis_movement [1,1] - │ │ │ ├── min_max_motor_positions [1,1] - │ │ │ └── parameters [0,*] - │ │ │ ├── increment [0,1] - │ │ │ ├── initial_stroke_length [0,1] - │ │ │ ├── offset [0,1] - │ │ │ ├── pivot_radius [0,1] - │ │ │ └── initial_angle [0,1] - │ │ └── actuator_2 - │ │ └── ... - ├── heliostat2 - │ └── ... - └── ... + ├── heliostats [1,1] (attribute: layout = "columnar") + │ ├── names [N] + │ ├── id [N] + │ ├── position [N,4] + │ ├── surface [0,1] + │ │ ├── heliostat_indices [N_s] + │ │ ├── control_points [N_s,F,U,V,3] + │ │ ├── degrees [N_s,F,2] + │ │ ├── position [N_s,F,4] + │ │ └── canting [N_s,F,2,4] + │ ├── kinematics [0,1] + │ │ ├── heliostat_indices [N_k] + │ │ ├── type [N_k] + │ │ ├── initial_orientation [N_k,4] + │ │ └── deviations [0,1] + │ │ ├── first_joint_translation_e [N_k] + │ │ ├── first_joint_tilt_n [N_k] + │ │ └── ... + │ └── actuator [0,1] + │ ├── heliostat_indices [N_a] + │ ├── names [N_a,A] + │ ├── type [N_a,A] + │ ├── clockwise_axis_movement [N_a,A] + │ ├── min_max_motor_positions [N_a,A,2] + │ └── parameters [0,1] + │ ├── increment [N_a,A] + │ ├── initial_stroke_length [N_a,A] + │ ├── offset [N_a,A] + │ ├── pivot_radius [N_a,A] + │ └── initial_angle [N_a,A] └── prototypes [1,1] ├── surface [1,1] │ └── facets [1,*] @@ -176,3 +169,67 @@ These five elements result in an ``ARTIST`` scenario HDF5 file with the followin │ └── initial_angle [0,1] └── actuator_2 └── ... + +The heliostats are stored *column by column*: each heliostat parameter is a single dataset holding the values of all +heliostats, with the heliostats along the first dimension. This keeps the number of HDF5 objects independent of the +number of heliostats, so that even fields with thousands of heliostats are saved and loaded within seconds. Here, +``N`` is the number of heliostats, ``N_s``, ``N_k``, and ``N_a`` are the numbers of heliostats with an individual +surface, kinematics, and actuators, ``F`` is the number of facets, ``U`` and ``V`` are the numbers of control points +per facet, and ``A`` is the number of actuators per heliostat. + +- Heliostats are sorted by name, and all ``[N]`` datasets follow this order. +- Individual surfaces, kinematics, and actuators are only stored for the heliostats that have them. + ``heliostat_indices`` lists the rows of these heliostats in ``names``; all other heliostats use the prototypes. +- Missing kinematics deviations and actuator parameters are stored as ``NaN`` and default to zero when loading. +- All individual surfaces must have the same shape, and all heliostats with individual actuators must have the same + number of actuators. If this is not the case, the ``H5ScenarioGenerator`` warns and stores the heliostats in the + legacy layout described below. + +.. note:: + + **Legacy heliostat layout (ARTIST 2.0.0 and earlier).** Scenarios created with ``ARTIST`` 2.0.0 or earlier store + one HDF5 group per heliostat instead, with every parameter as a separate dataset. All other parts of the scenario + are identical: + + .. code-block:: console + + heliostats [1,*] + ├── heliostat1 + │ ├── id [1,1] + │ ├── position [1,1] + │ ├── surface [0,1] + │ │ └── facets [1,*] + │ │ ├── facet_1 + │ │ │ ├── control_points [1,1] + │ │ │ ├── degrees [1,1] + │ │ │ ├── position [1,1] + │ │ │ └── canting [1,1] + │ │ └── ... + │ ├── kinematics [0,1] + │ │ ├── type [1,1] + │ │ ├── initial_orientation [1,1] + │ │ └── deviations [0,1] + │ │ └── ... + │ └── actuator [0,*] + │ ├── actuator_1 + │ │ ├── type [1,1] + │ │ ├── clockwise_axis_movement [1,1] + │ │ ├── min_max_motor_positions [1,1] + │ │ └── parameters [0,*] + │ │ └── ... + │ └── ... + ├── heliostat2 + │ └── ... + └── ... + + ``ARTIST`` still loads these scenarios and yields exactly the same scenario as from the columnar layout, but logs + a warning, as loading is much slower for large heliostat fields. Existing scenarios can be converted without + regenerating them: + + .. code-block:: python + + from artist.io import convert_scenario_to_columnar + + convert_scenario_to_columnar("scenario.h5", "scenario_columnar.h5") + + ``ARTIST`` 2.0.0 and earlier cannot load scenarios in the columnar layout. diff --git a/pyproject.toml b/pyproject.toml index 11cfdf5b2..66f4d4db6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ packages = ["artist"] [project] name = "artist-csp" -version = "2.0.0" +version = "2.1.0" authors = [ { name="ARTIST Consortium", email="artist@lists.kit.edu" }, ] diff --git a/tests/field/heliostat_field_state.py b/tests/field/heliostat_field_state.py new file mode 100644 index 000000000..242535e12 --- /dev/null +++ b/tests/field/heliostat_field_state.py @@ -0,0 +1,70 @@ +from typing import Any + +import torch + +from artist.field.heliostat_group import HeliostatGroup + + +def heliostat_group_state(heliostat_group: HeliostatGroup) -> dict[str, Any]: + """ + Collect the loaded state of a heliostat group. + + Buffers that are only filled once heliostats are activated are uninitialized after loading and are left out. + + Parameters + ---------- + heliostat_group : HeliostatGroup + The heliostat group. + + Returns + ------- + dict[str, Any] + The heliostat names and all loaded tensors of the group, its kinematics, and its actuators. + """ + state: dict[str, Any] = { + "type": type(heliostat_group).__name__, + "names": list(heliostat_group.names), + } + for prefix, component in ( + ("", heliostat_group), + ("kinematics.", heliostat_group.kinematics), + ("actuators.", heliostat_group.kinematics.actuators), + ): + for key, value in vars(component).items(): + if ( + torch.is_tensor(value) + and not key.startswith("active_") + and key != "preferred_reflection_directions" + ): + state[f"{prefix}{key}"] = value.detach().cpu() + return state + + +def assert_states_equal( + expected_states: list[dict[str, Any]], states: list[dict[str, Any]] +) -> None: + """ + Assert that two lists of heliostat group states are identical. + + Parameters + ---------- + expected_states : list[dict[str, Any]] + The expected heliostat group states. + states : list[dict[str, Any]] + The heliostat group states to check. + + Raises + ------ + AssertionError + If the states differ. + """ + assert len(states) == len(expected_states) + for expected_state, state in zip(expected_states, states): + assert state.keys() == expected_state.keys() + for key, expected_value in expected_state.items(): + if torch.is_tensor(expected_value): + torch.testing.assert_close( + state[key], expected_value, rtol=0, atol=0, msg=key + ) + else: + assert state[key] == expected_value, key diff --git a/tests/field/test_heliostat_field.py b/tests/field/test_heliostat_field.py index 9d6cad145..c324f4a52 100644 --- a/tests/field/test_heliostat_field.py +++ b/tests/field/test_heliostat_field.py @@ -1,12 +1,17 @@ +import pathlib from unittest import mock import h5py import pytest import torch +from artist import ARTIST_ROOT from artist.field import HeliostatField +from artist.io import h5_columnar_heliostats +from artist.scenario import Scenario from artist.util import constants from artist.util.config import SurfaceConfig +from tests.field.heliostat_field_state import assert_states_equal, heliostat_group_state @pytest.mark.parametrize( @@ -43,9 +48,10 @@ def test_heliostat_field_load_from_hdf5_errors( prototype_actuators: dict[str, str | torch.Tensor], error: str, device: torch.device, + tmp_path: pathlib.Path, ) -> None: """ - Test the heliostat field loaded from HDF5 method. + Test that loading heliostats without individual parameters requires the prototypes. Parameters ---------- @@ -59,22 +65,38 @@ def test_heliostat_field_load_from_hdf5_errors( The expected error message. device : torch.device The device on which to initialize tensors. + tmp_path : pathlib.Path + A temporary directory. Raises ------ AssertionError If test does not complete as expected. """ - mock_h5_file = mock.MagicMock(spec=h5py.File) - mock_group_heliostats = mock.MagicMock() - mock_group_heliostats.keys.return_value = ["heliostat_1"] - mock_h5_file.__getitem__.return_value = mock_group_heliostats - mock_group_heliostats.__len__.return_value = 1 - mock_group_heliostats.keys.return_value = ["heliostats"] + with h5py.File(tmp_path / "scenario.h5", "w") as scenario_file: + h5_columnar_heliostats.write_columnar_heliostats( + scenario_file.create_group(constants.heliostat_key), + h5_columnar_heliostats.heliostat_columns( + [ + ( + "heliostat_1", + { + constants.heliostat_id: 1, + constants.heliostat_position: torch.tensor( + [0.0, 0.0, 0.0, 1.0] + ), + }, + ) + ] + ), + ) - with pytest.raises(ValueError) as exc_info: + with ( + h5py.File(tmp_path / "scenario.h5", "r") as scenario_file, + pytest.raises(ValueError) as exc_info, + ): HeliostatField.from_hdf5( - config_file=mock_h5_file, + config_file=scenario_file, prototype_surface=prototype_surface, prototype_kinematics=prototype_kinematics, prototype_actuators=prototype_actuators, @@ -82,3 +104,79 @@ def test_heliostat_field_load_from_hdf5_errors( device=device, ) assert error in str(exc_info.value) + + +@pytest.mark.parametrize( + "scenario_name", + [ + "test_scenario_paint_four_heliostats", + "test_scenario_paint_mix_ideal_prototype_deflectometry", + ], +) +def test_surface_evaluation_chunk_size_does_not_change_result( + scenario_name: str, +) -> None: + """ + Test that evaluating the surfaces in smaller chunks loads exactly the same heliostat field. + + Parameters + ---------- + scenario_name : str + The name of the test scenario. + + Raises + ------ + AssertionError + If test does not complete as expected. + """ + scenario_path = ( + pathlib.Path(ARTIST_ROOT) / f"tests/data/scenarios/{scenario_name}.h5" + ) + states_per_chunk_size = [] + for surface_evaluation_chunk_size in (1024, 1, 3): + with h5py.File(scenario_path, "r") as scenario_file: + scenario = Scenario.load_scenario_from_hdf5( + scenario_file=scenario_file, + number_of_surface_points_per_facet=torch.tensor([10, 10]), + device=torch.device("cpu"), + surface_evaluation_chunk_size=surface_evaluation_chunk_size, + ) + states_per_chunk_size.append( + [ + heliostat_group_state(heliostat_group) + for heliostat_group in scenario.heliostat_field.heliostat_groups + ] + ) + + for states in states_per_chunk_size[1:]: + assert_states_equal(states_per_chunk_size[0], states) + + +@pytest.mark.parametrize("surface_evaluation_chunk_size", [0, -1]) +def test_surface_evaluation_chunk_size_must_be_positive( + surface_evaluation_chunk_size: int, +) -> None: + """ + Test that a non-positive surface evaluation chunk size is rejected. + + Parameters + ---------- + surface_evaluation_chunk_size : int + The invalid chunk size. + + Raises + ------ + AssertionError + If test does not complete as expected. + """ + with pytest.raises(ValueError) as exc_info: + HeliostatField.from_hdf5( + config_file=mock.MagicMock(), + prototype_surface=mock.MagicMock(), + prototype_kinematics={}, + prototype_actuators={}, + number_of_surface_points_per_facet=torch.tensor([10, 10]), + device=torch.device("cpu"), + surface_evaluation_chunk_size=surface_evaluation_chunk_size, + ) + assert "The surface evaluation chunk size must be positive" in str(exc_info.value) diff --git a/tests/io/test_h5_columnar_heliostats.py b/tests/io/test_h5_columnar_heliostats.py new file mode 100644 index 000000000..f836e9743 --- /dev/null +++ b/tests/io/test_h5_columnar_heliostats.py @@ -0,0 +1,545 @@ +import logging +import pathlib +import shutil +from typing import Any + +import h5py +import numpy as np +import pytest +import torch +from pytest_mock import MockerFixture + +from artist import ARTIST_ROOT +from artist.io import h5_columnar_heliostats, paint_scenario_parser +from artist.io.h5_columnar_heliostats import ColumnarLayoutError +from artist.scenario import H5ScenarioGenerator, Scenario +from artist.util import constants +from artist.util.config import LightSourceConfig, LightSourceListConfig +from tests.field.heliostat_field_state import assert_states_equal, heliostat_group_state + +scenarios_path = pathlib.Path(ARTIST_ROOT) / "tests/data/scenarios" + + +def load_states(path: pathlib.Path) -> list[dict[str, Any]]: + """ + Load a scenario on the CPU and collect the state of its heliostat groups. + + Parameters + ---------- + path : pathlib.Path + The path of the scenario. + + Returns + ------- + list[dict[str, Any]] + The state of each heliostat group. + """ + with h5py.File(path, "r") as scenario_file: + scenario = Scenario.load_scenario_from_hdf5( + scenario_file=scenario_file, + number_of_surface_points_per_facet=torch.tensor([10, 10]), + device=torch.device("cpu"), + ) + return [ + heliostat_group_state(heliostat_group) + for heliostat_group in scenario.heliostat_field.heliostat_groups + ] + + +@pytest.mark.parametrize( + "scenario_name", + [ + "test_blocking", + "test_scenario_paint_four_heliostats", + "test_scenario_paint_four_heliostats_ideal", + "test_scenario_paint_mix_ideal_prototype_deflectometry", + "test_scenario_paint_single_heliostat", + "test_scenario_stral_single_heliostat", + ], +) +def test_converted_scenario_loads_identically( + scenario_name: str, tmp_path: pathlib.Path +) -> None: + """ + Test that a scenario converted to the columnar layout loads exactly as the legacy scenario. + + Parameters + ---------- + scenario_name : str + The name of the test scenario. + tmp_path : pathlib.Path + A temporary directory. + + Raises + ------ + AssertionError + If test does not complete as expected. + """ + legacy_path = scenarios_path / f"{scenario_name}.h5" + columnar_path = tmp_path / "columnar.h5" + + h5_columnar_heliostats.convert_scenario_to_columnar(legacy_path, columnar_path) + + with ( + h5py.File(legacy_path, "r") as legacy_file, + h5py.File(columnar_path, "r") as columnar_file, + ): + assert dict(columnar_file.attrs) == dict(legacy_file.attrs) + assert set(columnar_file.keys()) == set(legacy_file.keys()) + assert h5_columnar_heliostats.is_columnar( + columnar_file[constants.heliostat_key] + ) + assert_states_equal(load_states(legacy_path), load_states(columnar_path)) + + +def test_missing_parameters_load_identically( + tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture +) -> None: + """ + Test that missing kinematics deviations and actuator parameters default to zero and are reported in both layouts. + + Parameters + ---------- + tmp_path : pathlib.Path + A temporary directory. + caplog : pytest.LogCaptureFixture + The captured log. + + Raises + ------ + AssertionError + If test does not complete as expected. + """ + legacy_path = tmp_path / "legacy.h5" + columnar_path = tmp_path / "columnar.h5" + shutil.copy(scenarios_path / "test_scenario_paint_four_heliostats.h5", legacy_path) + with h5py.File(legacy_path, "a") as scenario_file: + heliostats = scenario_file[constants.heliostat_key] + names = list(heliostats.keys()) + del heliostats[f"{names[1]}/kinematics/deviations/first_joint_tilt_n"] + del heliostats[f"{names[2]}/kinematics/deviations"] + del heliostats[f"{names[1]}/actuator/actuator_1/parameters/offset"] + h5_columnar_heliostats.convert_scenario_to_columnar(legacy_path, columnar_path) + + warnings = [] + for path in (legacy_path, columnar_path): + caplog.clear() + with caplog.at_level(logging.WARNING, logger="artist.field.heliostat_field"): + states = load_states(path) + warnings.append( + [ + record.getMessage() + for record in caplog.records + if "legacy layout" not in record.getMessage() + ] + ) + if path == legacy_path: + legacy_states = states + else: + assert_states_equal(legacy_states, states) + + assert warnings[0] == warnings[1] + assert any( + f"first_joint_tilt_n set for 2 heliostat(s) ({names[1]}, {names[2]})" in message + for message in warnings[0] + ) + assert any( + f"offset set for actuator_1 on 1 heliostat(s) ({names[1]})" in message + for message in warnings[0] + ) + + +def test_legacy_layout_warning( + tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture +) -> None: + """ + Test that loading the legacy layout warns once, and loading the columnar layout does not warn. + + Parameters + ---------- + tmp_path : pathlib.Path + A temporary directory. + caplog : pytest.LogCaptureFixture + The captured log. + + Raises + ------ + AssertionError + If test does not complete as expected. + """ + legacy_path = scenarios_path / "test_scenario_paint_four_heliostats.h5" + columnar_path = tmp_path / "columnar.h5" + h5_columnar_heliostats.convert_scenario_to_columnar(legacy_path, columnar_path) + + with caplog.at_level(logging.WARNING): + load_states(legacy_path) + assert caplog.text.count("legacy layout") == 1 + assert "convert_scenario_to_columnar" in caplog.text + + caplog.clear() + with caplog.at_level(logging.WARNING): + load_states(columnar_path) + assert "legacy layout" not in caplog.text + + +def heliostat( + name: str, number_of_actuators: int = 2, **parameters: Any +) -> tuple[str, dict[str, Any]]: + """ + Create the flattened parameters of a minimal heliostat. + + Parameters + ---------- + name : str + The heliostat name. + number_of_actuators : int + The number of actuators (default is 2). + **parameters : Any + Parameters to add or, if None, to remove. + + Returns + ------- + tuple[str, dict[str, Any]] + The heliostat name and its flattened parameters. + """ + flat_parameters: dict[str, Any] = { + "id": 1, + "position": torch.tensor([0.0, 1.0, 0.0, 1.0]), + "kinematics/type": "rigid_body", + "kinematics/initial_orientation": torch.tensor([0.0, -1.0, 0.0, 0.0]), + "kinematics/deviations/first_joint_tilt_n": torch.tensor(0.1), + } + for actuator in range(number_of_actuators): + flat_parameters.update( + { + f"actuator/actuator_{actuator}/type": "linear", + f"actuator/actuator_{actuator}/clockwise_axis_movement": True, + f"actuator/actuator_{actuator}/min_max_motor_positions": torch.tensor( + [0, 100] + ), + f"actuator/actuator_{actuator}/parameters/offset": torch.tensor(0.2), + } + ) + for key, value in parameters.items(): + key = key.replace("__", "/") + if value is None: + del flat_parameters[key] + else: + flat_parameters[key] = value + return name, flat_parameters + + +@pytest.mark.parametrize( + "heliostats, message", + [ + ([], "no heliostats"), + ([heliostat("a"), heliostat("a")], "must be unique"), + ([heliostat("a", id=None)], "has no id"), + ([heliostat("a", kinematics__type=None)], "has no kinematics/type"), + ([heliostat("a", unknown=1.0)], "contains the parameter unknown"), + ( + [heliostat("a"), heliostat("b", position=torch.tensor([0.0, 1.0, 0.0]))], + "position", + ), + ( + [heliostat("a", kinematics__deviations__first_joint_tilt_n=torch.ones(2))], + "non-scalar kinematics/deviations/first_joint_tilt_n", + ), + ( + [heliostat("a"), heliostat("b", number_of_actuators=3)], + "numbers of actuators", + ), + ], +) +def test_heliostat_columns_errors( + heliostats: list[tuple[str, dict[str, Any]]], message: str +) -> None: + """ + Test that heliostats the columnar layout cannot store raise an error instead of being stored incompletely. + + Parameters + ---------- + heliostats : list[tuple[str, dict[str, Any]]] + The heliostats. + message : str + A part of the expected error message. + + Raises + ------ + AssertionError + If test does not complete as expected. + """ + with pytest.raises(ColumnarLayoutError) as exc_info: + h5_columnar_heliostats.heliostat_columns(heliostats) + assert message in str(exc_info.value) + + +def test_heliostat_columns_missing_parameters_are_nan() -> None: + """ + Test that parameters missing for some heliostats are stored as NaN and heliostats are sorted by name. + + Raises + ------ + AssertionError + If test does not complete as expected. + """ + columns = h5_columnar_heliostats.heliostat_columns( + [ + heliostat("b", kinematics__deviations__first_joint_tilt_n=None), + heliostat("a", actuator__actuator_1__parameters__offset=None), + ] + ) + + assert columns[constants.names].tolist() == ["a", "b"] + np.testing.assert_array_equal( + columns["kinematics/deviations/first_joint_tilt_n"], + np.array([np.float32(0.1), np.nan]), + ) + np.testing.assert_array_equal( + columns["actuator/parameters/offset"], + np.array([[np.float32(0.2), np.nan], [np.float32(0.2), np.float32(0.2)]]), + ) + + +@pytest.mark.parametrize( + "dataset, value, message", + [ + ( + "kinematics/type", + "invalid", + "The kinematics type: invalid is not yet implemented!", + ), + ( + "actuator/type", + "invalid", + "The actuator type: invalid is not yet implemented!", + ), + ( + "actuator/type", + ["linear", "ideal"], + "all actuators for a given heliostat must have the same type", + ), + ], +) +def test_invalid_columnar_heliostats( + dataset: str, value: Any, message: str, tmp_path: pathlib.Path +) -> None: + """ + Test that invalid heliostat parameters in the columnar layout raise the same errors as in the legacy layout. + + Parameters + ---------- + dataset : str + The dataset to overwrite. + value : Any + The value to write into each row of the dataset. + message : str + A part of the expected error message. + tmp_path : pathlib.Path + A temporary directory. + + Raises + ------ + AssertionError + If test does not complete as expected. + """ + columnar_path = tmp_path / "columnar.h5" + h5_columnar_heliostats.convert_scenario_to_columnar( + scenarios_path / "test_scenario_paint_four_heliostats.h5", columnar_path + ) + with h5py.File(columnar_path, "a") as scenario_file: + heliostats = scenario_file[constants.heliostat_key] + heliostats[dataset][...] = np.broadcast_to( + np.array(value, dtype=h5py.string_dtype()), heliostats[dataset].shape + ) + + with pytest.raises(ValueError) as exc_info: + load_states(columnar_path) + assert message in str(exc_info.value) + + +def test_wrong_number_of_actuators(tmp_path: pathlib.Path) -> None: + """ + Test that heliostats with a wrong number of actuators for their kinematics raise an error. + + Parameters + ---------- + tmp_path : pathlib.Path + A temporary directory. + + Raises + ------ + AssertionError + If test does not complete as expected. + """ + columnar_path = tmp_path / "columnar.h5" + h5_columnar_heliostats.convert_scenario_to_columnar( + scenarios_path / "test_scenario_paint_four_heliostats.h5", columnar_path + ) + with h5py.File(columnar_path, "a") as scenario_file: + del scenario_file[constants.heliostat_key] + h5_columnar_heliostats.write_columnar_heliostats( + scenario_file.create_group(constants.heliostat_key), + h5_columnar_heliostats.heliostat_columns( + [heliostat("a", number_of_actuators=3)] + ), + ) + + with pytest.raises(ValueError) as exc_info: + load_states(columnar_path) + assert "Expected 2 actuators, found 3 actuator(s)." in str(exc_info.value) + + +def test_legacy_surfaces_of_different_shapes(tmp_path: pathlib.Path) -> None: + """ + Test that a legacy scenario whose heliostat groups have surfaces of different shapes still loads. + + Such heliostats cannot be arranged in one set of columns, so they are read one by one. + + Parameters + ---------- + tmp_path : pathlib.Path + A temporary directory. + + Raises + ------ + AssertionError + If test does not complete as expected. + """ + legacy_path = tmp_path / "legacy.h5" + shutil.copy(scenarios_path / "test_scenario_paint_four_heliostats.h5", legacy_path) + with h5py.File(legacy_path, "a") as scenario_file: + for heliostat_config in scenario_file[constants.heliostat_key].values(): + if heliostat_config["actuator/actuator_0/type"].asstr()[()] == "linear": + for facet in heliostat_config["surface/facets"].values(): + control_points = facet["control_points"][::2, ::2] + del facet["control_points"] + facet["control_points"] = control_points + + states = load_states(legacy_path) + assert [ + (state["names"], tuple(state["nurbs_control_points"].shape)) for state in states + ] == [ + (["AA28", "AC43"], (2, 4, 20, 20, 3)), + (["AA31", "AA39"], (2, 4, 10, 10, 3)), + ] + + +def test_convert_scenario_to_columnar_errors(tmp_path: pathlib.Path) -> None: + """ + Test that converting into the input file or converting a columnar scenario again raises an error. + + Parameters + ---------- + tmp_path : pathlib.Path + A temporary directory. + + Raises + ------ + AssertionError + If test does not complete as expected. + """ + legacy_path = scenarios_path / "test_scenario_paint_single_heliostat.h5" + columnar_path = tmp_path / "columnar.h5" + + with pytest.raises(ValueError, match="different file"): + h5_columnar_heliostats.convert_scenario_to_columnar(legacy_path, legacy_path) + + h5_columnar_heliostats.convert_scenario_to_columnar(legacy_path, columnar_path) + with pytest.raises(ValueError, match="already stored in the columnar layout"): + h5_columnar_heliostats.convert_scenario_to_columnar( + columnar_path, tmp_path / "again.h5" + ) + + +def test_generated_columnar_scenario_loads_like_legacy_scenario( + tmp_path: pathlib.Path, mocker: MockerFixture +) -> None: + """ + Test that a scenario generated from PAINT data loads identically in the columnar and in the legacy layout. + + Some heliostats use the prototypes, so that the scenario mixes individual and prototype parameters. + + Parameters + ---------- + tmp_path : pathlib.Path + A temporary directory. + mocker : MockerFixture + A pytest-mock fixture used to force the legacy layout. + + Raises + ------ + AssertionError + If test does not complete as expected. + """ + device = torch.device("cpu") + field_data = pathlib.Path(ARTIST_ROOT) / "tests/data/field_data" + ( + power_plant_config, + target_area_list_planar_config, + target_area_list_cylindrical_config, + ) = paint_scenario_parser.extract_paint_tower_measurements( + tower_measurements_path=field_data / "tower-measurements.json", device=device + ) + heliostat_list_config, prototype_config = ( + paint_scenario_parser.extract_paint_heliostats_ideal_surface( + paths=[ + (name, field_data / "AA39-heliostat-properties.json") + for name in ("AA39", "AB01", "AC02", "AD03", "AE04") + ], + power_plant_position=power_plant_config.power_plant_position, + device=device, + ) + ) + heliostat_list_config.heliostat_list[1].surface = None + heliostat_list_config.heliostat_list[2].kinematics = None + heliostat_list_config.heliostat_list[3].actuators = None + heliostat_list_config.heliostat_list[4].position = torch.tensor( + [10.0, 20.0, 0.0, 1.0] + ) + light_source_list_config = LightSourceListConfig( + light_source_list=[ + LightSourceConfig( + light_source_key="sun", + light_source_type=constants.sun_key, + number_of_rays=10, + distribution_type=constants.light_source_distribution_is_normal, + mean=0.0, + covariance=4.3681e-06, + ) + ] + ) + + def generate(path: pathlib.Path) -> None: + H5ScenarioGenerator( + file_path=path, + power_plant_config=power_plant_config, + target_area_list_planar_config=target_area_list_planar_config, + target_area_list_cylindrical_config=target_area_list_cylindrical_config, + light_source_list_config=light_source_list_config, + heliostat_list_config=heliostat_list_config, + prototype_config=prototype_config, + ).generate_scenario() + + generate(tmp_path / "columnar.h5") + mocker.patch.object( + h5_columnar_heliostats, + "heliostat_columns", + side_effect=ColumnarLayoutError("forced legacy layout"), + ) + generate(tmp_path / "legacy.h5") + mocker.stopall() + + with ( + h5py.File(tmp_path / "columnar.h5", "r") as columnar_file, + h5py.File(tmp_path / "legacy.h5", "r") as legacy_file, + ): + assert h5_columnar_heliostats.is_columnar( + columnar_file[constants.heliostat_key] + ) + assert not h5_columnar_heliostats.is_columnar( + legacy_file[constants.heliostat_key] + ) + assert_states_equal( + load_states(tmp_path / "legacy.h5"), load_states(tmp_path / "columnar.h5") + ) diff --git a/tests/scenario/test_h5_scenario_generator.py b/tests/scenario/test_h5_scenario_generator.py index 49652a770..4932116c5 100644 --- a/tests/scenario/test_h5_scenario_generator.py +++ b/tests/scenario/test_h5_scenario_generator.py @@ -1,9 +1,13 @@ +import logging import pathlib +import types import h5py import pytest +import torch from pytest_mock import MockerFixture +from artist.io import h5_columnar_heliostats from artist.scenario import H5ScenarioGenerator from artist.util import constants from artist.util.config import ( @@ -42,7 +46,10 @@ def scenario_generator(mocker: MockerFixture) -> H5ScenarioGenerator: } mock_prototype_config.create_prototype_dict.return_value = {"param5": "abc"} mock_heliostat_list_config.create_heliostat_list_dict.return_value = { - "param6": "xyz" + "heliostat_1": { + constants.heliostat_id: 1, + constants.heliostat_position: torch.tensor([0.0, 1.0, 0.0, 1.0]), + } } mocker.patch.object( @@ -107,12 +114,51 @@ def test_generate_scenario( constants.target_area_cylindrical_key: ["param3"], constants.light_source_key: ["param4"], constants.prototype_key: ["param5"], - constants.heliostat_key: ["param6"], } for prefix, keys in expected_datasets.items(): for key in keys: assert f"{prefix}/{key}" in f + heliostats = f[constants.heliostat_key] + assert h5_columnar_heliostats.is_columnar(heliostats) + assert heliostats[constants.names].asstr()[()].tolist() == ["heliostat_1"] + assert heliostats[constants.heliostat_position][()].tolist() == [ + [0.0, 1.0, 0.0, 1.0] + ] + + +def test_generate_scenario_falls_back_to_legacy_layout( + scenario_generator: H5ScenarioGenerator, + tmp_path: pathlib.Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test that heliostats the columnar layout cannot store are saved in the legacy layout with a warning.""" + scenario_generator.file_path = tmp_path / "scenario.h5" + # Positions of different shapes cannot be stacked into one dataset. + scenario_generator.heliostat_list_config.create_heliostat_list_dict.return_value = { # type: ignore[attr-defined] + "heliostat_1": { + constants.heliostat_id: 1, + constants.heliostat_position: torch.tensor([0.0, 1.0, 0.0, 1.0]), + }, + "heliostat_2": { + constants.heliostat_id: 2, + constants.heliostat_position: torch.tensor([0.0, 1.0, 0.0]), + }, + } + scenario_generator._flatten_dict = types.MethodType( # type: ignore[method-assign] + H5ScenarioGenerator._flatten_dict, scenario_generator + ) + scenario_generator._include_parameters = H5ScenarioGenerator._include_parameters # type: ignore[method-assign] + + with caplog.at_level(logging.WARNING): + scenario_generator.generate_scenario() + + assert "Storing the heliostats in the legacy layout" in caplog.text + with h5py.File(scenario_generator.file_path, "r") as f: + heliostats = f[constants.heliostat_key] + assert not h5_columnar_heliostats.is_columnar(heliostats) + assert heliostats[f"heliostat_2/{constants.heliostat_position}"].shape == (3,) + def test_generate_scenario_invalid_parent_path_raises(mocker: MockerFixture) -> None: """Test that generator init raises if output parent directory does not exist.""" diff --git a/tests/util/test_config.py b/tests/util/test_config.py index ab4b76f31..1d3ae07f6 100644 --- a/tests/util/test_config.py +++ b/tests/util/test_config.py @@ -186,7 +186,7 @@ def facet() -> FacetConfig: facet_key="facet_1", control_points=torch.zeros((3, 3)), degrees=torch.tensor([2, 2]), - translation_vector=torch.zeros(3), + facet_translations=torch.zeros(3), canting=torch.zeros(2), )