From a997b6b2aba8de6090b109220c3cbf8bb0899e36 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Wed, 10 Jun 2026 11:16:36 +0200 Subject: [PATCH 01/11] Add dataset option to get_era5 --- pvlib/iotools/era5.py | 7 ++++++- tests/iotools/test_era5.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/pvlib/iotools/era5.py b/pvlib/iotools/era5.py index 525848bb27..dd87061aeb 100644 --- a/pvlib/iotools/era5.py +++ b/pvlib/iotools/era5.py @@ -58,6 +58,7 @@ def _m_to_cm(m): def get_era5(latitude, longitude, start, end, variables, api_key, + dataset="reanalysis-era5-single-levels-timeseries", map_variables=True, timeout=60, url='https://cds.climate.copernicus.eu/api/retrieve/v1/'): """ @@ -84,6 +85,10 @@ def get_era5(latitude, longitude, start, end, variables, api_key, See [1]_ for additional options. api_key : str ECMWF CDS API key. + dataset : str, default "reanalysis-era5-single-levels-timeseries" + The dataset to query. May be either + "reanalysis-era5-single-levels-timeseries" or + "reanalysis-era5-land-timeseries". map_variables : bool, default True When true, renames columns of the DataFrame to pvlib variable names where applicable. Also converts units of some variables. See variable @@ -137,7 +142,7 @@ def _to_utc_dt_notz(dt): "data_format": "csv" } } - slug = "processes/reanalysis-era5-single-levels-timeseries/execution" + slug = f"processes/{dataset}/execution" response = requests.post(url + slug, json=params, headers=headers, timeout=timeout) submission_response = response.json() diff --git a/tests/iotools/test_era5.py b/tests/iotools/test_era5.py index 452097c83c..2c37f76bb8 100644 --- a/tests/iotools/test_era5.py +++ b/tests/iotools/test_era5.py @@ -48,6 +48,17 @@ def test_get_era5(params, expected): assert isinstance(meta['jobID'], str) +@requires_ecmwf_credentials +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_era5_land(params, expected): + params['dataset'] = "reanalysis-era5-land-timeseries" + df, meta = pvlib.iotools.get_era5(**params) + assert meta['longitude'] == -80.0 + assert meta['latitude'] == 40.0 + assert pd.testing.assert_index_equal(df.index, expected.index) + + @requires_ecmwf_credentials @pytest.mark.remote_data @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) From 7de716cd3e7ad7e065eb60dba4e49486cee0c315 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Wed, 10 Jun 2026 12:02:54 +0200 Subject: [PATCH 02/11] Update era5-land test to use np.isclose --- tests/iotools/test_era5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/iotools/test_era5.py b/tests/iotools/test_era5.py index 2c37f76bb8..1863e4132c 100644 --- a/tests/iotools/test_era5.py +++ b/tests/iotools/test_era5.py @@ -43,8 +43,8 @@ def expected(): def test_get_era5(params, expected): df, meta = pvlib.iotools.get_era5(**params) pd.testing.assert_frame_equal(df, expected, check_freq=False, atol=0.1) - assert meta['longitude'] == -80.0 - assert meta['latitude'] == 40.0 + assert np.isclose(meta['longitude'], -80.0) + assert np.isclose(meta['latitude'], 40.0) assert isinstance(meta['jobID'], str) From 97f1ee48c70094459b0fe112b35cc64380dd3e28 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Wed, 10 Jun 2026 13:37:31 +0200 Subject: [PATCH 03/11] Add numpy import to test_era5.py --- tests/iotools/test_era5.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/iotools/test_era5.py b/tests/iotools/test_era5.py index 1863e4132c..b27b7ab88e 100644 --- a/tests/iotools/test_era5.py +++ b/tests/iotools/test_era5.py @@ -3,6 +3,7 @@ """ import pandas as pd +import numpy as np import pytest import pvlib import requests From fce831e2c19feb2dce62168100289c7e45d6c4df Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Wed, 10 Jun 2026 15:04:31 +0200 Subject: [PATCH 04/11] Implement suggested changes from review --- pvlib/iotools/era5.py | 8 +++++--- tests/iotools/test_era5.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pvlib/iotools/era5.py b/pvlib/iotools/era5.py index dd87061aeb..c96ff1550e 100644 --- a/pvlib/iotools/era5.py +++ b/pvlib/iotools/era5.py @@ -66,9 +66,10 @@ def get_era5(latitude, longitude, start, end, variables, api_key, A CDS API key is needed to access this API. Register for one at [1]_. - This API [2]_ provides a subset of the full ERA5 dataset. See [3]_ for - the available variables. Data are available on a 0.25° x 0.25° grid. - + This API [2]_ provides a subset of parameters of the full ERA5 datasets, + see [3]_ for available variables. A comparison of ERA5 and ERA5-land is + available in [4]_. + Parameters ---------- latitude : float @@ -116,6 +117,7 @@ def get_era5(latitude, longitude, start, end, variables, api_key, .. [1] https://cds.climate.copernicus.eu/ .. [2] https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels-timeseries?tab=overview .. [3] https://confluence.ecmwf.int/pages/viewpage.action?pageId=505390919 + .. [4] https://confluence.ecmwf.int/display/CKB/The+family+of+ERA5+datasets """ # noqa: E501 def _to_utc_dt_notz(dt): diff --git a/tests/iotools/test_era5.py b/tests/iotools/test_era5.py index b27b7ab88e..532b38f9c3 100644 --- a/tests/iotools/test_era5.py +++ b/tests/iotools/test_era5.py @@ -55,8 +55,8 @@ def test_get_era5(params, expected): def test_get_era5_land(params, expected): params['dataset'] = "reanalysis-era5-land-timeseries" df, meta = pvlib.iotools.get_era5(**params) - assert meta['longitude'] == -80.0 - assert meta['latitude'] == 40.0 + assert np.isclose(meta['longitude'], -80.0) + assert np.isclose(meta['latitude'], 40.0) assert pd.testing.assert_index_equal(df.index, expected.index) From 3dfbac5be5b8861de29c4c48b3f2b2018bd9487d Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Wed, 10 Jun 2026 15:07:06 +0200 Subject: [PATCH 05/11] Add whatsnew entry --- docs/sphinx/source/whatsnew/v0.15.2.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.15.2.rst b/docs/sphinx/source/whatsnew/v0.15.2.rst index ed237f4720..11f584df44 100644 --- a/docs/sphinx/source/whatsnew/v0.15.2.rst +++ b/docs/sphinx/source/whatsnew/v0.15.2.rst @@ -39,12 +39,13 @@ Enhancements ~~~~~~~~~~~~ * Add the ``front_side_fraction`` parameter to :py:func:`pvlib.snow.loss_townsend` to support Townsend snow-loss - workflows for bifacial systems. (:issue:`2755`, :pull:`2756`) - -* Added mapping of the parameter ``"albedo"`` in + workflows for bifacial systems. (:issue:`2755`, :pull:`2756`) +* Add mapping of the parameter ``"albedo"`` in :py:func:`~pvlib.iotools.get_nasa_power` when ``map_variables=True`` (:pull:`2753`) - +* Add ``dataset`` parameter to :py:func:`~pvlib.iotools.get_era5` + allowing for choosing between ERA5 and ERA5-Land datasets. + (pull:`2780`) Documentation ~~~~~~~~~~~~~ From c7eadddef5286f5737624f859c6ab1cf1cfe0e89 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Wed, 10 Jun 2026 15:08:16 +0200 Subject: [PATCH 06/11] Fix linter --- pvlib/iotools/era5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/era5.py b/pvlib/iotools/era5.py index c96ff1550e..c3d6a253ee 100644 --- a/pvlib/iotools/era5.py +++ b/pvlib/iotools/era5.py @@ -69,7 +69,7 @@ def get_era5(latitude, longitude, start, end, variables, api_key, This API [2]_ provides a subset of parameters of the full ERA5 datasets, see [3]_ for available variables. A comparison of ERA5 and ERA5-land is available in [4]_. - + Parameters ---------- latitude : float From 03553695a242d588cb1c8a4bff2dff398bface63 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Thu, 11 Jun 2026 10:28:22 +0200 Subject: [PATCH 07/11] Apply suggestions from code review Co-authored-by: Echedey Luis <80125792+echedey-ls@users.noreply.github.com> --- pvlib/iotools/era5.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pvlib/iotools/era5.py b/pvlib/iotools/era5.py index c3d6a253ee..c2a3198d35 100644 --- a/pvlib/iotools/era5.py +++ b/pvlib/iotools/era5.py @@ -86,10 +86,10 @@ def get_era5(latitude, longitude, start, end, variables, api_key, See [1]_ for additional options. api_key : str ECMWF CDS API key. - dataset : str, default "reanalysis-era5-single-levels-timeseries" + dataset : str, default ``"reanalysis-era5-single-levels-timeseries"`` The dataset to query. May be either - "reanalysis-era5-single-levels-timeseries" or - "reanalysis-era5-land-timeseries". + ``"reanalysis-era5-single-levels-timeseries"`` or + ``"reanalysis-era5-land-timeseries"``. See [4]_ for details. map_variables : bool, default True When true, renames columns of the DataFrame to pvlib variable names where applicable. Also converts units of some variables. See variable From 6cc698ff3f27f0fc5e3e01055a7a7d1305e07fdb Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:01:09 +0200 Subject: [PATCH 08/11] Loop over multiple returned files --- docs/sphinx/source/whatsnew/v0.16.2.rst | 5 +++- pvlib/iotools/era5.py | 39 ++++++++++++++++++------- tests/iotools/test_era5.py | 7 ++++- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.16.2.rst b/docs/sphinx/source/whatsnew/v0.16.2.rst index f704525a12..4a11076582 100644 --- a/docs/sphinx/source/whatsnew/v0.16.2.rst +++ b/docs/sphinx/source/whatsnew/v0.16.2.rst @@ -18,7 +18,9 @@ Bug fixes Enhancements ~~~~~~~~~~~~ - +* Add ``dataset`` parameter to :py:func:`~pvlib.iotools.get_era5` + allowing for choosing between ERA5 and ERA5-Land datasets. + (:pull:`2780`) Documentation ~~~~~~~~~~~~~ @@ -42,4 +44,5 @@ Maintenance Contributors ~~~~~~~~~~~~ +* Adam R. Jensen (:ghuser:`AdamRJensen`) diff --git a/pvlib/iotools/era5.py b/pvlib/iotools/era5.py index 31725e97d3..7206740e57 100644 --- a/pvlib/iotools/era5.py +++ b/pvlib/iotools/era5.py @@ -115,6 +115,20 @@ def get_era5(latitude, longitude, start, end, variables, api_key, meta : dict Metadata. + Notes + ----- + Data are returned for the grid point nearest to the requested location. + The ERA5 dataset is available on a 0.25° x 0.25° grid, whereas ERA5-Land + is available on a finer 0.1° x 0.1° grid [4]_. + + ERA5-Land only covers land surfaces. For locations near the coast, the + nearest grid point may lie over water, in which case ERA5-Land returns + only missing values (NaN) without raising an error. Similarly, for ERA5 + the nearest grid point may be over water and therefore not + representative of conditions on land, e.g., for air temperature. Check + the returned ``meta['latitude']`` and ``meta['longitude']``, and + consider adjusting the requested coordinates slightly inland. + References ---------- .. [1] https://cds.climate.copernicus.eu/ @@ -189,23 +203,28 @@ def _to_utc_dt_notz(dt): results_response = response.json() download_url = results_response['asset']['value']['href'] - # Step 4: finally, download our dataset. it's a zipfile of one CSV + # Step 4: finally, download our dataset. it's a zipfile of CSVs; some + # datasets (e.g., ERA5-Land) return one CSV per group of parameters response = requests.get(download_url, timeout=timeout) zipbuffer = BytesIO(response.content) archive = zipfile.ZipFile(zipbuffer) - filename = archive.filelist[0].filename - csvbuffer = StringIO(archive.read(filename).decode('utf-8')) - df = pd.read_csv(csvbuffer) + filenames = [file.filename for file in archive.filelist] + dfs = [] + for filename in filenames: + csvbuffer = StringIO(archive.read(filename).decode('utf-8')) + dfs.append(pd.read_csv(csvbuffer)) # and parse into the usual formats metadata = submission_response['metadata'] # include messages from ECMWF metadata['jobID'] = job_id - if not df.empty: - metadata['latitude'] = df['latitude'].values[0] - metadata['longitude'] = df['longitude'].values[0] - - df.index = pd.to_datetime(df['valid_time']).dt.tz_localize('UTC') - df = df.drop(columns=['valid_time', 'latitude', 'longitude']) + if not dfs[0].empty: + metadata['latitude'] = dfs[0]['latitude'].values[0] + metadata['longitude'] = dfs[0]['longitude'].values[0] + + for i, df in enumerate(dfs): + df.index = pd.to_datetime(df['valid_time']).dt.tz_localize('UTC') + dfs[i] = df.drop(columns=['valid_time', 'latitude', 'longitude']) + df = pd.concat(dfs, axis=1) if map_variables: # convert units and rename diff --git a/tests/iotools/test_era5.py b/tests/iotools/test_era5.py index 5e601e800c..6e0703be28 100644 --- a/tests/iotools/test_era5.py +++ b/tests/iotools/test_era5.py @@ -60,10 +60,15 @@ def test_get_era5(params, expected): @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) def test_get_era5_land(params, expected): params['dataset'] = "reanalysis-era5-land-timeseries" + # ghi and temp_air belong to different parameter groups and are + # returned by ERA5-Land as separate CSV files (bhi is not available) + params['variables'] = ['ghi', 'temp_air'] df, meta = pvlib.iotools.get_era5(**params) assert np.isclose(meta['longitude'], -80.0) assert np.isclose(meta['latitude'], 40.0) - assert pd.testing.assert_index_equal(df.index, expected.index) + pd.testing.assert_index_equal(df.index, expected.index) + assert set(df.columns) == {'ghi', 'temp_air'} + assert df.notna().all().all() @requires_ecmwf_credentials From 82441a26127cc1381f2b16a8432f1f102015dfc1 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:02:08 +0200 Subject: [PATCH 09/11] Remove old whatsnew entry --- docs/sphinx/source/whatsnew/v0.15.2.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.15.2.rst b/docs/sphinx/source/whatsnew/v0.15.2.rst index 0b4349e5d0..9cf352004f 100644 --- a/docs/sphinx/source/whatsnew/v0.15.2.rst +++ b/docs/sphinx/source/whatsnew/v0.15.2.rst @@ -39,10 +39,6 @@ Enhancements * Add the ``front_side_fraction`` parameter to :py:func:`pvlib.snow.loss_townsend` to support Townsend snow-loss workflows for bifacial systems. (:issue:`2755`, :pull:`2756`) -* Add ``dataset`` parameter to :py:func:`~pvlib.iotools.get_era5` - allowing for choosing between ERA5 and ERA5-Land datasets. - (pull:`2780`) - workflows for bifacial systems. (:issue:`2755`, :pull:`2756`) * Add the following parameters to :py:func:`~pvlib.iotools.get_nasa_power` when ``map_variables=True``: ``temp_dew``, ``precipitable_water``, ``relative_humidity``, ``ghi_extra``, ``dhi_clear``, ``longwave_down``, From 8a7a2d98010cffeef816901a321c187dc0e9f6cc Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:02:40 +0200 Subject: [PATCH 10/11] Lint --- docs/sphinx/source/whatsnew/v0.15.2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sphinx/source/whatsnew/v0.15.2.rst b/docs/sphinx/source/whatsnew/v0.15.2.rst index 9cf352004f..cdcf0395f1 100644 --- a/docs/sphinx/source/whatsnew/v0.15.2.rst +++ b/docs/sphinx/source/whatsnew/v0.15.2.rst @@ -38,7 +38,7 @@ Enhancements ~~~~~~~~~~~~ * Add the ``front_side_fraction`` parameter to :py:func:`pvlib.snow.loss_townsend` to support Townsend snow-loss - workflows for bifacial systems. (:issue:`2755`, :pull:`2756`) + workflows for bifacial systems. (:issue:`2755`, :pull:`2756`) * Add the following parameters to :py:func:`~pvlib.iotools.get_nasa_power` when ``map_variables=True``: ``temp_dew``, ``precipitable_water``, ``relative_humidity``, ``ghi_extra``, ``dhi_clear``, ``longwave_down``, From b43d6260897f364156455e5fcb91f5fbbeea0356 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:03:11 +0200 Subject: [PATCH 11/11] Restore whatsnew --- docs/sphinx/source/whatsnew/v0.15.2.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/sphinx/source/whatsnew/v0.15.2.rst b/docs/sphinx/source/whatsnew/v0.15.2.rst index cdcf0395f1..97383a5723 100644 --- a/docs/sphinx/source/whatsnew/v0.15.2.rst +++ b/docs/sphinx/source/whatsnew/v0.15.2.rst @@ -44,6 +44,7 @@ Enhancements ``relative_humidity``, ``ghi_extra``, ``dhi_clear``, ``longwave_down``, and ``albedo``. (:issue:`2731`, :pull:`2753`, :pull:`2762`) + Documentation ~~~~~~~~~~~~~ * Clarify that :py:func:`pvlib.soiling.hsu` has an implicit minimum