diff --git a/docs/sphinx/source/whatsnew/v0.16.2.rst b/docs/sphinx/source/whatsnew/v0.16.2.rst index f704525a12..4a11076582 100644 --- a/docs/sphinx/source/whatsnew/v0.16.2.rst +++ b/docs/sphinx/source/whatsnew/v0.16.2.rst @@ -18,7 +18,9 @@ Bug fixes Enhancements ~~~~~~~~~~~~ - +* Add ``dataset`` parameter to :py:func:`~pvlib.iotools.get_era5` + allowing for choosing between ERA5 and ERA5-Land datasets. + (:pull:`2780`) Documentation ~~~~~~~~~~~~~ @@ -42,4 +44,5 @@ Maintenance Contributors ~~~~~~~~~~~~ +* Adam R. Jensen (:ghuser:`AdamRJensen`) diff --git a/pvlib/iotools/era5.py b/pvlib/iotools/era5.py index 5465039536..7206740e57 100644 --- a/pvlib/iotools/era5.py +++ b/pvlib/iotools/era5.py @@ -61,6 +61,7 @@ def _m_to_cm(m): def get_era5(latitude, longitude, start, end, variables, api_key, + dataset="reanalysis-era5-single-levels-timeseries", map_variables=True, timeout=60, url='https://cds.climate.copernicus.eu/api/retrieve/v1/'): """ @@ -68,8 +69,9 @@ def get_era5(latitude, longitude, start, end, variables, api_key, A CDS API key is needed to access this API. Register for one at [1]_. - This API [2]_ provides a subset of the full ERA5 dataset. See [3]_ for - the available variables. Data are available on a 0.25° x 0.25° grid. + This API [2]_ provides a subset of parameters of the full ERA5 datasets, + see [3]_ for available variables. A comparison of ERA5 and ERA5-land is + available in [4]_. Parameters ---------- @@ -87,6 +89,10 @@ def get_era5(latitude, longitude, start, end, variables, api_key, See [1]_ for additional options. api_key : str ECMWF CDS API key. + dataset : str, default ``"reanalysis-era5-single-levels-timeseries"`` + The dataset to query. May be either + ``"reanalysis-era5-single-levels-timeseries"`` or + ``"reanalysis-era5-land-timeseries"``. See [4]_ for details. map_variables : bool, default True When true, renames columns of the DataFrame to pvlib variable names where applicable. Also converts units of some variables. See variable @@ -109,11 +115,26 @@ def get_era5(latitude, longitude, start, end, variables, api_key, meta : dict Metadata. + Notes + ----- + Data are returned for the grid point nearest to the requested location. + The ERA5 dataset is available on a 0.25° x 0.25° grid, whereas ERA5-Land + is available on a finer 0.1° x 0.1° grid [4]_. + + ERA5-Land only covers land surfaces. For locations near the coast, the + nearest grid point may lie over water, in which case ERA5-Land returns + only missing values (NaN) without raising an error. Similarly, for ERA5 + the nearest grid point may be over water and therefore not + representative of conditions on land, e.g., for air temperature. Check + the returned ``meta['latitude']`` and ``meta['longitude']``, and + consider adjusting the requested coordinates slightly inland. + References ---------- .. [1] https://cds.climate.copernicus.eu/ .. [2] https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels-timeseries?tab=overview .. [3] https://confluence.ecmwf.int/pages/viewpage.action?pageId=505390919 + .. [4] https://confluence.ecmwf.int/display/CKB/The+family+of+ERA5+datasets """ # noqa: E501 def _to_utc_dt_notz(dt): @@ -140,7 +161,7 @@ def _to_utc_dt_notz(dt): "data_format": "csv" } } - slug = "processes/reanalysis-era5-single-levels-timeseries/execution" + slug = f"processes/{dataset}/execution" response = requests.post(url + slug, json=params, headers=headers, timeout=timeout) submission_response = response.json() @@ -182,23 +203,28 @@ def _to_utc_dt_notz(dt): results_response = response.json() download_url = results_response['asset']['value']['href'] - # Step 4: finally, download our dataset. it's a zipfile of one CSV + # Step 4: finally, download our dataset. it's a zipfile of CSVs; some + # datasets (e.g., ERA5-Land) return one CSV per group of parameters response = requests.get(download_url, timeout=timeout) zipbuffer = BytesIO(response.content) archive = zipfile.ZipFile(zipbuffer) - filename = archive.filelist[0].filename - csvbuffer = StringIO(archive.read(filename).decode('utf-8')) - df = pd.read_csv(csvbuffer) + filenames = [file.filename for file in archive.filelist] + dfs = [] + for filename in filenames: + csvbuffer = StringIO(archive.read(filename).decode('utf-8')) + dfs.append(pd.read_csv(csvbuffer)) # and parse into the usual formats metadata = submission_response['metadata'] # include messages from ECMWF metadata['jobID'] = job_id - if not df.empty: - metadata['latitude'] = df['latitude'].values[0] - metadata['longitude'] = df['longitude'].values[0] - - df.index = pd.to_datetime(df['valid_time']).dt.tz_localize('UTC') - df = df.drop(columns=['valid_time', 'latitude', 'longitude']) + if not dfs[0].empty: + metadata['latitude'] = dfs[0]['latitude'].values[0] + metadata['longitude'] = dfs[0]['longitude'].values[0] + + for i, df in enumerate(dfs): + df.index = pd.to_datetime(df['valid_time']).dt.tz_localize('UTC') + dfs[i] = df.drop(columns=['valid_time', 'latitude', 'longitude']) + df = pd.concat(dfs, axis=1) if map_variables: # convert units and rename diff --git a/tests/iotools/test_era5.py b/tests/iotools/test_era5.py index 8ef1602fea..6e0703be28 100644 --- a/tests/iotools/test_era5.py +++ b/tests/iotools/test_era5.py @@ -3,6 +3,7 @@ """ import pandas as pd +import numpy as np import pytest import pvlib import requests @@ -49,11 +50,27 @@ def expected(): def test_get_era5(params, expected): df, meta = pvlib.iotools.get_era5(**params) pd.testing.assert_frame_equal(df, expected, check_freq=False, atol=0.1) - assert meta['longitude'] == -80.0 - assert meta['latitude'] == 40.0 + assert np.isclose(meta['longitude'], -80.0) + assert np.isclose(meta['latitude'], 40.0) assert isinstance(meta['jobID'], str) +@requires_ecmwf_credentials +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_era5_land(params, expected): + params['dataset'] = "reanalysis-era5-land-timeseries" + # ghi and temp_air belong to different parameter groups and are + # returned by ERA5-Land as separate CSV files (bhi is not available) + params['variables'] = ['ghi', 'temp_air'] + df, meta = pvlib.iotools.get_era5(**params) + assert np.isclose(meta['longitude'], -80.0) + assert np.isclose(meta['latitude'], 40.0) + pd.testing.assert_index_equal(df.index, expected.index) + assert set(df.columns) == {'ghi', 'temp_air'} + assert df.notna().all().all() + + @requires_ecmwf_credentials @pytest.mark.remote_data @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)