Skip to content

set default midl target to 14 #158

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sahiljhawar merged 1 commit into from
Oct 16, 2026
Merged

set default midl target to 14 #158

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions swvo/io/solar_wind/midl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ def download_and_process(
start_time: datetime,
end_time: datetime,
*,
target: float | int | str = "L1",
target: float | int | str = 14,
method: Literal["ballistic", "mhd"] = "ballistic",
) -> None:
"""
Expand All @@ -190,9 +190,10 @@ def download_and_process(
end_time : datetime
End time of the data to download. Must be timezone-aware.
target : float | int | str, optional
Location at which MIDL returns the solar wind: ``"L1"`` (default) for unpropagated L1
observations, or a distance in Earth radii along the Sun-Earth line for data MIDL has
already propagated.
Location at which MIDL returns the solar wind: a distance in Earth radii along the
Sun-Earth line for data MIDL has already propagated (default 14, the distance used in
the MIDL paper for comparison against OMNI), or ``"L1"`` for unpropagated L1
observations.
method : Literal["ballistic", "mhd"], optional
MIDL propagation method, either ``"ballistic"`` (default) or ``"mhd"``. Only relevant
if `target` is not ``"L1"``. MHD targets must be integer Earth radii in ``[-70, 70]``.
Expand Down Expand Up @@ -383,7 +384,7 @@ def read(
download: bool = False,
propagation: bool = False,
*,
target: float | int | str = "L1",
target: float | int | str = 14,
method: Literal["ballistic", "mhd"] = "ballistic",
) -> pd.DataFrame:
"""
Expand All @@ -401,9 +402,10 @@ def read(
Propagate the data from L1 to near-Earth, defaults to False. Ignored if `target` is
not ``"L1"``, since MIDL has then already propagated the data.
target : float | int | str, optional
Location at which MIDL returns the solar wind: ``"L1"`` (default) for unpropagated L1
observations, or a distance in Earth radii along the Sun-Earth line for data MIDL has
already propagated.
Location at which MIDL returns the solar wind: a distance in Earth radii along the
Sun-Earth line for data MIDL has already propagated (default 14, the distance used in
the MIDL paper for comparison against OMNI), or ``"L1"`` for unpropagated L1
observations.
method : Literal["ballistic", "mhd"], optional
MIDL propagation method, either ``"ballistic"`` (default) or ``"mhd"``. Only relevant
if `target` is not ``"L1"``. MHD targets must be integer Earth radii in ``[-70, 70]``.
Expand Down Expand Up @@ -433,7 +435,7 @@ def read(

if propagation and target != "L1":
logger.warning(
f"Ignoring `propagation` because MIDL data is already propagated to {target_token}",
f"MIDL data is already propagated with target={target_token}, `propagation` is not needed and is ignored",
)
propagation = False

Expand Down
4 changes: 4 additions & 0 deletions swvo/io/solar_wind/read_solar_wind_from_multiple_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -382,8 +382,12 @@ def _read_historical_model(

"""
logger.info(f"Reading {model.LABEL} from {start_time} to {end_time}")
# OMNI is already propagated, so no L1 propagation is applied.
if isinstance(model, SWOMNI):
data_one_model = model.read(start_time, end_time, download=download)
# MIDL is requested already propagated to 14 Re (comparable to OMNI), so `propagation` is not needed.
elif isinstance(model, SWMIDL):
data_one_model = model.read(start_time, end_time, download=download, target=14)
else:
data_one_model = model.read(start_time, end_time, download=download, propagation=True)

Expand Down
57 changes: 25 additions & 32 deletions tests/io/solar_wind/test_midl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#
# SPDX-License-Identifier: Apache-2.0

import inspect
import os
import shutil
from datetime import datetime, timezone
Expand Down Expand Up @@ -90,8 +91,7 @@ def test_initialization_without_env_var(self):
@pytest.mark.parametrize(
"target,method,token",
[
("L1", "ballistic", "L1"),
("l1", "ballistic", "L1"),
(14, "ballistic", "14Re"),
(32, "ballistic", "32Re"),
(20.25, "ballistic", "20p25Re"),
(30, "mhd", "mhd_030Re"),
Expand Down Expand Up @@ -144,26 +144,26 @@ def test_get_processed_file_list(self, midl_instance):
start_time = datetime(2020, 1, 1, tzinfo=timezone.utc)
end_time = datetime(2020, 12, 31, tzinfo=timezone.utc)

file_paths, time_intervals = midl_instance._get_processed_file_list(start_time, end_time, "L1")
file_paths, time_intervals = midl_instance._get_processed_file_list(start_time, end_time, "14Re")

assert len(file_paths) == 366
assert all(str(path).startswith(str(DATA_DIR)) for path in file_paths)
assert all(path.name.startswith("MIDL_SW_L1_") for path in file_paths)
assert file_paths[0] == DATA_DIR / "2020/01" / "MIDL_SW_L1_20200101.csv"
assert all(path.name.startswith("MIDL_SW_14Re_") for path in file_paths)
assert file_paths[0] == DATA_DIR / "2020/01" / "MIDL_SW_14Re_20200101.csv"
assert len(time_intervals) == 366
assert all(isinstance(interval, tuple) for interval in time_intervals)

def test_source_urls(self, midl_instance):
urls = midl_instance._source_urls(
datetime(2023, 12, 20, tzinfo=timezone.utc),
datetime(2024, 2, 3, tzinfo=timezone.utc),
"L1",
"14Re",
)

assert urls == [
f"{SWMIDL.URL}/2023/12/202312_L1.csv",
f"{SWMIDL.URL}/2024/01/202401_L1.csv",
f"{SWMIDL.URL}/2024/02/202402_L1.csv",
f"{SWMIDL.URL}/2023/12/202312_14Re.csv",
f"{SWMIDL.URL}/2024/01/202401_14Re.csv",
f"{SWMIDL.URL}/2024/02/202402_14Re.csv",
]

mhd_urls = midl_instance._source_urls(
Expand All @@ -184,18 +184,24 @@ def test_download_and_process(self, midl_instance, sample_dataset):

args, kwargs = mock_load.call_args
# MIDL slices a naive index, so the UTC bounds must reach it without a timezone.
assert args == (start_time.replace(tzinfo=None), end_time.replace(tzinfo=None), "L1")
assert args == (start_time.replace(tzinfo=None), end_time.replace(tzinfo=None), 14)
assert all(arg.tzinfo is None for arg in args[:2])
assert kwargs == {"method": "ballistic", "coords": "GSM", "orbital_motion": False}

expected_file = DATA_DIR / "2024/03" / "MIDL_SW_L1_20240315.csv"
expected_file = DATA_DIR / "2024/03" / "MIDL_SW_14Re_20240315.csv"
assert expected_file.exists()

data = pd.read_csv(expected_file)
assert len(data) == 4
assert all(field in data.columns for field in EXPECTED_COLUMNS)
assert not any(column in data.columns for column in ("X", "B_source", "B_interp"))
assert midl_instance.url == f"{SWMIDL.URL}/2024/03/202403_L1.csv"
assert midl_instance.url == f"{SWMIDL.URL}/2024/03/202403_14Re.csv"

def test_default_target_is_14_re(self):
for method in (SWMIDL.download_and_process, SWMIDL.read):
defaults = inspect.signature(method).parameters
assert defaults["target"].default == 14
assert defaults["method"].default == "ballistic"

def test_download_and_process_splits_days(self, midl_instance):
start_time = datetime(2024, 3, 15, 23, 58, tzinfo=timezone.utc)
Expand All @@ -205,8 +211,8 @@ def test_download_and_process_splits_days(self, midl_instance):
with patch("midl.load", return_value=dataset):
midl_instance.download_and_process(start_time, end_time)

first_day = DATA_DIR / "2024/03" / "MIDL_SW_L1_20240315.csv"
second_day = DATA_DIR / "2024/03" / "MIDL_SW_L1_20240316.csv"
first_day = DATA_DIR / "2024/03" / "MIDL_SW_14Re_20240315.csv"
second_day = DATA_DIR / "2024/03" / "MIDL_SW_14Re_20240316.csv"

assert len(pd.read_csv(first_day)) == 2
assert len(pd.read_csv(second_day)) == 2
Expand All @@ -220,7 +226,7 @@ def test_download_and_process_merges_existing(self, midl_instance):
with patch("midl.load", return_value=make_dataset(second_start, 3)):
midl_instance.download_and_process(second_start, second_start.replace(hour=2, minute=2))

file_path = DATA_DIR / "2024/03" / "MIDL_SW_L1_20240315.csv"
file_path = DATA_DIR / "2024/03" / "MIDL_SW_14Re_20240315.csv"
data = pd.read_csv(file_path)

assert len(data) == 6
Expand Down Expand Up @@ -295,7 +301,7 @@ def test_read_with_existing_data(self, midl_instance):

data = midl_instance.read(start_time, end_time)

file_path = DATA_DIR / "2024/03" / "MIDL_SW_L1_20240315.csv"
file_path = DATA_DIR / "2024/03" / "MIDL_SW_14Re_20240315.csv"
assert isinstance(data, pd.DataFrame)
assert len(data) == 1440
assert all(col in data.columns for col in EXPECTED_COLUMNS)
Expand All @@ -314,27 +320,14 @@ def test_read_with_download(self, midl_instance, sample_dataset):
assert mock_load.called
assert data.loc[start_time, "bavg"] == pytest.approx(13.0)

def test_with_propagation(self, midl_instance):
start_time = datetime(2024, 3, 15, tzinfo=timezone.utc)
end_time = datetime(2024, 3, 16, tzinfo=timezone.utc)

with patch("midl.load", return_value=make_dataset(start_time, 2 * 1440)):
midl_instance.download_and_process(start_time, end_time)

data = midl_instance.read(start_time, end_time, propagation=True)

assert isinstance(data, pd.DataFrame)
assert all(col in data.columns for col in EXPECTED_COLUMNS)
assert data.index.is_monotonic_increasing
assert any(data["file_name"] == "propagated from previous MIDL file")

def test_propagation_skipped_for_propagated_target(self, midl_instance, caplog):
start_time = datetime(2024, 3, 15, tzinfo=timezone.utc)
end_time = datetime(2024, 3, 16, tzinfo=timezone.utc)

data = midl_instance.read(start_time, end_time, target=32, propagation=True)
data = midl_instance.read(start_time, end_time, propagation=True)

assert "already propagated" in caplog.text
assert "already propagated with target=14Re" in caplog.text
assert "`propagation` is not needed" in caplog.text
# No -1 day shift was applied either, so the output starts exactly at start_time
# instead of the day before, and carries no propagation provenance.
assert data.index[0] == start_time
Expand Down
12 changes: 5 additions & 7 deletions tests/io/solar_wind/test_read_solar_wind_from_multiple_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,17 +372,15 @@ def read_ace(self, read_start, read_end, *, download=False, propagation=False):

def test_midl_is_read_as_a_historical_model(self, tmp_path, expected_columns):
"""SWMIDL passed in the model order is accepted and read like the other historical
models: propagated from L1 and labelled `midl` in the `model` column."""
models: read at the default target (14 Re), without further propagation, and labelled `midl`
in the `model` column."""
start_time = datetime(2024, 11, 22, tzinfo=timezone.utc)
end_time = datetime(2024, 11, 23, tzinfo=timezone.utc)

midl_model = SWMIDL(tmp_path / "midl")

# One MIDL day file per day, plus the preceding day the propagation shift reaches back to.
# The speed ramps slightly across the window: a strictly constant speed would shift every
# sample by the same amount and `sw_mag_propagation` would round half the minutes onto
# each other, leaving an alternating grid that says nothing about the model wiring.
for day in pd.date_range(start_time - timedelta(days=1), end_time, freq="D"):
# One MIDL day file per day, already propagated, so no preceding day is needed.
for day in pd.date_range(start_time, end_time, freq="D"):
index = pd.date_range(day, day + timedelta(hours=24) - timedelta(minutes=1), freq="1min", tz="UTC")
index.name = "t"
speed = np.linspace(400.0, 401.0, len(index))
Expand All @@ -399,7 +397,7 @@ def test_midl_is_read_as_a_historical_model(self, tmp_path, expected_columns):
},
index=index,
)
path = midl_model.data_dir / day.strftime("%Y/%m") / midl_model._file_name(day, "L1")
path = midl_model.data_dir / day.strftime("%Y/%m") / midl_model._file_name(day, "14Re")
path.parent.mkdir(parents=True, exist_ok=True)
df.to_csv(path)

Expand Down
Loading