From 8f5bb15060c0898422281afeda74d331486c9efc Mon Sep 17 00:00:00 2001 From: hamid-arian Date: Sun, 23 Aug 2026 17:07:36 -0400 Subject: [PATCH 1/6] release: prepare RiskLabAI 3.0.0 --- LICENSE | 6 +- MANIFEST.in | 15 + PACKAGE_FILES.json | 786 +++ PACKAGE_INSPECTION.md | 56 + PUBLIC_API.json | 4474 +++++++++++++++++ README.md | 202 +- RELEASE_NOTES_NEXT_MAJOR.md | 29 + TEST_INVENTORY.json | 576 +++ docs/causal_factor_analysis.md | 49 + docs/compatibility.md | 110 + examples/causal_factor_analysis_quickstart.py | 37 + pyproject.toml | 177 +- src/RiskLabAI/__init__.py | 53 + src/RiskLabAI/_deprecation.py | 123 + src/RiskLabAI/backtest/__init__.py | 210 + src/RiskLabAI/backtest/advanced_bet_sizing.py | 227 + .../backtest_overfitting_simulation.py | 1560 ++++++ src/RiskLabAI/backtest/backtest_statistics.py | 300 ++ .../backtest/backtest_synthetic_data.py | 99 + src/RiskLabAI/backtest/bet_sizing.py | 509 ++ src/RiskLabAI/backtest/multiple_testing.py | 193 + src/RiskLabAI/backtest/ou_trading_rules.py | 311 ++ .../backtest/probabilistic_sharpe_ratio.py | 144 + .../probability_of_backtest_overfitting.py | 156 + src/RiskLabAI/backtest/robust_statistics.py | 289 ++ src/RiskLabAI/backtest/sharpe_inference.py | 200 + src/RiskLabAI/backtest/strategy_risk.py | 379 ++ .../backtest/test_set_overfitting.py | 320 ++ src/RiskLabAI/backtest/validation/__init__.py | 55 + .../adaptive_combinatorial_purged.py | 472 ++ .../validation/bagged_combinatorial_purged.py | 247 + .../validation/combinatorial_purged.py | 413 ++ .../validation/cross_validator_controller.py | 38 + .../validation/cross_validator_factory.py | 74 + .../validation/cross_validator_interface.py | 235 + src/RiskLabAI/backtest/validation/kfold.py | 359 ++ .../backtest/validation/leakage_aware_hpo.py | 240 + .../backtest/validation/path_adaptive_cpcv.py | 219 + .../backtest/validation/path_bagged_cpcv.py | 137 + .../backtest/validation/purged_kfold.py | 543 ++ .../backtest/validation/walk_forward.py | 183 + .../causal_factor_analysis/__init__.py | 135 + .../causal_factor_analysis/factor_mirage.py | 399 ++ .../graph_identification.py | 1051 ++++ .../causal_factor_analysis/optimizer.py | 212 + .../causal_factor_analysis/protocol.py | 1717 +++++++ .../specification_diagnostics.py | 267 + .../treatment_effects.py | 328 ++ src/RiskLabAI/cluster/__init__.py | 27 + src/RiskLabAI/cluster/clustering.py | 462 ++ src/RiskLabAI/controller/__init__.py | 2 + src/RiskLabAI/controller/bars_initializer.py | 299 ++ .../controller/data_structure_controller.py | 188 + src/RiskLabAI/core/__init__.py | 67 + src/RiskLabAI/core/_builtins.py | 195 + src/RiskLabAI/core/base.py | 193 + src/RiskLabAI/core/registry.py | 402 ++ src/RiskLabAI/data/__init__.py | 34 + src/RiskLabAI/data/denoise/__init__.py | 39 + src/RiskLabAI/data/denoise/denoising.py | 377 ++ src/RiskLabAI/data/denoise/nercome.py | 133 + .../data/differentiation/__init__.py | 7 + .../adaptive_differentiation.py | 268 + .../data/differentiation/differentiation.py | 399 ++ src/RiskLabAI/data/distance/__init__.py | 2 + src/RiskLabAI/data/distance/codependence.py | 188 + .../data/distance/distance_metric.py | 304 ++ src/RiskLabAI/data/labeling/__init__.py | 49 + .../data/labeling/financial_labels.py | 137 + src/RiskLabAI/data/labeling/labeling.py | 453 ++ src/RiskLabAI/data/structures/__init__.py | 35 + .../data/structures/abstract_bars.py | 252 + .../structures/abstract_imbalance_bars.py | 178 + .../abstract_information_driven_bars.py | 129 + .../data/structures/abstract_run_bars.py | 261 + .../hardened_information_driven_bars.py | 147 + .../data/structures/imbalance_bars.py | 134 + src/RiskLabAI/data/structures/run_bars.py | 132 + .../data/structures/standard_bars.py | 87 + src/RiskLabAI/data/structures/time_bars.py | 135 + src/RiskLabAI/data/synthetic_data/__init__.py | 35 + .../synthetic_data/drift_burst_hypothesis.py | 94 + .../data/synthetic_data/simulation.py | 103 + .../synthetic_controlled_environment.py | 431 ++ src/RiskLabAI/data/weights/__init__.py | 24 + src/RiskLabAI/data/weights/sample_weights.py | 234 + src/RiskLabAI/ensemble/__init__.py | 24 + .../ensemble/bagging_classifier_accuracy.py | 45 + .../ensemble/empirical_bagging_accuracy.py | 377 ++ src/RiskLabAI/features/__init__.py | 6 + .../features/entropy_features/__init__.py | 31 + .../entropy_features/bias_corrected.py | 293 ++ .../features/entropy_features/kontoyiannis.py | 109 + .../features/entropy_features/lempel_ziv.py | 41 + .../features/entropy_features/plug_in.py | 39 + .../features/entropy_features/pmf.py | 64 + .../features/entropy_features/shannon.py | 34 + .../features/feature_importance/__init__.py | 66 + .../clustered_feature_importance_mda.py | 157 + .../clustered_feature_importance_mdi.py | 108 + .../feature_importance/debiased_importance.py | 277 + .../feature_importance_controller.py | 42 + .../feature_importance_factory.py | 50 + .../feature_importance_mda.py | 135 + .../feature_importance_mdi.py | 88 + .../feature_importance_sfi.py | 119 + .../feature_importance_strategy.py | 37 + .../generate_synthetic_data.py | 80 + .../feature_importance/orthogonal_features.py | 99 + .../feature_importance/weighted_tau.py | 36 + .../microstructural_features/__init__.py | 29 + .../bekker_parkinson_volatility_estimator.py | 77 + .../corwin_schultz.py | 135 + .../features/microstructural_features/edge.py | 181 + .../features/structural_breaks/__init__.py | 41 + .../structural_breaks/pelt_change_points.py | 98 + .../structural_breaks/structural_breaks.py | 708 +++ .../volatility_robust_sadf.py | 126 + src/RiskLabAI/hpc/__init__.py | 27 + src/RiskLabAI/hpc/hpc.py | 379 ++ src/RiskLabAI/optimization/__init__.py | 48 + src/RiskLabAI/optimization/hedging.py | 69 + src/RiskLabAI/optimization/hrp.py | 215 + .../optimization/hyper_parameter_tuning.py | 197 + src/RiskLabAI/optimization/nco.py | 130 + src/RiskLabAI/pde/__init__.py | 75 + src/RiskLabAI/pde/equation.py | 316 ++ src/RiskLabAI/pde/model.py | 372 ++ src/RiskLabAI/pde/solver.py | 412 ++ src/RiskLabAI/utils/__init__.py | 108 + src/RiskLabAI/utils/constants.py | 87 + src/RiskLabAI/utils/ewma.py | 74 + .../momentum_mean_reverting_strategy_sides.py | 70 + src/RiskLabAI/utils/progress.py | 68 + src/RiskLabAI/utils/publication_plots.py | 192 + src/RiskLabAI/utils/update_figure_layout.py | 56 + test/backtest/test_technical_indicators.py | 119 + .../test_cross_validator_controller.py | 2 +- .../test_cross_validator_factory.py | 2 +- .../test_cfi_book_graph_examples.py | 104 + .../test_factor_mirage.py | 408 ++ .../test_graph_identification.py | 963 ++++ .../test_minimum_variance.py | 425 ++ test/causal_factor_analysis/test_protocol.py | 1173 +++++ .../test_specification_diagnostics.py | 260 + .../test_treatment_effects.py | 180 + .../test_synthetic_controlled_environment.py | 2 + .../test_feature_importance.py | 2 +- .../test_hyper_parameter_tuning.py | 2 +- test/test_deprecation.py | 14 +- 150 files changed, 35631 insertions(+), 222 deletions(-) create mode 100644 MANIFEST.in create mode 100644 PACKAGE_FILES.json create mode 100644 PACKAGE_INSPECTION.md create mode 100644 PUBLIC_API.json create mode 100644 RELEASE_NOTES_NEXT_MAJOR.md create mode 100644 TEST_INVENTORY.json create mode 100644 docs/causal_factor_analysis.md create mode 100644 docs/compatibility.md create mode 100644 examples/causal_factor_analysis_quickstart.py create mode 100644 src/RiskLabAI/__init__.py create mode 100644 src/RiskLabAI/_deprecation.py create mode 100644 src/RiskLabAI/backtest/__init__.py create mode 100644 src/RiskLabAI/backtest/advanced_bet_sizing.py create mode 100644 src/RiskLabAI/backtest/backtest_overfitting_simulation.py create mode 100644 src/RiskLabAI/backtest/backtest_statistics.py create mode 100644 src/RiskLabAI/backtest/backtest_synthetic_data.py create mode 100644 src/RiskLabAI/backtest/bet_sizing.py create mode 100644 src/RiskLabAI/backtest/multiple_testing.py create mode 100644 src/RiskLabAI/backtest/ou_trading_rules.py create mode 100644 src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py create mode 100644 src/RiskLabAI/backtest/probability_of_backtest_overfitting.py create mode 100644 src/RiskLabAI/backtest/robust_statistics.py create mode 100644 src/RiskLabAI/backtest/sharpe_inference.py create mode 100644 src/RiskLabAI/backtest/strategy_risk.py create mode 100644 src/RiskLabAI/backtest/test_set_overfitting.py create mode 100644 src/RiskLabAI/backtest/validation/__init__.py create mode 100644 src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py create mode 100644 src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py create mode 100644 src/RiskLabAI/backtest/validation/combinatorial_purged.py create mode 100644 src/RiskLabAI/backtest/validation/cross_validator_controller.py create mode 100644 src/RiskLabAI/backtest/validation/cross_validator_factory.py create mode 100644 src/RiskLabAI/backtest/validation/cross_validator_interface.py create mode 100644 src/RiskLabAI/backtest/validation/kfold.py create mode 100644 src/RiskLabAI/backtest/validation/leakage_aware_hpo.py create mode 100644 src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py create mode 100644 src/RiskLabAI/backtest/validation/path_bagged_cpcv.py create mode 100644 src/RiskLabAI/backtest/validation/purged_kfold.py create mode 100644 src/RiskLabAI/backtest/validation/walk_forward.py create mode 100644 src/RiskLabAI/causal_factor_analysis/__init__.py create mode 100644 src/RiskLabAI/causal_factor_analysis/factor_mirage.py create mode 100644 src/RiskLabAI/causal_factor_analysis/graph_identification.py create mode 100644 src/RiskLabAI/causal_factor_analysis/optimizer.py create mode 100644 src/RiskLabAI/causal_factor_analysis/protocol.py create mode 100644 src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py create mode 100644 src/RiskLabAI/causal_factor_analysis/treatment_effects.py create mode 100644 src/RiskLabAI/cluster/__init__.py create mode 100644 src/RiskLabAI/cluster/clustering.py create mode 100644 src/RiskLabAI/controller/__init__.py create mode 100644 src/RiskLabAI/controller/bars_initializer.py create mode 100644 src/RiskLabAI/controller/data_structure_controller.py create mode 100644 src/RiskLabAI/core/__init__.py create mode 100644 src/RiskLabAI/core/_builtins.py create mode 100644 src/RiskLabAI/core/base.py create mode 100644 src/RiskLabAI/core/registry.py create mode 100644 src/RiskLabAI/data/__init__.py create mode 100644 src/RiskLabAI/data/denoise/__init__.py create mode 100644 src/RiskLabAI/data/denoise/denoising.py create mode 100644 src/RiskLabAI/data/denoise/nercome.py create mode 100644 src/RiskLabAI/data/differentiation/__init__.py create mode 100644 src/RiskLabAI/data/differentiation/adaptive_differentiation.py create mode 100644 src/RiskLabAI/data/differentiation/differentiation.py create mode 100644 src/RiskLabAI/data/distance/__init__.py create mode 100644 src/RiskLabAI/data/distance/codependence.py create mode 100644 src/RiskLabAI/data/distance/distance_metric.py create mode 100644 src/RiskLabAI/data/labeling/__init__.py create mode 100644 src/RiskLabAI/data/labeling/financial_labels.py create mode 100644 src/RiskLabAI/data/labeling/labeling.py create mode 100644 src/RiskLabAI/data/structures/__init__.py create mode 100644 src/RiskLabAI/data/structures/abstract_bars.py create mode 100644 src/RiskLabAI/data/structures/abstract_imbalance_bars.py create mode 100644 src/RiskLabAI/data/structures/abstract_information_driven_bars.py create mode 100644 src/RiskLabAI/data/structures/abstract_run_bars.py create mode 100644 src/RiskLabAI/data/structures/hardened_information_driven_bars.py create mode 100644 src/RiskLabAI/data/structures/imbalance_bars.py create mode 100644 src/RiskLabAI/data/structures/run_bars.py create mode 100644 src/RiskLabAI/data/structures/standard_bars.py create mode 100644 src/RiskLabAI/data/structures/time_bars.py create mode 100644 src/RiskLabAI/data/synthetic_data/__init__.py create mode 100644 src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py create mode 100644 src/RiskLabAI/data/synthetic_data/simulation.py create mode 100644 src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py create mode 100644 src/RiskLabAI/data/weights/__init__.py create mode 100644 src/RiskLabAI/data/weights/sample_weights.py create mode 100644 src/RiskLabAI/ensemble/__init__.py create mode 100644 src/RiskLabAI/ensemble/bagging_classifier_accuracy.py create mode 100644 src/RiskLabAI/ensemble/empirical_bagging_accuracy.py create mode 100644 src/RiskLabAI/features/__init__.py create mode 100644 src/RiskLabAI/features/entropy_features/__init__.py create mode 100644 src/RiskLabAI/features/entropy_features/bias_corrected.py create mode 100644 src/RiskLabAI/features/entropy_features/kontoyiannis.py create mode 100644 src/RiskLabAI/features/entropy_features/lempel_ziv.py create mode 100644 src/RiskLabAI/features/entropy_features/plug_in.py create mode 100644 src/RiskLabAI/features/entropy_features/pmf.py create mode 100644 src/RiskLabAI/features/entropy_features/shannon.py create mode 100644 src/RiskLabAI/features/feature_importance/__init__.py create mode 100644 src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py create mode 100644 src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py create mode 100644 src/RiskLabAI/features/feature_importance/debiased_importance.py create mode 100644 src/RiskLabAI/features/feature_importance/feature_importance_controller.py create mode 100644 src/RiskLabAI/features/feature_importance/feature_importance_factory.py create mode 100644 src/RiskLabAI/features/feature_importance/feature_importance_mda.py create mode 100644 src/RiskLabAI/features/feature_importance/feature_importance_mdi.py create mode 100644 src/RiskLabAI/features/feature_importance/feature_importance_sfi.py create mode 100644 src/RiskLabAI/features/feature_importance/feature_importance_strategy.py create mode 100644 src/RiskLabAI/features/feature_importance/generate_synthetic_data.py create mode 100644 src/RiskLabAI/features/feature_importance/orthogonal_features.py create mode 100644 src/RiskLabAI/features/feature_importance/weighted_tau.py create mode 100644 src/RiskLabAI/features/microstructural_features/__init__.py create mode 100644 src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py create mode 100644 src/RiskLabAI/features/microstructural_features/corwin_schultz.py create mode 100644 src/RiskLabAI/features/microstructural_features/edge.py create mode 100644 src/RiskLabAI/features/structural_breaks/__init__.py create mode 100644 src/RiskLabAI/features/structural_breaks/pelt_change_points.py create mode 100644 src/RiskLabAI/features/structural_breaks/structural_breaks.py create mode 100644 src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py create mode 100644 src/RiskLabAI/hpc/__init__.py create mode 100644 src/RiskLabAI/hpc/hpc.py create mode 100644 src/RiskLabAI/optimization/__init__.py create mode 100644 src/RiskLabAI/optimization/hedging.py create mode 100644 src/RiskLabAI/optimization/hrp.py create mode 100644 src/RiskLabAI/optimization/hyper_parameter_tuning.py create mode 100644 src/RiskLabAI/optimization/nco.py create mode 100644 src/RiskLabAI/pde/__init__.py create mode 100644 src/RiskLabAI/pde/equation.py create mode 100644 src/RiskLabAI/pde/model.py create mode 100644 src/RiskLabAI/pde/solver.py create mode 100644 src/RiskLabAI/utils/__init__.py create mode 100644 src/RiskLabAI/utils/constants.py create mode 100644 src/RiskLabAI/utils/ewma.py create mode 100644 src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py create mode 100644 src/RiskLabAI/utils/progress.py create mode 100644 src/RiskLabAI/utils/publication_plots.py create mode 100644 src/RiskLabAI/utils/update_figure_layout.py create mode 100644 test/backtest/test_technical_indicators.py create mode 100644 test/causal_factor_analysis/test_cfi_book_graph_examples.py create mode 100644 test/causal_factor_analysis/test_factor_mirage.py create mode 100644 test/causal_factor_analysis/test_graph_identification.py create mode 100644 test/causal_factor_analysis/test_minimum_variance.py create mode 100644 test/causal_factor_analysis/test_protocol.py create mode 100644 test/causal_factor_analysis/test_specification_diagnostics.py create mode 100644 test/causal_factor_analysis/test_treatment_effects.py diff --git a/LICENSE b/LICENSE index 421ea80..587ef53 100644 --- a/LICENSE +++ b/LICENSE @@ -1,12 +1,8 @@ BSD 3-Clause License -Copyright (c) 2022, RiskLab AI (risklab.ai) +Copyright (c) 2022-2026, Hamid Arian All rights reserved. -RiskLabAI includes codes and algorithms from Marcos López de Prado's books -"Advances in Financial Machine Learning" and -"Machine Learning for Asset Managers". - Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..fbb55a8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,15 @@ +include LICENSE +include README.md +include RELEASE_NOTES_NEXT_MAJOR.md +include PUBLIC_API.json +include TEST_INVENTORY.json +include PACKAGE_FILES.json +include PACKAGE_INSPECTION.md +recursive-include src *.py +recursive-include test *.py +recursive-include docs *.md +recursive-include examples *.py +global-exclude __pycache__ +global-exclude *.py[cod] +global-exclude .DS_Store +global-exclude Thumbs.db diff --git a/PACKAGE_FILES.json b/PACKAGE_FILES.json new file mode 100644 index 0000000..98bfac2 --- /dev/null +++ b/PACKAGE_FILES.json @@ -0,0 +1,786 @@ +{ + "schema_version": 1, + "document_type": "RISKLABAI_PYTHON_INTENDED_PACKAGE_FILES", + "product": "RiskLabAI", + "language": "Python", + "version": "3.0.0", + "artifact_types": [ + "wheel", + "source_distribution" + ], + "candidate_file_count": 215, + "candidate_files": [ + "LICENSE", + "MANIFEST.in", + "PACKAGE_FILES.json", + "PACKAGE_IDENTITY.json", + "PACKAGE_INSPECTION.md", + "PUBLIC_API.json", + "README.md", + "RELEASE_NOTES_NEXT_MAJOR.md", + "TEST_INVENTORY.json", + "docs/causal_factor_analysis.md", + "docs/compatibility.md", + "examples/causal_factor_analysis_quickstart.py", + "pyproject.toml", + "src/RiskLabAI/__init__.py", + "src/RiskLabAI/_deprecation.py", + "src/RiskLabAI/backtest/__init__.py", + "src/RiskLabAI/backtest/advanced_bet_sizing.py", + "src/RiskLabAI/backtest/backtest_overfitting_simulation.py", + "src/RiskLabAI/backtest/backtest_statistics.py", + "src/RiskLabAI/backtest/backtest_synthetic_data.py", + "src/RiskLabAI/backtest/bet_sizing.py", + "src/RiskLabAI/backtest/multiple_testing.py", + "src/RiskLabAI/backtest/ou_trading_rules.py", + "src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py", + "src/RiskLabAI/backtest/probability_of_backtest_overfitting.py", + "src/RiskLabAI/backtest/robust_statistics.py", + "src/RiskLabAI/backtest/sharpe_inference.py", + "src/RiskLabAI/backtest/strategy_risk.py", + "src/RiskLabAI/backtest/test_set_overfitting.py", + "src/RiskLabAI/backtest/validation/__init__.py", + "src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/cross_validator_controller.py", + "src/RiskLabAI/backtest/validation/cross_validator_factory.py", + "src/RiskLabAI/backtest/validation/cross_validator_interface.py", + "src/RiskLabAI/backtest/validation/kfold.py", + "src/RiskLabAI/backtest/validation/leakage_aware_hpo.py", + "src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py", + "src/RiskLabAI/backtest/validation/path_bagged_cpcv.py", + "src/RiskLabAI/backtest/validation/purged_kfold.py", + "src/RiskLabAI/backtest/validation/walk_forward.py", + "src/RiskLabAI/causal_factor_analysis/__init__.py", + "src/RiskLabAI/causal_factor_analysis/factor_mirage.py", + "src/RiskLabAI/causal_factor_analysis/graph_identification.py", + "src/RiskLabAI/causal_factor_analysis/optimizer.py", + "src/RiskLabAI/causal_factor_analysis/protocol.py", + "src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py", + "src/RiskLabAI/causal_factor_analysis/treatment_effects.py", + "src/RiskLabAI/cluster/__init__.py", + "src/RiskLabAI/cluster/clustering.py", + "src/RiskLabAI/controller/__init__.py", + "src/RiskLabAI/controller/bars_initializer.py", + "src/RiskLabAI/controller/data_structure_controller.py", + "src/RiskLabAI/core/__init__.py", + "src/RiskLabAI/core/_builtins.py", + "src/RiskLabAI/core/base.py", + "src/RiskLabAI/core/registry.py", + "src/RiskLabAI/data/__init__.py", + "src/RiskLabAI/data/denoise/__init__.py", + "src/RiskLabAI/data/denoise/denoising.py", + "src/RiskLabAI/data/denoise/nercome.py", + "src/RiskLabAI/data/differentiation/__init__.py", + "src/RiskLabAI/data/differentiation/adaptive_differentiation.py", + "src/RiskLabAI/data/differentiation/differentiation.py", + "src/RiskLabAI/data/distance/__init__.py", + "src/RiskLabAI/data/distance/codependence.py", + "src/RiskLabAI/data/distance/distance_metric.py", + "src/RiskLabAI/data/labeling/__init__.py", + "src/RiskLabAI/data/labeling/financial_labels.py", + "src/RiskLabAI/data/labeling/labeling.py", + "src/RiskLabAI/data/structures/__init__.py", + "src/RiskLabAI/data/structures/abstract_bars.py", + "src/RiskLabAI/data/structures/abstract_imbalance_bars.py", + "src/RiskLabAI/data/structures/abstract_information_driven_bars.py", + "src/RiskLabAI/data/structures/abstract_run_bars.py", + "src/RiskLabAI/data/structures/hardened_information_driven_bars.py", + "src/RiskLabAI/data/structures/imbalance_bars.py", + "src/RiskLabAI/data/structures/run_bars.py", + "src/RiskLabAI/data/structures/standard_bars.py", + "src/RiskLabAI/data/structures/time_bars.py", + "src/RiskLabAI/data/synthetic_data/__init__.py", + "src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py", + "src/RiskLabAI/data/synthetic_data/simulation.py", + "src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py", + "src/RiskLabAI/data/weights/__init__.py", + "src/RiskLabAI/data/weights/sample_weights.py", + "src/RiskLabAI/ensemble/__init__.py", + "src/RiskLabAI/ensemble/bagging_classifier_accuracy.py", + "src/RiskLabAI/ensemble/empirical_bagging_accuracy.py", + "src/RiskLabAI/features/__init__.py", + "src/RiskLabAI/features/entropy_features/__init__.py", + "src/RiskLabAI/features/entropy_features/bias_corrected.py", + "src/RiskLabAI/features/entropy_features/kontoyiannis.py", + "src/RiskLabAI/features/entropy_features/lempel_ziv.py", + "src/RiskLabAI/features/entropy_features/plug_in.py", + "src/RiskLabAI/features/entropy_features/pmf.py", + "src/RiskLabAI/features/entropy_features/shannon.py", + "src/RiskLabAI/features/feature_importance/__init__.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/debiased_importance.py", + "src/RiskLabAI/features/feature_importance/feature_importance_controller.py", + "src/RiskLabAI/features/feature_importance/feature_importance_factory.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_sfi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_strategy.py", + "src/RiskLabAI/features/feature_importance/generate_synthetic_data.py", + "src/RiskLabAI/features/feature_importance/orthogonal_features.py", + "src/RiskLabAI/features/feature_importance/weighted_tau.py", + "src/RiskLabAI/features/microstructural_features/__init__.py", + "src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py", + "src/RiskLabAI/features/microstructural_features/corwin_schultz.py", + "src/RiskLabAI/features/microstructural_features/edge.py", + "src/RiskLabAI/features/structural_breaks/__init__.py", + "src/RiskLabAI/features/structural_breaks/pelt_change_points.py", + "src/RiskLabAI/features/structural_breaks/structural_breaks.py", + "src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py", + "src/RiskLabAI/hpc/__init__.py", + "src/RiskLabAI/hpc/hpc.py", + "src/RiskLabAI/optimization/__init__.py", + "src/RiskLabAI/optimization/hedging.py", + "src/RiskLabAI/optimization/hrp.py", + "src/RiskLabAI/optimization/hyper_parameter_tuning.py", + "src/RiskLabAI/optimization/nco.py", + "src/RiskLabAI/pde/__init__.py", + "src/RiskLabAI/pde/equation.py", + "src/RiskLabAI/pde/model.py", + "src/RiskLabAI/pde/solver.py", + "src/RiskLabAI/utils/__init__.py", + "src/RiskLabAI/utils/constants.py", + "src/RiskLabAI/utils/ewma.py", + "src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py", + "src/RiskLabAI/utils/progress.py", + "src/RiskLabAI/utils/publication_plots.py", + "src/RiskLabAI/utils/update_figure_layout.py", + "test/backtest/test_advanced_bet_sizing.py", + "test/backtest/test_backtest_overfitting_simulation.py", + "test/backtest/test_backtest_statistics.py", + "test/backtest/test_backtest_synthetic_data.py", + "test/backtest/test_bet_sizing.py", + "test/backtest/test_multiple_testing.py", + "test/backtest/test_ou_trading_rules.py", + "test/backtest/test_probabilistic_sharpe_ratio.py", + "test/backtest/test_probability_of_backtest_overfitting.py", + "test/backtest/test_robust_statistics.py", + "test/backtest/test_sharpe_inference.py", + "test/backtest/test_strategy_risk.py", + "test/backtest/test_technical_indicators.py", + "test/backtest/test_test_set_overfitting.py", + "test/backtest/validation/test_adaptive_combinatorial_purged.py", + "test/backtest/validation/test_bagged_combinatorial_purged.py", + "test/backtest/validation/test_combinatorial_purged.py", + "test/backtest/validation/test_cross_validator_controller.py", + "test/backtest/validation/test_cross_validator_factory.py", + "test/backtest/validation/test_kfold.py", + "test/backtest/validation/test_leakage_aware_hpo.py", + "test/backtest/validation/test_path_adaptive_cpcv.py", + "test/backtest/validation/test_path_bagged_cpcv.py", + "test/backtest/validation/test_purged_kfold.py", + "test/backtest/validation/test_walk_forward.py", + "test/causal_factor_analysis/test_cfi_book_graph_examples.py", + "test/causal_factor_analysis/test_factor_mirage.py", + "test/causal_factor_analysis/test_graph_identification.py", + "test/causal_factor_analysis/test_minimum_variance.py", + "test/causal_factor_analysis/test_protocol.py", + "test/causal_factor_analysis/test_specification_diagnostics.py", + "test/causal_factor_analysis/test_treatment_effects.py", + "test/cluster/test_clustering.py", + "test/controller/test_bars_initializer.py", + "test/controller/test_data_structure_controller.py", + "test/core/test_base.py", + "test/core/test_builtin_parity.py", + "test/core/test_registry.py", + "test/data/denoise/test_denoising.py", + "test/data/denoise/test_nercome.py", + "test/data/differentiation/test_adaptive_differentiation.py", + "test/data/differentiation/test_differentiation.py", + "test/data/distance/test_codependence.py", + "test/data/distance/test_distance_metric.py", + "test/data/labeling/test_financial_labels.py", + "test/data/labeling/test_labeling.py", + "test/data/structures/test_hardened_information_driven_bars.py", + "test/data/structures/test_imbalance_bars.py", + "test/data/structures/test_run_bars.py", + "test/data/structures/test_standard_bars.py", + "test/data/structures/test_time_bars.py", + "test/data/synthetic_data/test_drift_burst_hypothesis.py", + "test/data/synthetic_data/test_synthetic_controlled_environment.py", + "test/data/weights/test_sample_weights.py", + "test/ensemble/test_bagging_classifier_accuracy.py", + "test/features/entropy_features/test_entropy.py", + "test/features/feature_importance/test_debiased_importance.py", + "test/features/feature_importance/test_feature_importance.py", + "test/features/feature_importance/test_generate_synthetic_data.py", + "test/features/feature_importance/test_orthogonal_features.py", + "test/features/feature_importance/test_weighted_tau.py", + "test/features/microstructural_features/test_edge.py", + "test/features/microstructural_features/test_microstructure.py", + "test/features/structural_breaks/test_pelt_change_points.py", + "test/features/structural_breaks/test_structural_breaks.py", + "test/features/structural_breaks/test_volatility_robust_sadf.py", + "test/hpc/test_hpc.py", + "test/optimization/test_hedging.py", + "test/optimization/test_hrp.py", + "test/optimization/test_hyper_parameter_tuning.py", + "test/optimization/test_nco.py", + "test/pde/test_pde_solver.py", + "test/test_consolidation.py", + "test/test_deprecation.py", + "test/test_performance.py", + "test/utils/test_ewma.py", + "test/utils/test_momentum_mean_reverting_strategy_sides.py", + "test/utils/test_progress.py" + ], + "local_control_files_excluded_from_artifacts": [ + "PACKAGE_IDENTITY.json" + ], + "release_source_file_count": 214, + "release_source_files": [ + "LICENSE", + "MANIFEST.in", + "PACKAGE_FILES.json", + "PACKAGE_INSPECTION.md", + "PUBLIC_API.json", + "README.md", + "RELEASE_NOTES_NEXT_MAJOR.md", + "TEST_INVENTORY.json", + "docs/causal_factor_analysis.md", + "docs/compatibility.md", + "examples/causal_factor_analysis_quickstart.py", + "pyproject.toml", + "src/RiskLabAI/__init__.py", + "src/RiskLabAI/_deprecation.py", + "src/RiskLabAI/backtest/__init__.py", + "src/RiskLabAI/backtest/advanced_bet_sizing.py", + "src/RiskLabAI/backtest/backtest_overfitting_simulation.py", + "src/RiskLabAI/backtest/backtest_statistics.py", + "src/RiskLabAI/backtest/backtest_synthetic_data.py", + "src/RiskLabAI/backtest/bet_sizing.py", + "src/RiskLabAI/backtest/multiple_testing.py", + "src/RiskLabAI/backtest/ou_trading_rules.py", + "src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py", + "src/RiskLabAI/backtest/probability_of_backtest_overfitting.py", + "src/RiskLabAI/backtest/robust_statistics.py", + "src/RiskLabAI/backtest/sharpe_inference.py", + "src/RiskLabAI/backtest/strategy_risk.py", + "src/RiskLabAI/backtest/test_set_overfitting.py", + "src/RiskLabAI/backtest/validation/__init__.py", + "src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/cross_validator_controller.py", + "src/RiskLabAI/backtest/validation/cross_validator_factory.py", + "src/RiskLabAI/backtest/validation/cross_validator_interface.py", + "src/RiskLabAI/backtest/validation/kfold.py", + "src/RiskLabAI/backtest/validation/leakage_aware_hpo.py", + "src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py", + "src/RiskLabAI/backtest/validation/path_bagged_cpcv.py", + "src/RiskLabAI/backtest/validation/purged_kfold.py", + "src/RiskLabAI/backtest/validation/walk_forward.py", + "src/RiskLabAI/causal_factor_analysis/__init__.py", + "src/RiskLabAI/causal_factor_analysis/factor_mirage.py", + "src/RiskLabAI/causal_factor_analysis/graph_identification.py", + "src/RiskLabAI/causal_factor_analysis/optimizer.py", + "src/RiskLabAI/causal_factor_analysis/protocol.py", + "src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py", + "src/RiskLabAI/causal_factor_analysis/treatment_effects.py", + "src/RiskLabAI/cluster/__init__.py", + "src/RiskLabAI/cluster/clustering.py", + "src/RiskLabAI/controller/__init__.py", + "src/RiskLabAI/controller/bars_initializer.py", + "src/RiskLabAI/controller/data_structure_controller.py", + "src/RiskLabAI/core/__init__.py", + "src/RiskLabAI/core/_builtins.py", + "src/RiskLabAI/core/base.py", + "src/RiskLabAI/core/registry.py", + "src/RiskLabAI/data/__init__.py", + "src/RiskLabAI/data/denoise/__init__.py", + "src/RiskLabAI/data/denoise/denoising.py", + "src/RiskLabAI/data/denoise/nercome.py", + "src/RiskLabAI/data/differentiation/__init__.py", + "src/RiskLabAI/data/differentiation/adaptive_differentiation.py", + "src/RiskLabAI/data/differentiation/differentiation.py", + "src/RiskLabAI/data/distance/__init__.py", + "src/RiskLabAI/data/distance/codependence.py", + "src/RiskLabAI/data/distance/distance_metric.py", + "src/RiskLabAI/data/labeling/__init__.py", + "src/RiskLabAI/data/labeling/financial_labels.py", + "src/RiskLabAI/data/labeling/labeling.py", + "src/RiskLabAI/data/structures/__init__.py", + "src/RiskLabAI/data/structures/abstract_bars.py", + "src/RiskLabAI/data/structures/abstract_imbalance_bars.py", + "src/RiskLabAI/data/structures/abstract_information_driven_bars.py", + "src/RiskLabAI/data/structures/abstract_run_bars.py", + "src/RiskLabAI/data/structures/hardened_information_driven_bars.py", + "src/RiskLabAI/data/structures/imbalance_bars.py", + "src/RiskLabAI/data/structures/run_bars.py", + "src/RiskLabAI/data/structures/standard_bars.py", + "src/RiskLabAI/data/structures/time_bars.py", + "src/RiskLabAI/data/synthetic_data/__init__.py", + "src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py", + "src/RiskLabAI/data/synthetic_data/simulation.py", + "src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py", + "src/RiskLabAI/data/weights/__init__.py", + "src/RiskLabAI/data/weights/sample_weights.py", + "src/RiskLabAI/ensemble/__init__.py", + "src/RiskLabAI/ensemble/bagging_classifier_accuracy.py", + "src/RiskLabAI/ensemble/empirical_bagging_accuracy.py", + "src/RiskLabAI/features/__init__.py", + "src/RiskLabAI/features/entropy_features/__init__.py", + "src/RiskLabAI/features/entropy_features/bias_corrected.py", + "src/RiskLabAI/features/entropy_features/kontoyiannis.py", + "src/RiskLabAI/features/entropy_features/lempel_ziv.py", + "src/RiskLabAI/features/entropy_features/plug_in.py", + "src/RiskLabAI/features/entropy_features/pmf.py", + "src/RiskLabAI/features/entropy_features/shannon.py", + "src/RiskLabAI/features/feature_importance/__init__.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/debiased_importance.py", + "src/RiskLabAI/features/feature_importance/feature_importance_controller.py", + "src/RiskLabAI/features/feature_importance/feature_importance_factory.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_sfi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_strategy.py", + "src/RiskLabAI/features/feature_importance/generate_synthetic_data.py", + "src/RiskLabAI/features/feature_importance/orthogonal_features.py", + "src/RiskLabAI/features/feature_importance/weighted_tau.py", + "src/RiskLabAI/features/microstructural_features/__init__.py", + "src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py", + "src/RiskLabAI/features/microstructural_features/corwin_schultz.py", + "src/RiskLabAI/features/microstructural_features/edge.py", + "src/RiskLabAI/features/structural_breaks/__init__.py", + "src/RiskLabAI/features/structural_breaks/pelt_change_points.py", + "src/RiskLabAI/features/structural_breaks/structural_breaks.py", + "src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py", + "src/RiskLabAI/hpc/__init__.py", + "src/RiskLabAI/hpc/hpc.py", + "src/RiskLabAI/optimization/__init__.py", + "src/RiskLabAI/optimization/hedging.py", + "src/RiskLabAI/optimization/hrp.py", + "src/RiskLabAI/optimization/hyper_parameter_tuning.py", + "src/RiskLabAI/optimization/nco.py", + "src/RiskLabAI/pde/__init__.py", + "src/RiskLabAI/pde/equation.py", + "src/RiskLabAI/pde/model.py", + "src/RiskLabAI/pde/solver.py", + "src/RiskLabAI/utils/__init__.py", + "src/RiskLabAI/utils/constants.py", + "src/RiskLabAI/utils/ewma.py", + "src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py", + "src/RiskLabAI/utils/progress.py", + "src/RiskLabAI/utils/publication_plots.py", + "src/RiskLabAI/utils/update_figure_layout.py", + "test/backtest/test_advanced_bet_sizing.py", + "test/backtest/test_backtest_overfitting_simulation.py", + "test/backtest/test_backtest_statistics.py", + "test/backtest/test_backtest_synthetic_data.py", + "test/backtest/test_bet_sizing.py", + "test/backtest/test_multiple_testing.py", + "test/backtest/test_ou_trading_rules.py", + "test/backtest/test_probabilistic_sharpe_ratio.py", + "test/backtest/test_probability_of_backtest_overfitting.py", + "test/backtest/test_robust_statistics.py", + "test/backtest/test_sharpe_inference.py", + "test/backtest/test_strategy_risk.py", + "test/backtest/test_technical_indicators.py", + "test/backtest/test_test_set_overfitting.py", + "test/backtest/validation/test_adaptive_combinatorial_purged.py", + "test/backtest/validation/test_bagged_combinatorial_purged.py", + "test/backtest/validation/test_combinatorial_purged.py", + "test/backtest/validation/test_cross_validator_controller.py", + "test/backtest/validation/test_cross_validator_factory.py", + "test/backtest/validation/test_kfold.py", + "test/backtest/validation/test_leakage_aware_hpo.py", + "test/backtest/validation/test_path_adaptive_cpcv.py", + "test/backtest/validation/test_path_bagged_cpcv.py", + "test/backtest/validation/test_purged_kfold.py", + "test/backtest/validation/test_walk_forward.py", + "test/causal_factor_analysis/test_cfi_book_graph_examples.py", + "test/causal_factor_analysis/test_factor_mirage.py", + "test/causal_factor_analysis/test_graph_identification.py", + "test/causal_factor_analysis/test_minimum_variance.py", + "test/causal_factor_analysis/test_protocol.py", + "test/causal_factor_analysis/test_specification_diagnostics.py", + "test/causal_factor_analysis/test_treatment_effects.py", + "test/cluster/test_clustering.py", + "test/controller/test_bars_initializer.py", + "test/controller/test_data_structure_controller.py", + "test/core/test_base.py", + "test/core/test_builtin_parity.py", + "test/core/test_registry.py", + "test/data/denoise/test_denoising.py", + "test/data/denoise/test_nercome.py", + "test/data/differentiation/test_adaptive_differentiation.py", + "test/data/differentiation/test_differentiation.py", + "test/data/distance/test_codependence.py", + "test/data/distance/test_distance_metric.py", + "test/data/labeling/test_financial_labels.py", + "test/data/labeling/test_labeling.py", + "test/data/structures/test_hardened_information_driven_bars.py", + "test/data/structures/test_imbalance_bars.py", + "test/data/structures/test_run_bars.py", + "test/data/structures/test_standard_bars.py", + "test/data/structures/test_time_bars.py", + "test/data/synthetic_data/test_drift_burst_hypothesis.py", + "test/data/synthetic_data/test_synthetic_controlled_environment.py", + "test/data/weights/test_sample_weights.py", + "test/ensemble/test_bagging_classifier_accuracy.py", + "test/features/entropy_features/test_entropy.py", + "test/features/feature_importance/test_debiased_importance.py", + "test/features/feature_importance/test_feature_importance.py", + "test/features/feature_importance/test_generate_synthetic_data.py", + "test/features/feature_importance/test_orthogonal_features.py", + "test/features/feature_importance/test_weighted_tau.py", + "test/features/microstructural_features/test_edge.py", + "test/features/microstructural_features/test_microstructure.py", + "test/features/structural_breaks/test_pelt_change_points.py", + "test/features/structural_breaks/test_structural_breaks.py", + "test/features/structural_breaks/test_volatility_robust_sadf.py", + "test/hpc/test_hpc.py", + "test/optimization/test_hedging.py", + "test/optimization/test_hrp.py", + "test/optimization/test_hyper_parameter_tuning.py", + "test/optimization/test_nco.py", + "test/pde/test_pde_solver.py", + "test/test_consolidation.py", + "test/test_deprecation.py", + "test/test_performance.py", + "test/utils/test_ewma.py", + "test/utils/test_momentum_mean_reverting_strategy_sides.py", + "test/utils/test_progress.py" + ], + "runtime_source_file_count": 124, + "runtime_source_files": [ + "src/RiskLabAI/__init__.py", + "src/RiskLabAI/_deprecation.py", + "src/RiskLabAI/backtest/__init__.py", + "src/RiskLabAI/backtest/advanced_bet_sizing.py", + "src/RiskLabAI/backtest/backtest_overfitting_simulation.py", + "src/RiskLabAI/backtest/backtest_statistics.py", + "src/RiskLabAI/backtest/backtest_synthetic_data.py", + "src/RiskLabAI/backtest/bet_sizing.py", + "src/RiskLabAI/backtest/multiple_testing.py", + "src/RiskLabAI/backtest/ou_trading_rules.py", + "src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py", + "src/RiskLabAI/backtest/probability_of_backtest_overfitting.py", + "src/RiskLabAI/backtest/robust_statistics.py", + "src/RiskLabAI/backtest/sharpe_inference.py", + "src/RiskLabAI/backtest/strategy_risk.py", + "src/RiskLabAI/backtest/test_set_overfitting.py", + "src/RiskLabAI/backtest/validation/__init__.py", + "src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/cross_validator_controller.py", + "src/RiskLabAI/backtest/validation/cross_validator_factory.py", + "src/RiskLabAI/backtest/validation/cross_validator_interface.py", + "src/RiskLabAI/backtest/validation/kfold.py", + "src/RiskLabAI/backtest/validation/leakage_aware_hpo.py", + "src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py", + "src/RiskLabAI/backtest/validation/path_bagged_cpcv.py", + "src/RiskLabAI/backtest/validation/purged_kfold.py", + "src/RiskLabAI/backtest/validation/walk_forward.py", + "src/RiskLabAI/causal_factor_analysis/__init__.py", + "src/RiskLabAI/causal_factor_analysis/factor_mirage.py", + "src/RiskLabAI/causal_factor_analysis/graph_identification.py", + "src/RiskLabAI/causal_factor_analysis/optimizer.py", + "src/RiskLabAI/causal_factor_analysis/protocol.py", + "src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py", + "src/RiskLabAI/causal_factor_analysis/treatment_effects.py", + "src/RiskLabAI/cluster/__init__.py", + "src/RiskLabAI/cluster/clustering.py", + "src/RiskLabAI/controller/__init__.py", + "src/RiskLabAI/controller/bars_initializer.py", + "src/RiskLabAI/controller/data_structure_controller.py", + "src/RiskLabAI/core/__init__.py", + "src/RiskLabAI/core/_builtins.py", + "src/RiskLabAI/core/base.py", + "src/RiskLabAI/core/registry.py", + "src/RiskLabAI/data/__init__.py", + "src/RiskLabAI/data/denoise/__init__.py", + "src/RiskLabAI/data/denoise/denoising.py", + "src/RiskLabAI/data/denoise/nercome.py", + "src/RiskLabAI/data/differentiation/__init__.py", + "src/RiskLabAI/data/differentiation/adaptive_differentiation.py", + "src/RiskLabAI/data/differentiation/differentiation.py", + "src/RiskLabAI/data/distance/__init__.py", + "src/RiskLabAI/data/distance/codependence.py", + "src/RiskLabAI/data/distance/distance_metric.py", + "src/RiskLabAI/data/labeling/__init__.py", + "src/RiskLabAI/data/labeling/financial_labels.py", + "src/RiskLabAI/data/labeling/labeling.py", + "src/RiskLabAI/data/structures/__init__.py", + "src/RiskLabAI/data/structures/abstract_bars.py", + "src/RiskLabAI/data/structures/abstract_imbalance_bars.py", + "src/RiskLabAI/data/structures/abstract_information_driven_bars.py", + "src/RiskLabAI/data/structures/abstract_run_bars.py", + "src/RiskLabAI/data/structures/hardened_information_driven_bars.py", + "src/RiskLabAI/data/structures/imbalance_bars.py", + "src/RiskLabAI/data/structures/run_bars.py", + "src/RiskLabAI/data/structures/standard_bars.py", + "src/RiskLabAI/data/structures/time_bars.py", + "src/RiskLabAI/data/synthetic_data/__init__.py", + "src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py", + "src/RiskLabAI/data/synthetic_data/simulation.py", + "src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py", + "src/RiskLabAI/data/weights/__init__.py", + "src/RiskLabAI/data/weights/sample_weights.py", + "src/RiskLabAI/ensemble/__init__.py", + "src/RiskLabAI/ensemble/bagging_classifier_accuracy.py", + "src/RiskLabAI/ensemble/empirical_bagging_accuracy.py", + "src/RiskLabAI/features/__init__.py", + "src/RiskLabAI/features/entropy_features/__init__.py", + "src/RiskLabAI/features/entropy_features/bias_corrected.py", + "src/RiskLabAI/features/entropy_features/kontoyiannis.py", + "src/RiskLabAI/features/entropy_features/lempel_ziv.py", + "src/RiskLabAI/features/entropy_features/plug_in.py", + "src/RiskLabAI/features/entropy_features/pmf.py", + "src/RiskLabAI/features/entropy_features/shannon.py", + "src/RiskLabAI/features/feature_importance/__init__.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/debiased_importance.py", + "src/RiskLabAI/features/feature_importance/feature_importance_controller.py", + "src/RiskLabAI/features/feature_importance/feature_importance_factory.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_sfi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_strategy.py", + "src/RiskLabAI/features/feature_importance/generate_synthetic_data.py", + "src/RiskLabAI/features/feature_importance/orthogonal_features.py", + "src/RiskLabAI/features/feature_importance/weighted_tau.py", + "src/RiskLabAI/features/microstructural_features/__init__.py", + "src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py", + "src/RiskLabAI/features/microstructural_features/corwin_schultz.py", + "src/RiskLabAI/features/microstructural_features/edge.py", + "src/RiskLabAI/features/structural_breaks/__init__.py", + "src/RiskLabAI/features/structural_breaks/pelt_change_points.py", + "src/RiskLabAI/features/structural_breaks/structural_breaks.py", + "src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py", + "src/RiskLabAI/hpc/__init__.py", + "src/RiskLabAI/hpc/hpc.py", + "src/RiskLabAI/optimization/__init__.py", + "src/RiskLabAI/optimization/hedging.py", + "src/RiskLabAI/optimization/hrp.py", + "src/RiskLabAI/optimization/hyper_parameter_tuning.py", + "src/RiskLabAI/optimization/nco.py", + "src/RiskLabAI/pde/__init__.py", + "src/RiskLabAI/pde/equation.py", + "src/RiskLabAI/pde/model.py", + "src/RiskLabAI/pde/solver.py", + "src/RiskLabAI/utils/__init__.py", + "src/RiskLabAI/utils/constants.py", + "src/RiskLabAI/utils/ewma.py", + "src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py", + "src/RiskLabAI/utils/progress.py", + "src/RiskLabAI/utils/publication_plots.py", + "src/RiskLabAI/utils/update_figure_layout.py" + ], + "test_file_count": 78, + "test_files": [ + "test/backtest/test_advanced_bet_sizing.py", + "test/backtest/test_backtest_overfitting_simulation.py", + "test/backtest/test_backtest_statistics.py", + "test/backtest/test_backtest_synthetic_data.py", + "test/backtest/test_bet_sizing.py", + "test/backtest/test_multiple_testing.py", + "test/backtest/test_ou_trading_rules.py", + "test/backtest/test_probabilistic_sharpe_ratio.py", + "test/backtest/test_probability_of_backtest_overfitting.py", + "test/backtest/test_robust_statistics.py", + "test/backtest/test_sharpe_inference.py", + "test/backtest/test_strategy_risk.py", + "test/backtest/test_technical_indicators.py", + "test/backtest/test_test_set_overfitting.py", + "test/backtest/validation/test_adaptive_combinatorial_purged.py", + "test/backtest/validation/test_bagged_combinatorial_purged.py", + "test/backtest/validation/test_combinatorial_purged.py", + "test/backtest/validation/test_cross_validator_controller.py", + "test/backtest/validation/test_cross_validator_factory.py", + "test/backtest/validation/test_kfold.py", + "test/backtest/validation/test_leakage_aware_hpo.py", + "test/backtest/validation/test_path_adaptive_cpcv.py", + "test/backtest/validation/test_path_bagged_cpcv.py", + "test/backtest/validation/test_purged_kfold.py", + "test/backtest/validation/test_walk_forward.py", + "test/causal_factor_analysis/test_cfi_book_graph_examples.py", + "test/causal_factor_analysis/test_factor_mirage.py", + "test/causal_factor_analysis/test_graph_identification.py", + "test/causal_factor_analysis/test_minimum_variance.py", + "test/causal_factor_analysis/test_protocol.py", + "test/causal_factor_analysis/test_specification_diagnostics.py", + "test/causal_factor_analysis/test_treatment_effects.py", + "test/cluster/test_clustering.py", + "test/controller/test_bars_initializer.py", + "test/controller/test_data_structure_controller.py", + "test/core/test_base.py", + "test/core/test_builtin_parity.py", + "test/core/test_registry.py", + "test/data/denoise/test_denoising.py", + "test/data/denoise/test_nercome.py", + "test/data/differentiation/test_adaptive_differentiation.py", + "test/data/differentiation/test_differentiation.py", + "test/data/distance/test_codependence.py", + "test/data/distance/test_distance_metric.py", + "test/data/labeling/test_financial_labels.py", + "test/data/labeling/test_labeling.py", + "test/data/structures/test_hardened_information_driven_bars.py", + "test/data/structures/test_imbalance_bars.py", + "test/data/structures/test_run_bars.py", + "test/data/structures/test_standard_bars.py", + "test/data/structures/test_time_bars.py", + "test/data/synthetic_data/test_drift_burst_hypothesis.py", + "test/data/synthetic_data/test_synthetic_controlled_environment.py", + "test/data/weights/test_sample_weights.py", + "test/ensemble/test_bagging_classifier_accuracy.py", + "test/features/entropy_features/test_entropy.py", + "test/features/feature_importance/test_debiased_importance.py", + "test/features/feature_importance/test_feature_importance.py", + "test/features/feature_importance/test_generate_synthetic_data.py", + "test/features/feature_importance/test_orthogonal_features.py", + "test/features/feature_importance/test_weighted_tau.py", + "test/features/microstructural_features/test_edge.py", + "test/features/microstructural_features/test_microstructure.py", + "test/features/structural_breaks/test_pelt_change_points.py", + "test/features/structural_breaks/test_structural_breaks.py", + "test/features/structural_breaks/test_volatility_robust_sadf.py", + "test/hpc/test_hpc.py", + "test/optimization/test_hedging.py", + "test/optimization/test_hrp.py", + "test/optimization/test_hyper_parameter_tuning.py", + "test/optimization/test_nco.py", + "test/pde/test_pde_solver.py", + "test/test_consolidation.py", + "test/test_deprecation.py", + "test/test_performance.py", + "test/utils/test_ewma.py", + "test/utils/test_momentum_mean_reverting_strategy_sides.py", + "test/utils/test_progress.py" + ], + "wheel_python_source_members": [ + "src/RiskLabAI/__init__.py", + "src/RiskLabAI/_deprecation.py", + "src/RiskLabAI/backtest/__init__.py", + "src/RiskLabAI/backtest/advanced_bet_sizing.py", + "src/RiskLabAI/backtest/backtest_overfitting_simulation.py", + "src/RiskLabAI/backtest/backtest_statistics.py", + "src/RiskLabAI/backtest/backtest_synthetic_data.py", + "src/RiskLabAI/backtest/bet_sizing.py", + "src/RiskLabAI/backtest/multiple_testing.py", + "src/RiskLabAI/backtest/ou_trading_rules.py", + "src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py", + "src/RiskLabAI/backtest/probability_of_backtest_overfitting.py", + "src/RiskLabAI/backtest/robust_statistics.py", + "src/RiskLabAI/backtest/sharpe_inference.py", + "src/RiskLabAI/backtest/strategy_risk.py", + "src/RiskLabAI/backtest/test_set_overfitting.py", + "src/RiskLabAI/backtest/validation/__init__.py", + "src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/combinatorial_purged.py", + "src/RiskLabAI/backtest/validation/cross_validator_controller.py", + "src/RiskLabAI/backtest/validation/cross_validator_factory.py", + "src/RiskLabAI/backtest/validation/cross_validator_interface.py", + "src/RiskLabAI/backtest/validation/kfold.py", + "src/RiskLabAI/backtest/validation/leakage_aware_hpo.py", + "src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py", + "src/RiskLabAI/backtest/validation/path_bagged_cpcv.py", + "src/RiskLabAI/backtest/validation/purged_kfold.py", + "src/RiskLabAI/backtest/validation/walk_forward.py", + "src/RiskLabAI/causal_factor_analysis/__init__.py", + "src/RiskLabAI/causal_factor_analysis/factor_mirage.py", + "src/RiskLabAI/causal_factor_analysis/graph_identification.py", + "src/RiskLabAI/causal_factor_analysis/optimizer.py", + "src/RiskLabAI/causal_factor_analysis/protocol.py", + "src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py", + "src/RiskLabAI/causal_factor_analysis/treatment_effects.py", + "src/RiskLabAI/cluster/__init__.py", + "src/RiskLabAI/cluster/clustering.py", + "src/RiskLabAI/controller/__init__.py", + "src/RiskLabAI/controller/bars_initializer.py", + "src/RiskLabAI/controller/data_structure_controller.py", + "src/RiskLabAI/core/__init__.py", + "src/RiskLabAI/core/_builtins.py", + "src/RiskLabAI/core/base.py", + "src/RiskLabAI/core/registry.py", + "src/RiskLabAI/data/__init__.py", + "src/RiskLabAI/data/denoise/__init__.py", + "src/RiskLabAI/data/denoise/denoising.py", + "src/RiskLabAI/data/denoise/nercome.py", + "src/RiskLabAI/data/differentiation/__init__.py", + "src/RiskLabAI/data/differentiation/adaptive_differentiation.py", + "src/RiskLabAI/data/differentiation/differentiation.py", + "src/RiskLabAI/data/distance/__init__.py", + "src/RiskLabAI/data/distance/codependence.py", + "src/RiskLabAI/data/distance/distance_metric.py", + "src/RiskLabAI/data/labeling/__init__.py", + "src/RiskLabAI/data/labeling/financial_labels.py", + "src/RiskLabAI/data/labeling/labeling.py", + "src/RiskLabAI/data/structures/__init__.py", + "src/RiskLabAI/data/structures/abstract_bars.py", + "src/RiskLabAI/data/structures/abstract_imbalance_bars.py", + "src/RiskLabAI/data/structures/abstract_information_driven_bars.py", + "src/RiskLabAI/data/structures/abstract_run_bars.py", + "src/RiskLabAI/data/structures/hardened_information_driven_bars.py", + "src/RiskLabAI/data/structures/imbalance_bars.py", + "src/RiskLabAI/data/structures/run_bars.py", + "src/RiskLabAI/data/structures/standard_bars.py", + "src/RiskLabAI/data/structures/time_bars.py", + "src/RiskLabAI/data/synthetic_data/__init__.py", + "src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py", + "src/RiskLabAI/data/synthetic_data/simulation.py", + "src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py", + "src/RiskLabAI/data/weights/__init__.py", + "src/RiskLabAI/data/weights/sample_weights.py", + "src/RiskLabAI/ensemble/__init__.py", + "src/RiskLabAI/ensemble/bagging_classifier_accuracy.py", + "src/RiskLabAI/ensemble/empirical_bagging_accuracy.py", + "src/RiskLabAI/features/__init__.py", + "src/RiskLabAI/features/entropy_features/__init__.py", + "src/RiskLabAI/features/entropy_features/bias_corrected.py", + "src/RiskLabAI/features/entropy_features/kontoyiannis.py", + "src/RiskLabAI/features/entropy_features/lempel_ziv.py", + "src/RiskLabAI/features/entropy_features/plug_in.py", + "src/RiskLabAI/features/entropy_features/pmf.py", + "src/RiskLabAI/features/entropy_features/shannon.py", + "src/RiskLabAI/features/feature_importance/__init__.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/debiased_importance.py", + "src/RiskLabAI/features/feature_importance/feature_importance_controller.py", + "src/RiskLabAI/features/feature_importance/feature_importance_factory.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mda.py", + "src/RiskLabAI/features/feature_importance/feature_importance_mdi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_sfi.py", + "src/RiskLabAI/features/feature_importance/feature_importance_strategy.py", + "src/RiskLabAI/features/feature_importance/generate_synthetic_data.py", + "src/RiskLabAI/features/feature_importance/orthogonal_features.py", + "src/RiskLabAI/features/feature_importance/weighted_tau.py", + "src/RiskLabAI/features/microstructural_features/__init__.py", + "src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py", + "src/RiskLabAI/features/microstructural_features/corwin_schultz.py", + "src/RiskLabAI/features/microstructural_features/edge.py", + "src/RiskLabAI/features/structural_breaks/__init__.py", + "src/RiskLabAI/features/structural_breaks/pelt_change_points.py", + "src/RiskLabAI/features/structural_breaks/structural_breaks.py", + "src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py", + "src/RiskLabAI/hpc/__init__.py", + "src/RiskLabAI/hpc/hpc.py", + "src/RiskLabAI/optimization/__init__.py", + "src/RiskLabAI/optimization/hedging.py", + "src/RiskLabAI/optimization/hrp.py", + "src/RiskLabAI/optimization/hyper_parameter_tuning.py", + "src/RiskLabAI/optimization/nco.py", + "src/RiskLabAI/pde/__init__.py", + "src/RiskLabAI/pde/equation.py", + "src/RiskLabAI/pde/model.py", + "src/RiskLabAI/pde/solver.py", + "src/RiskLabAI/utils/__init__.py", + "src/RiskLabAI/utils/constants.py", + "src/RiskLabAI/utils/ewma.py", + "src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py", + "src/RiskLabAI/utils/progress.py", + "src/RiskLabAI/utils/publication_plots.py", + "src/RiskLabAI/utils/update_figure_layout.py" + ], + "generated_metadata_members": { + "policy": "Generated package-manager metadata is not pre-enumerated by filename here; the future artifact inspector must validate its normalized field set and reject undeclared payload files." + } +} diff --git a/PACKAGE_INSPECTION.md b/PACKAGE_INSPECTION.md new file mode 100644 index 0000000..08fc6c4 --- /dev/null +++ b/PACKAGE_INSPECTION.md @@ -0,0 +1,56 @@ +# Future Python package inspection specification + +This specification applies only after a separate human authorization creates a +temporary RiskLabAI 3.0.0 wheel and source distribution. It does not authorize +building, installing, uploading, publishing, tagging, or releasing anything. + +## Inputs + +The inspector must use the final, hash-locked source tree and compare it with +`PACKAGE_FILES.json`, `PUBLIC_API.json`, `TEST_INVENTORY.json`, and +`pyproject.toml`. It must reject a changed input, an undecodable file, an +unresolved link, or a concurrent mutation before considering an artifact. + +## Archive safety and membership + +For both archive formats, reject absolute paths, drive-qualified paths, +backslashes, traversal components, duplicate paths, case-fold or Unicode +normalization collisions, control characters, trailing dots or spaces, +alternate data streams, links, device files, and unexpected native binaries. + +The wheel may contain only the 124 declared Python runtime modules plus the +standard metadata directory created for RiskLabAI 3.0.0. Tests, examples, +local readiness controls, caches, bytecode, credentials, and undeclared data +must not be present in the wheel. The source distribution must contain exactly +the release-source members declared in `PACKAGE_FILES.json` plus only the +backend-generated metadata members that the inspection record enumerates and +hashes explicitly. + +## Metadata + +The normalized package metadata must agree exactly with `pyproject.toml`: +name RiskLabAI, version 3.0.0, Python 3.12 through 3.14, BSD-3-Clause, the +declared maintainer and contact, repository, issue, and documentation links, +all required dependency clauses, and every optional group. No dynamic version, +entry point, plugin, executable script, or undeclared dependency is permitted. + +Every wheel record must have a valid cryptographic digest and size, except its +own permitted empty digest fields. The license and long description must match +the reviewed source bytes. + +## Runtime checks + +In fresh, isolated environments for every supported Python and valid NumPy +lane, verify the exact module origins, `RiskLabAI.__version__ == "3.0.0"`, the +13 root exports, all module and package exports in `PUBLIC_API.json`, and the +complete 57-feature causal API. Run the exact package-scoped test inventory and +the valid optional-feature lanes. Confirm that optional groups do not become +base dependencies and that changepoints alone is unavailable on Python 3.14. + +## Final review + +Scan every archive member name and every decoded public text member against the +release hygiene denylist. Record archive hashes, member hashes, metadata, +runtime origins, test results, supported environments, and every deviation. +Any deviation keeps the artifact rejected. Human release, publication, and +version-control authority remain separate decisions after inspection. diff --git a/PUBLIC_API.json b/PUBLIC_API.json new file mode 100644 index 0000000..fbdf042 --- /dev/null +++ b/PUBLIC_API.json @@ -0,0 +1,4474 @@ +{ + "schema_version": 1, + "document_type": "RISKLABAI_PYTHON_PUBLIC_API_INVENTORY", + "product": "RiskLabAI", + "language": "Python", + "version": "3.0.0", + "inventory_rule": { + "root_and_package_exports": "Literal __all__ when present; otherwise imported public bindings in package initializers.", + "module_definitions": "Every non-underscore top-level function or class and every public uppercase constant is frozen conservatively." + }, + "root_exports": [ + "__version__", + "backtest", + "causal_factor_analysis", + "cluster", + "controller", + "core", + "data", + "ensemble", + "features", + "hpc", + "optimization", + "pde", + "utils" + ], + "runtime_module_count": 124, + "runtime_modules": [ + { + "path": "src/RiskLabAI/__init__.py", + "module": "RiskLabAI", + "module_kind": "package", + "sha256": "70093B30E9B7B92F555E40FB0DF936092A661BC88CA032549F1982624147A4AC", + "size": 1095, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/_deprecation.py", + "module": "RiskLabAI._deprecation", + "module_kind": "module", + "sha256": "F44F03C24759B46E69B0BD0241B51FF34454A9AAA1CD9A0498364284D5FB9498", + "size": 3889, + "defined_public_names": [ + "deprecated_alias", + "deprecated_class", + "warn_deprecated_name" + ], + "definitions": [ + { + "name": "deprecated_alias", + "kind": "function", + "signature": "(new_func: _F, old_name: str, *, removed_in: str)" + }, + { + "name": "deprecated_class", + "kind": "function", + "signature": "(new_cls: type, old_name: str, *, removed_in: str)" + }, + { + "name": "warn_deprecated_name", + "kind": "function", + "signature": "(old_name: str, new_name: str, *, removed_in: str)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/__init__.py", + "module": "RiskLabAI.backtest", + "module_kind": "package", + "sha256": "6AE9F8763745BD3331A53D77441E4AB76BA51727916F8516C3181CC7F88590FB", + "size": 6381, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/backtest/advanced_bet_sizing.py", + "module": "RiskLabAI.backtest.advanced_bet_sizing", + "module_kind": "module", + "sha256": "E50F8275DC235F7DA777146D0715B945BAA3C7B91AEDA1FBDAA210D44C4A099F", + "size": 10017, + "defined_public_names": [ + "PlattCalibrator", + "distributionally_robust_kelly_fraction", + "expected_calibration_error", + "kelly_bet_fraction" + ], + "definitions": [ + { + "name": "kelly_bet_fraction", + "kind": "function", + "signature": "(probability, payout_ratio: float=1.0, max_fraction: float=1.0)" + }, + { + "name": "expected_calibration_error", + "kind": "function", + "signature": "(probabilities, outcomes, n_bins: int=10)" + }, + { + "name": "PlattCalibrator", + "kind": "class", + "signature": "(self)" + }, + { + "name": "distributionally_robust_kelly_fraction", + "kind": "function", + "signature": "(probability_estimate, n_observations: int, payout_ratio: float=1.0, max_fraction: float=1.0, confidence_z: float=1.0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/backtest_overfitting_simulation.py", + "module": "RiskLabAI.backtest.backtest_overfitting_simulation", + "module_kind": "module", + "sha256": "F435D84F28A7ACFB3237D329BE0FF82B7F60B1729810E4B8936A324CAA8EF595", + "size": 61737, + "defined_public_names": [ + "backtest_overfitting_simulation_financial_metrics_rank_correlation", + "backtest_overfitting_simulation_model_complexity", + "backtest_overfitting_simulation_results", + "expected_shortfall", + "financial_features_backtest_overfitting_simulation", + "format_cpu_info", + "generate_random_data", + "get_cpu_info", + "measure_all_cv_computational_requirements", + "measure_computational_requirements", + "measure_cpcv_parallelization", + "measure_cpcv_scalability", + "noised_backtest_overfitting_simulation", + "overall_backtest_overfitting_simulation", + "overall_novel_methods_backtest_overfitting_simulation", + "sharpe_ratio", + "sortino_ratio", + "temporal_backtest_overfitting_simulation", + "time_temporal_backtest_overfitting_simulation", + "varying_embargo_backtest_overfitting_simulation" + ], + "definitions": [ + { + "name": "financial_features_backtest_overfitting_simulation", + "kind": "function", + "signature": "(prices: pd.Series, noise_scale: float=0.0, random_state: Optional[int]=None)" + }, + { + "name": "backtest_overfitting_simulation_results", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], cross_validators: dict[str, Any], noise_scale: float=0.0, random_state: int=None, n_jobs: int=1)" + }, + { + "name": "overall_backtest_overfitting_simulation", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], step_risk_free_rate: float, noise_scale: float=0.0, random_state: int=None, n_jobs: int=1)" + }, + { + "name": "temporal_backtest_overfitting_simulation", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], step_risk_free_rate: float, overfitting_partitions_length: int, n_jobs: int=1)" + }, + { + "name": "time_temporal_backtest_overfitting_simulation", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], step_risk_free_rate: float, overfitting_partitions_duration: str='A', n_jobs: int=1)" + }, + { + "name": "varying_embargo_backtest_overfitting_simulation", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], step_risk_free_rate: float, embargo_values: list[float], n_jobs: int=1)" + }, + { + "name": "sharpe_ratio", + "kind": "function", + "signature": "(returns, risk_free_rate=0)" + }, + { + "name": "sortino_ratio", + "kind": "function", + "signature": "(returns, risk_free_rate=0)" + }, + { + "name": "expected_shortfall", + "kind": "function", + "signature": "(returns, step_risk_free_rate, confidence_level=0.05)" + }, + { + "name": "backtest_overfitting_simulation_financial_metrics_rank_correlation", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], step_risk_free_rate: float, n_jobs: int=1)" + }, + { + "name": "backtest_overfitting_simulation_model_complexity", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, Any], step_risk_free_rate: float, n_jobs: int=1)" + }, + { + "name": "noised_backtest_overfitting_simulation", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], step_risk_free_rate: float, noise_scales: list[float], random_state: int=None, n_jobs: int=1)" + }, + { + "name": "overall_novel_methods_backtest_overfitting_simulation", + "kind": "function", + "signature": "(prices: pd.Series, strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], models: dict[str, dict[str, Any]], step_risk_free_rate: float, noise_scale: float=0.0, random_state: int=None, n_jobs: int=1)" + }, + { + "name": "get_cpu_info", + "kind": "function", + "signature": "()" + }, + { + "name": "format_cpu_info", + "kind": "function", + "signature": "(cpu_info)" + }, + { + "name": "generate_random_data", + "kind": "function", + "signature": "(n_samples: int, n_features: int)" + }, + { + "name": "measure_computational_requirements", + "kind": "function", + "signature": "(cross_validator, model, data, target, weights, n_jobs: int=1)" + }, + { + "name": "measure_all_cv_computational_requirements", + "kind": "function", + "signature": "(cross_validators: dict[str, Any], n_samples: int=40 * 252, n_features: int=22, n_jobs: int=1, n_repeats: int=30)" + }, + { + "name": "measure_cpcv_parallelization", + "kind": "function", + "signature": "(n_samples: int=40 * 252, n_features: int=22, n_repeats: int=30, n_jobs_list: list[int]=_DEFAULT_N_JOBS_LIST)" + }, + { + "name": "measure_cpcv_scalability", + "kind": "function", + "signature": "(sample_sizes: list[int], feature_sizes: list[int], n_repeats: int=1, n_jobs: int=1)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/backtest_statistics.py", + "module": "RiskLabAI.backtest.backtest_statistics", + "module_kind": "module", + "sha256": "D6365C332C46BE6E6A176B39D3CC70DA170301362A974609613FA370D710E643", + "size": 9157, + "defined_public_names": [ + "bet_timing", + "calculate_hhi", + "calculate_hhi_concentration", + "calculate_holding_period", + "compute_drawdowns_time_under_water", + "sharpe_ratio" + ], + "definitions": [ + { + "name": "sharpe_ratio", + "kind": "function", + "signature": "(returns: np.ndarray, risk_free_rate: float=0.0)" + }, + { + "name": "bet_timing", + "kind": "function", + "signature": "(target_positions: pd.Series)" + }, + { + "name": "calculate_holding_period", + "kind": "function", + "signature": "(target_positions: pd.Series)" + }, + { + "name": "calculate_hhi", + "kind": "function", + "signature": "(bet_returns: pd.Series)" + }, + { + "name": "calculate_hhi_concentration", + "kind": "function", + "signature": "(returns: pd.Series)" + }, + { + "name": "compute_drawdowns_time_under_water", + "kind": "function", + "signature": "(pnl_series: pd.Series, dollars: bool=False)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/backtest_synthetic_data.py", + "module": "RiskLabAI.backtest.backtest_synthetic_data", + "module_kind": "module", + "sha256": "9ACDA6D9D81702F611CBC4EBE11899C31463DA899709D9975931F592CD59AF39", + "size": 3523, + "defined_public_names": [ + "synthetic_back_testing" + ], + "definitions": [ + { + "name": "synthetic_back_testing", + "kind": "function", + "signature": "(forecast: float, half_life: float, sigma: float, n_iteration: int=100000, maximum_holding_period: int=100, profit_taking_range: np.ndarray=_DEFAULT_PROFIT_TAKING_RANGE, stop_loss_range: np.ndarray=_DEFAULT_STOP_LOSS_RANGE, seed: int=0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/bet_sizing.py", + "module": "RiskLabAI.backtest.bet_sizing", + "module_kind": "module", + "sha256": "40704B327DF86A3901B684D6085DDC503947B5641E081BFCBB9B80CBBE1BE889", + "size": 15577, + "defined_public_names": [ + "average_bet_sizes", + "avg_active_signals", + "bet_size_sigmoid", + "compute_sigmoid_width", + "discrete_signal", + "generate_signal", + "inverse_price", + "limit_price", + "mp_avg_active_signals", + "probability_bet_size", + "strategy_bet_sizing", + "target_position" + ], + "definitions": [ + { + "name": "probability_bet_size", + "kind": "function", + "signature": "(probabilities: np.ndarray, sides: np.ndarray)" + }, + { + "name": "average_bet_sizes", + "kind": "function", + "signature": "(price_dates: np.ndarray, start_dates: np.ndarray, end_dates: np.ndarray, bet_sizes: np.ndarray)" + }, + { + "name": "strategy_bet_sizing", + "kind": "function", + "signature": "(price_timestamps: pd.Index, times: pd.Series, sides: pd.Series, probabilities: pd.Series)" + }, + { + "name": "avg_active_signals", + "kind": "function", + "signature": "(signals: pd.DataFrame, n_threads: int)" + }, + { + "name": "mp_avg_active_signals", + "kind": "function", + "signature": "(signals: pd.DataFrame, molecule: list)" + }, + { + "name": "discrete_signal", + "kind": "function", + "signature": "(signal: pd.Series, step_size: float)" + }, + { + "name": "generate_signal", + "kind": "function", + "signature": "(events: pd.DataFrame, step_size: float, probability: pd.Series, prediction: pd.Series, n_classes: int, n_threads: int)" + }, + { + "name": "bet_size_sigmoid", + "kind": "function", + "signature": "(w: float, x: float)" + }, + { + "name": "target_position", + "kind": "function", + "signature": "(w: float, f: float, actual_price: float, maximum_position_size: int)" + }, + { + "name": "inverse_price", + "kind": "function", + "signature": "(f: float, w: float, m: float)" + }, + { + "name": "limit_price", + "kind": "function", + "signature": "(target_position_size: int, current_position: int, f: float, w: float, maximum_position_size: int)" + }, + { + "name": "compute_sigmoid_width", + "kind": "function", + "signature": "(x: float, m: float)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/multiple_testing.py", + "module": "RiskLabAI.backtest.multiple_testing", + "module_kind": "module", + "sha256": "8B867AE2529D3A354CE0847D9C08E90DDE29E9E20042C0888683EBB98B89AAA8", + "size": 7841, + "defined_public_names": [ + "benjamini_hochberg_yekutieli_adjusted_p_values", + "haircut_sharpe_ratios", + "holm_adjusted_p_values", + "sharpe_ratio_p_values" + ], + "definitions": [ + { + "name": "sharpe_ratio_p_values", + "kind": "function", + "signature": "(sharpe_ratios: np.ndarray, number_of_returns: int)" + }, + { + "name": "holm_adjusted_p_values", + "kind": "function", + "signature": "(p_values: np.ndarray)" + }, + { + "name": "benjamini_hochberg_yekutieli_adjusted_p_values", + "kind": "function", + "signature": "(p_values: np.ndarray)" + }, + { + "name": "haircut_sharpe_ratios", + "kind": "function", + "signature": "(sharpe_ratios: np.ndarray, number_of_returns: int, method: str='holm', significance_level: float=0.05)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/ou_trading_rules.py", + "module": "RiskLabAI.backtest.ou_trading_rules", + "module_kind": "module", + "sha256": "D48DFA47F2C98EF791E66AA7724C29153813F7742CD244E88FAAFD6062231742", + "size": 12532, + "defined_public_names": [ + "fit_ornstein_uhlenbeck", + "hit_upper_probability", + "mean_exit_time", + "optimal_ou_trading_rule", + "ou_rule_metrics", + "stationary_std", + "theta_from_half_life" + ], + "definitions": [ + { + "name": "theta_from_half_life", + "kind": "function", + "signature": "(half_life: float)" + }, + { + "name": "stationary_std", + "kind": "function", + "signature": "(theta: float, sigma: float)" + }, + { + "name": "hit_upper_probability", + "kind": "function", + "signature": "(entry_gap: float, profit_take: float, stop_loss: float, theta: float, sigma: float)" + }, + { + "name": "mean_exit_time", + "kind": "function", + "signature": "(entry_gap: float, profit_take: float, stop_loss: float, theta: float, sigma: float)" + }, + { + "name": "ou_rule_metrics", + "kind": "function", + "signature": "(profit_take: float, stop_loss: float, theta: float, sigma: float, entry_gap: float, cost: float=0.0)" + }, + { + "name": "optimal_ou_trading_rule", + "kind": "function", + "signature": "(theta: float, sigma: float, entry_gap: float, cost: float=0.0, bounds: tuple=(0.25, 4.0))" + }, + { + "name": "fit_ornstein_uhlenbeck", + "kind": "function", + "signature": "(series: np.ndarray, dt: float=1.0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py", + "module": "RiskLabAI.backtest.probabilistic_sharpe_ratio", + "module_kind": "module", + "sha256": "3CD56F7C02E0D42CE04E395737DB6039FAEA617B367E6BFBC2F0A74768FA7B49", + "size": 4538, + "defined_public_names": [ + "benchmark_sharpe_ratio", + "probabilistic_sharpe_ratio" + ], + "definitions": [ + { + "name": "probabilistic_sharpe_ratio", + "kind": "function", + "signature": "(observed_sharpe_ratio: float, benchmark_sharpe_ratio: float, number_of_returns: int, skewness_of_returns: float=0.0, kurtosis_of_returns: float=3.0, return_test_statistic: bool=False)" + }, + { + "name": "benchmark_sharpe_ratio", + "kind": "function", + "signature": "(sharpe_ratio_estimates: list[float])" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/probability_of_backtest_overfitting.py", + "module": "RiskLabAI.backtest.probability_of_backtest_overfitting", + "module_kind": "module", + "sha256": "C14BB1E1179AAC1D61D56ACF6BF282BF2B9593B143A43E357DC7D9181C7DA373", + "size": 5085, + "defined_public_names": [ + "performance_evaluation", + "probability_of_backtest_overfitting" + ], + "definitions": [ + { + "name": "performance_evaluation", + "kind": "function", + "signature": "(train_partition: np.ndarray, test_partition: np.ndarray, n_strategies: int, metric: Callable[[np.ndarray, float], float], risk_free_return: float)" + }, + { + "name": "probability_of_backtest_overfitting", + "kind": "function", + "signature": "(performances: np.ndarray, n_partitions: int=16, risk_free_return: float=0.0, metric: Optional[Callable[[np.ndarray, float], float]]=None, n_jobs: int=1)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/robust_statistics.py", + "module": "RiskLabAI.backtest.robust_statistics", + "module_kind": "module", + "sha256": "6144EAD8326C07605EF783AFC41AC49884A4DB3BD56F6D3DA8CFBE93AD3C0F29", + "size": 11865, + "defined_public_names": [ + "conditional_expected_drawdown", + "sharpe_difference_test" + ], + "definitions": [ + { + "name": "conditional_expected_drawdown", + "kind": "function", + "signature": "(returns: np.ndarray, horizon: int, alpha: float=0.9)" + }, + { + "name": "sharpe_difference_test", + "kind": "function", + "signature": "(returns_a: np.ndarray, returns_b: np.ndarray, method: str='ledoit_wolf', block_length: int | None=None, n_boot: int=1000, bandwidth: int | None=None, random_state: int | None=0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/sharpe_inference.py", + "module": "RiskLabAI.backtest.sharpe_inference", + "module_kind": "module", + "sha256": "CB3DE49BA2230604868ECAB11B4AA5C1C55AE4BCF47FF6ACD8FE2C71B4839E2C", + "size": 7799, + "defined_public_names": [ + "lplz_sharpe_inference", + "newey_west_automatic_lag", + "newey_west_long_run_variance", + "sharpe_ratio_influence_function" + ], + "definitions": [ + { + "name": "sharpe_ratio_influence_function", + "kind": "function", + "signature": "(returns: np.ndarray)" + }, + { + "name": "newey_west_long_run_variance", + "kind": "function", + "signature": "(series: np.ndarray, lag: int)" + }, + { + "name": "newey_west_automatic_lag", + "kind": "function", + "signature": "(number_of_returns: int)" + }, + { + "name": "lplz_sharpe_inference", + "kind": "function", + "signature": "(returns: np.ndarray, confidence_level: float=0.95, lag: Optional[int]=None, null_sharpe_ratio: float=0.0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/strategy_risk.py", + "module": "RiskLabAI.backtest.strategy_risk", + "module_kind": "module", + "sha256": "00D0E6574F81E68051A95CD762158E7BA52BB8109D8E47FB723C02C06C476447", + "size": 10670, + "defined_public_names": [ + "bin_frequency", + "binomial_sharpe_ratio", + "calculate_strategy_risk", + "failure_probability", + "implied_precision", + "mix_gaussians", + "sharpe_ratio_trials", + "target_sharpe_ratio_symbolic" + ], + "definitions": [ + { + "name": "sharpe_ratio_trials", + "kind": "function", + "signature": "(p: float, n_run: int)" + }, + { + "name": "target_sharpe_ratio_symbolic", + "kind": "function", + "signature": "()" + }, + { + "name": "implied_precision", + "kind": "function", + "signature": "(stop_loss: float, profit_taking: float, frequency: float, target_sharpe_ratio: float)" + }, + { + "name": "bin_frequency", + "kind": "function", + "signature": "(stop_loss: float, profit_taking: float, precision: float, target_sharpe_ratio: float)" + }, + { + "name": "binomial_sharpe_ratio", + "kind": "function", + "signature": "(stop_loss: float, profit_taking: float, frequency: float, probability: float)" + }, + { + "name": "mix_gaussians", + "kind": "function", + "signature": "(mu1: float, mu2: float, sigma1: float, sigma2: float, probability: float, n_obs: int)" + }, + { + "name": "failure_probability", + "kind": "function", + "signature": "(returns: np.ndarray, frequency: float, target_sharpe_ratio: float)" + }, + { + "name": "calculate_strategy_risk", + "kind": "function", + "signature": "(mu1: float, mu2: float, sigma1: float, sigma2: float, probability: float, n_obs: int, frequency: float, target_sharpe_ratio: float)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/test_set_overfitting.py", + "module": "RiskLabAI.backtest.test_set_overfitting", + "module_kind": "module", + "sha256": "2C813D80E85ED2717F4C3B991244B23DEE1253807C8FFA9EA71A36F37E9DDED0", + "size": 8865, + "defined_public_names": [ + "estimated_sharpe_ratio_z_statistics", + "expected_max_sharpe_ratio", + "generate_max_sharpe_ratios", + "mean_std_error", + "strategy_type1_error_probability", + "strategy_type2_error_probability", + "theta_for_type2_error" + ], + "definitions": [ + { + "name": "expected_max_sharpe_ratio", + "kind": "function", + "signature": "(n_trials: int, mean_sharpe_ratio: float, std_sharpe_ratio: float)" + }, + { + "name": "generate_max_sharpe_ratios", + "kind": "function", + "signature": "(n_sims: int, n_trials_list: list[int], std_sharpe_ratio: float, mean_sharpe_ratio: float)" + }, + { + "name": "mean_std_error", + "kind": "function", + "signature": "(n_sims0: int, n_sims1: int, n_trials: list[int], std_sharpe_ratio: float=1.0, mean_sharpe_ratio: float=0.0)" + }, + { + "name": "estimated_sharpe_ratio_z_statistics", + "kind": "function", + "signature": "(sharpe_ratio: float, t: int, true_sharpe_ratio: float=0.0, skew: float=0.0, kurt: int=3)" + }, + { + "name": "strategy_type1_error_probability", + "kind": "function", + "signature": "(z: float, k: int=1)" + }, + { + "name": "theta_for_type2_error", + "kind": "function", + "signature": "(sharpe_ratio: float, t: int, true_sharpe_ratio: float, skew: float=0.0, kurt: int=3)" + }, + { + "name": "strategy_type2_error_probability", + "kind": "function", + "signature": "(alpha_k: float, k: int, theta: float)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/__init__.py", + "module": "RiskLabAI.backtest.validation", + "module_kind": "package", + "sha256": "503E9653E49B6606D1534CD677A731F1D5EB07563ED39AE62B64CF39F4BED5B1", + "size": 1951, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py", + "module": "RiskLabAI.backtest.validation.adaptive_combinatorial_purged", + "module_kind": "module", + "sha256": "B466BD0C43297F0654467E1A91B135CCB7958CD856A1A45BAFBCDBE372F7FAA0", + "size": 18395, + "defined_public_names": [ + "AdaptiveCombinatorialPurged" + ], + "definitions": [ + { + "name": "AdaptiveCombinatorialPurged", + "kind": "class", + "signature": "(self, n_splits: int, n_test_groups: int, times: Union[pd.Series, dict[str, pd.Series]], embargo: float=0, n_subsplits: int=3, external_feature: Union[pd.Series, dict[str, pd.Series]]=None, lower_quantile: float=0.25, upper_quantile: float=0.75, subtract_border_adjustments: bool=True)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py", + "module": "RiskLabAI.backtest.validation.bagged_combinatorial_purged", + "module_kind": "module", + "sha256": "A881EDEBB301C8C5184248AA37EDCCE94159A9314BA5369026E0C423C5C73B12", + "size": 9472, + "defined_public_names": [ + "BaggedCombinatorialPurged" + ], + "definitions": [ + { + "name": "BaggedCombinatorialPurged", + "kind": "class", + "signature": "(self, n_splits: int, n_test_groups: int, times: Union[pd.Series, dict[str, pd.Series]], embargo: float=0, classifier: bool=True, n_estimators: int=10, max_samples: float=1.0, max_features: float=1.0, bootstrap: bool=True, bootstrap_features: bool=False, random_state: int=None)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/combinatorial_purged.py", + "module": "RiskLabAI.backtest.validation.combinatorial_purged", + "module_kind": "module", + "sha256": "8CC827C13CE4DE02A70C5A942A663CC2B704BD0EF5B251CDE78F59CDB9F375DB", + "size": 14461, + "defined_public_names": [ + "CombinatorialPurged" + ], + "definitions": [ + { + "name": "CombinatorialPurged", + "kind": "class", + "signature": "(self, n_splits: int, n_test_groups: int, times: Union[pd.Series, dict[str, pd.Series]], embargo: float=0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/cross_validator_controller.py", + "module": "RiskLabAI.backtest.validation.cross_validator_controller", + "module_kind": "module", + "sha256": "FE635E694640176AA96872AAE26046E4707C0F5FBE8F8AE79E2197BC0A4D6C51", + "size": 1233, + "defined_public_names": [ + "CrossValidatorController" + ], + "definitions": [ + { + "name": "CrossValidatorController", + "kind": "class", + "signature": "(self, validator_type: str, **kwargs: Any)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/cross_validator_factory.py", + "module": "RiskLabAI.backtest.validation.cross_validator_factory", + "module_kind": "module", + "sha256": "EAEECAE7A1F73789F72BA9E5FBF03FE48CC20487AAFF5404613B82F41DE96C77", + "size": 2650, + "defined_public_names": [ + "CrossValidatorFactory" + ], + "definitions": [ + { + "name": "CrossValidatorFactory", + "kind": "class", + "signature": null + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/cross_validator_interface.py", + "module": "RiskLabAI.backtest.validation.cross_validator_interface", + "module_kind": "module", + "sha256": "9BB6CFBFF867C8F863CF0CA0B1FF6384463A4385D88C2FD1FEFFA0775B85FD0D", + "size": 8202, + "defined_public_names": [ + "CrossValidator" + ], + "definitions": [ + { + "name": "CrossValidator", + "kind": "class", + "signature": null + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/kfold.py", + "module": "RiskLabAI.backtest.validation.kfold", + "module_kind": "module", + "sha256": "BBA92A01B9EA320C89DB7CEBFED147EDBDDAE9CB7FE2D550FF2F2ECFE7ACC15B", + "size": 12431, + "defined_public_names": [ + "KFold" + ], + "definitions": [ + { + "name": "KFold", + "kind": "class", + "signature": "(self, n_splits: int, shuffle: bool=False, random_seed: int=None)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/leakage_aware_hpo.py", + "module": "RiskLabAI.backtest.validation.leakage_aware_hpo", + "module_kind": "module", + "sha256": "A8D16874C56CFDB725B92906F5A08A1605DA93B263EDA210B3B561310E188410", + "size": 9891, + "defined_public_names": [ + "deflated_sharpe_gate", + "leakage_aware_hpo" + ], + "definitions": [ + { + "name": "leakage_aware_hpo", + "kind": "function", + "signature": "(suggest_params: Callable, estimator_factory: Callable, x, y, times, n_trials: int=50, sampler: str='tpe', n_splits: int=5, embargo: float=0.0, scoring: Callable | None=None, random_state: int=0)" + }, + { + "name": "deflated_sharpe_gate", + "kind": "function", + "signature": "(out_of_sample_returns: np.ndarray, n_trials: int, trial_sharpe_std: float, threshold: float=0.5)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py", + "module": "RiskLabAI.backtest.validation.path_adaptive_cpcv", + "module_kind": "module", + "sha256": "4A1BCEDEA466B9C05165C282740CC1EDBB96174629CC377276432C3325C2E3B6", + "size": 10541, + "defined_public_names": [ + "adaptive_probability_of_backtest_overfitting", + "estimate_volatility_regimes" + ], + "definitions": [ + { + "name": "estimate_volatility_regimes", + "kind": "function", + "signature": "(performances: np.ndarray, n_regimes: int=2, window: int | None=None)" + }, + { + "name": "adaptive_probability_of_backtest_overfitting", + "kind": "function", + "signature": "(performances: np.ndarray, n_partitions: int=16, target_fraction: float=0.25, risk_free_return: float=0.0, metric: Callable[[np.ndarray, float], float] | None=None, n_regimes: int=2)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/path_bagged_cpcv.py", + "module": "RiskLabAI.backtest.validation.path_bagged_cpcv", + "module_kind": "module", + "sha256": "CB9FDF560D59DD7E035DD194DC1F78EDB392A9072ADE4959F6D0B55D3EFD5B83", + "size": 5682, + "defined_public_names": [ + "bagged_probability_of_backtest_overfitting", + "moving_block_bootstrap_indices" + ], + "definitions": [ + { + "name": "moving_block_bootstrap_indices", + "kind": "function", + "signature": "(n_observations: int, block_size: int, rng: np.random.Generator)" + }, + { + "name": "bagged_probability_of_backtest_overfitting", + "kind": "function", + "signature": "(performances: np.ndarray, n_partitions: int=16, n_bag: int=30, block_size: Optional[int]=None, risk_free_return: float=0.0, metric: Optional[Callable[[np.ndarray, float], float]]=None, seed: int=0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/purged_kfold.py", + "module": "RiskLabAI.backtest.validation.purged_kfold", + "module_kind": "module", + "sha256": "86802CB8E32ECDD092A522FBAC196C2BA5F05FAAAF4227091C705EDBD2DBFCD8", + "size": 19810, + "defined_public_names": [ + "PurgedKFold" + ], + "definitions": [ + { + "name": "PurgedKFold", + "kind": "class", + "signature": "(self, n_splits: int, times: Union[pd.Series, dict[str, pd.Series]], embargo: float=0)" + } + ] + }, + { + "path": "src/RiskLabAI/backtest/validation/walk_forward.py", + "module": "RiskLabAI.backtest.validation.walk_forward", + "module_kind": "module", + "sha256": "47C0786EAF7B5F4D774B24537ADEEDC9E80AAC3604FA3D14C52A17864FEC9B7A", + "size": 6474, + "defined_public_names": [ + "WalkForward" + ], + "definitions": [ + { + "name": "WalkForward", + "kind": "class", + "signature": "(self, n_splits: int=5, max_train_size: Optional[int]=None, gap: int=0)" + } + ] + }, + { + "path": "src/RiskLabAI/causal_factor_analysis/__init__.py", + "module": "RiskLabAI.causal_factor_analysis", + "module_kind": "package", + "sha256": "6B6153F1D4C87E8FA6DEDF123CFD27DA3AAD23ED1DEDE271684DFC93E71A3F3C", + "size": 4290, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/causal_factor_analysis/factor_mirage.py", + "module": "RiskLabAI.causal_factor_analysis.factor_mirage", + "module_kind": "module", + "sha256": "5E5C44C8F25E926E5C1528D623BAA6886548E32C1308DB65B5AFB907988C522B", + "size": 14673, + "defined_public_names": [ + "ColliderCoefficients", + "ColliderDiagnostics", + "StrategyPerformance", + "collider_factor_return", + "collider_forecast_return", + "collider_model_diagnostics", + "collider_overcontrolled_coefficients", + "confounder_factor_return", + "confounder_forecast_return", + "confounder_undercontrolled_coefficient" + ], + "definitions": [ + { + "name": "StrategyPerformance", + "kind": "class", + "signature": null + }, + { + "name": "ColliderCoefficients", + "kind": "class", + "signature": null + }, + { + "name": "ColliderDiagnostics", + "kind": "class", + "signature": null + }, + { + "name": "confounder_undercontrolled_coefficient", + "kind": "function", + "signature": "(beta: Real, gamma: Real, delta: Real)" + }, + { + "name": "confounder_factor_return", + "kind": "function", + "signature": "(x: Real, z: Real, beta: Real, gamma: Real, delta_estimated: Real, *, delta_realized: Optional[Real]=None)" + }, + { + "name": "confounder_forecast_return", + "kind": "function", + "signature": "(beta: Real, gamma: Real, delta_estimated: Real, *, delta_realized: Optional[Real]=None)" + }, + { + "name": "collider_overcontrolled_coefficients", + "kind": "function", + "signature": "(beta: Real, gamma: Real, delta: Real)" + }, + { + "name": "collider_factor_return", + "kind": "function", + "signature": "(x: Real, collider_proxy: Real, beta: Real, gamma: Real, delta: Real)" + }, + { + "name": "collider_forecast_return", + "kind": "function", + "signature": "(beta: Real, gamma: Real, delta: Real)" + }, + { + "name": "collider_model_diagnostics", + "kind": "function", + "signature": "(beta: Real, gamma: Real, delta: Real, n_observations: Integral)" + } + ] + }, + { + "path": "src/RiskLabAI/causal_factor_analysis/graph_identification.py", + "module": "RiskLabAI.causal_factor_analysis.graph_identification", + "module_kind": "module", + "sha256": "E34CED76FDA172EB688B7F45D610F3297332AE43D24639860325CE38E0A180BE", + "size": 37121, + "defined_public_names": [ + "BackdoorAdjustmentEvidence", + "CausalDAG", + "DSeparationEvidence", + "FrontdoorAdjustmentEvidence", + "InstrumentEvidence", + "NodeRoleEvidence", + "PathEvidence", + "causal_role_evidence", + "check_backdoor_adjustment_set", + "check_frontdoor_adjustment_set", + "check_instrument", + "d_separation", + "minimal_backdoor_adjustment_sets", + "minimal_frontdoor_adjustment_sets" + ], + "definitions": [ + { + "name": "CausalDAG", + "kind": "class", + "signature": null + }, + { + "name": "PathEvidence", + "kind": "class", + "signature": null + }, + { + "name": "DSeparationEvidence", + "kind": "class", + "signature": null + }, + { + "name": "BackdoorAdjustmentEvidence", + "kind": "class", + "signature": null + }, + { + "name": "FrontdoorAdjustmentEvidence", + "kind": "class", + "signature": null + }, + { + "name": "InstrumentEvidence", + "kind": "class", + "signature": null + }, + { + "name": "NodeRoleEvidence", + "kind": "class", + "signature": null + }, + { + "name": "d_separation", + "kind": "function", + "signature": "(dag: CausalDAG, left: Iterable[str], right: Iterable[str], conditioned: Iterable[str]=(), *, max_paths: Optional[int]=_DEFAULT_MAX_PATHS, max_path_states: Optional[int]=_DEFAULT_MAX_PATH_STATES)" + }, + { + "name": "check_backdoor_adjustment_set", + "kind": "function", + "signature": "(dag: CausalDAG, treatment: str, outcome: str, adjustment_set: Iterable[str]=(), *, max_paths: Optional[int]=_DEFAULT_MAX_PATHS, max_path_states: Optional[int]=_DEFAULT_MAX_PATH_STATES)" + }, + { + "name": "minimal_backdoor_adjustment_sets", + "kind": "function", + "signature": "(dag: CausalDAG, treatment: str, outcome: str, *, max_candidates: Optional[int]=_DEFAULT_MAX_CANDIDATES)" + }, + { + "name": "check_frontdoor_adjustment_set", + "kind": "function", + "signature": "(dag: CausalDAG, treatment: str, outcome: str, mediator_set: Iterable[str], *, max_paths: Optional[int]=_DEFAULT_MAX_PATHS, max_path_states: Optional[int]=_DEFAULT_MAX_PATH_STATES)" + }, + { + "name": "minimal_frontdoor_adjustment_sets", + "kind": "function", + "signature": "(dag: CausalDAG, treatment: str, outcome: str, *, max_candidates: Optional[int]=_DEFAULT_MAX_CANDIDATES)" + }, + { + "name": "check_instrument", + "kind": "function", + "signature": "(dag: CausalDAG, instrument: str, treatment: str, outcome: str, conditioned: Iterable[str]=())" + }, + { + "name": "causal_role_evidence", + "kind": "function", + "signature": "(dag: CausalDAG, treatment: str, outcome: str, node: str, *, max_paths: Optional[int]=_DEFAULT_MAX_PATHS, max_path_states: Optional[int]=_DEFAULT_MAX_PATH_STATES)" + } + ] + }, + { + "path": "src/RiskLabAI/causal_factor_analysis/optimizer.py", + "module": "RiskLabAI.causal_factor_analysis.optimizer", + "module_kind": "module", + "sha256": "9C15ABD41FE232686553E37911A4F123C9FD745F877BC52C03E35CC0D36101A8", + "size": 8555, + "defined_public_names": [ + "minimum_variance_factor_weights" + ], + "definitions": [ + { + "name": "minimum_variance_factor_weights", + "kind": "function", + "signature": "(covariance: ArrayLike, factor_exposures: ArrayLike, target_exposures: ArrayLike)" + } + ] + }, + { + "path": "src/RiskLabAI/causal_factor_analysis/protocol.py", + "module": "RiskLabAI.causal_factor_analysis.protocol", + "module_kind": "module", + "sha256": "75CC2191038FAEBBDE7AD5AB38B8842FF9B9FE3772E732FB4F0764B83C432F4D", + "size": 63829, + "defined_public_names": [ + "BacktestStage", + "CausalAdjustmentSetStage", + "CausalDiscoveryStage", + "CausalExplanatoryAndPredictivePowerStage", + "CausalFactorProtocolReport", + "CausalPortfolioConstructionStage", + "EventHorizon", + "FoldEvidence", + "MultipleTestingAdjustmentsStage", + "ValidationEvidence", + "VariableSelectionStage", + "validate_causal_factor_protocol" + ], + "definitions": [ + { + "name": "EventHorizon", + "kind": "class", + "signature": null + }, + { + "name": "FoldEvidence", + "kind": "class", + "signature": null + }, + { + "name": "ValidationEvidence", + "kind": "class", + "signature": null + }, + { + "name": "VariableSelectionStage", + "kind": "class", + "signature": null + }, + { + "name": "CausalDiscoveryStage", + "kind": "class", + "signature": null + }, + { + "name": "CausalAdjustmentSetStage", + "kind": "class", + "signature": null + }, + { + "name": "CausalExplanatoryAndPredictivePowerStage", + "kind": "class", + "signature": null + }, + { + "name": "CausalPortfolioConstructionStage", + "kind": "class", + "signature": null + }, + { + "name": "BacktestStage", + "kind": "class", + "signature": null + }, + { + "name": "MultipleTestingAdjustmentsStage", + "kind": "class", + "signature": null + }, + { + "name": "CausalFactorProtocolReport", + "kind": "class", + "signature": null + }, + { + "name": "validate_causal_factor_protocol", + "kind": "function", + "signature": "(report: CausalFactorProtocolReport)" + } + ] + }, + { + "path": "src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py", + "module": "RiskLabAI.causal_factor_analysis.specification_diagnostics", + "module_kind": "module", + "sha256": "EB8066295C2C6374E817AE2A1EA4459443CF76B1F84692CB11EEC6B66A687429", + "size": 9661, + "defined_public_names": [ + "PopulationRegressionDiagnostics", + "RegressionDiagnostics", + "SpecificationExperimentResult", + "SpecificationPopulationResult", + "collider_population_diagnostics", + "collider_specification_experiment", + "confounded_mediator_population_diagnostics", + "confounded_mediator_specification_experiment", + "fork_population_diagnostics", + "fork_specification_experiment" + ], + "definitions": [ + { + "name": "PopulationRegressionDiagnostics", + "kind": "class", + "signature": null + }, + { + "name": "RegressionDiagnostics", + "kind": "class", + "signature": null + }, + { + "name": "SpecificationPopulationResult", + "kind": "class", + "signature": null + }, + { + "name": "SpecificationExperimentResult", + "kind": "class", + "signature": null + }, + { + "name": "fork_population_diagnostics", + "kind": "function", + "signature": "()" + }, + { + "name": "collider_population_diagnostics", + "kind": "function", + "signature": "()" + }, + { + "name": "confounded_mediator_population_diagnostics", + "kind": "function", + "signature": "()" + }, + { + "name": "fork_specification_experiment", + "kind": "function", + "signature": "(sample_size: Integral=5000, seed: Integral=0)" + }, + { + "name": "collider_specification_experiment", + "kind": "function", + "signature": "(sample_size: Integral=5000, seed: Integral=0)" + }, + { + "name": "confounded_mediator_specification_experiment", + "kind": "function", + "signature": "(sample_size: Integral=5000, seed: Integral=0)" + } + ] + }, + { + "path": "src/RiskLabAI/causal_factor_analysis/treatment_effects.py", + "module": "RiskLabAI.causal_factor_analysis.treatment_effects", + "module_kind": "module", + "sha256": "692B945C0B44FA3B87FE23957B4400E717F4D27903ACBC12003C44528A6F6919", + "size": 11888, + "defined_public_names": [ + "DifferenceInDifferencesEstimate", + "TreatmentEffectDecomposition", + "average_treatment_effect", + "backdoor_adjusted_average_treatment_effect", + "backdoor_adjusted_expectation", + "difference_in_differences", + "frontdoor_adjusted_probability", + "linear_instrumental_variable_effect", + "randomized_mean_difference", + "treatment_effect_decomposition" + ], + "definitions": [ + { + "name": "TreatmentEffectDecomposition", + "kind": "class", + "signature": null + }, + { + "name": "DifferenceInDifferencesEstimate", + "kind": "class", + "signature": null + }, + { + "name": "average_treatment_effect", + "kind": "function", + "signature": "(treated_interventional_mean: Real, control_interventional_mean: Real)" + }, + { + "name": "treatment_effect_decomposition", + "kind": "function", + "signature": "(observed_treated_mean: Real, observed_control_mean: Real, counterfactual_control_mean_for_treated: Real)" + }, + { + "name": "randomized_mean_difference", + "kind": "function", + "signature": "(treated_outcomes: Sequence[Real], control_outcomes: Sequence[Real])" + }, + { + "name": "difference_in_differences", + "kind": "function", + "signature": "(treated_before: Real, treated_after: Real, control_before: Real, control_after: Real)" + }, + { + "name": "backdoor_adjusted_expectation", + "kind": "function", + "signature": "(conditional_outcome_means: Sequence[Real], adjustment_probabilities: Sequence[Real])" + }, + { + "name": "backdoor_adjusted_average_treatment_effect", + "kind": "function", + "signature": "(treated_conditional_means: Sequence[Real], control_conditional_means: Sequence[Real], adjustment_probabilities: Sequence[Real])" + }, + { + "name": "frontdoor_adjusted_probability", + "kind": "function", + "signature": "(mediator_probabilities_given_treatment: Sequence[Real], outcome_probabilities_given_mediator_and_treatment: Sequence[Sequence[Real]], treatment_probabilities: Sequence[Real])" + }, + { + "name": "linear_instrumental_variable_effect", + "kind": "function", + "signature": "(outcome_instrument_covariance: Real, treatment_instrument_covariance: Real)" + } + ] + }, + { + "path": "src/RiskLabAI/cluster/__init__.py", + "module": "RiskLabAI.cluster", + "module_kind": "package", + "sha256": "CC1993BE2778895842431BD777BFEF3B620341E6DB13B4BEDB5A58891CAF14DC", + "size": 669, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/cluster/clustering.py", + "module": "RiskLabAI.cluster.clustering", + "module_kind": "module", + "sha256": "C9ADF94D36C3B1257D33E0DBC7D38243C4876BF1921B72A9092FE81A745610B2", + "size": 14275, + "defined_public_names": [ + "cluster_k_means_base", + "cluster_k_means_top", + "covariance_to_correlation", + "make_new_outputs", + "random_block_correlation", + "random_block_covariance", + "random_covariance_sub" + ], + "definitions": [ + { + "name": "covariance_to_correlation", + "kind": "function", + "signature": "(covariance: np.ndarray)" + }, + { + "name": "cluster_k_means_base", + "kind": "function", + "signature": "(correlation: pd.DataFrame, max_clusters: int=10, iterations: int=10, random_state: Optional[int]=None)" + }, + { + "name": "make_new_outputs", + "kind": "function", + "signature": "(correlation: pd.DataFrame, clusters_1: dict[int, list[str]], clusters_2: dict[int, list[str]])" + }, + { + "name": "cluster_k_means_top", + "kind": "function", + "signature": "(correlation: pd.DataFrame, max_clusters: Optional[int]=None, iterations: int=10, random_state: Optional[int]=None)" + }, + { + "name": "random_covariance_sub", + "kind": "function", + "signature": "(n_observations: int, n_columns: int, sigma: float, random_state: Optional[int]=None)" + }, + { + "name": "random_block_covariance", + "kind": "function", + "signature": "(n_columns: int, n_blocks: int, block_size_min: int=1, sigma: float=1.0, random_state: Optional[int]=None)" + }, + { + "name": "random_block_correlation", + "kind": "function", + "signature": "(n_columns: int, n_blocks: int, random_state: Optional[int]=None, block_size_min: int=1)" + } + ] + }, + { + "path": "src/RiskLabAI/controller/__init__.py", + "module": "RiskLabAI.controller", + "module_kind": "package", + "sha256": "875DCD9EC172311DF8E9DF665C60102BDC0F56218A5CDDE0760E2A081E369230", + "size": 75, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/controller/bars_initializer.py", + "module": "RiskLabAI.controller.bars_initializer", + "module_kind": "module", + "sha256": "28284D4877900DE9D3582C2B821FC66173E474E1F62C95F27385190C4A88426F", + "size": 13999, + "defined_public_names": [ + "BarsInitializerController" + ], + "definitions": [ + { + "name": "BarsInitializerController", + "kind": "class", + "signature": "(self)" + } + ] + }, + { + "path": "src/RiskLabAI/controller/data_structure_controller.py", + "module": "RiskLabAI.controller.data_structure_controller", + "module_kind": "module", + "sha256": "237843F3AD293C366607FA555C218F4F3CD8ACCD881F784030B3AA6B12BC8E23", + "size": 6102, + "defined_public_names": [ + "BAR_COLUMNS", + "Controller" + ], + "definitions": [ + { + "name": "Controller", + "kind": "class", + "signature": "(self)" + } + ] + }, + { + "path": "src/RiskLabAI/core/__init__.py", + "module": "RiskLabAI.core", + "module_kind": "package", + "sha256": "1F57377F2CDCD8958910376A2615F45A23FD96954FDE05FFBEDC8FDC20AD1DE8", + "size": 1933, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/core/_builtins.py", + "module": "RiskLabAI.core._builtins", + "module_kind": "module", + "sha256": "5D754E23EF41F580D0A4B5CB635DFA95A5A1AB20E3A500B1D4B9F9A75EB333A7", + "size": 7247, + "defined_public_names": [ + "BARS", + "BET_SIZERS", + "CROSS_VALIDATORS", + "FEATURE_IMPORTANCE", + "LABELERS", + "PORTFOLIO_OPTIMIZERS", + "REGISTRIES", + "get_registry", + "list_components" + ], + "definitions": [ + { + "name": "get_registry", + "kind": "function", + "signature": "(family: str)" + }, + { + "name": "list_components", + "kind": "function", + "signature": "()" + } + ] + }, + { + "path": "src/RiskLabAI/core/base.py", + "module": "RiskLabAI.core.base", + "module_kind": "module", + "sha256": "47F3B61B748629A7AFBB1209A701A8F6614F2474EA9919B298B4574D3E757C5E", + "size": 6270, + "defined_public_names": [ + "BaseBetSizer", + "BaseLabeler", + "BasePortfolioOptimizer", + "Estimator" + ], + "definitions": [ + { + "name": "Estimator", + "kind": "class", + "signature": null + }, + { + "name": "BaseLabeler", + "kind": "class", + "signature": null + }, + { + "name": "BaseBetSizer", + "kind": "class", + "signature": null + }, + { + "name": "BasePortfolioOptimizer", + "kind": "class", + "signature": null + } + ] + }, + { + "path": "src/RiskLabAI/core/registry.py", + "module": "RiskLabAI.core.registry", + "module_kind": "module", + "sha256": "2A48E47D33ED1E2EE074DEE4933C363884433226EB42CA4BFCA949E33AAC1016", + "size": 14263, + "defined_public_names": [ + "Registry" + ], + "definitions": [ + { + "name": "Registry", + "kind": "class", + "signature": "(self, name: str, *, base: type | None=None)" + } + ] + }, + { + "path": "src/RiskLabAI/data/__init__.py", + "module": "RiskLabAI.data", + "module_kind": "package", + "sha256": "F238157F6B54DB5E12E2F910821431BED049AB6E99665B64AD81F95EB7B1488B", + "size": 664, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/denoise/__init__.py", + "module": "RiskLabAI.data.denoise", + "module_kind": "package", + "sha256": "90FC750BE16266C6625CCA5423D69A3FA734A66B5E81CC294CD7B8450DF36A87", + "size": 824, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/denoise/denoising.py", + "module": "RiskLabAI.data.denoise.denoising", + "module_kind": "module", + "sha256": "411FFBD20F902E19426506D6C5472570960F9091E67E3D0F8626DE09E63B8B92", + "size": 11538, + "defined_public_names": [ + "corr_to_cov", + "cov_to_corr", + "denoise_cov", + "denoised_corr", + "find_max_eval", + "fit_kde", + "marcenko_pastur_pdf", + "optimal_portfolio", + "optimal_portfolio_denoised", + "pca" + ], + "definitions": [ + { + "name": "marcenko_pastur_pdf", + "kind": "function", + "signature": "(variance: float, q: float, num_points: int=1000)" + }, + { + "name": "fit_kde", + "kind": "function", + "signature": "(observations: np.ndarray, bandwidth: float=0.01, kernel: str='gaussian')" + }, + { + "name": "find_max_eval", + "kind": "function", + "signature": "(eigenvalues: np.ndarray, q: float, bandwidth: float)" + }, + { + "name": "denoised_corr", + "kind": "function", + "signature": "(eigenvalues: np.ndarray, eigenvectors: np.ndarray, num_facts: int)" + }, + { + "name": "pca", + "kind": "function", + "signature": "(matrix: np.ndarray)" + }, + { + "name": "cov_to_corr", + "kind": "function", + "signature": "(cov: np.ndarray)" + }, + { + "name": "corr_to_cov", + "kind": "function", + "signature": "(corr: np.ndarray, std: np.ndarray)" + }, + { + "name": "denoise_cov", + "kind": "function", + "signature": "(cov0: np.ndarray, q: float, bandwidth: float=0.01)" + }, + { + "name": "optimal_portfolio", + "kind": "function", + "signature": "(cov: np.ndarray, mu: Optional[np.ndarray]=None)" + }, + { + "name": "optimal_portfolio_denoised", + "kind": "function", + "signature": "(cov: np.ndarray, q: float, mu: Optional[np.ndarray]=None, bandwidth: float=0.01)" + } + ] + }, + { + "path": "src/RiskLabAI/data/denoise/nercome.py", + "module": "RiskLabAI.data.denoise.nercome", + "module_kind": "module", + "sha256": "81FEA0C543026F0C927DBC54CAC60DA5F3829DAFD84EAB788B59F7F8462BAFBB", + "size": 6386, + "defined_public_names": [ + "nercome_denoised_covariance" + ], + "definitions": [ + { + "name": "nercome_denoised_covariance", + "kind": "function", + "signature": "(returns: np.ndarray, n_splits: int=50, split_fraction: float=2.0 / 3.0, random_state: int | None=None)" + } + ] + }, + { + "path": "src/RiskLabAI/data/differentiation/__init__.py", + "module": "RiskLabAI.data.differentiation", + "module_kind": "package", + "sha256": "51D73513B8108EF8835E5113E337F336DD62CBC4006BBBD486B50623401B670B", + "size": 196, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/differentiation/adaptive_differentiation.py", + "module": "RiskLabAI.data.differentiation.adaptive_differentiation", + "module_kind": "module", + "sha256": "BD56143C44222FBACDF490FA366063CA04559FD180FF4B62E667F9288139F610", + "size": 11374, + "defined_public_names": [ + "DEFAULT_ADF_MAXLAG", + "DEFAULT_ADF_SIGNIFICANCE", + "DEFAULT_CV_THRESHOLDS", + "DEFAULT_DELTA_GRID", + "adaptive_differencing_order", + "adaptive_fractional_difference", + "rescaled_range_hurst", + "wavelet_variance_hurst" + ], + "definitions": [ + { + "name": "wavelet_variance_hurst", + "kind": "function", + "signature": "(series: np.ndarray, wavelet: str='db2', skip_fine: int=1)" + }, + { + "name": "rescaled_range_hurst", + "kind": "function", + "signature": "(series: np.ndarray)" + }, + { + "name": "adaptive_differencing_order", + "kind": "function", + "signature": "(increments: np.ndarray)" + }, + { + "name": "adaptive_fractional_difference", + "kind": "function", + "signature": "(series: pd.Series, delta_grid: list | None=None, cv_thresholds: tuple=DEFAULT_CV_THRESHOLDS, adf_significance: float=DEFAULT_ADF_SIGNIFICANCE, adf_maxlag: int=DEFAULT_ADF_MAXLAG)" + } + ] + }, + { + "path": "src/RiskLabAI/data/differentiation/differentiation.py", + "module": "RiskLabAI.data.differentiation.differentiation", + "module_kind": "module", + "sha256": "FD6A1B4247CC3F7B6C794866CD1809770460BD2AFC08375F2F96B02AF0D79E02", + "size": 12258, + "defined_public_names": [ + "calculate_weights_ffd", + "calculate_weights_std", + "find_optimal_ffd_simple", + "fractional_difference_fixed", + "fractional_difference_fixed_single", + "fractional_difference_std", + "fractionally_differentiated_log_price", + "plot_weights" + ], + "definitions": [ + { + "name": "calculate_weights_std", + "kind": "function", + "signature": "(degree: float, size: int)" + }, + { + "name": "calculate_weights_ffd", + "kind": "function", + "signature": "(degree: float, threshold: float=1e-05)" + }, + { + "name": "fractional_difference_std", + "kind": "function", + "signature": "(series: pd.DataFrame, degree: float, threshold: float=0.01)" + }, + { + "name": "fractional_difference_fixed", + "kind": "function", + "signature": "(series: pd.DataFrame, degree: float, threshold: float=1e-05)" + }, + { + "name": "fractional_difference_fixed_single", + "kind": "function", + "signature": "(series: pd.Series, degree: float, threshold: float=1e-05)" + }, + { + "name": "plot_weights", + "kind": "function", + "signature": "(degree_range: tuple[float, float], number_degrees: int, size: int, ax: Optional['plt.Axes']=None)" + }, + { + "name": "find_optimal_ffd_simple", + "kind": "function", + "signature": "(input_series: pd.DataFrame, p_value_threshold: float=0.05)" + }, + { + "name": "fractionally_differentiated_log_price", + "kind": "function", + "signature": "(input_series: pd.Series, threshold: float=1e-05, step: float=0.01, p_value_threshold: float=0.05)" + } + ] + }, + { + "path": "src/RiskLabAI/data/distance/__init__.py", + "module": "RiskLabAI.data.distance", + "module_kind": "package", + "sha256": "5BA3546F05199F587C42C2C8A4E80B541616EA3F80A93654B7F2E8D8F548F296", + "size": 102, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/distance/codependence.py", + "module": "RiskLabAI.data.distance.codependence", + "module_kind": "module", + "sha256": "DF6E672A5D25468FCE63B36C6AFAAF55B6AD14486E34D4CC42D84D7B43F278DA", + "size": 7241, + "defined_public_names": [ + "distance_correlation", + "ksg_mutual_information" + ], + "definitions": [ + { + "name": "ksg_mutual_information", + "kind": "function", + "signature": "(x: np.ndarray, y: np.ndarray, k: int=4, random_state: int | None=0)" + }, + { + "name": "distance_correlation", + "kind": "function", + "signature": "(x: np.ndarray, y: np.ndarray)" + } + ] + }, + { + "path": "src/RiskLabAI/data/distance/distance_metric.py", + "module": "RiskLabAI.data.distance.distance_metric", + "module_kind": "module", + "sha256": "6399EFDB1E12A82ED3BB6831FF6F985E0EAB18D2420D82670CF1599189D51E73", + "size": 8377, + "defined_public_names": [ + "calculate_cross_entropy", + "calculate_distance", + "calculate_kullback_leibler_divergence", + "calculate_mutual_information", + "calculate_number_of_bins", + "calculate_variation_of_information", + "calculate_variation_of_information_extended" + ], + "definitions": [ + { + "name": "calculate_variation_of_information", + "kind": "function", + "signature": "(x: np.ndarray, y: np.ndarray, bins: int, norm: bool=False)" + }, + { + "name": "calculate_number_of_bins", + "kind": "function", + "signature": "(num_observations: int, correlation: Optional[float]=None)" + }, + { + "name": "calculate_variation_of_information_extended", + "kind": "function", + "signature": "(x: np.ndarray, y: np.ndarray, norm: bool=False)" + }, + { + "name": "calculate_mutual_information", + "kind": "function", + "signature": "(x: np.ndarray, y: np.ndarray, norm: bool=False)" + }, + { + "name": "calculate_distance", + "kind": "function", + "signature": "(dependence: np.ndarray, metric: str='angular')" + }, + { + "name": "calculate_kullback_leibler_divergence", + "kind": "function", + "signature": "(p: np.ndarray, q: np.ndarray)" + }, + { + "name": "calculate_cross_entropy", + "kind": "function", + "signature": "(p: np.ndarray, q: np.ndarray)" + } + ] + }, + { + "path": "src/RiskLabAI/data/labeling/__init__.py", + "module": "RiskLabAI.data.labeling", + "module_kind": "package", + "sha256": "E84DB73F915081EF60F6C8441D10BFE5E759037EE7D42945B1AD2FBB77A10CC3", + "size": 1193, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/labeling/financial_labels.py", + "module": "RiskLabAI.data.labeling.financial_labels", + "module_kind": "module", + "sha256": "A2471BEAFFB28F2339D05DAED1811C4384C702CC0602638230D0FC03A4BE6A9C", + "size": 4424, + "defined_public_names": [ + "calculate_t_value_linear_regression", + "find_trend_using_trend_scanning" + ], + "definitions": [ + { + "name": "calculate_t_value_linear_regression", + "kind": "function", + "signature": "(prices: pd.Series)" + }, + { + "name": "find_trend_using_trend_scanning", + "kind": "function", + "signature": "(molecule: pd.Index, close: pd.Series, span: tuple[int, int])" + } + ] + }, + { + "path": "src/RiskLabAI/data/labeling/labeling.py", + "module": "RiskLabAI.data.labeling.labeling", + "module_kind": "module", + "sha256": "36687544C68B3016B5E4CFDE12ADE104971F45F79D4AC9FEB99BEB298F8E1418", + "size": 14680, + "defined_public_names": [ + "cusum_filter_events_dynamic_threshold", + "daily_volatility_with_log_returns", + "meta_events", + "meta_labeling", + "symmetric_cusum_filter", + "triple_barrier", + "vertical_barrier" + ], + "definitions": [ + { + "name": "cusum_filter_events_dynamic_threshold", + "kind": "function", + "signature": "(prices: pd.Series, threshold: pd.Series)" + }, + { + "name": "symmetric_cusum_filter", + "kind": "function", + "signature": "(prices: pd.Series, threshold: float)" + }, + { + "name": "daily_volatility_with_log_returns", + "kind": "function", + "signature": "(close: pd.Series, span: int=100)" + }, + { + "name": "vertical_barrier", + "kind": "function", + "signature": "(close: pd.Series, time_events: pd.DatetimeIndex, number_days: int)" + }, + { + "name": "triple_barrier", + "kind": "function", + "signature": "(close: pd.Series, events: pd.DataFrame, ptsl: list[float], molecule: list[pd.Timestamp])" + }, + { + "name": "meta_events", + "kind": "function", + "signature": "(close: pd.Series, time_events: pd.DatetimeIndex, ptsl: list[float], target: pd.Series, return_min: float, num_threads: int, vertical_barrier_times: Optional[pd.Series]=None, side: Optional[pd.Series]=None)" + }, + { + "name": "meta_labeling", + "kind": "function", + "signature": "(events: pd.DataFrame, close: pd.Series)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/__init__.py", + "module": "RiskLabAI.data.structures", + "module_kind": "package", + "sha256": "F0E79621D4AF2B9E8C8BE1EBEF26EBE66536A5A3A50CF09E1A16BAEA94834398", + "size": 873, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/structures/abstract_bars.py", + "module": "RiskLabAI.data.structures.abstract_bars", + "module_kind": "module", + "sha256": "F775F93AE69A634E22EA324A0AB8603E729786CB72A684741D2111E090DA6014", + "size": 7436, + "defined_public_names": [ + "AbstractBars" + ], + "definitions": [ + { + "name": "AbstractBars", + "kind": "class", + "signature": "(self, bar_type: str)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/abstract_imbalance_bars.py", + "module": "RiskLabAI.data.structures.abstract_imbalance_bars", + "module_kind": "module", + "sha256": "350535B7AAF4E997E19242E07991EA7AB22BB41C041490D2243F25B2D4D90111", + "size": 6388, + "defined_public_names": [ + "AbstractImbalanceBars" + ], + "definitions": [ + { + "name": "AbstractImbalanceBars", + "kind": "class", + "signature": "(self, bar_type: str, window_size_for_expected_n_ticks_estimation: Optional[int], window_size_for_expected_imbalance_estimation: int, initial_estimate_of_expected_n_ticks_in_bar: int, analyse_thresholds: bool)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/abstract_information_driven_bars.py", + "module": "RiskLabAI.data.structures.abstract_information_driven_bars", + "module_kind": "module", + "sha256": "2035EA576F53B0CF6B7A4A44CEB2DDC64E531BC91E8F6C98ADACF7578A11306A", + "size": 4150, + "defined_public_names": [ + "AbstractInformationDrivenBars" + ], + "definitions": [ + { + "name": "AbstractInformationDrivenBars", + "kind": "class", + "signature": "(self, bar_type: str, window_size_for_expected_n_ticks_estimation: Optional[int], initial_estimate_of_expected_n_ticks_in_bar: int, window_size_for_expected_imbalance_estimation: int)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/abstract_run_bars.py", + "module": "RiskLabAI.data.structures.abstract_run_bars", + "module_kind": "module", + "sha256": "51DDD1B7C662E2FB2D749C5C8483782C76EF00584ACC36FD8314787239DAFFED", + "size": 10204, + "defined_public_names": [ + "AbstractRunBars" + ], + "definitions": [ + { + "name": "AbstractRunBars", + "kind": "class", + "signature": "(self, bar_type: str, window_size_for_expected_n_ticks_estimation: Optional[int], window_size_for_expected_imbalance_estimation: int, initial_estimate_of_expected_n_ticks_in_bar: int, analyse_thresholds: bool)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/hardened_information_driven_bars.py", + "module": "RiskLabAI.data.structures.hardened_information_driven_bars", + "module_kind": "module", + "sha256": "48BCDD8B73D5DED1815505696C36D56528ECBAED3A295EDCE31DE56AB2B3B8EB", + "size": 6781, + "defined_public_names": [ + "HardenedExpectedImbalanceBars", + "HardenedExpectedRunBars" + ], + "definitions": [ + { + "name": "HardenedExpectedImbalanceBars", + "kind": "class", + "signature": "(self, *args, min_ticks: int, max_ticks: int, **kwargs)" + }, + { + "name": "HardenedExpectedRunBars", + "kind": "class", + "signature": "(self, *args, min_ticks: int, max_ticks: int, **kwargs)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/imbalance_bars.py", + "module": "RiskLabAI.data.structures.imbalance_bars", + "module_kind": "module", + "sha256": "D355A01123732210788FAB14EDC8D68ABC6FA4E1035AC5BD636BD11DD4E210AC", + "size": 4741, + "defined_public_names": [ + "ExpectedImbalanceBars", + "FixedImbalanceBars" + ], + "definitions": [ + { + "name": "ExpectedImbalanceBars", + "kind": "class", + "signature": "(self, bar_type: str, window_size_for_expected_n_ticks_estimation: int=10000, initial_estimate_of_expected_n_ticks_in_bar: int=20000, window_size_for_expected_imbalance_estimation: int=10000, expected_ticks_number_bounds: Optional[tuple[float, float]]=None, analyse_thresholds: bool=False)" + }, + { + "name": "FixedImbalanceBars", + "kind": "class", + "signature": "(self, bar_type: str, initial_estimate_of_expected_n_ticks_in_bar: int=20000, window_size_for_expected_imbalance_estimation: int=10000, analyse_thresholds: bool=False, window_size_for_expected_n_ticks_estimation: Optional[int]=None)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/run_bars.py", + "module": "RiskLabAI.data.structures.run_bars", + "module_kind": "module", + "sha256": "7FE0F23DF5BE39B475073CA69FABD6FDE7126EC6E4289C2D84652D930D251900", + "size": 4515, + "defined_public_names": [ + "ExpectedRunBars", + "FixedRunBars" + ], + "definitions": [ + { + "name": "ExpectedRunBars", + "kind": "class", + "signature": "(self, bar_type: str, window_size_for_expected_n_ticks_estimation: int, initial_estimate_of_expected_n_ticks_in_bar: int, window_size_for_expected_imbalance_estimation: int, expected_ticks_number_bounds: Optional[tuple[float, float]]=None, analyse_thresholds: bool=False)" + }, + { + "name": "FixedRunBars", + "kind": "class", + "signature": "(self, bar_type: str, initial_estimate_of_expected_n_ticks_in_bar: int, window_size_for_expected_imbalance_estimation: int, analyse_thresholds: bool=False, window_size_for_expected_n_ticks_estimation: Optional[int]=None)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/standard_bars.py", + "module": "RiskLabAI.data.structures.standard_bars", + "module_kind": "module", + "sha256": "C4142C10BEE79EF96E8A9DA31F1444EE062933D587630EB567E0A5CE3C81C362", + "size": 2669, + "defined_public_names": [ + "StandardBars" + ], + "definitions": [ + { + "name": "StandardBars", + "kind": "class", + "signature": "(self, bar_type: str, threshold: float=50000)" + } + ] + }, + { + "path": "src/RiskLabAI/data/structures/time_bars.py", + "module": "RiskLabAI.data.structures.time_bars", + "module_kind": "module", + "sha256": "F9126A81DA595A4E1D87400164C09071833222CD9C1546D69164F643D0AC582F", + "size": 4544, + "defined_public_names": [ + "TimeBars" + ], + "definitions": [ + { + "name": "TimeBars", + "kind": "class", + "signature": "(self, resolution_type: str, resolution_units: int)" + } + ] + }, + { + "path": "src/RiskLabAI/data/synthetic_data/__init__.py", + "module": "RiskLabAI.data.synthetic_data", + "module_kind": "package", + "sha256": "BE3B0444592D9B45C9439E57210EB713C761D4B21CE497609B2B63E955F70768", + "size": 857, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py", + "module": "RiskLabAI.data.synthetic_data.drift_burst_hypothesis", + "module_kind": "module", + "sha256": "3A613F5FE50555654556236EF409BC4F7FB83252CDF3502CA60D46EB29F4BC4C", + "size": 3115, + "defined_public_names": [ + "drift_volatility_burst" + ], + "definitions": [ + { + "name": "drift_volatility_burst", + "kind": "function", + "signature": "(bubble_length: int, a_before: float, a_after: float, b_before: float, b_after: float, alpha: float, beta: float, explosion_filter_width: float=0.1)" + } + ] + }, + { + "path": "src/RiskLabAI/data/synthetic_data/simulation.py", + "module": "RiskLabAI.data.synthetic_data.simulation", + "module_kind": "module", + "sha256": "09394D81D758211140D894B5CB4CB6289EACEC862638006C8A7D9A79FD16DC55", + "size": 3392, + "defined_public_names": [ + "form_block_matrix", + "form_true_matrix", + "random_cov", + "simulates_cov_mu" + ], + "definitions": [ + { + "name": "random_cov", + "kind": "function", + "signature": "(num_columns: int, num_factors: int)" + }, + { + "name": "form_block_matrix", + "kind": "function", + "signature": "(n_blocks: int, block_size: int, block_correlation: float)" + }, + { + "name": "form_true_matrix", + "kind": "function", + "signature": "(n_blocks: int, block_size: int, block_correlation: float)" + }, + { + "name": "simulates_cov_mu", + "kind": "function", + "signature": "(mu0: np.ndarray, cov0: np.ndarray, n_obs: int, shrink: bool=False)" + } + ] + }, + { + "path": "src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py", + "module": "RiskLabAI.data.synthetic_data.synthetic_controlled_environment", + "module_kind": "module", + "sha256": "6D5DFCF4C4430A7BAE367A79CB156D3D6DF3D24457C1D60B68A1A4D556CF4AD4", + "size": 12391, + "defined_public_names": [ + "align_params_length", + "compute_log_returns", + "generate_prices_from_regimes", + "heston_merton_log_returns", + "parallel_generate_prices" + ], + "definitions": [ + { + "name": "compute_log_returns", + "kind": "function", + "signature": "(n_steps: int, mu_vector: np.ndarray, kappa_vector: np.ndarray, theta_vector: np.ndarray, xi_vector: np.ndarray, dw_stock: np.ndarray, dw_vol: np.ndarray, jump_comp: np.ndarray, poisson_var: np.ndarray, dt: float, sqrt_dt: float, lambda_vector: np.ndarray, m_vector: np.ndarray, v_vector: np.ndarray, regime_change: np.ndarray)" + }, + { + "name": "heston_merton_log_returns", + "kind": "function", + "signature": "(total_time: float, n_steps: int, mu_vector: np.ndarray, kappa_vector: np.ndarray, theta_vector: np.ndarray, xi_vector: np.ndarray, rho_vector: np.ndarray, lambda_vector: np.ndarray, m_vector: np.ndarray, v_vector: np.ndarray, regime_change: np.ndarray, random_state: Optional[int]=None)" + }, + { + "name": "align_params_length", + "kind": "function", + "signature": "(regime_params: RegimeParams)" + }, + { + "name": "generate_prices_from_regimes", + "kind": "function", + "signature": "(regimes: dict[str, RegimeParams], transition_matrix: np.ndarray, total_time: float, n_steps: int, random_state: Optional[int]=None)" + }, + { + "name": "parallel_generate_prices", + "kind": "function", + "signature": "(number_of_paths: int, regimes: dict[str, RegimeParams], transition_matrix: np.ndarray, total_time: float, n_steps: int, random_state: Optional[int]=None, n_jobs: int=1)" + } + ] + }, + { + "path": "src/RiskLabAI/data/weights/__init__.py", + "module": "RiskLabAI.data.weights", + "module_kind": "package", + "sha256": "7275D7101AE1F1DC977EF7B6C53819139AC9F46F629742178AA0AA453390A523", + "size": 621, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/data/weights/sample_weights.py", + "module": "RiskLabAI.data.weights.sample_weights", + "module_kind": "module", + "sha256": "A1AEBA7541663238C20B3B7A76B8706AB62D6860CD171DB6000E2EF92C492724", + "size": 7225, + "defined_public_names": [ + "calculate_average_uniqueness", + "calculate_time_decay", + "expand_label_for_meta_labeling", + "sample_weight_absolute_return_meta_labeling" + ], + "definitions": [ + { + "name": "expand_label_for_meta_labeling", + "kind": "function", + "signature": "(close_index: pd.Index, timestamp: pd.Series, molecule: pd.Index)" + }, + { + "name": "calculate_average_uniqueness", + "kind": "function", + "signature": "(index_matrix: pd.DataFrame)" + }, + { + "name": "sample_weight_absolute_return_meta_labeling", + "kind": "function", + "signature": "(timestamp: pd.Series, price: pd.Series, molecule: pd.Index)" + }, + { + "name": "calculate_time_decay", + "kind": "function", + "signature": "(weight: pd.Series, clf_last_weight: float=1.0)" + } + ] + }, + { + "path": "src/RiskLabAI/ensemble/__init__.py", + "module": "RiskLabAI.ensemble", + "module_kind": "package", + "sha256": "2301547AD658B8DD96941FD1098FBDA9AC3464E487F2A9F1811C6BD656F8F46A", + "size": 707, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/ensemble/bagging_classifier_accuracy.py", + "module": "RiskLabAI.ensemble.bagging_classifier_accuracy", + "module_kind": "module", + "sha256": "DF1570D491B4452412103BE1ECDC6B0C4CFDA3D9AAF55D9DDD7620B611134F92", + "size": 1300, + "defined_public_names": [ + "bagging_classifier_accuracy" + ], + "definitions": [ + { + "name": "bagging_classifier_accuracy", + "kind": "function", + "signature": "(N: int, p: float)" + } + ] + }, + { + "path": "src/RiskLabAI/ensemble/empirical_bagging_accuracy.py", + "module": "RiskLabAI.ensemble.empirical_bagging_accuracy", + "module_kind": "module", + "sha256": "E679640BE29A4986F0E67AF7F5A060894A72EDCD581C3F2B059D9C4177B7415F", + "size": 11622, + "defined_public_names": [ + "BaggingClassifierAccuracy", + "calculate_bootstrap_accuracy", + "plot_bootstrap_accuracy_distribution" + ], + "definitions": [ + { + "name": "BaggingClassifierAccuracy", + "kind": "class", + "signature": "(self, n_estimators: int=1000, max_samples: int=100, max_features: float=1.0, random_state: Optional[int]=None)" + }, + { + "name": "calculate_bootstrap_accuracy", + "kind": "function", + "signature": "(clf: BaggingClassifier, X: pd.DataFrame, y: pd.Series, n_bootstraps: int=1000)" + }, + { + "name": "plot_bootstrap_accuracy_distribution", + "kind": "function", + "signature": "(a_n_values: np.ndarray, a_n_mean: float, a_n_std: float, ax: Optional['plt.Axes']=None)" + } + ] + }, + { + "path": "src/RiskLabAI/features/__init__.py", + "module": "RiskLabAI.features", + "module_kind": "package", + "sha256": "8E6EF92F8D1E0E9BCA37E3436806A37308B8B3CB08C221481F4A1261327C5A18", + "size": 123, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/features/entropy_features/__init__.py", + "module": "RiskLabAI.features.entropy_features", + "module_kind": "package", + "sha256": "2DB4F80E1037C4562E8034FF3120C8DF41DE54D85C7F8E1CF04628BBA9A52374", + "size": 883, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/features/entropy_features/bias_corrected.py", + "module": "RiskLabAI.features.entropy_features.bias_corrected", + "module_kind": "module", + "sha256": "B6E66D401B569B26452EC1129A8FB5F0B150A990595BEEBFA92437457B1DAC1D", + "size": 12824, + "defined_public_names": [ + "grassberger_entropy", + "miller_madow_entropy", + "nsb_entropy" + ], + "definitions": [ + { + "name": "miller_madow_entropy", + "kind": "function", + "signature": "(message: str, approximate_word_length: int=1)" + }, + { + "name": "grassberger_entropy", + "kind": "function", + "signature": "(message: str, approximate_word_length: int=1)" + }, + { + "name": "nsb_entropy", + "kind": "function", + "signature": "(message: str, approximate_word_length: int=1, alphabet_size: Optional[int]=None, n_points: int=80)" + } + ] + }, + { + "path": "src/RiskLabAI/features/entropy_features/kontoyiannis.py", + "module": "RiskLabAI.features.entropy_features.kontoyiannis", + "module_kind": "module", + "sha256": "08FCF8EEEF1D9C4FFFC4F812A39B77F406050091ED287C9AE1A577E849998A4B", + "size": 2902, + "defined_public_names": [ + "kontoyiannis_entropy", + "longest_match_length" + ], + "definitions": [ + { + "name": "longest_match_length", + "kind": "function", + "signature": "(message: str, i: int, n: int)" + }, + { + "name": "kontoyiannis_entropy", + "kind": "function", + "signature": "(message: str, window: Optional[int]=None)" + } + ] + }, + { + "path": "src/RiskLabAI/features/entropy_features/lempel_ziv.py", + "module": "RiskLabAI.features.entropy_features.lempel_ziv", + "module_kind": "module", + "sha256": "295BFA3C0E3970C75C924C6D19E2D0B384E29A2B50AED7C8C08E1723A975BF4F", + "size": 1017, + "defined_public_names": [ + "lempel_ziv_entropy" + ], + "definitions": [ + { + "name": "lempel_ziv_entropy", + "kind": "function", + "signature": "(message: str)" + } + ] + }, + { + "path": "src/RiskLabAI/features/entropy_features/plug_in.py", + "module": "RiskLabAI.features.entropy_features.plug_in", + "module_kind": "module", + "sha256": "90DB453E95DD416771EAC11232E33584582C7FF1EA9B79DD441588185BADC483", + "size": 981, + "defined_public_names": [ + "plug_in_entropy_estimator" + ], + "definitions": [ + { + "name": "plug_in_entropy_estimator", + "kind": "function", + "signature": "(message: str, approximate_word_length: int=1)" + } + ] + }, + { + "path": "src/RiskLabAI/features/entropy_features/pmf.py", + "module": "RiskLabAI.features.entropy_features.pmf", + "module_kind": "module", + "sha256": "649221CC7406A3CF9607DE10CCA44C0F7D661E2B877209626704EBFF93F0BBC9", + "size": 1862, + "defined_public_names": [ + "ngram_counts", + "probability_mass_function" + ], + "definitions": [ + { + "name": "ngram_counts", + "kind": "function", + "signature": "(message: str, approximate_word_length: int)" + }, + { + "name": "probability_mass_function", + "kind": "function", + "signature": "(message: str, approximate_word_length: int)" + } + ] + }, + { + "path": "src/RiskLabAI/features/entropy_features/shannon.py", + "module": "RiskLabAI.features.entropy_features.shannon", + "module_kind": "module", + "sha256": "B594B49278CB1904E605CC44083B0A545F71E14B2BDDA6EF0A2BE03FAE2A2DA8", + "size": 723, + "defined_public_names": [ + "shannon_entropy" + ], + "definitions": [ + { + "name": "shannon_entropy", + "kind": "function", + "signature": "(message: str)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/__init__.py", + "module": "RiskLabAI.features.feature_importance", + "module_kind": "package", + "sha256": "AA2D447A39D488D12DBDAA7E54F158AC9F4E94F7839D4F0BE56BBE6F9E0EC27D", + "size": 2142, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py", + "module": "RiskLabAI.features.feature_importance.clustered_feature_importance_mda", + "module_kind": "module", + "sha256": "656BBC12FE44777CE8557AF6A0F3292DA11A69B23C0F812E10501E9AFF4E2BF1", + "size": 5518, + "defined_public_names": [ + "ClusteredFeatureImportanceMDA" + ], + "definitions": [ + { + "name": "ClusteredFeatureImportanceMDA", + "kind": "class", + "signature": "(self, classifier: object, clusters: dict[str, list[str]], n_splits: int=10, random_state: int=42)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py", + "module": "RiskLabAI.features.feature_importance.clustered_feature_importance_mdi", + "module_kind": "module", + "sha256": "965BFFF4A3EF79239B54C7DD5A09B7590C3EA1DF8F2B2F10695ADFA095CA45C7", + "size": 3511, + "defined_public_names": [ + "ClusteredFeatureImportanceMDI" + ], + "definitions": [ + { + "name": "ClusteredFeatureImportanceMDI", + "kind": "class", + "signature": "(self, classifier: BaseEnsemble, clusters: dict[str, list[str]])" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/debiased_importance.py", + "module": "RiskLabAI.features.feature_importance.debiased_importance", + "module_kind": "module", + "sha256": "49B8D749041A18283724E5563E1FBB70F4143BECE89C109EA70B2D1BC7BB5750", + "size": 11360, + "defined_public_names": [ + "conditional_predictive_impact", + "mdi_plus_importance" + ], + "definitions": [ + { + "name": "mdi_plus_importance", + "kind": "function", + "signature": "(x: pd.DataFrame, y: pd.Series, random_state: int=0, n_trees: int=_DEFAULT_TREES_DECOMP, classifier: RandomForestClassifier | None=None)" + }, + { + "name": "conditional_predictive_impact", + "kind": "function", + "signature": "(x: pd.DataFrame, y: pd.Series, random_state: int=0, n_splits: int=4, n_estimators: int=80, classifier: RandomForestClassifier | None=None)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/feature_importance_controller.py", + "module": "RiskLabAI.features.feature_importance.feature_importance_controller", + "module_kind": "module", + "sha256": "52907FFC70EE8654B29E2552FE03AD22D494DFFC9E4777021BE0E2A199E8966A", + "size": 1399, + "defined_public_names": [ + "FeatureImportanceController" + ], + "definitions": [ + { + "name": "FeatureImportanceController", + "kind": "class", + "signature": "(self, strategy_type: str, **kwargs: Any)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/feature_importance_factory.py", + "module": "RiskLabAI.features.feature_importance.feature_importance_factory", + "module_kind": "module", + "sha256": "5D44788D9C58D531D4913E7C9D8E4C9BCEB2A634ED3CF05EF60D882EE1084A26", + "size": 1620, + "defined_public_names": [ + "FeatureImportanceFactory" + ], + "definitions": [ + { + "name": "FeatureImportanceFactory", + "kind": "class", + "signature": null + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/feature_importance_mda.py", + "module": "RiskLabAI.features.feature_importance.feature_importance_mda", + "module_kind": "module", + "sha256": "5E22AE26672B9B3A608CC237791625386C5B500FFBF768AEEC5478154F727CF0", + "size": 4650, + "defined_public_names": [ + "FeatureImportanceMDA" + ], + "definitions": [ + { + "name": "FeatureImportanceMDA", + "kind": "class", + "signature": "(self, classifier: object, n_splits: int=10, random_state: int=42)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/feature_importance_mdi.py", + "module": "RiskLabAI.features.feature_importance.feature_importance_mdi", + "module_kind": "module", + "sha256": "82F93E0DA408EB9D4984856C3C432704DC8093ADC90FE9DC898D2109E7381A92", + "size": 2739, + "defined_public_names": [ + "FeatureImportanceMDI" + ], + "definitions": [ + { + "name": "FeatureImportanceMDI", + "kind": "class", + "signature": "(self, classifier: BaseEnsemble)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/feature_importance_sfi.py", + "module": "RiskLabAI.features.feature_importance.feature_importance_sfi", + "module_kind": "module", + "sha256": "67CB65A25ACA73457000C23AC4853A1D281CA3C2DB3CE6614C1262AF51B808BF", + "size": 3984, + "defined_public_names": [ + "FeatureImportanceSFI" + ], + "definitions": [ + { + "name": "FeatureImportanceSFI", + "kind": "class", + "signature": "(self, classifier: object, n_splits: int=10, scoring: str='log_loss')" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/feature_importance_strategy.py", + "module": "RiskLabAI.features.feature_importance.feature_importance_strategy", + "module_kind": "module", + "sha256": "E007852540710974A4C56089DB6A5DE46F11B19760690E400BAFCE154156BD1D", + "size": 894, + "defined_public_names": [ + "FeatureImportanceStrategy" + ], + "definitions": [ + { + "name": "FeatureImportanceStrategy", + "kind": "class", + "signature": null + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/generate_synthetic_data.py", + "module": "RiskLabAI.features.feature_importance.generate_synthetic_data", + "module_kind": "module", + "sha256": "67342708D90B9AC31BF31FF851E341D122600D7A75818F20CEFF31CB39E41747", + "size": 2535, + "defined_public_names": [ + "get_test_dataset" + ], + "definitions": [ + { + "name": "get_test_dataset", + "kind": "function", + "signature": "(n_features: int=100, n_informative: int=25, n_redundant: int=25, n_samples: int=10000, random_state: int=0, sigma_std: float=0.0)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/orthogonal_features.py", + "module": "RiskLabAI.features.feature_importance.orthogonal_features", + "module_kind": "module", + "sha256": "0250FFD2E9B7E2D1B5BFBF960D63EA32CBB0CD9335249222E37A66724B6B0FF9", + "size": 3140, + "defined_public_names": [ + "orthogonal_features" + ], + "definitions": [ + { + "name": "orthogonal_features", + "kind": "function", + "signature": "(features: pd.DataFrame, variance_threshold: float=0.95)" + } + ] + }, + { + "path": "src/RiskLabAI/features/feature_importance/weighted_tau.py", + "module": "RiskLabAI.features.feature_importance.weighted_tau", + "module_kind": "module", + "sha256": "706F53A5531A3B5089A9A252883558C7561C9B1B0DB6B251904EE80A8961C098", + "size": 974, + "defined_public_names": [ + "calculate_weighted_tau" + ], + "definitions": [ + { + "name": "calculate_weighted_tau", + "kind": "function", + "signature": "(feature_importances: np.ndarray, principal_component_ranks: np.ndarray)" + } + ] + }, + { + "path": "src/RiskLabAI/features/microstructural_features/__init__.py", + "module": "RiskLabAI.features.microstructural_features", + "module_kind": "package", + "sha256": "69C079797912E65103C77DD079B00F420C30479FEFEB7A3C0C3E70AF7BCC855A", + "size": 737, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py", + "module": "RiskLabAI.features.microstructural_features.bekker_parkinson_volatility_estimator", + "module_kind": "module", + "sha256": "BC606B1E368BC738846DCA08DE31C99A0A42D91F0CF5F8BB686EDE14021821FF", + "size": 2058, + "defined_public_names": [ + "bekker_parkinson_volatility_estimates", + "sigma_estimates" + ], + "definitions": [ + { + "name": "sigma_estimates", + "kind": "function", + "signature": "(beta: pd.Series, gamma: pd.Series)" + }, + { + "name": "bekker_parkinson_volatility_estimates", + "kind": "function", + "signature": "(high_prices: pd.Series, low_prices: pd.Series, window_span: int=20)" + } + ] + }, + { + "path": "src/RiskLabAI/features/microstructural_features/corwin_schultz.py", + "module": "RiskLabAI.features.microstructural_features.corwin_schultz", + "module_kind": "module", + "sha256": "9120D5D7F9763E017A896E5F7A3288F02112E1A72AA92C42717E6B2506070532", + "size": 3471, + "defined_public_names": [ + "alpha_estimates", + "beta_estimates", + "corwin_schultz_estimator", + "gamma_estimates" + ], + "definitions": [ + { + "name": "beta_estimates", + "kind": "function", + "signature": "(high_prices: pd.Series, low_prices: pd.Series, window_span: int)" + }, + { + "name": "gamma_estimates", + "kind": "function", + "signature": "(high_prices: pd.Series, low_prices: pd.Series)" + }, + { + "name": "alpha_estimates", + "kind": "function", + "signature": "(beta: pd.Series, gamma: pd.Series)" + }, + { + "name": "corwin_schultz_estimator", + "kind": "function", + "signature": "(high_prices: pd.Series, low_prices: pd.Series, window_span: int=20)" + } + ] + }, + { + "path": "src/RiskLabAI/features/microstructural_features/edge.py", + "module": "RiskLabAI.features.microstructural_features.edge", + "module_kind": "module", + "sha256": "D9C99CB172B1AA9B6F33AF39723D9BE4880214466F863F6177D7B028D7F73B61", + "size": 6251, + "defined_public_names": [ + "edge_estimator" + ], + "definitions": [ + { + "name": "edge_estimator", + "kind": "function", + "signature": "(open: _ArrayLike, high: _ArrayLike, low: _ArrayLike, close: _ArrayLike, sign: bool=False)" + } + ] + }, + { + "path": "src/RiskLabAI/features/structural_breaks/__init__.py", + "module": "RiskLabAI.features.structural_breaks", + "module_kind": "package", + "sha256": "46AA8C0C7DA4955E2A6397B7FCCC51B6D48C3A7016E9413A4D356EDF22D69C2F", + "size": 1122, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/features/structural_breaks/pelt_change_points.py", + "module": "RiskLabAI.features.structural_breaks.pelt_change_points", + "module_kind": "module", + "sha256": "1F913C5EE9BEF2144D687068F0526C8A0F0AE5A9B0A51991E5EC686701307070", + "size": 4218, + "defined_public_names": [ + "pelt_change_points" + ], + "definitions": [ + { + "name": "pelt_change_points", + "kind": "function", + "signature": "(series, model: str='normal', penalty_multiplier: float=3.0, min_size: int=10, jump: int=5)" + } + ] + }, + { + "path": "src/RiskLabAI/features/structural_breaks/structural_breaks.py", + "module": "RiskLabAI.features.structural_breaks.structural_breaks", + "module_kind": "module", + "sha256": "9F5E690FDAC43FFE12728DBF696DF2B250235B192604F8CA9DDE9BC922E14656", + "size": 26104, + "defined_public_names": [ + "compute_beta", + "get_bsadf_sequence", + "get_bsadf_statistic", + "get_bubble_episodes", + "get_expanding_window_adf", + "get_gsadf_statistic", + "get_sadf_sequence", + "lag_dataframe", + "prepare_data", + "psy_minimum_window", + "simulate_psy_critical_values" + ], + "definitions": [ + { + "name": "lag_dataframe", + "kind": "function", + "signature": "(market_data: pd.DataFrame, lags: Union[int, list[int]])" + }, + { + "name": "prepare_data", + "kind": "function", + "signature": "(log_price_series: pd.Series, constant: str, lags: int)" + }, + { + "name": "compute_beta", + "kind": "function", + "signature": "(y_window: np.ndarray, x_window: np.ndarray)" + }, + { + "name": "get_expanding_window_adf", + "kind": "function", + "signature": "(log_price: pd.Series, min_sample_length: int, constant: str, lags: int)" + }, + { + "name": "get_bsadf_statistic", + "kind": "function", + "signature": "(log_price: pd.Series, min_sample_length: int, constant: str, lags: int)" + }, + { + "name": "psy_minimum_window", + "kind": "function", + "signature": "(sample_length: int)" + }, + { + "name": "get_sadf_sequence", + "kind": "function", + "signature": "(log_price: pd.Series, min_sample_length: int, constant: str='c', lags: int=0)" + }, + { + "name": "get_bsadf_sequence", + "kind": "function", + "signature": "(log_price: pd.Series, min_sample_length: int, constant: str='c', lags: int=0)" + }, + { + "name": "get_gsadf_statistic", + "kind": "function", + "signature": "(log_price: pd.Series, min_sample_length: int, constant: str='c', lags: int=0)" + }, + { + "name": "get_bubble_episodes", + "kind": "function", + "signature": "(statistic_sequence: pd.Series, critical_value_sequence: Union[pd.Series, np.ndarray, float], min_duration: int=1)" + }, + { + "name": "simulate_psy_critical_values", + "kind": "function", + "signature": "(sample_length: int, min_sample_length: Optional[int]=None, constant: str='c', lags: int=0, n_simulations: int=2000, level: float=0.95, seed: int=0)" + } + ] + }, + { + "path": "src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py", + "module": "RiskLabAI.features.structural_breaks.volatility_robust_sadf", + "module_kind": "module", + "sha256": "69B7F026E05E66DE4755E1A591602A59AD3B5CFE49B29DE03724F37919BDD27E", + "size": 5904, + "defined_public_names": [ + "volatility_robust_sadf" + ], + "definitions": [ + { + "name": "volatility_robust_sadf", + "kind": "function", + "signature": "(log_price: np.ndarray | pd.Series, min_length: int | None=None, n_bootstrap: int=99, random_state: int | None=None)" + } + ] + }, + { + "path": "src/RiskLabAI/hpc/__init__.py", + "module": "RiskLabAI.hpc", + "module_kind": "package", + "sha256": "C08896EA7791033B286FAB38AFDBD8EC5A79F562316AA470DD071F90C6F51384", + "size": 511, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/hpc/hpc.py", + "module": "RiskLabAI.hpc.hpc", + "module_kind": "module", + "sha256": "B5AEFF775459A0AD8A5AD0E10B0806E7FC08137F1CCD0CD85966FFF60D03D1C5", + "size": 11685, + "defined_public_names": [ + "expand_call", + "linear_partitions", + "mp_pandas_obj", + "nested_partitions", + "parallel_run", + "process_jobs", + "process_jobs_sequential", + "report_progress" + ], + "definitions": [ + { + "name": "parallel_run", + "kind": "function", + "signature": "(func: Callable[..., Any], iterable: Iterable[Any], num_cpus: int=-1, lin_partition: bool=False, **kwargs)" + }, + { + "name": "report_progress", + "kind": "function", + "signature": "(job_number: int, total_jobs: int, start_time: float, task: str)" + }, + { + "name": "expand_call", + "kind": "function", + "signature": "(kargs: dict[str, Any])" + }, + { + "name": "process_jobs", + "kind": "function", + "signature": "(jobs: list[dict[str, Any]], task: Optional[str]=None, num_threads: int=-1)" + }, + { + "name": "process_jobs_sequential", + "kind": "function", + "signature": "(jobs: list[dict[str, Any]])" + }, + { + "name": "linear_partitions", + "kind": "function", + "signature": "(num_atoms: int, num_threads: int)" + }, + { + "name": "nested_partitions", + "kind": "function", + "signature": "(num_atoms: int, num_threads: int, upper_triangle: bool=False)" + }, + { + "name": "mp_pandas_obj", + "kind": "function", + "signature": "(func: Callable[..., pd.Series], pandas_object: tuple[str, pd.Index], num_threads: int=-1, mp_batches: int=1, linear_partition: bool=True, **kwargs: Any)" + } + ] + }, + { + "path": "src/RiskLabAI/optimization/__init__.py", + "module": "RiskLabAI.optimization", + "module_kind": "package", + "sha256": "E67CA20C00441769F46EA11BFDC7113646556FF45EFD63C1E748CC2BA93805A1", + "size": 1257, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/optimization/hedging.py", + "module": "RiskLabAI.optimization.hedging", + "module_kind": "module", + "sha256": "4A87276BD9FBAF8726F8646E57C6B27E6274C3481692532A04F67CC0B206E149", + "size": 1962, + "defined_public_names": [ + "pca_weights" + ], + "definitions": [ + { + "name": "pca_weights", + "kind": "function", + "signature": "(cov: np.ndarray, risk_distribution: Optional[np.ndarray]=None, risk_target: float=1.0)" + } + ] + }, + { + "path": "src/RiskLabAI/optimization/hrp.py", + "module": "RiskLabAI.optimization.hrp", + "module_kind": "module", + "sha256": "5A5F6F3D3145C9F7870CD1FEE15BD88D11A7C7E707D4A6EDCF83F9F43F92AED1", + "size": 6323, + "defined_public_names": [ + "cluster_variance", + "distance_corr", + "hrp", + "inverse_variance_weights", + "quasi_diagonal", + "recursive_bisection" + ], + "definitions": [ + { + "name": "inverse_variance_weights", + "kind": "function", + "signature": "(covariance_matrix: pd.DataFrame)" + }, + { + "name": "cluster_variance", + "kind": "function", + "signature": "(covariance_matrix: pd.DataFrame, clustered_items: list[str])" + }, + { + "name": "quasi_diagonal", + "kind": "function", + "signature": "(linkage_matrix: np.ndarray)" + }, + { + "name": "recursive_bisection", + "kind": "function", + "signature": "(covariance_matrix: pd.DataFrame, sorted_items: list[str])" + }, + { + "name": "distance_corr", + "kind": "function", + "signature": "(corr_matrix: np.ndarray)" + }, + { + "name": "hrp", + "kind": "function", + "signature": "(cov: pd.DataFrame, corr: pd.DataFrame)" + } + ] + }, + { + "path": "src/RiskLabAI/optimization/hyper_parameter_tuning.py", + "module": "RiskLabAI.optimization.hyper_parameter_tuning", + "module_kind": "module", + "sha256": "93619B3E22168D701F2DF2E17ABA0134B3E3BF1345C2B6CF234E14A9A5083A33", + "size": 6347, + "defined_public_names": [ + "MyPipeline", + "SampleWeightedPipeline", + "clf_hyper_fit" + ], + "definitions": [ + { + "name": "SampleWeightedPipeline", + "kind": "class", + "signature": null + }, + { + "name": "clf_hyper_fit", + "kind": "function", + "signature": "(feature_data: pd.DataFrame, label: pd.Series, times: pd.Series, pipe_clf: Pipeline, param_grid: dict[str, Any], validator_type: str='purgedkfold', validator_params: Optional[dict[str, Any]]=None, bagging: Optional[list[Union[int, float]]]=None, rnd_search_iter: int=0, n_jobs: int=-1, **fit_params)" + }, + { + "name": "MyPipeline", + "kind": "class", + "signature": "(self, steps, *, memory=None, verbose=False)" + } + ] + }, + { + "path": "src/RiskLabAI/optimization/nco.py", + "module": "RiskLabAI.optimization.nco", + "module_kind": "module", + "sha256": "96D9A482B49D9E784B6790B46E28EFE706E3C63A262521D721BD75C46A59085A", + "size": 4122, + "defined_public_names": [ + "get_optimal_portfolio_weights", + "get_optimal_portfolio_weights_nco" + ], + "definitions": [ + { + "name": "get_optimal_portfolio_weights", + "kind": "function", + "signature": "(covariance: np.ndarray, mu: Optional[np.ndarray]=None)" + }, + { + "name": "get_optimal_portfolio_weights_nco", + "kind": "function", + "signature": "(covariance: np.ndarray, mu: Optional[np.ndarray]=None, number_clusters: Optional[int]=None)" + } + ] + }, + { + "path": "src/RiskLabAI/pde/__init__.py", + "module": "RiskLabAI.pde", + "module_kind": "package", + "sha256": "C7B95E7E7DF3A7BDD666D940EE776A5D8349ADE4CEE33D8FC453C7C56189C4CA", + "size": 1833, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/pde/equation.py", + "module": "RiskLabAI.pde.equation", + "module_kind": "module", + "sha256": "4A9A073721FAAC4FFAC3513A61B08586027204248EE8A47F184E5F174DD86E06", + "size": 10999, + "defined_public_names": [ + "BlackScholesBarenblatt", + "Equation", + "HJBLQ", + "PricingDefaultRisk", + "PricingDiffRate" + ], + "definitions": [ + { + "name": "Equation", + "kind": "class", + "signature": "(self, eqn_config: dict)" + }, + { + "name": "PricingDefaultRisk", + "kind": "class", + "signature": "(self, eqn_config: dict)" + }, + { + "name": "HJBLQ", + "kind": "class", + "signature": "(self, eqn_config: dict)" + }, + { + "name": "BlackScholesBarenblatt", + "kind": "class", + "signature": "(self, eqn_config: dict)" + }, + { + "name": "PricingDiffRate", + "kind": "class", + "signature": "(self, eqn_config: dict)" + } + ] + }, + { + "path": "src/RiskLabAI/pde/model.py", + "module": "RiskLabAI.pde.model", + "module_kind": "module", + "sha256": "98F8C874C57C04E28883BB9B4CFD526C0D1622E27C84F723E5883D30CBFF7001", + "size": 12264, + "defined_public_names": [ + "DeepBSDE", + "DeepTimeSetTransformer", + "FBSNNNetwork", + "ISAB", + "MAB", + "Net1", + "PMA", + "SAB", + "TimeDependentNetwork", + "TimeDependentNetworkMonteCarlo", + "TimeNet", + "TimeNetForSet" + ], + "definitions": [ + { + "name": "MAB", + "kind": "class", + "signature": "(self, dim_q: int, dim_k: int, dim_v: int, num_heads: int, ln: bool=False)" + }, + { + "name": "SAB", + "kind": "class", + "signature": "(self, dim_in: int, dim_out: int, num_heads: int, ln: bool=False)" + }, + { + "name": "ISAB", + "kind": "class", + "signature": "(self, dim_in: int, dim_out: int, num_heads: int, num_inds: int, ln: bool=False)" + }, + { + "name": "PMA", + "kind": "class", + "signature": "(self, dim: int, num_heads: int, num_seeds: int, ln: bool=False)" + }, + { + "name": "TimeNetForSet", + "kind": "class", + "signature": "(self, in_features: int=1, out_features: int=64)" + }, + { + "name": "DeepTimeSetTransformer", + "kind": "class", + "signature": "(self, input_dim: int)" + }, + { + "name": "TimeNet", + "kind": "class", + "signature": "(self, output_dim: int)" + }, + { + "name": "Net1", + "kind": "class", + "signature": "(self, input_dim: int, output_dim: int)" + }, + { + "name": "FBSNNNetwork", + "kind": "class", + "signature": "(self, layer_sizes: list[int])" + }, + { + "name": "DeepBSDE", + "kind": "class", + "signature": "(self, layer_sizes: list[int])" + }, + { + "name": "TimeDependentNetwork", + "kind": "class", + "signature": "(self, indim: int, layersize: list[int], outdim: int)" + }, + { + "name": "TimeDependentNetworkMonteCarlo", + "kind": "class", + "signature": "(self, indim: int, layersize: list[int], outdim: int, sigma: float)" + } + ] + }, + { + "path": "src/RiskLabAI/pde/solver.py", + "module": "RiskLabAI.pde.solver", + "module_kind": "module", + "sha256": "A7F2A65833339E5E9258F165B0AC0BA9F37130061F456C12465C81179B740756", + "size": 13859, + "defined_public_names": [ + "FBSDESolver", + "FBSNNSolver", + "initialize_weights" + ], + "definitions": [ + { + "name": "initialize_weights", + "kind": "function", + "signature": "(m: nn.Module)" + }, + { + "name": "FBSDESolver", + "kind": "class", + "signature": "(self, pde: Equation, layer_sizes: list[int], learning_rate: float, solving_method: str, device: torch.device)" + }, + { + "name": "FBSNNSolver", + "kind": "class", + "signature": "(self, pde: Equation, layer_sizes: list[int], learning_rate: float, device: torch.device)" + } + ] + }, + { + "path": "src/RiskLabAI/utils/__init__.py", + "module": "RiskLabAI.utils", + "module_kind": "package", + "sha256": "DE1F4F4832425C9B8FE275BF48A9621ABA3525BB219F43C9D9E38DBBCEAE0582", + "size": 3409, + "defined_public_names": [], + "definitions": [] + }, + { + "path": "src/RiskLabAI/utils/constants.py", + "module": "RiskLabAI.utils.constants", + "module_kind": "module", + "sha256": "A151D28E3AC9C3860C823F60097360532230864FF049ADDB5524ABBD99A88ABA", + "size": 3088, + "defined_public_names": [ + "BUY_TICKS_NUMBER", + "CLOSE_PRICE", + "CUMULATIVE_BUY_THETA", + "CUMULATIVE_BUY_VOLUME", + "CUMULATIVE_DOLLAR", + "CUMULATIVE_SELL_THETA", + "CUMULATIVE_SELL_VOLUME", + "CUMULATIVE_THETA", + "CUMULATIVE_TICKS", + "CUMULATIVE_VOLUME", + "DATE_TIME", + "EXPECTED_BUY_IMBALANCE", + "EXPECTED_BUY_TICKS_PROPORTION", + "EXPECTED_IMBALANCE", + "EXPECTED_IMBALANCE_WINDOW", + "EXPECTED_SELL_IMBALANCE", + "EXPECTED_TICKS_NUMBER", + "HIGH_PRICE", + "LOW_PRICE", + "N_PREVIOUS_BARS_FOR_EXPECTED_N_TICKS_ESTIMATION", + "N_TICKS_ON_BAR_FORMATION", + "OPEN_PRICE", + "PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST", + "PREVIOUS_BARS_N_TICKS_LIST", + "PREVIOUS_TICK_IMBALANCES_BUY_LIST", + "PREVIOUS_TICK_IMBALANCES_LIST", + "PREVIOUS_TICK_IMBALANCES_SELL_LIST", + "PREVIOUS_TICK_RULE", + "THRESHOLD", + "TICK_NUMBER", + "TIMESTAMP" + ], + "definitions": [] + }, + { + "path": "src/RiskLabAI/utils/ewma.py", + "module": "RiskLabAI.utils.ewma", + "module_kind": "module", + "sha256": "31C124A43EA1CD2BACE221F76C3EB3518E71C19D6BF547AF20D555DC8ACD97FC", + "size": 2173, + "defined_public_names": [ + "ewma" + ], + "definitions": [ + { + "name": "ewma", + "kind": "function", + "signature": "(array: np.ndarray, window: int)" + } + ] + }, + { + "path": "src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py", + "module": "RiskLabAI.utils.momentum_mean_reverting_strategy_sides", + "module_kind": "module", + "sha256": "F19E1691AD40DF94640AEFBFA06431B39538444139277568B8A6AE247FB48509", + "size": 2156, + "defined_public_names": [ + "determine_strategy_side" + ], + "definitions": [ + { + "name": "determine_strategy_side", + "kind": "function", + "signature": "(prices: pd.Series, fast_window: int=20, slow_window: int=50, exponential: bool=False, mean_reversion: bool=False)" + } + ] + }, + { + "path": "src/RiskLabAI/utils/progress.py", + "module": "RiskLabAI.utils.progress", + "module_kind": "module", + "sha256": "74B4D2EA57526348055B657EB254D92606C00C7A49DBC2B122D5DA34A44F75D2", + "size": 1945, + "defined_public_names": [ + "progress_bar" + ], + "definitions": [ + { + "name": "progress_bar", + "kind": "function", + "signature": "(current_progress: int, total_progress: int, start_time: float, bar_length: int=20)" + } + ] + }, + { + "path": "src/RiskLabAI/utils/publication_plots.py", + "module": "RiskLabAI.utils.publication_plots", + "module_kind": "module", + "sha256": "B00FD12382E692BB3F74F2B37EFFAB1D95BC3641679F588BBB60FB0C203764D8", + "size": 5850, + "defined_public_names": [ + "THEMES", + "apply_plot_style", + "finalize_plot", + "setup_publication_style" + ], + "definitions": [ + { + "name": "setup_publication_style", + "kind": "function", + "signature": "(theme: str='light', quality: int=300, save_plots: bool=False, save_dir: str='figs')" + }, + { + "name": "apply_plot_style", + "kind": "function", + "signature": "(ax: plt.Axes, title: str, xlabel: str, ylabel: str, legend_title: Optional[str]=None)" + }, + { + "name": "finalize_plot", + "kind": "function", + "signature": "(fig: fig.Figure, filename: str)" + } + ] + }, + { + "path": "src/RiskLabAI/utils/update_figure_layout.py", + "module": "RiskLabAI.utils.update_figure_layout", + "module_kind": "module", + "sha256": "8C5C6DAEFE392D1AD59AF598B1D8D76732FD4B0A7A437AB3C4AB160CB1CDFFB2", + "size": 1324, + "defined_public_names": [ + "update_figure_layout" + ], + "definitions": [ + { + "name": "update_figure_layout", + "kind": "function", + "signature": "(fig: go.Figure, title: str, xaxis_title: str, yaxis_title: str, legend_x: float=1.0, legend_y: float=1.0)" + } + ] + } + ], + "package_export_surfaces": [ + { + "module": "RiskLabAI", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "__version__", + "backtest", + "causal_factor_analysis", + "cluster", + "controller", + "core", + "data", + "ensemble", + "features", + "hpc", + "optimization", + "pde", + "utils" + ] + }, + { + "module": "RiskLabAI.backtest", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "validation", + "PlattCalibrator", + "distributionally_robust_kelly_fraction", + "expected_calibration_error", + "kelly_bet_fraction", + "conditional_expected_drawdown", + "sharpe_difference_test", + "optimal_ou_trading_rule", + "ou_rule_metrics", + "fit_ornstein_uhlenbeck", + "bet_timing", + "calculate_holding_period", + "calculate_hhi_concentration", + "calculate_hhi", + "compute_drawdowns_time_under_water", + "synthetic_back_testing", + "probability_bet_size", + "average_bet_sizes", + "strategy_bet_sizing", + "avg_active_signals", + "mp_avg_active_signals", + "discrete_signal", + "generate_signal", + "bet_size_sigmoid", + "target_position", + "inverse_price", + "limit_price", + "compute_sigmoid_width", + "avgActiveSignals", + "mpAvgActiveSignals", + "discreteSignal", + "Signal", + "betSize", + "TPos", + "inversePrice", + "limitPrice", + "getW", + "sharpe_ratio_trials", + "target_sharpe_ratio_symbolic", + "implied_precision", + "bin_frequency", + "binomial_sharpe_ratio", + "mix_gaussians", + "failure_probability", + "calculate_strategy_risk", + "expected_max_sharpe_ratio", + "generate_max_sharpe_ratios", + "mean_std_error", + "estimated_sharpe_ratio_z_statistics", + "strategy_type1_error_probability", + "theta_for_type2_error", + "strategy_type2_error_probability", + "pbo_sharpe_ratio", + "performance_evaluation", + "probability_of_backtest_overfitting", + "probabilistic_sharpe_ratio", + "benchmark_sharpe_ratio", + "sharpe_ratio_p_values", + "holm_adjusted_p_values", + "benjamini_hochberg_yekutieli_adjusted_p_values", + "haircut_sharpe_ratios", + "sharpe_ratio_influence_function", + "newey_west_long_run_variance", + "newey_west_automatic_lag", + "lplz_sharpe_inference", + "overall_backtest_overfitting_simulation", + "temporal_backtest_overfitting_simulation", + "time_temporal_backtest_overfitting_simulation", + "varying_embargo_backtest_overfitting_simulation", + "backtest_overfitting_simulation_financial_metrics_rank_correlation", + "backtest_overfitting_simulation_model_complexity", + "noised_backtest_overfitting_simulation", + "overall_novel_methods_backtest_overfitting_simulation", + "measure_all_cv_computational_requirements", + "measure_cpcv_parallelization", + "measure_cpcv_scalability", + "get_cpu_info", + "format_cpu_info" + ] + }, + { + "module": "RiskLabAI.backtest.validation", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "CrossValidator", + "KFold", + "PurgedKFold", + "WalkForward", + "CombinatorialPurged", + "BaggedCombinatorialPurged", + "AdaptiveCombinatorialPurged", + "bagged_probability_of_backtest_overfitting", + "moving_block_bootstrap_indices", + "adaptive_probability_of_backtest_overfitting", + "estimate_volatility_regimes", + "leakage_aware_hpo", + "deflated_sharpe_gate", + "CrossValidatorFactory", + "CrossValidatorController" + ] + }, + { + "module": "RiskLabAI.causal_factor_analysis", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "BackdoorAdjustmentEvidence", + "BacktestStage", + "CausalDAG", + "CausalAdjustmentSetStage", + "ColliderCoefficients", + "ColliderDiagnostics", + "CausalDiscoveryStage", + "CausalExplanatoryAndPredictivePowerStage", + "CausalFactorProtocolReport", + "CausalPortfolioConstructionStage", + "DSeparationEvidence", + "DifferenceInDifferencesEstimate", + "EventHorizon", + "FoldEvidence", + "FrontdoorAdjustmentEvidence", + "InstrumentEvidence", + "MultipleTestingAdjustmentsStage", + "NodeRoleEvidence", + "PathEvidence", + "PopulationRegressionDiagnostics", + "RegressionDiagnostics", + "SpecificationExperimentResult", + "SpecificationPopulationResult", + "StrategyPerformance", + "TreatmentEffectDecomposition", + "ValidationEvidence", + "VariableSelectionStage", + "average_treatment_effect", + "backdoor_adjusted_average_treatment_effect", + "backdoor_adjusted_expectation", + "causal_role_evidence", + "check_backdoor_adjustment_set", + "check_frontdoor_adjustment_set", + "check_instrument", + "collider_factor_return", + "collider_forecast_return", + "collider_model_diagnostics", + "collider_overcontrolled_coefficients", + "collider_population_diagnostics", + "collider_specification_experiment", + "confounded_mediator_population_diagnostics", + "confounded_mediator_specification_experiment", + "confounder_factor_return", + "confounder_forecast_return", + "confounder_undercontrolled_coefficient", + "d_separation", + "difference_in_differences", + "fork_population_diagnostics", + "fork_specification_experiment", + "frontdoor_adjusted_probability", + "linear_instrumental_variable_effect", + "minimal_backdoor_adjustment_sets", + "minimal_frontdoor_adjustment_sets", + "minimum_variance_factor_weights", + "randomized_mean_difference", + "treatment_effect_decomposition", + "validate_causal_factor_protocol" + ] + }, + { + "module": "RiskLabAI.cluster", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "covariance_to_correlation", + "cluster_k_means_base", + "make_new_outputs", + "cluster_k_means_top", + "random_covariance_sub", + "random_block_covariance", + "random_block_correlation" + ] + }, + { + "module": "RiskLabAI.controller", + "export_rule": "IMPORTED_PUBLIC_BINDINGS", + "ordered_exports": [] + }, + { + "module": "RiskLabAI.core", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "Registry", + "REGISTRIES", + "get_registry", + "list_components", + "BARS", + "CROSS_VALIDATORS", + "FEATURE_IMPORTANCE", + "LABELERS", + "BET_SIZERS", + "PORTFOLIO_OPTIMIZERS", + "Estimator", + "BaseLabeler", + "BaseBetSizer", + "BasePortfolioOptimizer" + ] + }, + { + "module": "RiskLabAI.data", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "denoise", + "differentiation", + "distance", + "labeling", + "structures", + "synthetic_data", + "weights" + ] + }, + { + "module": "RiskLabAI.data.denoise", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "marcenko_pastur_pdf", + "fit_kde", + "find_max_eval", + "pca", + "denoised_corr", + "denoised_corr2", + "cov_to_corr", + "corr_to_cov", + "denoise_cov", + "optimal_portfolio", + "nercome_denoised_covariance" + ] + }, + { + "module": "RiskLabAI.data.differentiation", + "export_rule": "IMPORTED_PUBLIC_BINDINGS", + "ordered_exports": [ + "adaptive_differencing_order", + "adaptive_fractional_difference", + "rescaled_range_hurst", + "wavelet_variance_hurst" + ] + }, + { + "module": "RiskLabAI.data.distance", + "export_rule": "IMPORTED_PUBLIC_BINDINGS", + "ordered_exports": [ + "distance_correlation", + "ksg_mutual_information" + ] + }, + { + "module": "RiskLabAI.data.labeling", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "cusum_filter_events_dynamic_threshold", + "symmetric_cusum_filter", + "daily_volatility_with_log_returns", + "vertical_barrier", + "triple_barrier", + "meta_events", + "meta_labeling", + "lin_parts", + "process_jobs", + "expand_call", + "report_progress", + "calculate_t_value_linear_regression", + "find_trend_using_trend_scanning" + ] + }, + { + "module": "RiskLabAI.data.structures", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "StandardBars", + "TimeBars", + "ExpectedImbalanceBars", + "FixedImbalanceBars", + "ExpectedRunBars", + "FixedRunBars", + "HardenedExpectedImbalanceBars", + "HardenedExpectedRunBars" + ] + }, + { + "module": "RiskLabAI.data.synthetic_data", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "drift_volatility_burst", + "compute_log_returns", + "heston_merton_log_returns", + "align_params_length", + "generate_prices_from_regimes", + "parallel_generate_prices", + "random_cov", + "form_block_matrix", + "form_true_matrix", + "simulates_cov_mu" + ] + }, + { + "module": "RiskLabAI.data.weights", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "expand_label_for_meta_labeling", + "calculate_average_uniqueness", + "sample_weight_absolute_return_meta_labeling", + "calculate_time_decay" + ] + }, + { + "module": "RiskLabAI.ensemble", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "bagging_classifier_accuracy", + "BaggingClassifierAccuracy", + "calculate_bootstrap_accuracy", + "plot_bootstrap_accuracy_distribution" + ] + }, + { + "module": "RiskLabAI.features", + "export_rule": "IMPORTED_PUBLIC_BINDINGS", + "ordered_exports": [ + "entropy_features", + "feature_importance", + "microstructural_features", + "structural_breaks" + ] + }, + { + "module": "RiskLabAI.features.entropy_features", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "shannon_entropy", + "lempel_ziv_entropy", + "probability_mass_function", + "ngram_counts", + "plug_in_entropy_estimator", + "kontoyiannis_entropy", + "longest_match_length", + "miller_madow_entropy", + "grassberger_entropy", + "nsb_entropy" + ] + }, + { + "module": "RiskLabAI.features.feature_importance", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "FeatureImportanceStrategy", + "FeatureImportanceFactory", + "FeatureImportanceController", + "FeatureImportanceMDI", + "ClusteredFeatureImportanceMDI", + "FeatureImportanceMDA", + "ClusteredFeatureImportanceMDA", + "FeatureImportanceSFI", + "mdi_plus_importance", + "conditional_predictive_impact", + "cluster_k_means_top", + "random_block_correlation", + "form_true_matrix", + "simulates_cov_mu", + "get_test_dataset", + "orthogonal_features", + "calculate_weighted_tau" + ] + }, + { + "module": "RiskLabAI.features.microstructural_features", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "beta_estimates", + "gamma_estimates", + "alpha_estimates", + "corwin_schultz_estimator", + "sigma_estimates", + "bekker_parkinson_volatility_estimates", + "edge_estimator" + ] + }, + { + "module": "RiskLabAI.features.structural_breaks", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "lag_dataframe", + "prepare_data", + "compute_beta", + "get_expanding_window_adf", + "get_bsadf_statistic", + "psy_minimum_window", + "get_sadf_sequence", + "get_bsadf_sequence", + "get_gsadf_statistic", + "get_bubble_episodes", + "simulate_psy_critical_values", + "volatility_robust_sadf", + "pelt_change_points" + ] + }, + { + "module": "RiskLabAI.hpc", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "report_progress", + "expand_call", + "process_jobs", + "process_jobs_sequential", + "linear_partitions", + "nested_partitions", + "mp_pandas_obj", + "parallel_run" + ] + }, + { + "module": "RiskLabAI.optimization", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "cluster_variance", + "quasi_diagonal", + "recursive_bisection", + "hrp", + "get_optimal_portfolio_weights", + "get_optimal_portfolio_weights_nco", + "pca_weights", + "SampleWeightedPipeline", + "MyPipeline", + "clf_hyper_fit" + ] + }, + { + "module": "RiskLabAI.pde", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "Equation", + "PricingDefaultRisk", + "HJBLQ", + "BlackScholesBarenblatt", + "PricingDiffRate", + "TimeNet", + "Net1", + "MAB", + "SAB", + "ISAB", + "PMA", + "TimeNetForSet", + "DeepTimeSetTransformer", + "FBSNNNetwork", + "DeepBSDE", + "TimeDependentNetwork", + "TimeDependentNetworkMonteCarlo", + "initialize_weights", + "FBSDESolver", + "FBSNNSolver", + "FBSNNolver" + ] + }, + { + "module": "RiskLabAI.utils", + "export_rule": "EXPLICIT___ALL__", + "ordered_exports": [ + "DATE_TIME", + "TIMESTAMP", + "TICK_NUMBER", + "OPEN_PRICE", + "HIGH_PRICE", + "LOW_PRICE", + "CLOSE_PRICE", + "CUMULATIVE_TICKS", + "CUMULATIVE_DOLLAR", + "THRESHOLD", + "CUMULATIVE_VOLUME", + "CUMULATIVE_BUY_VOLUME", + "CUMULATIVE_SELL_VOLUME", + "CUMULATIVE_THETA", + "CUMULATIVE_BUY_THETA", + "CUMULATIVE_SELL_THETA", + "EXPECTED_IMBALANCE", + "EXPECTED_TICKS_NUMBER", + "EXPECTED_BUY_IMBALANCE", + "EXPECTED_SELL_IMBALANCE", + "EXPECTED_BUY_TICKS_PROPORTION", + "BUY_TICKS_NUMBER", + "N_TICKS_ON_BAR_FORMATION", + "PREVIOUS_TICK_RULE", + "EXPECTED_IMBALANCE_WINDOW", + "PREVIOUS_BARS_N_TICKS_LIST", + "PREVIOUS_TICK_IMBALANCES_LIST", + "PREVIOUS_TICK_IMBALANCES_BUY_LIST", + "PREVIOUS_TICK_IMBALANCES_SELL_LIST", + "PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST", + "N_PREVIOUS_BARS_FOR_EXPECTED_N_TICKS_ESTIMATION", + "ewma", + "compute_exponential_weighted_moving_average", + "progress_bar", + "determine_strategy_side", + "update_figure_layout", + "setup_publication_style", + "apply_plot_style", + "finalize_plot" + ] + } + ], + "causal_factor_analysis": { + "export_count": 57, + "ordered_exports": [ + "BackdoorAdjustmentEvidence", + "BacktestStage", + "CausalDAG", + "CausalAdjustmentSetStage", + "ColliderCoefficients", + "ColliderDiagnostics", + "CausalDiscoveryStage", + "CausalExplanatoryAndPredictivePowerStage", + "CausalFactorProtocolReport", + "CausalPortfolioConstructionStage", + "DSeparationEvidence", + "DifferenceInDifferencesEstimate", + "EventHorizon", + "FoldEvidence", + "FrontdoorAdjustmentEvidence", + "InstrumentEvidence", + "MultipleTestingAdjustmentsStage", + "NodeRoleEvidence", + "PathEvidence", + "PopulationRegressionDiagnostics", + "RegressionDiagnostics", + "SpecificationExperimentResult", + "SpecificationPopulationResult", + "StrategyPerformance", + "TreatmentEffectDecomposition", + "ValidationEvidence", + "VariableSelectionStage", + "average_treatment_effect", + "backdoor_adjusted_average_treatment_effect", + "backdoor_adjusted_expectation", + "causal_role_evidence", + "check_backdoor_adjustment_set", + "check_frontdoor_adjustment_set", + "check_instrument", + "collider_factor_return", + "collider_forecast_return", + "collider_model_diagnostics", + "collider_overcontrolled_coefficients", + "collider_population_diagnostics", + "collider_specification_experiment", + "confounded_mediator_population_diagnostics", + "confounded_mediator_specification_experiment", + "confounder_factor_return", + "confounder_forecast_return", + "confounder_undercontrolled_coefficient", + "d_separation", + "difference_in_differences", + "fork_population_diagnostics", + "fork_specification_experiment", + "frontdoor_adjusted_probability", + "linear_instrumental_variable_effect", + "minimal_backdoor_adjustment_sets", + "minimal_frontdoor_adjustment_sets", + "minimum_variance_factor_weights", + "randomized_mean_difference", + "treatment_effect_decomposition", + "validate_causal_factor_protocol" + ], + "ordered_export_sha256": "6EF5A71729FA52EF02B2275A9E10868F3284ADDDA3C21585AA9D3074BDB751F1", + "entries": [ + { + "name": "BackdoorAdjustmentEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.BackdoorAdjustmentEvidence", + "kind": "record_type", + "signature": "(treatment: ForwardRef('str'), outcome: ForwardRef('str'), adjustment_set: ForwardRef('NodeTuple'), all_observed: ForwardRef('bool'), endpoints_in_set: ForwardRef('NodeTuple'), descendants_in_set: ForwardRef('NodeTuple'), open_backdoor_paths: ForwardRef('tuple[PathEvidence, ...]'), admissible: ForwardRef('bool'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification", + "record_fields": [ + "treatment", + "outcome", + "adjustment_set", + "all_observed", + "endpoints_in_set", + "descendants_in_set", + "open_backdoor_paths", + "admissible" + ] + }, + { + "name": "BacktestStage", + "qualified_name": "RiskLabAI.causal_factor_analysis.BacktestStage", + "kind": "class", + "signature": "(method_labels: 'tuple[str, ...]', trial_family_id: 'str', declared_trial_ids: 'tuple[str, ...]', validation: 'Optional[ValidationEvidence]' = None, monte_carlo_dgp: 'Optional[str]' = None) -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "CausalDAG", + "qualified_name": "RiskLabAI.causal_factor_analysis.CausalDAG", + "kind": "record_type", + "signature": "(nodes, directed_edges, observed_nodes)", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification", + "record_fields": [ + "nodes", + "directed_edges", + "observed_nodes" + ] + }, + { + "name": "CausalAdjustmentSetStage", + "qualified_name": "RiskLabAI.causal_factor_analysis.CausalAdjustmentSetStage", + "kind": "class", + "signature": "(treatment: 'str', outcome: 'str', method_label: 'str', identified: 'bool', admissible_adjustment_sets: 'tuple[tuple[str, ...], ...]', selected_adjustment_set: 'tuple[str, ...]', confounders: 'tuple[str, ...]', descendants: 'tuple[str, ...]', mediators: 'tuple[str, ...]', colliders: 'tuple[str, ...]', instruments: 'tuple[str, ...]', control_justifications: 'tuple[tuple[str, str], ...]', open_backdoor_paths: 'tuple[str, ...]', frontdoor_criteria_satisfied: 'Optional[bool]' = None, instrument_relevance_satisfied: 'Optional[bool]' = None, instrument_exclusion_satisfied: 'Optional[bool]' = None, instrument_exogeneity_satisfied: 'Optional[bool]' = None) -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "ColliderCoefficients", + "qualified_name": "RiskLabAI.causal_factor_analysis.ColliderCoefficients", + "kind": "class", + "signature": "(beta_hat: 'float', theta_hat: 'float') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "ColliderDiagnostics", + "qualified_name": "RiskLabAI.causal_factor_analysis.ColliderDiagnostics", + "kind": "class", + "signature": "(residual_variance: 'float', outcome_variance: 'float', correct_r_squared: 'float', overcontrolled_r_squared: 'float', correct_adjusted_r_squared: 'float', overcontrolled_adjusted_r_squared: 'float', correct_beta_variance: 'float', overcontrolled_beta_variance: 'float', collider_coefficient_variance: 'float', correct_beta_t_statistic: 'float', overcontrolled_beta_t_statistic: 'float', collider_t_statistic: 'float', adjusted_r_squared_prefers_overcontrolled: 'bool', absolute_beta_t_prefers_overcontrolled: 'bool') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "CausalDiscoveryStage", + "qualified_name": "RiskLabAI.causal_factor_analysis.CausalDiscoveryStage", + "kind": "class", + "signature": "(method_labels: 'tuple[str, ...]', graph_id: 'str', graph_nodes: 'tuple[str, ...]', directed_edges: 'tuple[tuple[str, str], ...]', graph_kind: 'str', ambiguous_edges: 'tuple[tuple[str, str], ...]', assumptions: 'tuple[str, ...]') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "CausalExplanatoryAndPredictivePowerStage", + "qualified_name": "RiskLabAI.causal_factor_analysis.CausalExplanatoryAndPredictivePowerStage", + "kind": "class", + "signature": "(task_types: 'tuple[str, ...]', estimator_label: 'str', explanatory_metric_labels: 'tuple[str, ...]', predictive_metric_labels: 'tuple[str, ...]', explanatory_evidence_id: 'Optional[str]', predictive_evidence_id: 'Optional[str]', naive_benchmark_id: 'str', validation: 'ValidationEvidence', multiclass_encoding: 'Optional[str]' = None, averaging_method: 'Optional[str]' = None) -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "CausalFactorProtocolReport", + "qualified_name": "RiskLabAI.causal_factor_analysis.CausalFactorProtocolReport", + "kind": "class", + "signature": "(trial_family_id: 'str', declared_trial_ids: 'tuple[str, ...]', variable_selection: 'VariableSelectionStage', causal_discovery: 'CausalDiscoveryStage', causal_adjustment_set: 'CausalAdjustmentSetStage', causal_explanatory_and_predictive_power: 'CausalExplanatoryAndPredictivePowerStage', causal_portfolio_construction: 'CausalPortfolioConstructionStage', backtest: 'BacktestStage', multiple_testing_adjustments: 'MultipleTestingAdjustmentsStage') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "CausalPortfolioConstructionStage", + "qualified_name": "RiskLabAI.causal_factor_analysis.CausalPortfolioConstructionStage", + "kind": "class", + "signature": "(method_labels: 'tuple[str, ...]', causal_exposures: 'tuple[str, ...]', controlled_unintended_exposures: 'tuple[str, ...]', cost_model_id: 'str', constraint_set_id: 'str', economic_rationale: 'str', fragility_scenarios: 'tuple[str, ...]', transfer_coefficient: 'Real') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "DSeparationEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.DSeparationEvidence", + "kind": "record_type", + "signature": "(left: ForwardRef('NodeTuple'), right: ForwardRef('NodeTuple'), conditioned: ForwardRef('NodeTuple'), separated: ForwardRef('bool'), paths: ForwardRef('tuple[PathEvidence, ...]'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification", + "record_fields": [ + "left", + "right", + "conditioned", + "separated", + "paths" + ] + }, + { + "name": "DifferenceInDifferencesEstimate", + "qualified_name": "RiskLabAI.causal_factor_analysis.DifferenceInDifferencesEstimate", + "kind": "record_type", + "signature": "(treated_change: ForwardRef('float'), control_change: ForwardRef('float'), estimate: ForwardRef('float'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects", + "record_fields": [ + "treated_change", + "control_change", + "estimate" + ] + }, + { + "name": "EventHorizon", + "qualified_name": "RiskLabAI.causal_factor_analysis.EventHorizon", + "kind": "class", + "signature": "(observation_id: 'str', start: 'Real', end: 'Real') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "FoldEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.FoldEvidence", + "kind": "class", + "signature": "(fold_id: 'str', path_id: 'str', train_ids: 'tuple[str, ...]', test_ids: 'tuple[str, ...]', fit_ids: 'tuple[str, ...]', prediction_ids: 'tuple[str, ...]', refit_id: 'str', refit_components: 'tuple[str, ...]') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "FrontdoorAdjustmentEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.FrontdoorAdjustmentEvidence", + "kind": "record_type", + "signature": "(treatment: ForwardRef('str'), outcome: ForwardRef('str'), mediator_set: ForwardRef('NodeTuple'), all_observed: ForwardRef('bool'), endpoints_in_set: ForwardRef('NodeTuple'), directed_path_exists: ForwardRef('bool'), unintercepted_directed_paths: ForwardRef('tuple[PathEvidence, ...]'), open_treatment_mediator_backdoor_paths: ForwardRef('tuple[PathEvidence, ...]'), open_mediator_outcome_backdoor_paths_given_treatment: ForwardRef('tuple[PathEvidence, ...]'), graphically_admissible: ForwardRef('bool'), positivity_required: ForwardRef('bool'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification", + "record_fields": [ + "treatment", + "outcome", + "mediator_set", + "all_observed", + "endpoints_in_set", + "directed_path_exists", + "unintercepted_directed_paths", + "open_treatment_mediator_backdoor_paths", + "open_mediator_outcome_backdoor_paths_given_treatment", + "graphically_admissible", + "positivity_required" + ] + }, + { + "name": "InstrumentEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.InstrumentEvidence", + "kind": "record_type", + "signature": "(instrument: ForwardRef('str'), treatment: ForwardRef('str'), outcome: ForwardRef('str'), conditioned: ForwardRef('NodeTuple'), all_observed: ForwardRef('bool'), conditioning_disjoint: ForwardRef('bool'), conditioning_unaffected_by_treatment: ForwardRef('bool'), pearl_relevance: ForwardRef('bool'), pearl_exclusion_exogeneity: ForwardRef('bool'), pearl_graphical: ForwardRef('bool'), cfi_direct_relevance: ForwardRef('bool'), cfi_full_mediation: ForwardRef('bool'), cfi_exogeneity: ForwardRef('bool'), cfi_simple: ForwardRef('bool'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification", + "record_fields": [ + "instrument", + "treatment", + "outcome", + "conditioned", + "all_observed", + "conditioning_disjoint", + "conditioning_unaffected_by_treatment", + "pearl_relevance", + "pearl_exclusion_exogeneity", + "pearl_graphical", + "cfi_direct_relevance", + "cfi_full_mediation", + "cfi_exogeneity", + "cfi_simple" + ] + }, + { + "name": "MultipleTestingAdjustmentsStage", + "qualified_name": "RiskLabAI.causal_factor_analysis.MultipleTestingAdjustmentsStage", + "kind": "class", + "signature": "(method_labels: 'tuple[str, ...]', trial_family_id: 'str', declared_trial_ids: 'tuple[str, ...]', family_partitions: 'tuple[tuple[str, tuple[str, ...]], ...]', alpha: 'Real', backtests_independent: 'bool', p_value_estimator: 'Optional[str]' = None, time_dependence_model: 'Optional[str]' = None, p_value_inputs_assume_independence: 'Optional[bool]' = None, sharpe_variance: 'Optional[Real]' = None, effective_trials: 'Optional[Real]' = None, sample_length: 'Optional[Integral]' = None, skewness: 'Optional[Real]' = None, kurtosis: 'Optional[Real]' = None, selection_bias_evidence_id: 'Optional[str]' = None) -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "NodeRoleEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.NodeRoleEvidence", + "kind": "record_type", + "signature": "(node: ForwardRef('str'), treatment: ForwardRef('str'), outcome: ForwardRef('str'), collider_on_paths: ForwardRef('tuple[PathTuple, ...]'), noncollider_on_paths: ForwardRef('tuple[PathTuple, ...]'), mediator_on_directed_paths: ForwardRef('tuple[PathTuple, ...]'), backdoor_noncollider_on_paths: ForwardRef('tuple[PathTuple, ...]'), common_cause_paths: ForwardRef('tuple[PathTuple, ...]'), descendant_of_treatment: ForwardRef('bool'), ancestor_of_outcome: ForwardRef('bool'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification", + "record_fields": [ + "node", + "treatment", + "outcome", + "collider_on_paths", + "noncollider_on_paths", + "mediator_on_directed_paths", + "backdoor_noncollider_on_paths", + "common_cause_paths", + "descendant_of_treatment", + "ancestor_of_outcome" + ] + }, + { + "name": "PathEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.PathEvidence", + "kind": "record_type", + "signature": "(nodes: ForwardRef('PathTuple'), colliders: ForwardRef('NodeTuple'), noncolliders: ForwardRef('NodeTuple'), conditioned_noncolliders: ForwardRef('NodeTuple'), activated_colliders: ForwardRef('NodeTuple'), blocking_colliders: ForwardRef('NodeTuple'), is_open: ForwardRef('bool'), is_directed_from_left: ForwardRef('bool'), is_backdoor_from_left: ForwardRef('bool'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification", + "record_fields": [ + "nodes", + "colliders", + "noncolliders", + "conditioned_noncolliders", + "activated_colliders", + "blocking_colliders", + "is_open", + "is_directed_from_left", + "is_backdoor_from_left" + ] + }, + { + "name": "PopulationRegressionDiagnostics", + "qualified_name": "RiskLabAI.causal_factor_analysis.PopulationRegressionDiagnostics", + "kind": "record_type", + "signature": "(coefficient_names: ForwardRef('Tuple[str, ...]'), coefficients: ForwardRef('Tuple[float, ...]'), r_squared: ForwardRef('float'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics", + "record_fields": [ + "coefficient_names", + "coefficients", + "r_squared" + ] + }, + { + "name": "RegressionDiagnostics", + "qualified_name": "RiskLabAI.causal_factor_analysis.RegressionDiagnostics", + "kind": "record_type", + "signature": "(coefficient_names: ForwardRef('Tuple[str, ...]'), coefficients: ForwardRef('Tuple[float, ...]'), standard_errors: ForwardRef('Tuple[float, ...]'), t_statistics: ForwardRef('Tuple[float, ...]'), r_squared: ForwardRef('float'), adjusted_r_squared: ForwardRef('float'), residual_variance: ForwardRef('float'), n_observations: ForwardRef('int'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics", + "record_fields": [ + "coefficient_names", + "coefficients", + "standard_errors", + "t_statistics", + "r_squared", + "adjusted_r_squared", + "residual_variance", + "n_observations" + ] + }, + { + "name": "SpecificationExperimentResult", + "qualified_name": "RiskLabAI.causal_factor_analysis.SpecificationExperimentResult", + "kind": "record_type", + "signature": "(structure: ForwardRef('str'), conditioning_variable_role: ForwardRef('str'), unconditioned: ForwardRef('RegressionDiagnostics'), conditioned: ForwardRef('RegressionDiagnostics'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics", + "record_fields": [ + "structure", + "conditioning_variable_role", + "unconditioned", + "conditioned" + ] + }, + { + "name": "SpecificationPopulationResult", + "qualified_name": "RiskLabAI.causal_factor_analysis.SpecificationPopulationResult", + "kind": "record_type", + "signature": "(structure: ForwardRef('str'), conditioning_variable_role: ForwardRef('str'), unconditioned: ForwardRef('PopulationRegressionDiagnostics'), conditioned: ForwardRef('PopulationRegressionDiagnostics'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics", + "record_fields": [ + "structure", + "conditioning_variable_role", + "unconditioned", + "conditioned" + ] + }, + { + "name": "StrategyPerformance", + "qualified_name": "RiskLabAI.causal_factor_analysis.StrategyPerformance", + "kind": "class", + "signature": "(correct: 'float', misspecified: 'float') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "TreatmentEffectDecomposition", + "qualified_name": "RiskLabAI.causal_factor_analysis.TreatmentEffectDecomposition", + "kind": "record_type", + "signature": "(observed_difference: ForwardRef('float'), average_treatment_effect_on_treated: ForwardRef('float'), sample_selection_bias: ForwardRef('float'))", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects", + "record_fields": [ + "observed_difference", + "average_treatment_effect_on_treated", + "sample_selection_bias" + ] + }, + { + "name": "ValidationEvidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.ValidationEvidence", + "kind": "class", + "signature": "(method_labels: 'tuple[str, ...]', event_horizons: 'tuple[EventHorizon, ...]', folds: 'tuple[FoldEvidence, ...]', embargo: 'Real' = 0.0) -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "VariableSelectionStage", + "qualified_name": "RiskLabAI.causal_factor_analysis.VariableSelectionStage", + "kind": "class", + "signature": "(purpose: 'str', selected_variables: 'tuple[str, ...]', method_labels: 'tuple[str, ...]', overlapping_returns: 'bool', strong_time_dependence: 'bool', validation: 'Optional[ValidationEvidence]') -> None", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + }, + { + "name": "average_treatment_effect", + "qualified_name": "RiskLabAI.causal_factor_analysis.average_treatment_effect", + "kind": "function", + "signature": "(treated_interventional_mean: 'Real', control_interventional_mean: 'Real') -> 'float'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "backdoor_adjusted_average_treatment_effect", + "qualified_name": "RiskLabAI.causal_factor_analysis.backdoor_adjusted_average_treatment_effect", + "kind": "function", + "signature": "(treated_conditional_means: 'Sequence[Real]', control_conditional_means: 'Sequence[Real]', adjustment_probabilities: 'Sequence[Real]') -> 'float'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "backdoor_adjusted_expectation", + "qualified_name": "RiskLabAI.causal_factor_analysis.backdoor_adjusted_expectation", + "kind": "function", + "signature": "(conditional_outcome_means: 'Sequence[Real]', adjustment_probabilities: 'Sequence[Real]') -> 'float'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "causal_role_evidence", + "qualified_name": "RiskLabAI.causal_factor_analysis.causal_role_evidence", + "kind": "function", + "signature": "(dag: 'CausalDAG', treatment: 'str', outcome: 'str', node: 'str', *, max_paths: 'Optional[int]' = 10000, max_path_states: 'Optional[int]' = 100000) -> 'NodeRoleEvidence'", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification" + }, + { + "name": "check_backdoor_adjustment_set", + "qualified_name": "RiskLabAI.causal_factor_analysis.check_backdoor_adjustment_set", + "kind": "function", + "signature": "(dag: 'CausalDAG', treatment: 'str', outcome: 'str', adjustment_set: 'Iterable[str]' = (), *, max_paths: 'Optional[int]' = 10000, max_path_states: 'Optional[int]' = 100000) -> 'BackdoorAdjustmentEvidence'", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification" + }, + { + "name": "check_frontdoor_adjustment_set", + "qualified_name": "RiskLabAI.causal_factor_analysis.check_frontdoor_adjustment_set", + "kind": "function", + "signature": "(dag: 'CausalDAG', treatment: 'str', outcome: 'str', mediator_set: 'Iterable[str]', *, max_paths: 'Optional[int]' = 10000, max_path_states: 'Optional[int]' = 100000) -> 'FrontdoorAdjustmentEvidence'", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification" + }, + { + "name": "check_instrument", + "qualified_name": "RiskLabAI.causal_factor_analysis.check_instrument", + "kind": "function", + "signature": "(dag: 'CausalDAG', instrument: 'str', treatment: 'str', outcome: 'str', conditioned: 'Iterable[str]' = ()) -> 'InstrumentEvidence'", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification" + }, + { + "name": "collider_factor_return", + "qualified_name": "RiskLabAI.causal_factor_analysis.collider_factor_return", + "kind": "function", + "signature": "(x: 'Real', collider_proxy: 'Real', beta: 'Real', gamma: 'Real', delta: 'Real') -> 'StrategyPerformance'", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "collider_forecast_return", + "qualified_name": "RiskLabAI.causal_factor_analysis.collider_forecast_return", + "kind": "function", + "signature": "(beta: 'Real', gamma: 'Real', delta: 'Real') -> 'StrategyPerformance'", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "collider_model_diagnostics", + "qualified_name": "RiskLabAI.causal_factor_analysis.collider_model_diagnostics", + "kind": "function", + "signature": "(beta: 'Real', gamma: 'Real', delta: 'Real', n_observations: 'Integral') -> 'ColliderDiagnostics'", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "collider_overcontrolled_coefficients", + "qualified_name": "RiskLabAI.causal_factor_analysis.collider_overcontrolled_coefficients", + "kind": "function", + "signature": "(beta: 'Real', gamma: 'Real', delta: 'Real') -> 'ColliderCoefficients'", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "collider_population_diagnostics", + "qualified_name": "RiskLabAI.causal_factor_analysis.collider_population_diagnostics", + "kind": "function", + "signature": "() -> 'SpecificationPopulationResult'", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics" + }, + { + "name": "collider_specification_experiment", + "qualified_name": "RiskLabAI.causal_factor_analysis.collider_specification_experiment", + "kind": "function", + "signature": "(sample_size: 'Integral' = 5000, seed: 'Integral' = 0) -> 'SpecificationExperimentResult'", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics" + }, + { + "name": "confounded_mediator_population_diagnostics", + "qualified_name": "RiskLabAI.causal_factor_analysis.confounded_mediator_population_diagnostics", + "kind": "function", + "signature": "() -> 'SpecificationPopulationResult'", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics" + }, + { + "name": "confounded_mediator_specification_experiment", + "qualified_name": "RiskLabAI.causal_factor_analysis.confounded_mediator_specification_experiment", + "kind": "function", + "signature": "(sample_size: 'Integral' = 5000, seed: 'Integral' = 0) -> 'SpecificationExperimentResult'", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics" + }, + { + "name": "confounder_factor_return", + "qualified_name": "RiskLabAI.causal_factor_analysis.confounder_factor_return", + "kind": "function", + "signature": "(x: 'Real', z: 'Real', beta: 'Real', gamma: 'Real', delta_estimated: 'Real', *, delta_realized: 'Optional[Real]' = None) -> 'StrategyPerformance'", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "confounder_forecast_return", + "qualified_name": "RiskLabAI.causal_factor_analysis.confounder_forecast_return", + "kind": "function", + "signature": "(beta: 'Real', gamma: 'Real', delta_estimated: 'Real', *, delta_realized: 'Optional[Real]' = None) -> 'StrategyPerformance'", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "confounder_undercontrolled_coefficient", + "qualified_name": "RiskLabAI.causal_factor_analysis.confounder_undercontrolled_coefficient", + "kind": "function", + "signature": "(beta: 'Real', gamma: 'Real', delta: 'Real') -> 'float'", + "implementation_module": "RiskLabAI.causal_factor_analysis.factor_mirage" + }, + { + "name": "d_separation", + "qualified_name": "RiskLabAI.causal_factor_analysis.d_separation", + "kind": "function", + "signature": "(dag: 'CausalDAG', left: 'Iterable[str]', right: 'Iterable[str]', conditioned: 'Iterable[str]' = (), *, max_paths: 'Optional[int]' = 10000, max_path_states: 'Optional[int]' = 100000) -> 'DSeparationEvidence'", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification" + }, + { + "name": "difference_in_differences", + "qualified_name": "RiskLabAI.causal_factor_analysis.difference_in_differences", + "kind": "function", + "signature": "(treated_before: 'Real', treated_after: 'Real', control_before: 'Real', control_after: 'Real') -> 'DifferenceInDifferencesEstimate'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "fork_population_diagnostics", + "qualified_name": "RiskLabAI.causal_factor_analysis.fork_population_diagnostics", + "kind": "function", + "signature": "() -> 'SpecificationPopulationResult'", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics" + }, + { + "name": "fork_specification_experiment", + "qualified_name": "RiskLabAI.causal_factor_analysis.fork_specification_experiment", + "kind": "function", + "signature": "(sample_size: 'Integral' = 5000, seed: 'Integral' = 0) -> 'SpecificationExperimentResult'", + "implementation_module": "RiskLabAI.causal_factor_analysis.specification_diagnostics" + }, + { + "name": "frontdoor_adjusted_probability", + "qualified_name": "RiskLabAI.causal_factor_analysis.frontdoor_adjusted_probability", + "kind": "function", + "signature": "(mediator_probabilities_given_treatment: 'Sequence[Real]', outcome_probabilities_given_mediator_and_treatment: 'Sequence[Sequence[Real]]', treatment_probabilities: 'Sequence[Real]') -> 'float'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "linear_instrumental_variable_effect", + "qualified_name": "RiskLabAI.causal_factor_analysis.linear_instrumental_variable_effect", + "kind": "function", + "signature": "(outcome_instrument_covariance: 'Real', treatment_instrument_covariance: 'Real') -> 'float'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "minimal_backdoor_adjustment_sets", + "qualified_name": "RiskLabAI.causal_factor_analysis.minimal_backdoor_adjustment_sets", + "kind": "function", + "signature": "(dag: 'CausalDAG', treatment: 'str', outcome: 'str', *, max_candidates: 'Optional[int]' = 65536) -> 'tuple[NodeTuple, ...]'", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification" + }, + { + "name": "minimal_frontdoor_adjustment_sets", + "qualified_name": "RiskLabAI.causal_factor_analysis.minimal_frontdoor_adjustment_sets", + "kind": "function", + "signature": "(dag: 'CausalDAG', treatment: 'str', outcome: 'str', *, max_candidates: 'Optional[int]' = 65536) -> 'tuple[NodeTuple, ...]'", + "implementation_module": "RiskLabAI.causal_factor_analysis.graph_identification" + }, + { + "name": "minimum_variance_factor_weights", + "qualified_name": "RiskLabAI.causal_factor_analysis.minimum_variance_factor_weights", + "kind": "function", + "signature": "(covariance: 'ArrayLike', factor_exposures: 'ArrayLike', target_exposures: 'ArrayLike') -> 'NDArray[np.float64]'", + "implementation_module": "RiskLabAI.causal_factor_analysis.optimizer" + }, + { + "name": "randomized_mean_difference", + "qualified_name": "RiskLabAI.causal_factor_analysis.randomized_mean_difference", + "kind": "function", + "signature": "(treated_outcomes: 'Sequence[Real]', control_outcomes: 'Sequence[Real]') -> 'float'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "treatment_effect_decomposition", + "qualified_name": "RiskLabAI.causal_factor_analysis.treatment_effect_decomposition", + "kind": "function", + "signature": "(observed_treated_mean: 'Real', observed_control_mean: 'Real', counterfactual_control_mean_for_treated: 'Real') -> 'TreatmentEffectDecomposition'", + "implementation_module": "RiskLabAI.causal_factor_analysis.treatment_effects" + }, + { + "name": "validate_causal_factor_protocol", + "qualified_name": "RiskLabAI.causal_factor_analysis.validate_causal_factor_protocol", + "kind": "function", + "signature": "(report: 'CausalFactorProtocolReport') -> 'CausalFactorProtocolReport'", + "implementation_module": "RiskLabAI.causal_factor_analysis.protocol" + } + ] + } +} diff --git a/README.md b/README.md index cbaa6a2..0ba418b 100644 --- a/README.md +++ b/README.md @@ -1,113 +1,89 @@ -# RiskLabAI.py - -[![PyPI version](https://badge.fury.io/py/RiskLabAI.svg)](https://badge.fury.io/py/RiskLabAI) -[![CI](https://github.com/RiskLabAI/RiskLabAI.py/actions/workflows/ci.yml/badge.svg)](https://github.com/RiskLabAI/RiskLabAI.py/actions/workflows/ci.yml) - -A Python library for quantitative finance and financial machine learning, -implementing core methods from Marcos López de Prado's *Advances in Financial -Machine Learning* and *Machine Learning for Asset Managers*. - -The library provides implementations for: - -- **Financial data structures** — tick, volume, dollar, imbalance, and run bars -- **Labeling** — the triple-barrier method, meta-labeling, trend-scanning -- **Fractional differentiation** — standard and fixed-width window (FFD) -- **Sample weights**, **denoising** (Marčenko–Pastur), **distance metrics** -- **Cross-validation** — Purged K-Fold, Combinatorial Purged CV (+ adaptive/bagged), walk-forward -- **Feature importance** — MDI, MDA, SFI, and clustered variants -- **Portfolio optimization** — HRP, NCO, hedging -- **Backtest statistics** — PSR/DSR, PBO, strategy risk -- **Microstructure & entropy features**, **structural breaks**, and a Deep-BSDE PDE solver - -There is a companion Julia package, -[RiskLabAI.jl](https://github.com/RiskLabAI/RiskLabAI.jl), which mirrors this -API. - -## Installation - -```bash -pip install RiskLabAI -``` - -The base install is lightweight. Heavier, optional capabilities are available as -extras: - -| Extra | Installs | Enables | -|---|---|---| -| `RiskLabAI[pde]` | `torch` | the Deep-BSDE PDE solver (`RiskLabAI.pde`) | -| `RiskLabAI[plot]` | `matplotlib`, `seaborn`, `plotly` | plotting helpers | -| `RiskLabAI[synth]` | `quantecon` | synthetic-data utilities | -| `RiskLabAI[all]` | all of the above | everything | - -```bash -pip install "RiskLabAI[all]" -``` - -For development (editable install + tests), see -[`INSTALLATION.md`](INSTALLATION.md). - -## Quickstart - -Sample dollar/volume/tick bars from raw ticks: - -```python -from RiskLabAI.data.structures.standard_bars import StandardBars -from RiskLabAI.utils.constants import CUMULATIVE_DOLLAR - -# ticks: an iterable of (datetime, price, volume) -ticks = [ - ("2020-01-01 10:00:00", 100.0, 10), - ("2020-01-01 10:00:01", 101.0, 5), - ("2020-01-01 10:00:02", 100.0, 20), -] - -bars = StandardBars(bar_type=CUMULATIVE_DOLLAR, threshold=3000) -bar_list = bars.construct_bars_from_data(ticks) -# each bar: [date_time, idx, open, high, low, close, volume, -# buy_volume, sell_volume, ticks, dollar, threshold] -``` - -Discover and construct components by name through the extension registry: - -```python -import RiskLabAI.core as core - -core.list_components() # {family: [available keys]} -cv = core.CROSS_VALIDATORS.create("purgedkfold", n_splits=5, times=event_times) -``` - -## Logging - -RiskLabAI logs under the `"RiskLabAI"` logger and is silent by default. To see -progress and diagnostics, configure logging in your application: - -```python -import logging -logging.basicConfig(level=logging.INFO) -``` - -## Extending the library - -RiskLabAI is built to be extended with new models. The `RiskLabAI.core` layer -provides a component registry and base interfaces so a new bar type, labeler, -cross-validator, etc. can be registered and discovered without editing central -code. See [`EXTENDING.md`](EXTENDING.md) for a step-by-step guide with worked -examples. - -## Contributing - -Contributions are welcome. The project uses `pytest` for tests and -`black` + `ruff` for formatting/linting (run before opening a PR): - -```bash -pip install -e ".[all]" pytest black ruff -pytest -q --ignore=test/pde -black RiskLabAI test -ruff check RiskLabAI test -``` - -Please branch from `main`, keep changes focused, and update `CHANGELOG.md`. - -## License - -See [`LICENSE.txt`](LICENSE.txt). +# RiskLabAI 3.0.0 pre-package source + +Status: **local-only and blocked**. + +This directory is a temporary clean release nucleus for the next major +RiskLabAI release across Python and Julia. It is not a separate product. The +Python distribution and import namespace remain `RiskLabAI`; the Julia package +remains `RiskLabAI` with UUID `a72881da-fdaa-49c1-8962-99caf4ccfee8`. + +The distributable metadata contract is complete, but no wheel or source +distribution has been created, installed, published, registered, or released. +The approved target version is `3.0.0`. The source remains blocked until a +separately authorized temporary artifact is inspected and the owner separately +authorizes version-control and release actions. + +## Final Python source scope + +The `src` tree preserves all 117 runtime-source paths captured from the +current public Python `main` baseline and adds seven reviewed +`causal_factor_analysis` source files. The prior modules remain organized under +`backtest`, `cluster`, `controller`, `core`, `data`, `ensemble`, `features`, +`hpc`, `optimization`, `pde`, and `utils`. Their captured bytes are retained as +a continuity baseline; this does not clear later local changes or make the full +surface release-ready. + +The completed book-derived causal-factor namespace exposes 57 public concepts: + +- minimum-variance target-exposure allocation; +- published confounder and collider analytical diagnostics; +- immutable graphical-identification evidence and checks; and +- immutable evidence records for the seven-stage causal-factor protocol; +- randomized, stratified, difference-in-differences, and instrumental-variable + treatment-effect identities; and +- exact fork, collider, and confounded-mediator population diagnostics with + deterministic specification experiments. + +The complete preserved source library has passed its approved compatibility +matrix on CPython 3.12.13, 3.13.9, and 3.14.3. The base policy is NumPy +`>=2.2,<3`, tested at the lowest genuinely compatible and current NumPy 2.x +release for each interpreter. Feature-specific dependencies remain optional; +Numba does not constrain the non-Numba base, and the unavailable Python 3.14 +changepoint backend disables only that feature. + +## Scope and parity hold + +No path from the current public repository baseline has been deleted. The +completed 57-concept causal namespace is integrated here, and the Julia +candidate independently verifies the same 57 concepts. The full-library +runtime and dependency matrix, exact public API inventory, 78-file +package-scoped test inventory, distributable metadata, documentation, and +intended package file list are complete. The exact Python `3.0.0` and Julia +`1.0.0` targets are assigned. Artifact inspection and human-controlled +version-control, publication, and release decisions remain blocked. + +The graph routines report implications of a caller-supplied DAG; they do not +discover or certify that graph, prove positivity or instrument strength, or +estimate an effect. The protocol validator checks structural evidence records; +it does not perform the empirical stages or prove that caller declarations are +true. The analytical functions implement only the source-consistent public +results recorded by the publication audit; source-conflicted results remain +excluded. + +## Confirmed identity and stewardship + +- Product and package identity: `RiskLabAI` +- Intended license: BSD-3-Clause +- Rights holder and public maintainer: Hamid Arian +- Copyright: 2022-2026 +- Contact: arian@risklab.ai +- Python repository: https://github.com/RiskLabAI/RiskLabAI.py +- Python issues: https://github.com/RiskLabAI/RiskLabAI.py/issues +- Julia repository: https://github.com/RiskLabAI/RiskLabAI.jl +- Documentation: https://github.com/RiskLabAI/RiskLabAI.py#readme + +The rights holder has confirmed permission to publish and license every file +admitted to the clean public package. This confirmation does not remove the +artifact-inspection or human-authorization blocks. + +See `PACKAGE_IDENTITY.json` for the fail-closed machine-readable state. + +The complete Python contract is recorded in `pyproject.toml`, +`PUBLIC_API.json`, `TEST_INVENTORY.json`, and `PACKAGE_FILES.json`. The future +artifact checks are specified in `PACKAGE_INSPECTION.md`; that document is an +inspection plan, not build or release authorization. + +The 57-concept contract is documented in `docs/causal_factor_analysis.md`, and +runtime details are in `docs/compatibility.md`. A small deterministic example +is in `examples/causal_factor_analysis_quickstart.py`, and the blocked +next-major summary is in `RELEASE_NOTES_NEXT_MAJOR.md`. diff --git a/RELEASE_NOTES_NEXT_MAJOR.md b/RELEASE_NOTES_NEXT_MAJOR.md new file mode 100644 index 0000000..3bdece2 --- /dev/null +++ b/RELEASE_NOTES_NEXT_MAJOR.md @@ -0,0 +1,29 @@ +# RiskLabAI Python next-major release notes — blocked draft + +Target version: **3.0.0**. The owner approved this exact version, but assigning +it does not authorize a build, artifact, publication, upload, or release. + +The next major Python release preserves the prior public repository source +modules and adds the verified 57-concept `RiskLabAI.causal_factor_analysis` +namespace. It intentionally drops support for Python 3.9-3.11 and NumPy 1.x; +the supported policy is CPython 3.12-3.14 with NumPy `>=2.2,<3`. + +Compatibility repairs include NumPy 2 trapezoidal integration, public +scikit-learn bootstrap logic, optional Numba acceleration, lazy optional +dependencies, a Python-3.14-specific Joblib floor, and self-contained standard +technical indicators. Python 3.14 remains supported when the optional +changepoint backend is unavailable. + +Compatibility aliases that remain present in 3.0.0 now consistently warn of +removal in 4.0.0. No alias is silently deleted, and the warning contract is +locked by package-scoped tests. + +No source-conflicted causal result is admitted. The metadata contract is +complete; artifacts, publication, upload, and release remain blocked. + +The static distributable metadata, exact 124-module runtime inventory, +78-file package-scoped test inventory, public API inventory, documentation, +examples, dependency groups, and intended package file list are now frozen. +No package artifact has been created. The remaining gates are future artifact +inspection and separate human authorization for version-control, publication, +and release actions. diff --git a/TEST_INVENTORY.json b/TEST_INVENTORY.json new file mode 100644 index 0000000..720cf31 --- /dev/null +++ b/TEST_INVENTORY.json @@ -0,0 +1,576 @@ +{ + "schema_version": 1, + "document_type": "RISKLABAI_PYTHON_TEST_INVENTORY", + "product": "RiskLabAI", + "language": "Python", + "version": "3.0.0", + "runner": "pytest", + "file_count": 78, + "file_groups": { + "CAUSAL_FACTOR_ANALYSIS": 7, + "PRESERVED_LIBRARY": 70, + "TECHNICAL_INDICATOR_ORACLES": 1 + }, + "collected_cases_in_reference_full_feature_environment": { + "preserved_library": 343, + "causal_factor_analysis": 341, + "technical_indicator_oracles": 8, + "total": 692 + }, + "verified_required_matrix_cases_per_lane": { + "preserved_library_applicable": 331, + "causal_factor_analysis": 341, + "technical_indicator_oracles": 8, + "total_applicable": 680 + }, + "required_lane_count": 6, + "required_lane_case_total": 4080, + "files": [ + { + "path": "test/backtest/test_advanced_bet_sizing.py", + "size": 3755, + "sha256": "F6AF65AA18989A2C3BBB4B43698719227F2199B825366553A7D177D1642572C2", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 8 + }, + { + "path": "test/backtest/test_backtest_overfitting_simulation.py", + "size": 2489, + "sha256": "195EE1482CC4C4F1EC9346D1C8C0A600EBB14898D331BC7E37F0FB29D3E1A68A", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/backtest/test_backtest_statistics.py", + "size": 4198, + "sha256": "16185AE1BEC34431ECD85EE1BE4E1A49C622C9CDB8B59CA4427148B1B6C2BA0A", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/backtest/test_backtest_synthetic_data.py", + "size": 2147, + "sha256": "4E81268462F0321FDE82215768BC6ACAD07D5F70F881C5D74CCE0991A1BD6435", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 2 + }, + { + "path": "test/backtest/test_bet_sizing.py", + "size": 5392, + "sha256": "7CA2E228BD7421C87F15BFABFD6C79E529D239F34A685ACBF5AEE1620EE1898C", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/backtest/test_multiple_testing.py", + "size": 3034, + "sha256": "AA6E928AD7A211C4C3D38525CC6FAFF1DF4D5F513C8EA97927D3DEFBF1672802", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 7 + }, + { + "path": "test/backtest/test_ou_trading_rules.py", + "size": 4403, + "sha256": "12498715145DA0BF5CEC21CFBCFF8EA206B7F82C01CEC15F05292C94F47A1C15", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/backtest/test_probabilistic_sharpe_ratio.py", + "size": 3242, + "sha256": "A62D3309AE74178AA80065FF9CFF4A96CE33A701AEB2D6F54127074875F91CC7", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/backtest/test_probability_of_backtest_overfitting.py", + "size": 3715, + "sha256": "178F73E2D1E3175C51EDCC0D990DFA49FB21C2843E4CF15D3726E370A42DBCBA", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/backtest/test_robust_statistics.py", + "size": 4672, + "sha256": "4954FF5BB03F7E6B040CAE874D10E04EE937DB1EEA85E7F0719FCD01A3DBA0FC", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/backtest/test_sharpe_inference.py", + "size": 3456, + "sha256": "814021459A2904B6B5C5C9ADC90EB9E503D5437C1764CFDA431291132ED692E2", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/backtest/test_strategy_risk.py", + "size": 3708, + "sha256": "E9696D0CDA4EAAFF01F536AD31E611658B36236BCADB5F470F0949EE4CE33820", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/backtest/test_technical_indicators.py", + "size": 3947, + "sha256": "63D92133A4731E007902E5FD73344F4B2FD6D3238E3124E2B3BD2C44B967B38C", + "group": "TECHNICAL_INDICATOR_ORACLES", + "declared_test_functions": 5 + }, + { + "path": "test/backtest/test_test_set_overfitting.py", + "size": 3145, + "sha256": "C9E6DF7D60DA22FFC20FD9C4386B25A83DE9C2F24BAA7C2EC6985FF0B91949FC", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/backtest/validation/test_adaptive_combinatorial_purged.py", + "size": 3326, + "sha256": "E63E3168B1C24A68A7A9B97BD6AE86D653781CABBB75B612973D6E526F74A576", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/backtest/validation/test_bagged_combinatorial_purged.py", + "size": 2500, + "sha256": "36CD1050290F37C76B94776F7EF62D883205350373B2C251963EB3D8F424A8D5", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/backtest/validation/test_combinatorial_purged.py", + "size": 4996, + "sha256": "2A7176CB64BC3117914123472650C2E28675E127AF721507FEC708A977AB014E", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/backtest/validation/test_cross_validator_controller.py", + "size": 1836, + "sha256": "B27F524D6BFF839D9D11A2AF3C6F69C4E8725149DA3FF4BE7D47EE4ACFF78664", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/backtest/validation/test_cross_validator_factory.py", + "size": 3073, + "sha256": "059213105205BA8A4B1EADD6923C513980EB38E7D48AF2ED9261841819E7AD04", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 9 + }, + { + "path": "test/backtest/validation/test_kfold.py", + "size": 4242, + "sha256": "42FFA8774B3398D0B2D4F6ABCEEE8934D9A600AA7A2C859E56F958F3291B2409", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/backtest/validation/test_leakage_aware_hpo.py", + "size": 3941, + "sha256": "C36A724AA150622F41C083A6C4370AF51E791D5FB27A1109F2F3A7F52B4F1CB2", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/backtest/validation/test_path_adaptive_cpcv.py", + "size": 8956, + "sha256": "5DDCF270F911FD61C06898BFBBF32E524711F035DBE27044E3137EED3BBC57DF", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 10 + }, + { + "path": "test/backtest/validation/test_path_bagged_cpcv.py", + "size": 3965, + "sha256": "A040B21C906867C5CA0B196755CDC8C3F4F79032E9E6E5A045C779AEB43197B1", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/backtest/validation/test_purged_kfold.py", + "size": 5007, + "sha256": "C64CB49639B8AB7E3878AE0A541F42CE3D3853405E8763B9331F6A2A0EA3153D", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/backtest/validation/test_walk_forward.py", + "size": 4290, + "sha256": "DB6846CCB5C99C47B07F2C54C405C28F2E903E4D5EAB68405DAD4F7B37C27564", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/causal_factor_analysis/test_cfi_book_graph_examples.py", + "size": 3672, + "sha256": "6A840BBD03FC9C098455C0DAE641ACA279D6C8E8A4F70B040DB2A555D7E50078", + "group": "CAUSAL_FACTOR_ANALYSIS", + "declared_test_functions": 7 + }, + { + "path": "test/causal_factor_analysis/test_factor_mirage.py", + "size": 16394, + "sha256": "268DB559600AF1D56EDADEEB0FC20DCF642D4FF5AF4DBBBD4517CC5EE4F94DFF", + "group": "CAUSAL_FACTOR_ANALYSIS", + "declared_test_functions": 32 + }, + { + "path": "test/causal_factor_analysis/test_graph_identification.py", + "size": 35186, + "sha256": "64D386BDD80E76CEAA4762D429B5E3ADE083AB7E509BE348A7A632B9C11E4C35", + "group": "CAUSAL_FACTOR_ANALYSIS", + "declared_test_functions": 47 + }, + { + "path": "test/causal_factor_analysis/test_minimum_variance.py", + "size": 15983, + "sha256": "954E52279A50DADEAE6519B2CF6679B7AB4CE0C3F01CE70AC81AD829E9423F16", + "group": "CAUSAL_FACTOR_ANALYSIS", + "declared_test_functions": 32 + }, + { + "path": "test/causal_factor_analysis/test_protocol.py", + "size": 39100, + "sha256": "8781036062F8734967F115FA63A52F66A14552B1EA56CB9B422A71B1E25070B2", + "group": "CAUSAL_FACTOR_ANALYSIS", + "declared_test_functions": 43 + }, + { + "path": "test/causal_factor_analysis/test_specification_diagnostics.py", + "size": 9096, + "sha256": "DDED9E16AA02CBA5284BACE5B61E4CA53CBB1FBC2A72E665F289CD252E81930E", + "group": "CAUSAL_FACTOR_ANALYSIS", + "declared_test_functions": 10 + }, + { + "path": "test/causal_factor_analysis/test_treatment_effects.py", + "size": 5969, + "sha256": "0B67456180473825489748AD99098C8BFCE6D15293EFE136546785AC19FE836E", + "group": "CAUSAL_FACTOR_ANALYSIS", + "declared_test_functions": 12 + }, + { + "path": "test/cluster/test_clustering.py", + "size": 3269, + "sha256": "33F301B160E90B4FA6BA369524E6A3D30F3BB63887B4898D6DC6708CCD900A7A", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/controller/test_bars_initializer.py", + "size": 2484, + "sha256": "13B2E36A9D10E25284C3CDCE19F04320524623790BE64F79C851D31703D3724C", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/controller/test_data_structure_controller.py", + "size": 2833, + "sha256": "9510A0FBD3B714FE87742519A9200CBFD0D6CEB99C14A9ED369F51EA7E3AAA56", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/core/test_base.py", + "size": 2701, + "sha256": "0BFC387B4C7A94205708140A7485F1AF1D3BFA1FC9472A1898ACBCEABA2DFE64", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/core/test_builtin_parity.py", + "size": 5095, + "sha256": "560CB3D389459FEA8572E7C71B5F66563A1EF17C4545B782A989CD8A67FCAD64", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 8 + }, + { + "path": "test/core/test_registry.py", + "size": 5789, + "sha256": "C6F667A94DB9803219591A3E4E8F963E820E5508C996A9850386BD4E8A8FED03", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 18 + }, + { + "path": "test/data/denoise/test_denoising.py", + "size": 2818, + "sha256": "DAC2EA0D49203308445314A87BBBD73EDA875C197996D75F88C7F5074C4E9D33", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/data/denoise/test_nercome.py", + "size": 4263, + "sha256": "9B3799A2EB458510D869EE2624B525C2746D923E09BB11C5FB4EBDFA6385FA8C", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/data/differentiation/test_adaptive_differentiation.py", + "size": 3832, + "sha256": "63AF31E69F26F186EA26B531189FEEDFEC653055DC6FA954E714F92F0A419CA5", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/data/differentiation/test_differentiation.py", + "size": 3712, + "sha256": "C95ECCBBC869053624EB81A990F91003C835B4CCB1ABFEA56B01ECA347C54C33", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/data/distance/test_codependence.py", + "size": 3364, + "sha256": "1E206FE53D4C899011C33D2B08E5CA5D8A0CFAD47391E4EAD5A08FD24DEA78FF", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 7 + }, + { + "path": "test/data/distance/test_distance_metric.py", + "size": 2758, + "sha256": "C20662C35D3253A3C0157456B3385523DDF6042D74C0BD031CF0DAE168D57589", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/data/labeling/test_financial_labels.py", + "size": 2624, + "sha256": "5B194D19D2565708359FB005A22336DD9C82F6BEED562D8B5E91DCE08387F8FD", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 2 + }, + { + "path": "test/data/labeling/test_labeling.py", + "size": 4598, + "sha256": "571BEBC1BBD55A1E1F3F2163B79C1B5D2BB3F07F355A850B5E11106EA28B33E3", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/data/structures/test_hardened_information_driven_bars.py", + "size": 4756, + "sha256": "EC65CE1DDC9C43C72569E0E2087203309EC4C557F06FBDAA6DE4A88BF74C6650", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/data/structures/test_imbalance_bars.py", + "size": 3432, + "sha256": "95B11988BA1E13F67868EDB84E361290E8206214E67A2ADD368EC50A04035678", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 2 + }, + { + "path": "test/data/structures/test_run_bars.py", + "size": 2298, + "sha256": "AA84352C3434FA6BE5B36E1E04506FFFB04A71B723F52D1E76E3684AB76F5FD5", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/data/structures/test_standard_bars.py", + "size": 2573, + "sha256": "7FBDEE31397875FD3CD6AA0755F79376C8298C623EDCEFEC62FCBE02801FDE81", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/data/structures/test_time_bars.py", + "size": 1948, + "sha256": "777F07CBF7B1285660BE260CF28DAEDD8409119B580220789C06C571E83012A9", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/data/synthetic_data/test_drift_burst_hypothesis.py", + "size": 2496, + "sha256": "D36F02ED24979C2AC250DA8F78DCF00E84AC15629A66FF0F49B864DC7F29234F", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/data/synthetic_data/test_synthetic_controlled_environment.py", + "size": 2706, + "sha256": "3EDD684CD47C177AC3B14201D00F92D58B653F80015FBBF5FE8B199681A9CFF3", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/data/weights/test_sample_weights.py", + "size": 3920, + "sha256": "6A682B9C8273A9480CC40EDEF503F4ADB254FDE03A7C6846A6576B2844177DA5", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/ensemble/test_bagging_classifier_accuracy.py", + "size": 1055, + "sha256": "B9E9FCDEC62CCA57773D775874B1A49FE9FFDBA70AE75C1A5B44D21BE5F42F70", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/features/entropy_features/test_entropy.py", + "size": 8265, + "sha256": "EB0639713017FA205239818CF5EB33B37488D1F5015E41BE572DF9B3DC4B92A3", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 14 + }, + { + "path": "test/features/feature_importance/test_debiased_importance.py", + "size": 3135, + "sha256": "9CB7F3A10AADC4263C5A3BC18952A832C61DD1E106E7C6D97E44480D9B77E303", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/features/feature_importance/test_feature_importance.py", + "size": 4539, + "sha256": "1CC36BB1739EEDBA96BC9CFE6C784F4B10796AD8B7030D2423BFADDA83AC4BBC", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 6 + }, + { + "path": "test/features/feature_importance/test_generate_synthetic_data.py", + "size": 1297, + "sha256": "60E8710D8FB14D2DB9C8A8561B760A7C82E78A40B58D050DE4B454FAD9D09A19", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/features/feature_importance/test_orthogonal_features.py", + "size": 1377, + "sha256": "2FFF242081857CAE3677849ED9615F812C02C4F8142C2BE3F165EB7F6163E43F", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/features/feature_importance/test_weighted_tau.py", + "size": 717, + "sha256": "DE4E792D04BE710A70F0531FB3BA4CF6F4FD281CD1CCB0AF66775B2AFF9DEB5B", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/features/microstructural_features/test_edge.py", + "size": 2005, + "sha256": "E7F97BFC152F1ABC9CDA7ABACCAA96F424E2F5F60178AB358B282944AF7BB1CE", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/features/microstructural_features/test_microstructure.py", + "size": 2089, + "sha256": "FAF86E2B87ED6C052039270C8FDD500399B71AFFAB9B3833ABD01C2EB7065A93", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 2 + }, + { + "path": "test/features/structural_breaks/test_pelt_change_points.py", + "size": 2480, + "sha256": "41D6A781C71F971EDF58728FC9C7F963C751CA07B23B8ADA7F640542EBA80DEF", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/features/structural_breaks/test_structural_breaks.py", + "size": 9740, + "sha256": "3AB71A45767135BA7B3104135A86E45C301F2131FCEB190695FCAA7B5FD34872", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 14 + }, + { + "path": "test/features/structural_breaks/test_volatility_robust_sadf.py", + "size": 2732, + "sha256": "F098186FFD06AC96600F32986039804A177193313D1E237F2EE97264968E7BB8", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/hpc/test_hpc.py", + "size": 2213, + "sha256": "2E3414DD74A72FD86FED54059515E2A088F99F4C23492A1992A55A5F311F09A8", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/optimization/test_hedging.py", + "size": 2026, + "sha256": "C34C92CCD66E0B895140B2D45270AC94803AB408532888E04C6FABC9C444BC22", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 1 + }, + { + "path": "test/optimization/test_hrp.py", + "size": 3689, + "sha256": "08A03D26DDCA53A9793C46D76AFEC5EA2D1203B8A833895A93C29E0EBCD6A4C7", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/optimization/test_hyper_parameter_tuning.py", + "size": 3405, + "sha256": "C756502F9BA1FFC0EAFCF2F0EE3E73F500CB07D956378FFADE018CC610AEC650", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/optimization/test_nco.py", + "size": 1875, + "sha256": "07393543334EED0500954A475403AFDF68BFDB5D9AEC14768E5D21D91E41A2AE", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/pde/test_pde_solver.py", + "size": 1764, + "sha256": "9606F9460D87E1268B55A2EC5FC8AD7BE92B4E9DB64F219E0C444343CDB763DD", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 2 + }, + { + "path": "test/test_consolidation.py", + "size": 3427, + "sha256": "653D2C917A29D41030CE835E18736ED607BCBBC9E7A75B1A16FEFC286F391F8F", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 5 + }, + { + "path": "test/test_deprecation.py", + "size": 1746, + "sha256": "1318D33E6B9790C48F3E366568733FB14574D0D7ECCF7086D5EF5D8BE8545930", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/test_performance.py", + "size": 6986, + "sha256": "857264FBB2C1507995BB18CBD2E9081849D9C3F4EECA73F58FBC0371FAE42F73", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/utils/test_ewma.py", + "size": 920, + "sha256": "A75275626813BC8DFE6A0A792370FC1B013ED8A5398232468CE7E871137F70D6", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + }, + { + "path": "test/utils/test_momentum_mean_reverting_strategy_sides.py", + "size": 2321, + "sha256": "EEBBFAC83B87A3A2BF72C9D9B203DB11D8794D9CBE7CA5125DEBE141C10E3889", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 4 + }, + { + "path": "test/utils/test_progress.py", + "size": 1022, + "sha256": "2CC8FBD465B007948A09C0E8975288A13D46590113FB9DA6AA441C07C55F23BA", + "group": "PRESERVED_LIBRARY", + "declared_test_functions": 3 + } + ] +} diff --git a/docs/causal_factor_analysis.md b/docs/causal_factor_analysis.md new file mode 100644 index 0000000..fa3f013 --- /dev/null +++ b/docs/causal_factor_analysis.md @@ -0,0 +1,49 @@ +# Causal factor analysis + +The public causal API implements the source-consistent analytical scope of *Causal Factor Investing* and exposes 57 concepts. The Python namespace is `RiskLabAI.causal_factor_analysis`; the matching Julia namespace is `RiskLabAI.CausalFactorAnalysis`. + +## Public surface + +| Group | Records and functions | +|---|---| +| Optimizer (1) | `minimum_variance_factor_weights` | +| Factor-mirage analytics (10) | `StrategyPerformance`, `ColliderCoefficients`, `ColliderDiagnostics`, `confounder_undercontrolled_coefficient`, `confounder_factor_return`, `confounder_forecast_return`, `collider_overcontrolled_coefficients`, `collider_factor_return`, `collider_forecast_return`, `collider_model_diagnostics` | +| Protocol (12) | `EventHorizon`, `FoldEvidence`, `ValidationEvidence`, `VariableSelectionStage`, `CausalDiscoveryStage`, `CausalAdjustmentSetStage`, `CausalExplanatoryAndPredictivePowerStage`, `CausalPortfolioConstructionStage`, `BacktestStage`, `MultipleTestingAdjustmentsStage`, `CausalFactorProtocolReport`, `validate_causal_factor_protocol` | +| Graph identification (14) | `CausalDAG`, `PathEvidence`, `DSeparationEvidence`, `BackdoorAdjustmentEvidence`, `FrontdoorAdjustmentEvidence`, `InstrumentEvidence`, `NodeRoleEvidence`, `d_separation`, `check_backdoor_adjustment_set`, `minimal_backdoor_adjustment_sets`, `check_frontdoor_adjustment_set`, `minimal_frontdoor_adjustment_sets`, `check_instrument`, `causal_role_evidence` | +| Treatment effects (10) | `DifferenceInDifferencesEstimate`, `TreatmentEffectDecomposition`, `average_treatment_effect`, `backdoor_adjusted_average_treatment_effect`, `backdoor_adjusted_expectation`, `difference_in_differences`, `frontdoor_adjusted_probability`, `linear_instrumental_variable_effect`, `randomized_mean_difference`, `treatment_effect_decomposition` | +| Specification diagnostics (10) | `PopulationRegressionDiagnostics`, `RegressionDiagnostics`, `SpecificationExperimentResult`, `SpecificationPopulationResult`, `collider_population_diagnostics`, `collider_specification_experiment`, `confounded_mediator_population_diagnostics`, `confounded_mediator_specification_experiment`, `fork_population_diagnostics`, `fork_specification_experiment` | + +## Behavioral contract + +The optimizer requires a symmetric positive-definite covariance matrix and a full-column-rank exposure matrix. It rejects singular or numerically unresolved constraints rather than adding hidden regularization. + +Graph routines operate on a caller-supplied acyclic graph with an explicit observed-node set. They provide path evidence for d-separation, back-door, front-door, and instrument checks. These are graphical criteria, not causal-discovery procedures, positivity proofs, empirical instrument-strength tests, or claims that the supplied graph is true. Inclusion-minimal adjustment sets are not necessarily variance-optimal or minimum-cost sets. + +Treatment-effect functions evaluate explicit identification formulas from caller-supplied interventional, conditional, or grouped quantities. They do not infer that the identification assumptions hold in observed data. + +Protocol records are immutable evidence containers. The validator checks the declared seven-stage structure and fail-closed evidence rules; it does not execute empirical research or certify the truth of a declaration. + +Specification diagnostics expose exact population identities and deterministic experiments for fork, collider, and confounded-mediator structures. They are mathematical diagnostics rather than automated model selection. + +## Cross-language behavior + +Python returns immutable tuple-backed evidence records and NumPy arrays; Julia returns immutable structs and Julia vectors or matrices. Python raises `ValueError` for invalid analytical inputs; Julia raises `ArgumentError`. Canonical node and set ordering, graphical criteria, result fields, resource limits, numerical tolerances, and estimands otherwise match. + +Both implementations are tested against independent analytical results and graph oracles. Agreement between the languages is additional evidence, never the sole correctness test. + +## Exclusions + +Three later-source results remain excluded because their sources conflict: `CFA-CONF-01`, `CFA-COLL-01`, and `MIRAGE-COLL-SHIFT-01`. The clean namespace does not delegate to the excluded legacy causal implementation. + +See `../examples/causal_factor_analysis_quickstart.py` for a deterministic example and `compatibility.md` for the tested runtime policy. + +The exact ordered names, signatures, immutable record fields, implementation +modules, and source hashes are frozen in `../PUBLIC_API.json`. The seven causal +test files and their hashes are part of the exact package-scoped inventory in +`../TEST_INVENTORY.json`. + +Cross-language parity is defined over the exact 57-name set and behavior, not +over incidental export-list order. Python preserves its approved `__all__` +order; the Julia inventory uses canonical lexical order. Both language-native +digests and the canonical name-set digest are recorded in the blocked +pre-package contract. diff --git a/docs/compatibility.md b/docs/compatibility.md new file mode 100644 index 0000000..41f73bf --- /dev/null +++ b/docs/compatibility.md @@ -0,0 +1,110 @@ +# Python runtime and dependency support + +Status: **matrix verified; release blocked**. + +This policy applies to the blocked RiskLabAI `3.0.0` release candidate. The +assigned version does not authorize a build, package, upload, publication, or +release. + +## Supported interpreter and NumPy policy + +RiskLabAI supports standard GIL builds of CPython 3.12, 3.13, and 3.14. The +base numerical policy is `numpy>=2.2,<3`. A matrix lane uses only versions for +which compatible distributions actually exist; it never forces an older +NumPy source release onto a newer interpreter. + +| Lane | CPython | NumPy | Required-dependency position | +|---|---:|---:|---| +| 3.12 minimum | 3.12.13 | 2.2.0 | approved floors | +| 3.12 current | 3.12.13 | 2.5.2 | current compatible releases | +| 3.13 minimum | 3.13.9 | 2.2.0 | approved floors | +| 3.13 current | 3.13.9 | 2.5.2 | current compatible releases | +| 3.14 minimum | 3.14.3 | 2.3.2 | lowest compatible binary set | +| 3.14 current | 3.14.3 | 2.5.2 | current compatible releases | + +The NumPy lower bound was reconsidered at metadata freeze on 2026-08-21. NumPy +2.2 remains inside the two-year Scientific Python SPEC 0 support window through +2026-12-08 and has passed the approved Python 3.12 and 3.13 minimum lanes, so +`numpy>=2.2,<3` is retained for RiskLabAI 3.0.0. Python 3.14 resolves NumPy +2.3.2 or newer because that is its first tested compatible binary lane. A +future RiskLabAI release must reassess the floor rather than silently carrying +it forward. See https://scientific-python.org/specs/spec-0000/. + +The required dependency policy is: + +- pandas `>=2.3,<4`; +- SciPy `>=1.15,<2`; +- scikit-learn `>=1.6,<2`; +- statsmodels `>=0.14.6,<0.15`; +- Joblib `>=1.4,<2` on Python below 3.14 and `>=1.5.2,<2` on Python 3.14; +- PyWavelets `>=1.9,<2`. + +The Python 3.14 minimum lane resolves newer compatible SciPy and +scikit-learn releases where older releases do not publish compatible wheels. +This is an interpreter-specific resolution fact, not a reason to raise the +floor for Python 3.12 or 3.13. + +## Optional capabilities + +Optional dependencies do not constrain the lean base: + +- `speed`: Numba `>=0.67,<0.68`, with NumPy `<2.6` for this group only; +- `pde`: PyTorch `>=2.10,<3`; +- `synth`: QuantEcon `>=0.11.4,<0.12`; +- `hpo`: Optuna `>=4.9,<5`; +- `plot`: matplotlib `>=3.10,<4`, seaborn `>=0.13.2,<0.14`, and Plotly `>=6,<7`; +- `symbolic`: SymPy `>=1.14,<2`; +- `profile`: memory-profiler `==0.61.0` provisionally; +- `simulation`: tqdm `>=4.70,<5`; +- `changepoints`: ruptures `==1.1.10` on Python 3.12 and 3.13 only. + +Python 3.14 remains supported when ruptures is unavailable. Only +`pelt_change_points` is unavailable in that environment, and calling it raises +a direct dependency error. The rest of RiskLabAI remains importable. + +The financial-feature simulation no longer relies on the stale `ta` package. +ADX, RSI, CCI, stochastic momentum, ROC, ATR, and Ichimoku lines are computed +internally from standard definitions and have hand-calculated oracle tests. + +## Deprecation policy + +Public compatibility aliases retained in RiskLabAI 3.0.0 remain callable and +emit `DeprecationWarning`. Their removal target is 4.0.0. Package-scoped tests +bind both continued availability and the stated target; no preserved alias is +deleted in this release. + +## Matrix evidence + +Every lane passed all 341 frozen causal-factor tests. Across six lanes this is +2,046 causal assertions with no failures. + +The final self-contained package test tree collects exactly 692 tests: 343 +preserved-library tests, 341 causal-factor tests, and 8 independent technical- +indicator oracles. Its complete Python 3.12 current-dependency run succeeded +with 691 passes and one declared Windows platform skip. The package inventory +records both the collected suite and the 680-test applicable base matrix used +on every supported interpreter/dependency lane. + +The complete preserved base suite passed 331 applicable tests per lane. Five +platform or optional-capability cases were skipped by their declared guards; +the two QuantEcon-dependent cases were exercised separately in the `synth` +lanes. The new indicator-oracle suite passed 8 tests in every lane. Numba's +compiled and pure-Python paths agreed in every lane. PyTorch 2.10 and 2.13 both +passed the preserved PDE tests on all three interpreter lines. Plotting, +symbolic, profiling, HPO, and synthetic-data smoke tests passed in their +feature-specific environments. + +One real conditional floor was discovered: Joblib releases before 1.5.2 are +not reliable for the preserved Windows parallel-HPO path on Python 3.14. +Joblib 1.5.2 and 1.5.3 pass; older Python lines continue to pass with Joblib +1.4.x. + +The causal public design remains exactly 57 concepts. Compatibility repairs +did not import or reintroduce the excluded legacy causal implementation. + +## Remaining release gates + +The metadata, public surface, tests, and source allowlists are complete. No +package has been built or installed. Temporary artifact inspection and separate +human authorization for version-control, publication, and release remain +outstanding. diff --git a/examples/causal_factor_analysis_quickstart.py b/examples/causal_factor_analysis_quickstart.py new file mode 100644 index 0000000..c3800be --- /dev/null +++ b/examples/causal_factor_analysis_quickstart.py @@ -0,0 +1,37 @@ +"""Small, deterministic examples for the frozen causal-factor API.""" + +import numpy as np +import RiskLabAI + +from RiskLabAI.causal_factor_analysis import ( + CausalDAG, + average_treatment_effect, + check_backdoor_adjustment_set, + minimum_variance_factor_weights, +) + +assert RiskLabAI.__version__ == "3.0.0" + + +covariance = np.diag([1.0, 2.0, 4.0]) +factor_exposures = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]) +target_exposures = np.array([0.0, 1.0]) +weights = minimum_variance_factor_weights( + covariance, factor_exposures, target_exposures +) +np.testing.assert_allclose(weights, [-2.0 / 7.0, 5.0 / 7.0, 2.0 / 7.0]) + +effect = average_treatment_effect(3.5, 1.25) +assert effect == 2.25 + +dag = CausalDAG( + nodes=("T", "U", "Y"), + directed_edges=(("U", "T"), ("U", "Y"), ("T", "Y")), + observed_nodes=("T", "U", "Y"), +) +assert not check_backdoor_adjustment_set(dag, "T", "Y").admissible +assert check_backdoor_adjustment_set(dag, "T", "Y", ("U",)).admissible + +print("minimum-variance weights:", weights) +print("average treatment effect:", effect) +print("back-door adjustment set: ('U',)") diff --git a/pyproject.toml b/pyproject.toml index 88dacff..ce9a2ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,115 +1,106 @@ [build-system] -requires = ["setuptools>=61.0"] +requires = ["setuptools>=77", "wheel>=0.45"] build-backend = "setuptools.build_meta" [project] name = "RiskLabAI" -version = "2.0.3" +version = "3.0.0" +description = "Financial machine-learning and causal-factor analysis tools for research and model validation." +readme = "README.md" +requires-python = ">=3.12,<3.15" +license = "BSD-3-Clause" +license-files = ["LICENSE"] authors = [ - { name = "RiskLab", email = "arian@risklab.ai" }, + { name = "Hamid Arian", email = "arian@risklab.ai" }, ] -description = "Financial AI using Python, based on 'Advances in Financial Machine Learning' and 'Machine Learning for Asset Managers'." -readme = "README.md" -license = { file = "LICENSE" } -requires-python = ">=3.9" -classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: BSD License", - "Intended Audience :: Developers", - "Intended Audience :: Financial and Insurance Industry", - "Intended Audience :: Science/Research", - "Operating System :: OS Independent", - "Topic :: Office/Business :: Financial", - "Topic :: Scientific/Engineering :: Artificial Intelligence", +maintainers = [ + { name = "Hamid Arian", email = "arian@risklab.ai" }, ] keywords = [ - "finance", - "machine learning", - "quantitative finance", - "fintech", - "de prado", - "backtesting", + "causal inference", + "factor investing", + "financial machine learning", + "quantitative finance", + "risk management", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Financial and Insurance Industry", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Scientific/Engineering", ] - -# --- CORE DEPENDENCIES --- -# These are fast to install and essential for the library's main functions. dependencies = [ - "numpy>=1.22,<3", - "pandas>=1.5", - "scipy>=1.8", - "scikit-learn>=1.1", - "statsmodels>=0.13", - "numba>=0.57", - "joblib>=1.1", - "ta>=0.10", - "tqdm>=4.60", - "sympy>=1.10", + "numpy>=2.2,<3", + "pandas>=2.3,<4", + "scipy>=1.15,<2", + "scikit-learn>=1.6,<2", + "statsmodels>=0.14.6,<0.15", + "joblib>=1.4,<2; python_version < '3.14'", + "joblib>=1.5.2,<2; python_version >= '3.14'", + "PyWavelets>=1.9,<2", ] -# --- OPTIONAL DEPENDENCIES --- -# These are installed on-demand by users, e.g., pip install RiskLabAI[pde] [project.optional-dependencies] -pde = ["torch"] -synth = ["quantecon"] -plot = ["matplotlib", "seaborn" , "plotly"] -dev = [ - "yfinance", - "memory_profiler", - "jupyterlab", - "stable-baselines3", - "gymnasium", - "torchvision", # Often needed with torch - "torchaudio", # Often needed with torch - "pytest", - # Linters pinned to match CI exactly (.github/workflows/ci.yml). - "black==26.5.1", - "ruff==0.15.17", +speed = [ + "numba>=0.67,<0.68", + "numpy>=2.2,<2.6", ] -all = [ - "RiskLabAI[pde]", - "RiskLabAI[synth]", - "RiskLabAI[plot]", +pde = ["torch>=2.10,<3"] +synth = ["quantecon>=0.11.4,<0.12"] +hpo = ["optuna>=4.9,<5"] +plot = [ + "matplotlib>=3.10,<4", + "seaborn>=0.13.2,<0.14", + "plotly>=6,<7", ] - +symbolic = ["sympy>=1.14,<2"] +profile = ["memory-profiler==0.61.0"] +simulation = ["tqdm>=4.70,<5"] +changepoints = ["ruptures==1.1.10; python_version < '3.14'"] +test = ["pytest>=8.4,<10"] [project.urls] -Homepage = "https://github.com/RiskLabAI/RiskLabAI.py" -"Bug Tracker" = "https://github.com/RiskLabAI/RiskLabAI.py/issues" -Changelog = "https://github.com/RiskLabAI/RiskLabAI.py/blob/main/CHANGELOG.md" +Repository = "https://github.com/RiskLabAI/RiskLabAI.py" +Issues = "https://github.com/RiskLabAI/RiskLabAI.py/issues" +Documentation = "https://github.com/RiskLabAI/RiskLabAI.py#readme" -# Package only the library; the repo root also holds local-only appraisal -# harnesses under bench/, which must never be discovered or shipped. -[tool.setuptools.packages.find] -include = ["RiskLabAI*"] +[tool.setuptools] +package-dir = { "" = "src" } +packages = [ + "RiskLabAI", + "RiskLabAI.backtest", + "RiskLabAI.backtest.validation", + "RiskLabAI.causal_factor_analysis", + "RiskLabAI.cluster", + "RiskLabAI.controller", + "RiskLabAI.core", + "RiskLabAI.data", + "RiskLabAI.data.denoise", + "RiskLabAI.data.differentiation", + "RiskLabAI.data.distance", + "RiskLabAI.data.labeling", + "RiskLabAI.data.structures", + "RiskLabAI.data.synthetic_data", + "RiskLabAI.data.weights", + "RiskLabAI.ensemble", + "RiskLabAI.features", + "RiskLabAI.features.entropy_features", + "RiskLabAI.features.feature_importance", + "RiskLabAI.features.microstructural_features", + "RiskLabAI.features.structural_breaks", + "RiskLabAI.hpc", + "RiskLabAI.optimization", + "RiskLabAI.pde", + "RiskLabAI.utils", +] +include-package-data = false [tool.pytest.ini_options] testpaths = ["test"] -# Resolve `import RiskLabAI` from the repo root so a bare `pytest` works without -# an editable install (and regardless of any untracked bench/ working tree). -pythonpath = ["."] - -[tool.black] -line-length = 88 -target-version = ["py39"] - -[tool.ruff] -line-length = 88 -target-version = "py39" - -[tool.ruff.lint] -select = ["E", "F", "W", "I", "UP", "B"] -# E501 deferred until the mechanical black formatting pass lands -ignore = ["E501"] - -[tool.ruff.lint.per-file-ignores] -# Package __init__ files exist to re-export their submodules' public API. -"__init__.py" = ["F401", "F403", "F405"] -# pde/solver builds the networks from `from ...model import *`; the torch-only -# model module is intentionally left star-imported. -"RiskLabAI/pde/solver.py" = ["F403", "F405"] -# base.py re-exports the canonical interfaces lazily via module __getattr__, -# so the names listed in __all__ are resolved on access, not at definition. -"RiskLabAI/core/base.py" = ["F822"] -# importorskip("torch") must execute before the torch-dependent imports below -# it, so module-level imports cannot all sit at the top of the file. -"test/pde/test_pde_solver.py" = ["E402"] +addopts = "-ra --strict-config --strict-markers" diff --git a/src/RiskLabAI/__init__.py b/src/RiskLabAI/__init__.py new file mode 100644 index 0000000..2d2afd3 --- /dev/null +++ b/src/RiskLabAI/__init__.py @@ -0,0 +1,53 @@ +"""RiskLabAI financial machine-learning and causal-factor toolkit.""" + +from importlib import import_module as _import_module + +from . import causal_factor_analysis + +__version__ = "3.0.0" + +_LEGACY_SUBMODULES = frozenset( + { + "backtest", + "cluster", + "controller", + "core", + "data", + "ensemble", + "features", + "hpc", + "optimization", + "pde", + "utils", + } +) + +__all__ = [ + "__version__", + "backtest", + "causal_factor_analysis", + "cluster", + "controller", + "core", + "data", + "ensemble", + "features", + "hpc", + "optimization", + "pde", + "utils", +] + + +def __getattr__(name): + """Load only a declared legacy subpackage when it is first requested.""" + if name in _LEGACY_SUBMODULES: + module = _import_module(".{}".format(name), __name__) + globals()[name] = module + return module + raise AttributeError("module {!r} has no attribute {!r}".format(__name__, name)) + + +def __dir__(): + """Return the exact declared package surface.""" + return list(__all__) diff --git a/src/RiskLabAI/_deprecation.py b/src/RiskLabAI/_deprecation.py new file mode 100644 index 0000000..e3207e0 --- /dev/null +++ b/src/RiskLabAI/_deprecation.py @@ -0,0 +1,123 @@ +""" +Internal helpers for deprecating public names without breaking callers. + +Used by the 2.0.0 naming canon (see ``NAMING_CANON_2.0.0.md``): every renamed +function, class, or constant keeps working throughout the 2.0.x series and emits +a :class:`DeprecationWarning` naming its replacement. The aliases are scheduled +for removal in 4.0.0. + +Three mechanisms, one per kind of object: + +- :func:`deprecated_alias` wraps a renamed **function**. +- :func:`deprecated_class` subclasses a renamed **class**. +- :func:`warn_deprecated_name` is called from a module ``__getattr__`` (PEP 562) + for renamed **module-level names** such as constants. +""" + +from __future__ import annotations + +import functools +import warnings +from typing import Any, Callable, TypeVar + +__all__ = ["deprecated_alias", "deprecated_class", "warn_deprecated_name"] + +_F = TypeVar("_F", bound=Callable[..., Any]) + + +def deprecated_alias(new_func: _F, old_name: str, *, removed_in: str) -> _F: + """ + Return a thin wrapper that calls ``new_func`` but warns under the old name. + + Parameters + ---------- + new_func : Callable + The renamed (canonical) function. + old_name : str + The deprecated public name being kept alive. + removed_in : str + Version in which the alias will be removed (e.g. ``"4.0.0"``). + + Returns + ------- + Callable + A wrapper with ``__name__`` set to ``old_name`` that emits a + :class:`DeprecationWarning` and delegates to ``new_func``. + """ + + @functools.wraps(new_func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + warnings.warn( + f"{old_name}() is deprecated and will be removed in {removed_in}; " + f"use {new_func.__name__}() instead.", + DeprecationWarning, + stacklevel=2, + ) + return new_func(*args, **kwargs) + + wrapper.__name__ = old_name + wrapper.__qualname__ = old_name + wrapper.__doc__ = ( + f"Deprecated alias for :func:`{new_func.__name__}` " + f"(removed in {removed_in})." + ) + return wrapper # type: ignore[return-value] + + +def deprecated_class(new_cls: type, old_name: str, *, removed_in: str) -> type: + """ + Return a subclass of ``new_cls`` that warns when instantiated. + + The subclass behaves identically to ``new_cls`` but emits a + :class:`DeprecationWarning` on construction, naming the replacement. + + Parameters + ---------- + new_cls : type + The renamed (canonical) class. + old_name : str + The deprecated public class name being kept alive. + removed_in : str + Version in which the alias will be removed. + """ + + def __init__(self: Any, *args: Any, **kwargs: Any) -> None: + warnings.warn( + f"{old_name} is deprecated and will be removed in {removed_in}; " + f"use {new_cls.__name__} instead.", + DeprecationWarning, + stacklevel=2, + ) + new_cls.__init__(self, *args, **kwargs) + + shim = type(old_name, (new_cls,), {"__init__": __init__}) + shim.__doc__ = ( + f"Deprecated alias for :class:`{new_cls.__name__}` " + f"(removed in {removed_in})." + ) + shim.__module__ = new_cls.__module__ + return shim + + +def warn_deprecated_name(old_name: str, new_name: str, *, removed_in: str) -> None: + """ + Emit a deprecation warning for a renamed module-level name. + + Call this from a module's ``__getattr__`` (PEP 562) before returning the + value bound to ``new_name``. + + Parameters + ---------- + old_name : str + The deprecated name that was accessed. + new_name : str + The canonical replacement name. + removed_in : str + Version in which the old name will be removed. + """ + warnings.warn( + f"{old_name} is deprecated and will be removed in {removed_in}; " + f"use {new_name} instead.", + DeprecationWarning, + stacklevel=3, + ) diff --git a/src/RiskLabAI/backtest/__init__.py b/src/RiskLabAI/backtest/__init__.py new file mode 100644 index 0000000..614e601 --- /dev/null +++ b/src/RiskLabAI/backtest/__init__.py @@ -0,0 +1,210 @@ +""" +RiskLabAI Backtesting Module + +This package provides tools for: +- Backtest simulation +- Bet sizing and strategy risk analysis +- Overfitting detection (PBO, PSR) +- Advanced cross-validation (in the .validation submodule) +""" + +from . import validation +from .advanced_bet_sizing import ( + PlattCalibrator, + distributionally_robust_kelly_fraction, + expected_calibration_error, + kelly_bet_fraction, +) +from .backtest_overfitting_simulation import ( + backtest_overfitting_simulation_financial_metrics_rank_correlation, + backtest_overfitting_simulation_model_complexity, + format_cpu_info, + get_cpu_info, + measure_all_cv_computational_requirements, + measure_cpcv_parallelization, + measure_cpcv_scalability, + noised_backtest_overfitting_simulation, + # This file contains many functions, exporting the main ones + overall_backtest_overfitting_simulation, + overall_novel_methods_backtest_overfitting_simulation, + temporal_backtest_overfitting_simulation, + time_temporal_backtest_overfitting_simulation, + varying_embargo_backtest_overfitting_simulation, +) +from .backtest_statistics import ( + bet_timing, + calculate_hhi, + calculate_hhi_concentration, + calculate_holding_period, + compute_drawdowns_time_under_water, +) +from .backtest_statistics import ( + sharpe_ratio as pbo_sharpe_ratio, +) +from .backtest_synthetic_data import synthetic_back_testing +from .bet_sizing import ( + Signal, + TPos, + average_bet_sizes, + avg_active_signals, + avgActiveSignals, + bet_size_sigmoid, + betSize, + compute_sigmoid_width, + discrete_signal, + discreteSignal, + generate_signal, + getW, + inverse_price, + inversePrice, + limit_price, + limitPrice, + mp_avg_active_signals, + mpAvgActiveSignals, + probability_bet_size, + strategy_bet_sizing, + target_position, +) +from .multiple_testing import ( + benjamini_hochberg_yekutieli_adjusted_p_values, + haircut_sharpe_ratios, + holm_adjusted_p_values, + sharpe_ratio_p_values, +) +from .ou_trading_rules import ( + fit_ornstein_uhlenbeck, + optimal_ou_trading_rule, + ou_rule_metrics, +) +from .probabilistic_sharpe_ratio import ( + benchmark_sharpe_ratio, + probabilistic_sharpe_ratio, +) +from .probability_of_backtest_overfitting import ( + performance_evaluation, + probability_of_backtest_overfitting, +) +from .robust_statistics import ( + conditional_expected_drawdown, + sharpe_difference_test, +) +from .sharpe_inference import ( + lplz_sharpe_inference, + newey_west_automatic_lag, + newey_west_long_run_variance, + sharpe_ratio_influence_function, +) +from .strategy_risk import ( + bin_frequency, + binomial_sharpe_ratio, + calculate_strategy_risk, + failure_probability, + implied_precision, + mix_gaussians, + sharpe_ratio_trials, + target_sharpe_ratio_symbolic, +) +from .test_set_overfitting import ( + estimated_sharpe_ratio_z_statistics, + expected_max_sharpe_ratio, + generate_max_sharpe_ratios, + mean_std_error, + strategy_type1_error_probability, + strategy_type2_error_probability, + theta_for_type2_error, +) + +# Define what `from RiskLabAI.backtest import *` will import +__all__ = [ + "validation", + # from advanced_bet_sizing (Appraisal 18 admits) + "PlattCalibrator", + "distributionally_robust_kelly_fraction", + "expected_calibration_error", + "kelly_bet_fraction", + # from robust_statistics + ou_trading_rules (Appraisal 22, 23 admits) + "conditional_expected_drawdown", + "sharpe_difference_test", + "optimal_ou_trading_rule", + "ou_rule_metrics", + "fit_ornstein_uhlenbeck", + # from backtest_statistics + "bet_timing", + "calculate_holding_period", + "calculate_hhi_concentration", + "calculate_hhi", + "compute_drawdowns_time_under_water", + # from backtest_synthetic_data + "synthetic_back_testing", + # from bet_sizing (canonical 2.0.0 names) + "probability_bet_size", + "average_bet_sizes", + "strategy_bet_sizing", + "avg_active_signals", + "mp_avg_active_signals", + "discrete_signal", + "generate_signal", + "bet_size_sigmoid", + "target_position", + "inverse_price", + "limit_price", + "compute_sigmoid_width", + # from bet_sizing (deprecated aliases, removed in 4.0.0) + "avgActiveSignals", + "mpAvgActiveSignals", + "discreteSignal", + "Signal", + "betSize", + "TPos", + "inversePrice", + "limitPrice", + "getW", + # from strategy_risk + "sharpe_ratio_trials", + "target_sharpe_ratio_symbolic", + "implied_precision", + "bin_frequency", + "binomial_sharpe_ratio", + "mix_gaussians", + "failure_probability", + "calculate_strategy_risk", + # from test_set_overfitting + "expected_max_sharpe_ratio", + "generate_max_sharpe_ratios", + "mean_std_error", + "estimated_sharpe_ratio_z_statistics", + "strategy_type1_error_probability", + "theta_for_type2_error", + "strategy_type2_error_probability", + # from probability_of_backtest_overfitting + "pbo_sharpe_ratio", + "performance_evaluation", + "probability_of_backtest_overfitting", + # from probabilistic_sharpe_ratio + "probabilistic_sharpe_ratio", + "benchmark_sharpe_ratio", + # from multiple_testing + "sharpe_ratio_p_values", + "holm_adjusted_p_values", + "benjamini_hochberg_yekutieli_adjusted_p_values", + "haircut_sharpe_ratios", + # from sharpe_inference + "sharpe_ratio_influence_function", + "newey_west_long_run_variance", + "newey_west_automatic_lag", + "lplz_sharpe_inference", + # from backtest_overfitting_..._simulation + "overall_backtest_overfitting_simulation", + "temporal_backtest_overfitting_simulation", + "time_temporal_backtest_overfitting_simulation", + "varying_embargo_backtest_overfitting_simulation", + "backtest_overfitting_simulation_financial_metrics_rank_correlation", + "backtest_overfitting_simulation_model_complexity", + "noised_backtest_overfitting_simulation", + "overall_novel_methods_backtest_overfitting_simulation", + "measure_all_cv_computational_requirements", + "measure_cpcv_parallelization", + "measure_cpcv_scalability", + "get_cpu_info", + "format_cpu_info", +] diff --git a/src/RiskLabAI/backtest/advanced_bet_sizing.py b/src/RiskLabAI/backtest/advanced_bet_sizing.py new file mode 100644 index 0000000..5172948 --- /dev/null +++ b/src/RiskLabAI/backtest/advanced_bet_sizing.py @@ -0,0 +1,227 @@ +r""" +Advanced bet sizing: probability calibration before sizing, and distributionally-robust Kelly. + +de Prado sizes a bet from a predicted probability via a sigmoid (AFML ch.10, in `bet_sizing.py`). Two +admitted extensions sit alongside that baseline: + +- **Platt calibration before sizing** (Meyer-Barziy-Joubert 2023). A classifier's raw probability is + usually miscalibrated, so sizing straight from it over- or under-bets. Fitting a Platt (logistic) + calibrator on a held-out block and sizing from the calibrated probability recovers near-optimal sizing + when the input is miscalibrated, and is a safe no-op when it is already calibrated. +- **Distributionally-robust Kelly** (Sun-Boyd 2018). Point-estimate Kelly is fragile to estimation + error. Sizing at the pessimistic edge of a confidence box on the win probability gives near-optimal + growth with materially lower drawdown under estimation uncertainty, and converges to full Kelly as the + box collapses with abundant data. + +Admitted in Appraisal 18 (CONTRIBUTIONS_LEDGER 2026-06-27). Scope tags, verbatim from the verdict: + + Platt calibration: calibrate the meta-label probability before sizing when it is miscalibrated + (measurable by ECE); use Platt, not isotonic, which overfits well-calibrated inputs; the benefit is + largest under strong miscalibration and converges to no-op when already calibrated. + + DR-Kelly: prefer DR-Kelly over de Prado sigmoid / full Kelly when estimation uncertainty is material + (small samples, noisy edge): near-optimal growth with lower drawdown; converges to Kelly with + abundant data. + +Held-out confirmed (appraisals/18_results, HELDOUT.md): on the sealed high-miscalibration corner Platt +closes the growth gap to the Kelly optimum distinguishably more than the de Prado sigmoid (paired +0.0032 [0.0030, 0.0034]); on the sealed tight-drawdown / small-sample corner DR-Kelly has lower +from-initial drawdown than full Kelly (0.144 vs 0.409) with near-optimal growth. Real-data net-of-cost +confirmation for both is a tracked obligation (`REAL_DATA_FOLLOWUPS.md`). Isotonic calibration and +risk-constrained Kelly were NOT admitted (shelved). Evidence and caveats: appraisals/18_verdict.md. + +The DR-Kelly box-ambiguity worst case has a closed form (Kelly at the pessimistic probability, because a +long favorable bet's log-growth is monotone in the win probability), so no convex solver is required here; +``cvxpy`` (Apache-2.0) would only be needed for a general, non-box ambiguity set. + +References +---------- +Platt, J. (1999) Probabilistic outputs for support vector machines and comparisons to regularized + likelihood methods. Advances in Large Margin Classifiers. +Meyer, B., Barziy, I. and Joubert, J. F. (2023) Meta-labeling: calibration and the bet-sizing of + meta-label probabilities. +Sun, Q. and Boyd, S. (2018) Distributional robustness in Kelly gambling. (Distributionally-robust Kelly.) +Lopez de Prado, M. (2018) Advances in Financial Machine Learning, ch. 10 (the de Prado sigmoid baseline). +""" + +from __future__ import annotations + +import numpy as np +from sklearn.linear_model import LogisticRegression + + +def kelly_bet_fraction( + probability, + payout_ratio: float = 1.0, + max_fraction: float = 1.0, +) -> np.ndarray: + r""" + Growth-optimal Kelly fraction for a binary bet, clipped to ``[0, max_fraction]`` (long only). + + For a favorable bet that pays ``+payout_ratio`` per unit staked on a win (probability ``p``) and + ``-1`` on a loss, the Kelly fraction is :math:`f^* = (p(b+1) - 1)/b` with ``b = payout_ratio``. + + Parameters + ---------- + probability : float or np.ndarray + Win probability/probabilities. + payout_ratio : float, default 1.0 + Net odds ``b`` (even money by default). + max_fraction : float, default 1.0 + Upper clip on the fraction. + + Returns + ------- + np.ndarray + The Kelly fraction(s) in ``[0, max_fraction]``. + + Examples + -------- + >>> float(kelly_bet_fraction(0.6)) + 0.2 + """ + p = np.asarray(probability, dtype=float) + b = float(payout_ratio) + f = (p * (b + 1.0) - 1.0) / b + return np.clip(f, 0.0, max_fraction) + + +def expected_calibration_error(probabilities, outcomes, n_bins: int = 10) -> float: + r""" + Expected Calibration Error: the bin-mass-weighted gap between confidence and accuracy. + + Use this to decide whether calibration is worthwhile: a high ECE indicates the probabilities are + miscalibrated (the regime where :class:`PlattCalibrator` helps); a near-zero ECE means calibration + should be a no-op. + + Parameters + ---------- + probabilities : np.ndarray + Predicted probabilities in ``[0, 1]``. + outcomes : np.ndarray + Binary realized outcomes ``{0, 1}``. + n_bins : int, default 10 + Number of equal-width probability bins. + + Returns + ------- + float + The ECE in ``[0, 1]``. + """ + p = np.asarray(probabilities, dtype=float) + y = np.asarray(outcomes, dtype=float) + edges = np.linspace(0.0, 1.0, n_bins + 1) + n = p.size + ece = 0.0 + for i in range(n_bins): + lo, hi = edges[i], edges[i + 1] + mask = (p > lo) & (p <= hi) if i > 0 else (p >= lo) & (p <= hi) + if mask.sum() == 0: + continue + ece += (mask.sum() / n) * abs(y[mask].mean() - p[mask].mean()) + return float(ece) + + +class PlattCalibrator: + r""" + Platt scaling: a logistic recalibration of predicted probabilities, fit on a held-out block. + + Fits a logistic regression of the binary outcome on ``logit(p_hat)``; ``transform`` maps a raw + probability to the calibrated one. Prefer Platt over isotonic calibration: isotonic overfits the + calibration map and hurts already-calibrated inputs, whereas Platt's two-parameter logistic form is + a safe near-no-op when the input is already calibrated (the characterized failure mode in the + verdict). Calibrate before sizing (feed the calibrated probability into the de Prado sigmoid or a + Kelly sizer) when ECE indicates miscalibration. See the module docstring for the verbatim scope tag + and appraisals/18_verdict.md. + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> y = rng.integers(0, 2, 500) + >>> p_hat = np.clip(0.5 + 0.3 * (2 * y - 1) + 0.1 * rng.standard_normal(500), 0.01, 0.99) + >>> cal = PlattCalibrator().fit(p_hat, y) + >>> p_cal = cal.transform(p_hat) + >>> p_cal.shape == p_hat.shape + True + """ + + def __init__(self) -> None: + self._model: LogisticRegression | None = None + + @staticmethod + def _logit(p_hat) -> np.ndarray: + x = np.clip(np.asarray(p_hat, dtype=float), 1e-4, 1 - 1e-4) + return np.log(x / (1.0 - x)).reshape(-1, 1) + + def fit(self, predicted_probabilities, outcomes) -> PlattCalibrator: + """Fit the Platt logistic map on (predicted probability, binary outcome). Returns ``self``.""" + y = np.asarray(outcomes, dtype=int) + if np.unique(y).size < 2: + # Degenerate block (one class only): leave as an identity calibrator. + self._model = None + return self + model = LogisticRegression(C=1e6, solver="lbfgs", max_iter=1000) + model.fit(self._logit(predicted_probabilities), y) + self._model = model + return self + + def transform(self, predicted_probabilities) -> np.ndarray: + """Map raw probabilities to calibrated probabilities (identity if fit on one class only).""" + if self._model is None: + return np.asarray(predicted_probabilities, dtype=float) + return self._model.predict_proba(self._logit(predicted_probabilities))[:, 1] + + +def distributionally_robust_kelly_fraction( + probability_estimate, + n_observations: int, + payout_ratio: float = 1.0, + max_fraction: float = 1.0, + confidence_z: float = 1.0, +) -> np.ndarray: + r""" + Distributionally-robust Kelly fraction over a confidence box on the win probability (Sun-Boyd 2018). + + Builds a one-sided confidence box ``p_lo = p_hat - z * sqrt(p_hat (1 - p_hat) / n)`` and sizes with + growth-optimal Kelly at the pessimistic ``p_lo``. For a long favorable bet the worst-case log-growth + over the box is attained at the lowest win probability, so this closed form is the box-ambiguity + distributionally-robust optimum (no convex solver needed). As ``n`` grows the box collapses and the + fraction converges to full Kelly at ``p_hat``. + + Prefer DR-Kelly over de Prado sigmoid / full Kelly when estimation uncertainty is material (small + samples, noisy edge): near-optimal growth with lower drawdown; converges to Kelly with abundant data. + See the module docstring for the verbatim scope tag and appraisals/18_verdict.md. + + Parameters + ---------- + probability_estimate : float or np.ndarray + The estimated win probability ``p_hat``. + n_observations : int + The number of observations behind the estimate (drives the box width). + payout_ratio : float, default 1.0 + Net odds ``b``. + max_fraction : float, default 1.0 + Upper clip on the fraction. + confidence_z : float, default 1.0 + Box half-width in standard errors (1.0 ~ one-sigma pessimism). + + Returns + ------- + np.ndarray + The distributionally-robust Kelly fraction(s) in ``[0, max_fraction]``. + + Examples + -------- + >>> import numpy as np + >>> small = float(distributionally_robust_kelly_fraction(0.6, n_observations=40)) + >>> large = float(distributionally_robust_kelly_fraction(0.6, n_observations=100000)) + >>> small < large <= 0.2 + 1e-9 # converges up to full Kelly (0.2) as n grows + True + """ + p_hat = np.clip(np.asarray(probability_estimate, dtype=float), 1e-4, 1 - 1e-4) + standard_error = np.sqrt(p_hat * (1.0 - p_hat) / max(int(n_observations), 1)) + p_lo = np.clip(p_hat - confidence_z * standard_error, 1e-4, 1 - 1e-4) + return kelly_bet_fraction( + p_lo, payout_ratio=payout_ratio, max_fraction=max_fraction + ) diff --git a/src/RiskLabAI/backtest/backtest_overfitting_simulation.py b/src/RiskLabAI/backtest/backtest_overfitting_simulation.py new file mode 100644 index 0000000..2315e63 --- /dev/null +++ b/src/RiskLabAI/backtest/backtest_overfitting_simulation.py @@ -0,0 +1,1560 @@ +""" +Orchestrates complex backtest overfitting and simulation scenarios, +including PBO, DSR, and hardware performance profiling. + +## TODO: +- [ ] **Refactor Feature Generation:** Move hardcoded parameters in + `financial_features_backtest_overfitting_simulation` + (e.g., volatility=100, rolling=20, indicator windows) + to a configuration dictionary or function arguments. +- [ ] **Refactor Profiling:** Move hardware/performance functions + (`get_cpu_info`, `format_cpu_info`, + `measure_computational_requirements`, etc.) + to a separate `RiskLabAI.utils.profiling` module + to reduce this module's responsibilities. +""" + +import itertools +import subprocess +import time +import warnings +from math import ceil +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd +from scipy import stats as ss +from scipy.stats import kendalltau +from sklearn.linear_model import LogisticRegression + +from RiskLabAI.core import CROSS_VALIDATORS +from RiskLabAI.data.differentiation import fractionally_differentiated_log_price +from RiskLabAI.data.labeling import ( + cusum_filter_events_dynamic_threshold, + daily_volatility_with_log_returns, + meta_events, + meta_labeling, + vertical_barrier, +) +from RiskLabAI.data.weights import sample_weight_absolute_return_meta_labeling +from RiskLabAI.utils import determine_strategy_side + +from .bet_sizing import strategy_bet_sizing +from .probabilistic_sharpe_ratio import ( + benchmark_sharpe_ratio, + probabilistic_sharpe_ratio, +) +from .probability_of_backtest_overfitting import probability_of_backtest_overfitting + + +def _progress(iterable): + try: + from tqdm import tqdm + except ImportError: + return iterable + return tqdm(iterable) + + +def _wilder_change_average(values: np.ndarray, window: int) -> np.ndarray: + """Return Wilder's recursive average for quantities defined on price changes.""" + if not isinstance(window, int) or isinstance(window, bool) or window < 1: + raise ValueError("window must be a positive integer") + + array = np.asarray(values, dtype=float) + result = np.full(array.shape, np.nan, dtype=float) + if array.ndim != 1: + raise ValueError("values must be one-dimensional") + if array.size <= window: + return result + + seed = array[1 : window + 1] + if not np.isfinite(seed).all(): + return result + result[window] = float(seed.mean()) + for position in range(window + 1, array.size): + value = array[position] + if not np.isfinite(value): + result[position] = np.nan + elif np.isfinite(result[position - 1]): + result[position] = ( + result[position - 1] * (window - 1) + value + ) / window + return result + + +def _relative_strength_index(prices: pd.Series, window: int = 14) -> pd.Series: + values = prices.to_numpy(dtype=float, copy=False) + changes = np.empty(values.shape, dtype=float) + changes[0] = 0.0 + changes[1:] = np.diff(values) + average_gain = _wilder_change_average(np.maximum(changes, 0.0), window) + average_loss = _wilder_change_average(np.maximum(-changes, 0.0), window) + + denominator = average_gain + average_loss + result = np.full(values.shape, np.nan, dtype=float) + np.divide(100.0 * average_gain, denominator, out=result, where=denominator > 0.0) + result[(denominator == 0.0) & np.isfinite(denominator)] = 50.0 + return pd.Series(result, index=prices.index, dtype=float) + + +def _average_true_range(prices: pd.Series, window: int = 14) -> pd.Series: + values = prices.to_numpy(dtype=float, copy=False) + true_range = np.empty(values.shape, dtype=float) + true_range[0] = 0.0 + true_range[1:] = np.abs(np.diff(values)) + return pd.Series( + _wilder_change_average(true_range, window), + index=prices.index, + dtype=float, + ) + + +def _average_directional_index(prices: pd.Series, window: int = 14) -> pd.Series: + values = prices.to_numpy(dtype=float, copy=False) + changes = np.empty(values.shape, dtype=float) + changes[0] = 0.0 + changes[1:] = np.diff(values) + true_range = np.abs(changes) + positive_movement = np.maximum(changes, 0.0) + negative_movement = np.maximum(-changes, 0.0) + + average_range = _wilder_change_average(true_range, window) + average_positive = _wilder_change_average(positive_movement, window) + average_negative = _wilder_change_average(negative_movement, window) + positive_index = np.zeros(values.shape, dtype=float) + negative_index = np.zeros(values.shape, dtype=float) + np.divide( + 100.0 * average_positive, + average_range, + out=positive_index, + where=average_range > 0.0, + ) + np.divide( + 100.0 * average_negative, + average_range, + out=negative_index, + where=average_range > 0.0, + ) + direction_sum = positive_index + negative_index + directional_index = np.zeros(values.shape, dtype=float) + np.divide( + 100.0 * np.abs(positive_index - negative_index), + direction_sum, + out=directional_index, + where=direction_sum > 0.0, + ) + + result = np.full(values.shape, np.nan, dtype=float) + first = 2 * window - 1 + if values.size > first: + result[first] = float(directional_index[window : first + 1].mean()) + for position in range(first + 1, values.size): + result[position] = ( + result[position - 1] * (window - 1) + + directional_index[position] + ) / window + return pd.Series(result, index=prices.index, dtype=float) + + +def _commodity_channel_index(prices: pd.Series, window: int = 20) -> pd.Series: + rolling_mean = prices.rolling(window, min_periods=window).mean() + mean_deviation = prices.rolling(window, min_periods=window).apply( + lambda values: float(np.mean(np.abs(values - values.mean()))), raw=True + ) + denominator = 0.015 * mean_deviation + result = (prices - rolling_mean) / denominator + return result.where(denominator != 0.0, 0.0).astype(float) + + +def _stochastic_oscillator( + prices: pd.Series, window: int = 14, smooth_window: int = 3 +) -> tuple[pd.Series, pd.Series]: + rolling_low = prices.rolling(window, min_periods=window).min() + rolling_high = prices.rolling(window, min_periods=window).max() + price_range = rolling_high - rolling_low + oscillator = 100.0 * (prices - rolling_low) / price_range + oscillator = oscillator.where(price_range != 0.0, 50.0).astype(float) + signal = oscillator.rolling( + smooth_window, min_periods=smooth_window + ).mean() + return oscillator, signal + + +def _ichimoku_lines( + prices: pd.Series, + conversion_window: int = 9, + base_window: int = 26, + span_window: int = 52, +) -> tuple[pd.Series, pd.Series, pd.Series, pd.Series]: + def midpoint(window: int) -> pd.Series: + rolling = prices.rolling(window, min_periods=window) + return (rolling.max() + rolling.min()) / 2.0 + + conversion_line = midpoint(conversion_window) + base_line = midpoint(base_window) + span_a = (conversion_line + base_line) / 2.0 + span_b = midpoint(span_window) + return conversion_line, base_line, span_a, span_b + + +def _create_cross_validator(validator_type, **kwargs): + """ + Build a cross-validator from the core registry (the 2.0.0 single source). + + Mirrors the old ``_create_cross_validator(...)`` behaviour, + including dropping keyword arguments a given validator does not accept. + """ + return CROSS_VALIDATORS.create(validator_type, filter_unknown_kwargs=True, **kwargs) + + +def financial_features_backtest_overfitting_simulation( + prices: pd.Series, noise_scale: float = 0.0, random_state: Optional[int] = None +) -> pd.DataFrame: + """ + Create a DataFrame of financial features from the given price series. + + Args: + prices (pd.Series): Time series of asset prices. + noise_scale (float): Scale of Gaussian noise to be added to the features. Default is 0.0. + random_state (Optional[int]): Seed for random number generator. Default is None. + + Returns: + pd.DataFrame: DataFrame containing the computed financial features. + + Notes: + RSI, ATR, and ADX use Wilder smoothing. CCI uses the standard 0.015 + scaling constant; stochastic momentum uses a 14-period range and a + 3-period signal; ROC uses 12 periods; and Ichimoku uses 9, 26, and 52 + periods without a visual displacement. These formulas are implemented + locally so this preserved API does not depend on an unmaintained + technical-analysis package. + """ + rng = np.random.default_rng(random_state) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=RuntimeWarning) + # Features + features = pd.DataFrame() + features["FracDiff"] = fractionally_differentiated_log_price(prices) + features["Volatility"] = daily_volatility_with_log_returns(prices, 100) + features["Z-Score"] = (prices - prices.rolling(20).mean()) / prices.rolling( + 20 + ).std() + macd_line = np.log(prices.ewm(span=12).mean() / prices.ewm(span=26).mean()) + signal_line = macd_line.ewm(span=9).mean() + features["Log MACD Histogram"] = macd_line - signal_line + features["ADX"] = _average_directional_index(prices) + features["RSI"] = _relative_strength_index(prices) + features["CCI"] = _commodity_channel_index(prices) + stochastic, stochastic_signal = _stochastic_oscillator(prices) + features["Stochastic"] = stochastic + features["ROC"] = prices.pct_change(12) * 100.0 + features["ATR"] = _average_true_range(prices) + features["Log DPO"] = np.log( + prices.rolling(11).mean() / prices.rolling(20).mean() + ) + + # 1. MACD Crossovers: + features["MACD Position"] = 0 # default to no crossover + features.loc[features["Log MACD Histogram"] >= 0, "MACD Position"] = 1 + features.loc[features["Log MACD Histogram"] < 0, "MACD Position"] = -1 + + # 2. ADX Trend Strength: + features["ADX Strength"] = 0 + features.loc[features["ADX"] > 25, "ADX Strength"] = 1 + features.loc[features["ADX"] < 25, "ADX Strength"] = -1 + + # 3. RSI Overbought/Oversold: + features["RSI Signal"] = 0 + features.loc[features["RSI"] > 70, "RSI Signal"] = 1 + features.loc[features["RSI"] < 30, "RSI Signal"] = -1 + + # 4. CCI Overbought/Oversold: + features["CCI Signal"] = 0 + features.loc[features["CCI"] > 100, "CCI Signal"] = 1 + features.loc[features["CCI"] < -100, "CCI Signal"] = -1 + + # 5. Stochastic Oscillator Overbought/Oversold: + features["Stochastic Signal"] = 0 + features.loc[stochastic_signal > 80, "Stochastic Signal"] = 1 + features.loc[stochastic_signal < 20, "Stochastic Signal"] = -1 + + # 6. ROC Momentum Shifts: + features["ROC Momentum"] = 0 + features.loc[features["ROC"] > 0, "ROC Momentum"] = 1 # Positive momentum + features.loc[features["ROC"] < 0, "ROC Momentum"] = -1 # Negative momentum + tenkan_sen, kijun_sen, senkou_span_a, senkou_span_b = _ichimoku_lines( + prices + ) + + # 1. Kumo Breakouts + features["Kumo Breakout"] = 0 + features.loc[ + (prices > senkou_span_a) & (prices > senkou_span_b), "Kumo Breakout" + ] = 1 + features.loc[ + (prices < senkou_span_a) & (prices < senkou_span_b), "Kumo Breakout" + ] = -1 + + # 2. TK Crosses + features["TK Position"] = 0 + features.loc[tenkan_sen >= kijun_sen, "TK Position"] = 1 + features.loc[tenkan_sen < kijun_sen, "TK Position"] = -1 + + # 4. Price Relative to Kumo + features["Price Kumo Position"] = 0 + features.loc[ + (prices > senkou_span_a) & (prices > senkou_span_b), "Price Kumo Position" + ] = 1 + features.loc[ + (prices < senkou_span_a) & (prices < senkou_span_b), "Price Kumo Position" + ] = -1 + + # 5. Cloud Thickness + features["Cloud Thickness"] = np.log(senkou_span_a / senkou_span_b) + + # 6. Momentum Confirmation + features["Momentum Confirmation"] = 0 + features.loc[ + (tenkan_sen > senkou_span_a) & (prices > senkou_span_a), + "Momentum Confirmation", + ] = 1 + features.loc[ + (tenkan_sen < senkou_span_a) & (prices < senkou_span_a), + "Momentum Confirmation", + ] = -1 + + if noise_scale > 0.0: + for col in features.columns: + noise = rng.normal( + loc=0, scale=noise_scale * features[col].std(), size=features[col].shape + ) + features[col] += noise + + return features + + +def backtest_overfitting_simulation_results( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + cross_validators: dict[str, Any], + noise_scale: float = 0.0, + random_state: int = None, + n_jobs: int = 1, +) -> dict[str, list[dict[str, Any]]]: + """ + Conducts a simulation to evaluate the performance of trading strategies and models. + + This function simulates a trading environment to assess various cross-validation methods in the context of financial analytics. It uses a set of market regime parameters and machine learning models to backtest trading strategies and compute metrics indicative of overfitting. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior, including window sizes and flags for mean reversion. + models (dict): A collection of machine learning models and their associated parameters. + cross_validators (dict): A dictionary of cross-validation methods. + noise_scale (float): Scale of Gaussian noise to be added to the features. + random_state (int): Seed for random number generator. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Dict[str, List[Dict[str, Any]]]: A dictionary containing the results of the backtest for each cross-validation method tested. + """ + + volatility = daily_volatility_with_log_returns(prices, 100) + filter_threshold = 1.8 + moelcules = cusum_filter_events_dynamic_threshold( + np.log(prices), filter_threshold * volatility + ) + vertical_barriers = vertical_barrier(prices, moelcules, 20) + features = financial_features_backtest_overfitting_simulation( + prices, noise_scale=noise_scale, random_state=random_state + ) + + results = {cv: [] for cv in cross_validators.keys()} + + # Iterate over each strategy parameter combination + strategy_parameters_keys, strategy_parameters_values = zip( + *strategy_parameters.items() + ) + for strategy_parameters_value in itertools.product(*strategy_parameters_values): + strategy_params = dict(zip(strategy_parameters_keys, strategy_parameters_value)) + if ( + ( + strategy_params["fast_window"] == strategy_parameters["fast_window"][0] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][0] + ) + or ( + strategy_params["fast_window"] == strategy_parameters["fast_window"][1] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][1] + ) + or ( + strategy_params["fast_window"] == strategy_parameters["fast_window"][2] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][2] + ) + or ( + strategy_params["fast_window"] == strategy_parameters["fast_window"][3] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][3] + ) + ): + + strategy_sides = determine_strategy_side(prices, **strategy_params) + else: + continue + + triple_barrier_events = meta_events( + prices, + moelcules, + [0.5, 1.5], + volatility, + 0, + 1, + vertical_barriers, + strategy_sides, + ) + labels = meta_labeling(triple_barrier_events, prices) + sample_weights = sample_weight_absolute_return_meta_labeling( + triple_barrier_events["End Time"], prices, moelcules + ) + + index = ( + features.loc[moelcules] + .dropna() + .index.intersection(labels.dropna().index) + .intersection(sample_weights.dropna().index) + ) + data = features.loc[index] + target = labels.loc[index]["Label"] + weights = sample_weights.loc[index] + times = labels.loc[index]["End Time"] + external_feature = volatility.loc[index] + + # Iterate over each model and hyperparameter configuration + for model_name, model_details in models.items(): + model = model_details["Model"] + param_grid = model_details["Parameters"] + + # Generate all combinations of hyperparameters + model_keys, model_values = zip(*param_grid.items()) + for model_value in itertools.product(*model_values): + params = dict(zip(model_keys, model_value)) + model.set_params(**params) + + for cross_validator_type, cross_validator in cross_validators.items(): + if "times" in cross_validator.__dict__: + cross_validator.times = times + if "external_feature" in cross_validator.__dict__: + cross_validator.external_feature = external_feature + predictions = cross_validator.backtest_predictions( + model, + data, + target, + weights, + predict_probability=True, + n_jobs=n_jobs, + ) + probabilities = pd.Series( + np.vstack( + list(map(lambda x: x[:, 1], predictions.values())) + ).mean(axis=0), + times.index, + ).dropna() + positions = strategy_bet_sizing( + prices.index, + times.loc[probabilities.index], + strategy_sides[probabilities.index], + probabilities, + ) + strategy_log_returns = ( + np.log(prices).diff() * positions.shift() + ).dropna() + results[cross_validator_type].append( + { + "Trial Info.": { + "Strategy Parameters": strategy_params, + "Model Name": model_name, + "Model Parameters": params, + }, + "Returns": strategy_log_returns, + } + ) + + return results + + +def overall_backtest_overfitting_simulation( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + step_risk_free_rate: float, + noise_scale: float = 0.0, + random_state: int = None, + n_jobs: int = 1, +) -> tuple[dict[str, float], dict[str, float]]: + """ + Conducts an overall backtest overfitting simulation to calculate the metrics. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models and their associated parameters. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + noise_scale (float): Scale of Gaussian noise to be added to the features. + random_state (int): Seed for random number generator. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Tuple[Dict[str, float], Dict[str, float]]: A tuple containing two dictionaries, one for the Probability of Backtest Overfitting (PBO) and the other for the Deflated Sharpe Ratio (DSR), for each cross-validation method tested. + """ + + cross_validators = { + "Walk-Forward": _create_cross_validator( + "walkforward", + n_splits=4, + ), + "K-Fold": _create_cross_validator( + "kfold", + n_splits=4, + ), + "Purged K-Fold": _create_cross_validator( + "purgedkfold", n_splits=4, times=None, embargo=0.02 + ), + "Combinatorial Purged": _create_cross_validator( + "combinatorialpurged", n_splits=8, n_test_groups=2, times=None, embargo=0.02 + ), + } + + results = backtest_overfitting_simulation_results( + prices, + strategy_parameters, + models, + cross_validators, + noise_scale, + random_state, + n_jobs=n_jobs, + ) + + cv_deflated_sr = {} + cv_pbo = {} + + for cv, trials in results.items(): + performances = pd.concat([trial["Returns"] for trial in trials], axis=1) + + sharpe_ratios = performances.apply(lambda y: sharpe_ratio(y.values)) + benchmark_sr = benchmark_sharpe_ratio(sharpe_ratios) + best_strategy_index = sharpe_ratios.idxmax() + cv_deflated_sr[cv] = probabilistic_sharpe_ratio( + sharpe_ratios.loc[best_strategy_index], + benchmark_sr, + len(performances), + ss.skew(performances[best_strategy_index]), + ss.kurtosis(performances[best_strategy_index]), + return_test_statistic=True, + ) + + pbo, logit_values = probability_of_backtest_overfitting( + performances.values, risk_free_return=step_risk_free_rate + ) + cv_pbo[cv] = pbo + + return cv_pbo, cv_deflated_sr + + +def temporal_backtest_overfitting_simulation( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + step_risk_free_rate: float, + overfitting_partitions_length: int, + n_jobs: int = 1, +) -> tuple[dict[str, list[float]], dict[str, list[float]]]: + """ + Conducts a temporal backtest overfitting simulation to calculate the metrics in chunks. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models and their associated parameters. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + overfitting_partitions_length (int): The number of partitions to divide the dataset into for temporal overfitting analysis. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Tuple[Dict[str, List[float]], Dict[str, List[float]]]: A tuple containing two dictionaries, one for the Probability of Backtest Overfitting (PBO) and the other for the Deflated Sharpe Ratio (DSR), for each cross-validation method tested. + """ + cross_validators = { + "Walk-Forward": _create_cross_validator( + "walkforward", + n_splits=4, + ), + "K-Fold": _create_cross_validator( + "kfold", + n_splits=4, + ), + "Purged K-Fold": _create_cross_validator( + "purgedkfold", n_splits=4, times=None, embargo=0.02 + ), + "Combinatorial Purged": _create_cross_validator( + "combinatorialpurged", n_splits=8, n_test_groups=2, times=None, embargo=0.02 + ), + } + + results = backtest_overfitting_simulation_results( + prices, strategy_parameters, models, cross_validators, n_jobs=n_jobs + ) + + cv_deflated_sr = {cv: [] for cv in results.keys()} + cv_pbo = {cv: [] for cv in results.keys()} + + for cv, trials in results.items(): + performances = pd.concat([trial["Returns"] for trial in trials], axis=1) + + # Calculate the number of chunks using ceil + n_chunks = ceil(performances.shape[0] / overfitting_partitions_length) + for chunk in np.array_split(performances, n_chunks): + sharpe_ratios = chunk.apply(lambda y: sharpe_ratio(y.values)) + benchmark_sr = benchmark_sharpe_ratio(sharpe_ratios) + best_strategy_index = sharpe_ratios.idxmax() + + deflated_sr = probabilistic_sharpe_ratio( + sharpe_ratios.loc[best_strategy_index], + benchmark_sr, + len(chunk), + ss.skew(chunk[best_strategy_index]), + ss.kurtosis(chunk[best_strategy_index]), + return_test_statistic=True, + ) + cv_deflated_sr[cv].append(deflated_sr) + + pbo, logit_values = probability_of_backtest_overfitting( + chunk.values, risk_free_return=step_risk_free_rate, n_jobs=n_jobs + ) + cv_pbo[cv].append(pbo) + + return cv_pbo, cv_deflated_sr + + +def time_temporal_backtest_overfitting_simulation( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + step_risk_free_rate: float, + overfitting_partitions_duration: str = "A", # Annual grouping by default + n_jobs: int = 1, +) -> tuple[dict[str, pd.Series], dict[str, pd.Series]]: + """ + Conducts a time-temporal backtest overfitting simulation to calculate the metrics in time-indexed chunks. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models and their associated parameters. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + overfitting_partitions_duration (str): The frequency for time-based grouping to divide the dataset into for temporal overfitting analysis. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Tuple[Dict[str, pd.Series], Dict[str, pd.Series]]: A tuple containing two dictionaries, one for the Probability of Backtest Overfitting (PBO) and the other for the Deflated Sharpe Ratio (DSR), for each cross-validation method tested, indexed by time. + """ + cross_validators = { + "Walk-Forward": _create_cross_validator( + "walkforward", + n_splits=4, + ), + "K-Fold": _create_cross_validator( + "kfold", + n_splits=4, + ), + "Purged K-Fold": _create_cross_validator( + "purgedkfold", n_splits=4, times=None, embargo=0.02 + ), + "Combinatorial Purged": _create_cross_validator( + "combinatorialpurged", n_splits=8, n_test_groups=2, times=None, embargo=0.02 + ), + } + + results = backtest_overfitting_simulation_results( + prices, strategy_parameters, models, cross_validators, n_jobs=n_jobs + ) + + cv_deflated_sr = {cv: pd.Series(dtype=float) for cv in results.keys()} + cv_pbo = {cv: pd.Series(dtype=float) for cv in results.keys()} + + for cv, trials in results.items(): + performances = pd.concat([trial["Returns"] for trial in trials], axis=1) + + # Group by the specified duration and calculate metrics + grouped = performances.groupby(pd.Grouper(freq=overfitting_partitions_duration)) + for timestamp, chunk in grouped: + sharpe_ratios = chunk.apply(lambda y: sharpe_ratio(y.values)) + benchmark_sr = benchmark_sharpe_ratio(sharpe_ratios) + best_strategy_index = sharpe_ratios.idxmax() + + deflated_sr = probabilistic_sharpe_ratio( + sharpe_ratios.loc[best_strategy_index], + benchmark_sr, + len(chunk), + ss.skew(chunk[best_strategy_index]), + ss.kurtosis(chunk[best_strategy_index]), + return_test_statistic=True, + ) + cv_deflated_sr[cv].at[timestamp] = deflated_sr + + pbo, logit_values = probability_of_backtest_overfitting( + chunk.values, risk_free_return=step_risk_free_rate, n_jobs=1 + ) + cv_pbo[cv].at[timestamp] = pbo + + return cv_pbo, cv_deflated_sr + + +def varying_embargo_backtest_overfitting_simulation( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + step_risk_free_rate: float, + embargo_values: list[float], + n_jobs: int = 1, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Conducts a backtest overfitting simulation with varying embargo values to calculate the metrics. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models and their associated parameters. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + embargo_values (List[float]): List of embargo values to test. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Tuple[pd.DataFrame, pd.DataFrame]: Two DataFrames containing PBO and DSR values for each embargo value and cross-validation method. + """ + + volatility = daily_volatility_with_log_returns(prices, 100) + filter_threshold = 1.8 + moelcules = cusum_filter_events_dynamic_threshold( + np.log(prices), filter_threshold * volatility + ) + vertical_barriers = vertical_barrier(prices, moelcules, 20) + features = financial_features_backtest_overfitting_simulation(prices) + + cv_pbo_embargo = pd.DataFrame( + index=embargo_values, columns=["Purged K-Fold", "Combinatorial Purged"] + ) + cv_deflated_sr_embargo = pd.DataFrame( + index=embargo_values, columns=["Purged K-Fold", "Combinatorial Purged"] + ) + + for embargo in embargo_values: + results = { + "Purged K-Fold": [], + "Combinatorial Purged": [], + } + + cross_validators = { + "Purged K-Fold": _create_cross_validator( + "purgedkfold", n_splits=4, times=None, embargo=embargo + ), + "Combinatorial Purged": _create_cross_validator( + "combinatorialpurged", + n_splits=8, + n_test_groups=2, + times=None, + embargo=embargo, + ), + } + + # Iterate over each strategy parameter combination + strategy_parameters_keys, strategy_parameters_values = zip( + *strategy_parameters.items() + ) + for strategy_parameters_value in itertools.product(*strategy_parameters_values): + strategy_params = dict( + zip(strategy_parameters_keys, strategy_parameters_value) + ) + if ( + ( + strategy_params["fast_window"] + == strategy_parameters["fast_window"][0] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][0] + ) + or ( + strategy_params["fast_window"] + == strategy_parameters["fast_window"][1] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][1] + ) + or ( + strategy_params["fast_window"] + == strategy_parameters["fast_window"][2] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][2] + ) + or ( + strategy_params["fast_window"] + == strategy_parameters["fast_window"][3] + and strategy_params["slow_window"] + == strategy_parameters["slow_window"][3] + ) + ): + + strategy_sides = determine_strategy_side(prices, **strategy_params) + else: + continue + + triple_barrier_events = meta_events( + prices, + moelcules, + [0.5, 1.5], + volatility, + 0, + 1, + vertical_barriers, + strategy_sides, + ) + labels = meta_labeling(triple_barrier_events, prices) + sample_weights = sample_weight_absolute_return_meta_labeling( + triple_barrier_events["End Time"], prices, moelcules + ) + + index = ( + features.loc[moelcules] + .dropna() + .index.intersection(labels.dropna().index) + .intersection(sample_weights.dropna().index) + ) + data = features.loc[index] + target = labels.loc[index]["Label"] + weights = sample_weights.loc[index] + times = labels.loc[index]["End Time"] + + # Iterate over each model and hyperparameter configuration + for model_name, model_details in models.items(): + model = model_details["Model"] + param_grid = model_details["Parameters"] + + # Generate all combinations of hyperparameters + model_keys, model_values = zip(*param_grid.items()) + for model_value in itertools.product(*model_values): + params = dict(zip(model_keys, model_value)) + model.set_params(**params) + + for ( + cross_validator_type, + cross_validator, + ) in cross_validators.items(): + if "times" in cross_validator.__dict__: + cross_validator.times = times + predictions = cross_validator.backtest_predictions( + model, + data, + target, + weights, + predict_probability=True, + n_jobs=n_jobs, + ) + probabilities = pd.Series( + np.vstack( + list(map(lambda x: x[:, 1], predictions.values())) + ).mean(axis=0), + times.index, + ).dropna() + positions = strategy_bet_sizing( + prices.index, + times.loc[probabilities.index], + strategy_sides[probabilities.index], + probabilities, + ) + strategy_log_returns = ( + np.log(prices).diff() * positions.shift() + ).dropna() + results[cross_validator_type].append( + { + "Trial Info.": { + "Strategy Parameters": strategy_params, + "Model Name": model_name, + "Model Parameters": params, + }, + "Returns": strategy_log_returns, + } + ) + + cv_deflated_sr = {} + cv_pbo = {} + + for cv, trials in results.items(): + performances = pd.concat([trial["Returns"] for trial in trials], axis=1) + + sharpe_ratios = performances.apply(lambda y: sharpe_ratio(y.values)) + benchmark_sr = benchmark_sharpe_ratio(sharpe_ratios) + best_strategy_index = sharpe_ratios.idxmax() + cv_deflated_sr[cv] = probabilistic_sharpe_ratio( + sharpe_ratios.loc[best_strategy_index], + benchmark_sr, + len(performances), + ss.skew(performances[best_strategy_index]), + ss.kurtosis(performances[best_strategy_index]), + return_test_statistic=True, + ) + + pbo, logit_values = probability_of_backtest_overfitting( + performances.values, risk_free_return=step_risk_free_rate, n_jobs=1 + ) + cv_pbo[cv] = pbo + + for cv in results.keys(): + cv_pbo_embargo.loc[embargo, cv] = cv_pbo[cv] + cv_deflated_sr_embargo.loc[embargo, cv] = cv_deflated_sr[cv] + + return cv_pbo_embargo, cv_deflated_sr_embargo + + +def sharpe_ratio(returns, risk_free_rate=0): + """Calculate the Sharpe ratio of the given returns.""" + std = returns.std() + if std == 0: + return 0.0 + return (returns.mean() - risk_free_rate) / std + + +def sortino_ratio(returns, risk_free_rate=0): + """Calculate the Sortino ratio of the given returns.""" + downside_returns = returns[returns < risk_free_rate] + expected_return = returns.mean() - risk_free_rate + + if downside_returns.empty: + return np.inf if expected_return > 0 else 0.0 + + downside_risk = downside_returns.std() + + if downside_risk == 0: + return np.inf if expected_return > 0 else 0.0 + + return expected_return / downside_risk + + +def expected_shortfall(returns, step_risk_free_rate, confidence_level=0.05): + """Calculate the expected shortfall (conditional VaR) of the given returns.""" + var = np.percentile(returns, 100 * confidence_level) + es = returns[returns <= var].mean() + return es + + +def backtest_overfitting_simulation_financial_metrics_rank_correlation( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + step_risk_free_rate: float, + n_jobs: int = 1, +) -> pd.DataFrame: + """ + Conducts a backtest overfitting simulation and calculates the rank correlation of financial metrics. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models and their associated parameters. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + pd.DataFrame: DataFrame containing the rank correlations for each cross-validation method and each metric. + """ + # Run the backtest overfitting simulation to get results + cross_validators = { + "Walk-Forward": _create_cross_validator( + "walkforward", + n_splits=4, + ), + "K-Fold": _create_cross_validator( + "kfold", + n_splits=4, + ), + "Purged K-Fold": _create_cross_validator( + "purgedkfold", n_splits=4, times=None, embargo=0.02 + ), + "Combinatorial Purged": _create_cross_validator( + "combinatorialpurged", n_splits=8, n_test_groups=2, times=None, embargo=0.02 + ), + } + + results = backtest_overfitting_simulation_results( + prices, strategy_parameters, models, cross_validators, n_jobs=n_jobs + ) + + metrics = { + "Sharpe Ratio": sharpe_ratio, + "Sortino Ratio": sortino_ratio, + "Expected Shortfall": expected_shortfall, + } + + rank_correlations = {cv: {} for cv in results.keys()} + + for cv, trials in results.items(): + performances = pd.concat([trial["Returns"] for trial in trials], axis=1) + + # Split the data into two halves + midpoint = len(performances) // 2 + first_half = performances.iloc[:midpoint] + second_half = performances.iloc[midpoint:] + + # Calculate the metrics for each half + first_half_metrics = { + metric_name: first_half.apply( + lambda x, fn=metric_func: fn(x, step_risk_free_rate), axis=0 + ) + for metric_name, metric_func in metrics.items() + } + second_half_metrics = { + metric_name: second_half.apply( + lambda x, fn=metric_func: fn(x, step_risk_free_rate), axis=0 + ) + for metric_name, metric_func in metrics.items() + } + + # Rank the trials in each half + first_half_ranks = { + metric_name: metrics_values.rank() + for metric_name, metrics_values in first_half_metrics.items() + } + second_half_ranks = { + metric_name: metrics_values.rank() + for metric_name, metrics_values in second_half_metrics.items() + } + + # Calculate the rank correlation for each metric + for metric_name in metrics.keys(): + rank_corr, _ = kendalltau( + first_half_ranks[metric_name], second_half_ranks[metric_name] + ) + rank_correlations[cv][metric_name] = rank_corr + + # Create the final DataFrame + rank_corr_df = pd.DataFrame(rank_correlations) + + return rank_corr_df + + +def backtest_overfitting_simulation_model_complexity( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, Any], + step_risk_free_rate: float, + n_jobs: int = 1, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Conducts a backtest overfitting simulation to compare the PBO and DSR values of each CV method for simple and complex models. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Tuple[pd.DataFrame, pd.DataFrame]: Two DataFrames containing PBO and DSR values for each model and each CV method. + """ + # Initialize DataFrames to store results + pbo_df = pd.DataFrame(columns=["Combinatorial Purged"], index=models.keys()) + dsr_df = pd.DataFrame(columns=["Combinatorial Purged"], index=models.keys()) + + # Create features + volatility = daily_volatility_with_log_returns(prices, 100) + filter_threshold = 1.8 + moelcules = cusum_filter_events_dynamic_threshold( + np.log(prices), filter_threshold * volatility + ) + vertical_barriers = vertical_barrier(prices, moelcules, 20) + features = financial_features_backtest_overfitting_simulation(prices) + + for model_name, model in models.items(): + results = { + "Combinatorial Purged": [], + } + + # Iterate over each strategy parameter combination + strategy_parameters_keys, strategy_parameters_values = zip( + *strategy_parameters.items() + ) + for strategy_parameters_value in itertools.product(*strategy_parameters_values): + strategy_params = dict( + zip(strategy_parameters_keys, strategy_parameters_value) + ) + try: + strategy_sides = determine_strategy_side(prices, **strategy_params) + + except ValueError: + continue + + triple_barrier_events = meta_events( + prices, + moelcules, + [0.5, 1.5], + volatility, + 0, + 1, + vertical_barriers, + strategy_sides, + ) + labels = meta_labeling(triple_barrier_events, prices) + sample_weights = sample_weight_absolute_return_meta_labeling( + triple_barrier_events["End Time"], prices, moelcules + ) + + index = ( + features.loc[moelcules] + .dropna() + .index.intersection(labels.dropna().index) + .intersection(sample_weights.dropna().index) + ) + data = features.loc[index] + target = labels.loc[index]["Label"] + weights = sample_weights.loc[index] + times = labels.loc[index]["End Time"] + + cross_validators = { + "Combinatorial Purged": _create_cross_validator( + "combinatorialpurged", + n_splits=8, + n_test_groups=2, + times=times, + embargo=0.02, + ), + } + + for cross_validator_type, cross_validator in cross_validators.items(): + predictions = cross_validator.backtest_predictions( + model, + data, + target, + weights, + predict_probability=True, + n_jobs=n_jobs, + ) + probabilities = pd.Series( + np.vstack(list(map(lambda x: x[:, 1], predictions.values()))).mean( + axis=0 + ), + times.index, + ).dropna() + positions = strategy_bet_sizing( + prices.index, + times.loc[probabilities.index], + strategy_sides[probabilities.index], + probabilities, + ) + strategy_log_returns = ( + np.log(prices).diff() * positions.shift() + ).dropna() + results[cross_validator_type].append( + { + "Trial Info.": { + "Strategy Parameters": strategy_params, + "Model Name": model_name, + "Model Parameters": {}, + }, + "Returns": strategy_log_returns, + } + ) + + cv_deflated_sr = {} + cv_pbo = {} + + for cv, trials in results.items(): + performances = pd.concat([trial["Returns"] for trial in trials], axis=1) + + sharpe_ratios = performances.apply(lambda y: sharpe_ratio(y.values)) + benchmark_sr = benchmark_sharpe_ratio(sharpe_ratios) + best_strategy_index = sharpe_ratios.idxmax() + cv_deflated_sr[cv] = probabilistic_sharpe_ratio( + sharpe_ratios.loc[best_strategy_index], + benchmark_sr, + len(performances), + ss.skew(performances[best_strategy_index]), + ss.kurtosis(performances[best_strategy_index]), + return_test_statistic=True, + ) + + pbo, logit_values = probability_of_backtest_overfitting( + performances.values, risk_free_return=step_risk_free_rate, n_jobs=1 + ) + cv_pbo[cv] = pbo + + for cv in results.keys(): + pbo_df.loc[model_name, cv] = cv_pbo[cv] + dsr_df.loc[model_name, cv] = cv_deflated_sr[cv] + + return pbo_df, dsr_df + + +def noised_backtest_overfitting_simulation( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + step_risk_free_rate: float, + noise_scales: list[float], + random_state: int = None, + n_jobs: int = 1, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Conducts a noised backtest overfitting simulation to compare the new PBO/DSR values for different noise scales. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models and their associated parameters. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + noise_scales (List[float]): List of noise scale values to test. + random_state (int): Seed for random number generator. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Tuple[pd.DataFrame, pd.DataFrame]: Two DataFrames containing PBO and DSR values for each noise scale and each CV method. + """ + # Initialize DataFrames to store results + pbo_df = pd.DataFrame( + columns=["Walk-Forward", "K-Fold", "Purged K-Fold", "Combinatorial Purged"], + index=noise_scales, + ) + dsr_df = pd.DataFrame( + columns=["Walk-Forward", "K-Fold", "Purged K-Fold", "Combinatorial Purged"], + index=noise_scales, + ) + + for noise_scale in noise_scales: + # Perform the overall backtest overfitting simulation with the given noise scale + cv_pbo, cv_deflated_sr = overall_backtest_overfitting_simulation( + prices=prices, + strategy_parameters=strategy_parameters, + models=models, + step_risk_free_rate=step_risk_free_rate, + noise_scale=noise_scale, + random_state=random_state, + n_jobs=n_jobs, + ) + + # Store the results in the DataFrames + for cv in cv_pbo.keys(): + pbo_df.loc[noise_scale, cv] = cv_pbo[cv] + dsr_df.loc[noise_scale, cv] = cv_deflated_sr[cv] + + return pbo_df, dsr_df + + +def overall_novel_methods_backtest_overfitting_simulation( + prices: pd.Series, + strategy_parameters: dict[str, Union[list[int], list[float], list[bool]]], + models: dict[str, dict[str, Any]], + step_risk_free_rate: float, + noise_scale: float = 0.0, + random_state: int = None, + n_jobs: int = 1, +) -> tuple[dict[str, float], dict[str, float]]: + """ + Conducts an overall backtest overfitting simulation to calculate the metrics for the novel CPCV methods. + + Args: + prices (pd.Series): Time series of asset prices. + strategy_parameters (dict): Parameters dictating trading strategy behavior. + models (dict): A collection of machine learning models and their associated parameters. + step_risk_free_rate (float): The risk-free rate used in the simulation for Sharpe ratio calculations. + noise_scale (float): Scale of Gaussian noise to be added to the features. + random_state (int): Seed for random number generator. + n_jobs (int): The number of jobs to run in parallel for cross_validator.backtest_predictions. + + Returns: + Tuple[Dict[str, float], Dict[str, float]]: A tuple containing two dictionaries, one for the Probability of Backtest Overfitting (PBO) and the other for the Deflated Sharpe Ratio (DSR), for each cross-validation method tested. + """ + + cross_validators = { + "Combinatorial Purged": _create_cross_validator( + "combinatorialpurged", n_splits=8, n_test_groups=2, times=None, embargo=0.02 + ), + "Bagged Combinatorial Purged": _create_cross_validator( + "baggedcombinatorialpurged", + n_splits=8, + n_test_groups=2, + times=None, + embargo=0.02, + random_state=random_state, + ), + "Adaptive Combinatorial Purged": _create_cross_validator( + "adaptivecombinatorialpurged", + n_splits=8, + n_test_groups=2, + times=None, + embargo=0.02, + ), + } + + results = backtest_overfitting_simulation_results( + prices, + strategy_parameters, + models, + cross_validators, + noise_scale, + random_state, + n_jobs=n_jobs, + ) + + cv_deflated_sr = {} + cv_pbo = {} + + for cv, trials in results.items(): + performances = pd.concat([trial["Returns"] for trial in trials], axis=1) + + sharpe_ratios = performances.apply(lambda y: sharpe_ratio(y.values)) + benchmark_sr = benchmark_sharpe_ratio(sharpe_ratios) + best_strategy_index = sharpe_ratios.idxmax() + cv_deflated_sr[cv] = probabilistic_sharpe_ratio( + sharpe_ratios.loc[best_strategy_index], + benchmark_sr, + len(performances), + ss.skew(performances[best_strategy_index]), + ss.kurtosis(performances[best_strategy_index]), + return_test_statistic=True, + ) + + pbo, logit_values = probability_of_backtest_overfitting( + performances.values, risk_free_return=step_risk_free_rate + ) + cv_pbo[cv] = pbo + + return cv_pbo, cv_deflated_sr + + +def get_cpu_info(): + # Run the lscpu command + result = subprocess.run(["lscpu"], stdout=subprocess.PIPE) + # Decode the output from bytes to string + lscpu_output = result.stdout.decode("utf-8") + + # Parse the lscpu output + cpu_info = {} + for line in lscpu_output.split("\n"): + if line.strip(): + parts = line.split(":", 1) + if len(parts) == 2: + key, value = parts + cpu_info[key.strip()] = value.strip() + + # Extract useful information + useful_info = { + "Architecture": cpu_info.get("Architecture"), + "CPU op-mode(s)": cpu_info.get("CPU op-mode(s)"), + "CPU(s)": cpu_info.get("CPU(s)"), + "Thread(s) per core": cpu_info.get("Thread(s) per core"), + "Core(s) per socket": cpu_info.get("Core(s) per socket"), + "Socket(s)": cpu_info.get("Socket(s)"), + "Vendor ID": cpu_info.get("Vendor ID"), + "Model name": cpu_info.get("Model name"), + "CPU MHz": cpu_info.get("CPU MHz"), + "L1d cache": cpu_info.get("L1d cache"), + "L1i cache": cpu_info.get("L1i cache"), + "L2 cache": cpu_info.get("L2 cache"), + "L3 cache": cpu_info.get("L3 cache"), + } + + return useful_info + + +def format_cpu_info(cpu_info): + report = ( + f"Architecture: {cpu_info['Architecture']}\n" + f"CPU Operational Modes: {cpu_info['CPU op-mode(s)']}\n" + f"Total CPUs: {cpu_info['CPU(s)']}\n" + f"Threads per Core: {cpu_info['Thread(s) per core']}\n" + f"Cores per Socket: {cpu_info['Core(s) per socket']}\n" + f"Sockets: {cpu_info['Socket(s)']}\n" + f"Vendor ID: {cpu_info['Vendor ID']}\n" + f"Model Name: {cpu_info['Model name']}\n" + f"CPU MHz: {cpu_info['CPU MHz']}\n" + f"L1d Cache: {cpu_info['L1d cache']}\n" + f"L1i Cache: {cpu_info['L1i cache']}\n" + f"L2 Cache: {cpu_info['L2 cache']}\n" + f"L3 Cache: {cpu_info['L3 cache']}\n" + ) + return report + + +# Function to generate random data, target, weights, and times +def generate_random_data( + n_samples: int, n_features: int +) -> tuple[pd.DataFrame, pd.Series, np.ndarray, pd.Series]: + date_range = pd.date_range(start="1980-01-01", periods=n_samples, freq="1h") + data = pd.DataFrame( + np.random.randn(n_samples, n_features), + columns=[f"feature_{i}" for i in range(n_features)], + index=date_range, + ) + target = pd.Series(np.random.randint(0, 2, n_samples), index=date_range) + weights = pd.Series(np.random.rand(n_samples), index=date_range) + weights = weights / weights.sum() + times = pd.Series(date_range + pd.DateOffset(hours=3), index=date_range) + return data, target, weights, times + + +# Function to measure computational requirements +def measure_computational_requirements( + cross_validator, model, data, target, weights, n_jobs: int = 1 +) -> dict[str, Any]: + from memory_profiler import memory_usage # optional dependency: RiskLabAI[dev] + + start_time = time.time() + mem_usage = memory_usage( + ( + cross_validator.backtest_predictions, + (model, data, target, weights), + {"predict_probability": True, "n_jobs": n_jobs}, + ), + interval=0.1, + ) + end_time = time.time() + return {"execution_time": end_time - start_time, "memory_usage": max(mem_usage)} + + +# Main function to measure computational requirements for all CV methods +def measure_all_cv_computational_requirements( + cross_validators: dict[str, Any], + n_samples: int = 40 * 252, + n_features: int = 22, + n_jobs: int = 1, + n_repeats: int = 30, +) -> pd.DataFrame: + # Generate random data, target, weights, and times + data, target, weights, times = generate_random_data(n_samples, n_features) + + # Define the logistic regression model without regularization + model = LogisticRegression(penalty=None, solver="lbfgs", max_iter=10000) + + # Measure computational requirements for each CV method + results = { + cv_name: {"execution_time": [], "memory_usage": []} + for cv_name in cross_validators.keys() + } + + for _ in _progress(range(n_repeats)): + for cv_name, cross_validator in cross_validators.items(): + # print(f"Measuring computational requirements for {cv_name} (repeat {_ + 1}/{n_repeats})...") + # Update cross-validator with times if required + if "times" in cross_validator.__dict__: + cross_validator.times = times + result = measure_computational_requirements( + cross_validator, model, data, target, weights, n_jobs=n_jobs + ) + results[cv_name]["execution_time"].append(result["execution_time"]) + results[cv_name]["memory_usage"].append(result["memory_usage"]) + + # Calculate mean and standard deviation for each CV method + results_summary = {} + for cv_name, metrics in results.items(): + results_summary[cv_name] = { + "execution_time_mean": np.mean(metrics["execution_time"]), + "execution_time_std": np.std(metrics["execution_time"]), + "memory_usage_mean": np.mean(metrics["memory_usage"]), + "memory_usage_std": np.std(metrics["memory_usage"]), + } + + # Convert results to DataFrame + results_df = pd.DataFrame(results_summary).T + + return results_df + + +_DEFAULT_N_JOBS_LIST = list(range(1, 9)) + + +def measure_cpcv_parallelization( + n_samples: int = 40 * 252, + n_features: int = 22, + n_repeats: int = 30, + n_jobs_list: list[int] = _DEFAULT_N_JOBS_LIST, +) -> pd.DataFrame: + # Generate random data, target, weights, and times + data, target, weights, times = generate_random_data(n_samples, n_features) + + # Define the logistic regression model without regularization + model = LogisticRegression(penalty=None, solver="lbfgs", max_iter=10000) + + # Define the CPCV cross-validator + cpcv_cross_validator = _create_cross_validator( + "combinatorialpurged", n_splits=8, n_test_groups=2, times=times, embargo=0.02 + ) + + # Measure computational requirements with and without parallelization + results = {} + + for n_jobs in _progress(n_jobs_list): + key = f"n_jobs_{n_jobs}" + results[key] = {"execution_time": [], "memory_usage": []} + for _ in range(n_repeats): + result = measure_computational_requirements( + cpcv_cross_validator, model, data, target, weights, n_jobs=n_jobs + ) + results[key]["execution_time"].append(result["execution_time"]) + results[key]["memory_usage"].append(result["memory_usage"]) + + # Calculate mean and standard deviation for each setting + results_summary = {} + for key, metrics in results.items(): + results_summary[key] = { + "execution_time_mean": np.mean(metrics["execution_time"]), + "execution_time_std": np.std(metrics["execution_time"]), + "memory_usage_mean": np.mean(metrics["memory_usage"]), + "memory_usage_std": np.std(metrics["memory_usage"]), + } + + # Convert results to DataFrame + results_df = pd.DataFrame(results_summary).T + + return results_df + + +def measure_cpcv_scalability( + sample_sizes: list[int], + feature_sizes: list[int], + n_repeats: int = 1, + n_jobs: int = 1, +) -> pd.DataFrame: + # Define the logistic regression model without regularization + model = LogisticRegression(penalty=None, solver="lbfgs", max_iter=10000) + + execution_times = pd.DataFrame(index=sample_sizes, columns=feature_sizes) + memory_usages = pd.DataFrame(index=sample_sizes, columns=feature_sizes) + + for n_samples, n_features in itertools.product(sample_sizes, feature_sizes): + execution_time_list = [] + memory_usage_list = [] + + for _ in range(n_repeats): + # Generate random data, target, weights, and times + data, target, weights, times = generate_random_data(n_samples, n_features) + + # Define the CPCV cross-validator + cpcv_cross_validator = _create_cross_validator( + "combinatorialpurged", + n_splits=8, + n_test_groups=2, + times=times, + embargo=0.02, + ) + + # Measure computational requirements + result = measure_computational_requirements( + cpcv_cross_validator, model, data, target, weights, n_jobs=n_jobs + ) + execution_time_list.append(result["execution_time"]) + memory_usage_list.append(result["memory_usage"]) + + # Calculate mean execution time and memory usage + execution_times.loc[n_samples, n_features] = np.mean(execution_time_list) + memory_usages.loc[n_samples, n_features] = np.mean(memory_usage_list) + + return execution_times, memory_usages diff --git a/src/RiskLabAI/backtest/backtest_statistics.py b/src/RiskLabAI/backtest/backtest_statistics.py new file mode 100644 index 0000000..e7ce374 --- /dev/null +++ b/src/RiskLabAI/backtest/backtest_statistics.py @@ -0,0 +1,300 @@ +""" +Calculates various backtest statistics like holding period, +concentration, and drawdowns. + +## TODO: +- [ ] Optimize `calculate_holding_period` with Numba (@jit) + to improve performance, similar to the `sharpe_ratio` function. +- [ ] Add an `annualized_sharpe_ratio` helper function that + wraps the Numba `sharpe_ratio` and scales it by sqrt(freq). +""" + +import numpy as np +import pandas as pd + +try: + from numba import jit +except ImportError: + def jit(*decorator_args, **decorator_kwargs): + """Return the original function when optional acceleration is absent.""" + if ( + len(decorator_args) == 1 + and callable(decorator_args[0]) + and not decorator_kwargs + ): + return decorator_args[0] + + def decorator(function): + return function + + return decorator + + +@jit(nopython=True) +def sharpe_ratio(returns: np.ndarray, risk_free_rate: float = 0.0) -> float: + """ + Calculate the Sharpe Ratio (Numba-optimized). + + Parameters + ---------- + returns : np.ndarray + An array of returns. + risk_free_rate : float, default=0.0 + The risk-free rate. + + Returns + ------- + float + The calculated Sharpe Ratio. + """ + excess_returns = returns - risk_free_rate + std_dev = np.std(excess_returns) + + if std_dev == 0.0: + return 0.0 + + return np.mean(excess_returns) / std_dev + + +def bet_timing(target_positions: pd.Series) -> pd.Index: + """ + Determine the timestamps of bets, defined as when positions + are closed (return to zero) or flipped (sign change). + + Parameters + ---------- + target_positions : pd.Series + Time series of target positions. + + Returns + ------- + pd.Index + A DatetimeIndex of timestamps when bets are timed. + + Example + ------- + >>> dates = pd.to_datetime(['2020-01-01', '2020-01-02', '2020-01-03', \ + '2020-01-04', '2020-01-05', '2020-01-06']) + >>> positions = pd.Series([0, 1, 1, 0, -1, 0], index=dates) + >>> bet_timing(positions) + DatetimeIndex(['2020-01-04', '2020-01-05', '2020-01-06'], \ +dtype='datetime64[ns]', freq=None) + """ + # 1. Bets at points where position returns to 0 + zero_positions = target_positions[target_positions == 0].index + + # 2. Bets at points where position flips sign + # Find timestamps where t=0 and t-1 != 0 + lagged_non_zero = target_positions.shift(1).fillna(0) + lagged_non_zero = lagged_non_zero[lagged_non_zero != 0].index + bets = zero_positions.intersection(lagged_non_zero) + + # Find timestamps where sign flips, e.g., 1 to -1 + sign_flips = target_positions.iloc[1:] * target_positions.iloc[:-1].values + bets = bets.union(sign_flips[sign_flips < 0].index).sort_values() + + # 3. Add last timestamp if not already included + if target_positions.index[-1] not in bets: + bets = bets.append(pd.Index([target_positions.index[-1]])) + + return bets + + +def calculate_holding_period( + target_positions: pd.Series, +) -> tuple[pd.DataFrame, float]: + """ + Derive the average holding period in days. + + Uses the average entry time pairing algorithm. + + Parameters + ---------- + target_positions : pd.Series + Time series of target positions. + + Returns + ------- + Tuple[pd.DataFrame, float] + - hold_period_df: DataFrame with holding time ('dT') and + weight ('w') for each bet. + - mean_holding_period: The weighted-average holding period in days. + """ + hold_period_data = [] + time_entry = 0.0 # Average entry time + position_diff = target_positions.diff() + # Time difference in fractional days + time_diff = (target_positions.index - target_positions.index[0]) / np.timedelta64( + 1, "D" + ) + + for i in range(1, target_positions.shape[0]): + current_pos = target_positions.iloc[i] + prev_pos = target_positions.iloc[i - 1] + diff = position_diff.iloc[i] + + if diff * prev_pos >= 0: # Position increase or flat + if current_pos != 0: + # Update average entry time + time_entry = (time_entry * prev_pos + time_diff[i] * diff) / current_pos + else: # Position decrease or flip + if current_pos * prev_pos < 0: # Position flip + # Close old position + hold_period_data.append( + { + "index": target_positions.index[i], + "dT": time_diff[i] - time_entry, + "w": abs(prev_pos), + } + ) + time_entry = time_diff[i] # Reset entry time + else: # Position decrease (but not flip) + hold_period_data.append( + { + "index": target_positions.index[i], + "dT": time_diff[i] - time_entry, + "w": abs(diff), + } + ) + + if not hold_period_data: + return pd.DataFrame(columns=["dT", "w"]), np.nan + + hold_period_df = pd.DataFrame(hold_period_data).set_index("index") + + if hold_period_df["w"].sum() > 0: + mean_holding_period = ( + hold_period_df["dT"] * hold_period_df["w"] + ).sum() / hold_period_df["w"].sum() + else: + mean_holding_period = np.nan + + return hold_period_df, mean_holding_period + + +def calculate_hhi(bet_returns: pd.Series) -> float: + """ + Calculate the Herfindahl-Hirschman Index (HHI) for concentration. + + HHI is normalized to be between 0 and 1. + - 0 indicates a perfectly diversified portfolio. + - 1 indicates a fully concentrated portfolio. + + Parameters + ---------- + bet_returns : pd.Series + Series of returns for each bet. + + Returns + ------- + float + The normalized HHI value, or np.nan if undefined. + """ + if bet_returns.shape[0] <= 2: + return np.nan + + total_return = bet_returns.sum() + if total_return == 0: + return np.nan + + weights = bet_returns / total_return + hhi = (weights**2).sum() + + # Normalize HHI + n = bet_returns.shape[0] + hhi_normalized = (hhi - 1.0 / n) / (1.0 - 1.0 / n) + + return hhi_normalized + + +def calculate_hhi_concentration(returns: pd.Series) -> tuple[float, float, float]: + """ + Calculate HHI concentration for positive, negative, and monthly returns. + + Parameters + ---------- + returns : pd.Series + Time series of returns. + + Returns + ------- + Tuple[float, float, float] + - hhi_positive: HHI for positive returns. + - hhi_negative: HHI for negative returns. + - hhi_time: HHI for time-based (monthly) concentration. + """ + hhi_positive = calculate_hhi(returns[returns >= 0]) + hhi_negative = calculate_hhi(returns[returns < 0]) + + # Calculate time concentration (by month) + time_concentration = returns.groupby(pd.Grouper(freq="M")).count() + hhi_time = calculate_hhi(time_concentration) + + return hhi_positive, hhi_negative, hhi_time + + +def compute_drawdowns_time_under_water( + pnl_series: pd.Series, dollars: bool = False +) -> tuple[pd.Series, pd.Series]: + """ + Compute series of drawdowns and the time under water. + + Parameters + ---------- + pnl_series : pd.Series + Time series of returns or dollar PnL. + dollars : bool, default=False + If True, treat `pnl_series` as dollar PnL. + If False, treat as returns (e.g., 1.01, 0.99) and + calculate drawdowns as percentages. + + Returns + ------- + Tuple[pd.Series, pd.Series] + - drawdown: pd.Series indexed by start date, showing max drawdown. + - time_under_water: pd.Series indexed by start date, showing + duration of drawdown in fractional years. + """ + series_df = pnl_series.to_frame("PnL") + series_df["HWM"] = series_df["PnL"].expanding().max() + + # Group by High Water Mark (HWM) + groups = series_df.groupby("HWM") + drawdown_analysis_data = [] + + for hwm, group in groups: + if len(group) <= 1 or hwm == group["PnL"].min(): + continue # Skip groups with no drawdown + + drawdown_analysis_data.append( + { + "Start": group.index[0], + "Stop": group.index[-1], + "HWM": hwm, + "Min": group["PnL"].min(), + } + ) + + if not drawdown_analysis_data: + return pd.Series(dtype=float), pd.Series(dtype=float) + + drawdown_analysis_df = pd.DataFrame(drawdown_analysis_data) + drawdown_analysis_df = drawdown_analysis_df.set_index("Start") + + if dollars: + drawdown = drawdown_analysis_df["HWM"] - drawdown_analysis_df["Min"] + else: + # Percentage drawdown + drawdown = 1.0 - (drawdown_analysis_df["Min"] / drawdown_analysis_df["HWM"]) + + # Time under water in fractional years + time_under_water = ( + (drawdown_analysis_df["Stop"] - drawdown_analysis_df.index) + / np.timedelta64(1, "D") + / 365.25 + ) + + drawdown.index.name = "Datetime" + time_under_water.index.name = "Datetime" + + return drawdown, time_under_water diff --git a/src/RiskLabAI/backtest/backtest_synthetic_data.py b/src/RiskLabAI/backtest/backtest_synthetic_data.py new file mode 100644 index 0000000..c60abb1 --- /dev/null +++ b/src/RiskLabAI/backtest/backtest_synthetic_data.py @@ -0,0 +1,99 @@ +""" +Performs backtesting on synthetic price data generated using the +Ornstein-Uhlenbeck (OU) process. +""" + +from itertools import product +from random import gauss + +import numpy as np + +_DEFAULT_PROFIT_TAKING_RANGE = np.linspace(0.5, 10, 20) +_DEFAULT_STOP_LOSS_RANGE = np.linspace(0.5, 10, 20) + + +def synthetic_back_testing( + forecast: float, + half_life: float, + sigma: float, + n_iteration: int = 100000, + maximum_holding_period: int = 100, + profit_taking_range: np.ndarray = _DEFAULT_PROFIT_TAKING_RANGE, + stop_loss_range: np.ndarray = _DEFAULT_STOP_LOSS_RANGE, + seed: int = 0, +) -> list[tuple[float, float, float, float, float]]: + r""" + Perform backtesting on synthetic price data from an OU process. + + This function simulates multiple price paths based on the + Ornstein-Uhlenbeck (OU) process and tests a grid of + profit-taking (PT) and stop-loss (SL) levels. + + The OU process is given by: + .. math:: + P_t = (1 - \rho) F + \rho P_{t-1} + \sigma Z_t + + Where: + - \(P_t\) is the price at time t + - \(F\) is the forecast price (long-term mean) + - \(\rho\) is the autoregression coefficient + - \(\sigma\) is the standard deviation of noise + - \(Z_t \sim N(0, 1)\) is standard normal noise + + Parameters + ---------- + forecast : float + The forecasted price (long-term mean F). + half_life : float + The half-life of the mean-reversion process. + sigma : float + The standard deviation of the noise (\(\sigma\)). + n_iteration : int, default=100000 + Number of price paths to simulate for each PT/SL combination. + maximum_holding_period : int, default=100 + Maximum number of steps to hold the position. + profit_taking_range : np.ndarray, default=np.linspace(0.5, 10, 20) + An array of profit-taking levels to test. + stop_loss_range : np.ndarray, default=np.linspace(0.5, 10, 20) + An array of stop-loss levels to test. + seed : int, default=0 + The initial price for all simulations. + + Returns + ------- + List[Tuple[float, float, float, float, float]] + A list of tuples. Each tuple contains: + (profit_taking, stop_loss, mean_return, std_return, sharpe_ratio) + """ + # compute ρ coefficient from half-life + rho = 2 ** (-1.0 / half_life) + back_test_results = [] + + for profit_taking, stop_loss in product(profit_taking_range, stop_loss_range): + stop_returns = [] + for _ in range(n_iteration): + price, holding_period = float(seed), 0 # initial price + while True: + # Update price using O-U process + price = (1 - rho) * forecast + rho * price + sigma * gauss(0, 1) + gain = price - seed # compute gain from initial seed price + holding_period += 1 + + # Check stop conditions + if ( + gain > profit_taking + or gain < -stop_loss + or holding_period > maximum_holding_period + ): + stop_returns.append(gain) + break + + mean_return = np.mean(stop_returns) + std_return = np.std(stop_returns) + sharpe_ratio = mean_return / std_return if std_return > 0 else 0.0 + + back_test_results.append( + (profit_taking, stop_loss, mean_return, std_return, sharpe_ratio) + ) + + return back_test_results diff --git a/src/RiskLabAI/backtest/bet_sizing.py b/src/RiskLabAI/backtest/bet_sizing.py new file mode 100644 index 0000000..2badeea --- /dev/null +++ b/src/RiskLabAI/backtest/bet_sizing.py @@ -0,0 +1,509 @@ +""" +Functions for calculating bet size based on model probabilities and +other strategy parameters. + +Includes implementations from de Prado (2018). +""" + +import numpy as np +import pandas as pd +from scipy.stats import norm + +try: + from numba import jit +except ImportError: + def jit(*decorator_args, **decorator_kwargs): + """Return the original function when optional acceleration is absent.""" + if ( + len(decorator_args) == 1 + and callable(decorator_args[0]) + and not decorator_kwargs + ): + return decorator_args[0] + + def decorator(function): + return function + + return decorator + +from RiskLabAI._deprecation import deprecated_alias +from RiskLabAI.hpc import mp_pandas_obj + + +def probability_bet_size(probabilities: np.ndarray, sides: np.ndarray) -> np.ndarray: + r""" + Calculate the bet size based on probabilities and side. + + The bet size is derived from the probability of a correct bet, + transformed from [0, 1] to [-1, 1]. + + .. math:: + \text{bet_size} = \text{side} \times (2 \times \Phi(p) - 1) + + Where \(\Phi\) is the CDF of a standard normal distribution and + \(p\) is the probability from the meta-model. + + Parameters + ---------- + probabilities : np.ndarray + Array of probabilities (e.g., from a meta-model). + sides : np.ndarray + Array indicating the side of the bet (e.g., 1 for long, -1 for short). + + Returns + ------- + np.ndarray + Array of bet sizes, ranging from -1 to 1. + """ + return sides * (2 * norm.cdf(probabilities) - 1) + + +@jit(nopython=True) +def average_bet_sizes( + price_dates: np.ndarray, + start_dates: np.ndarray, + end_dates: np.ndarray, + bet_sizes: np.ndarray, +) -> np.ndarray: + """ + Compute average bet sizes for each date (Numba-optimized). + + This function calculates the concurrent average bet size for each + timestamp in `price_dates`. A bet is active at `price_dates[i]` if + `start_dates[j] <= price_dates[i] <= end_dates[j]`. + + Parameters + ---------- + price_dates : np.ndarray + Array of timestamps for which to calculate the average bet size. + Must be numerical (e.g., int64 from .to_numpy()). + start_dates : np.ndarray + Array of bet start timestamps (numerical). + end_dates : np.ndarray + Array of bet end timestamps (numerical). + bet_sizes : np.ndarray + Array of bet sizes corresponding to each (start, end) pair. + + Returns + ------- + np.ndarray + Array of average bet sizes for each date in `price_dates`. + """ + num_dates = len(price_dates) + avg_bet_sizes = np.zeros(num_dates) + + for i in range(num_dates): + total = 0.0 + count = 0 + for j in range(len(start_dates)): + if start_dates[j] <= price_dates[i] <= end_dates[j]: + total += bet_sizes[j] + count += 1 + if count > 0: + avg_bet_sizes[i] = total / count + + return avg_bet_sizes + + +def strategy_bet_sizing( + price_timestamps: pd.Index, + times: pd.Series, + sides: pd.Series, + probabilities: pd.Series, +) -> pd.Series: + """ + Calculate the average bet size for a trading strategy. + + This function orchestrates the calculation of bet sizes from + probabilities and then computes the concurrent average bet size + over the strategy's history. + + Parameters + ---------- + price_timestamps : pd.Index + The Index of the master price series, defining all timestamps + for the output. + times : pd.Series + Series with bet start times as indices and end times as values. + sides : pd.Series + Series indicating the side of the position (1 or -1). + probabilities : pd.Series + Series of probabilities (from meta-model) for each bet. + + Returns + ------- + pd.Series + Series of average bet sizes, indexed by `price_timestamps`. + """ + # Align inputs + common_index = times.index.intersection(sides.index).intersection( + probabilities.index + ) + if len(common_index) == 0: + return pd.Series(0.0, index=price_timestamps) + + _times = times.loc[common_index] + _sides = sides.loc[common_index] + _probabilities = probabilities.loc[common_index] + + # 1. Calculate individual bet sizes + bet_sizes_arr = probability_bet_size(_probabilities.to_numpy(), _sides.to_numpy()) + + # 2. Calculate concurrent average + avg_bet_sizes_arr = average_bet_sizes( + price_timestamps.to_numpy(), + _times.index.to_numpy(), + _times.values, + bet_sizes_arr, + ) + + return pd.Series(avg_bet_sizes_arr, index=price_timestamps) + + +# --- Bet-sizing functions from de Prado (2018), AFML Chapter 10. --- + + +def avg_active_signals(signals: pd.DataFrame, n_threads: int) -> pd.DataFrame: + """ + Calculate the average signal among active signals using parallel processing. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: SNIPPET 10.2 + + Parameters + ---------- + signals : pd.DataFrame + DataFrame with signal start times as index, and columns 't1' (end time) + and 'signal' (signal value). + n_threads : int + Number of threads to use for parallel execution via `mp_pandas_obj`. + + Returns + ------- + pd.DataFrame + DataFrame containing the average active signal at each time point. + """ + # 1) time points where signals change (either one starts or one ends) + time_points = set(signals["t1"].dropna().values) + time_points = time_points.union(signals.index.values) + time_points = list(time_points) + time_points.sort() + + # 2) call parallel function + out = mp_pandas_obj( + mp_avg_active_signals, + ("molecule", time_points), + n_threads, + signals=signals, + ) + return out + + +def mp_avg_active_signals(signals: pd.DataFrame, molecule: list) -> pd.Series: + """ + Worker function for `avg_active_signals`. + + At time `loc`, average signal among those still active. + Signal is active if: + a) issued before or at `loc` AND + b) `loc` before signal's endtime, or endtime is still unknown (NaT). + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: SNIPPET 10.2 + + Parameters + ---------- + signals : pd.DataFrame + DataFrame of signals (see `avgActiveSignals`). + molecule : list + The list of timestamps (timePoints) assigned to this worker. + + Returns + ------- + pd.Series + Series indexed by `loc` (timestamp) with the average signal value. + """ + # A signal i is active at `loc` iff start_i <= loc < t1_i. The average of + # active signals therefore equals (sum of `signal` over signals started by + # `loc`) minus (sum over signals ended by `loc`), divided by the analogous + # active count. Prefix sums + searchsorted compute this for every `loc` in + # O((n + m) log n) instead of the O(n * m) double scan, with identical + # values (verified in test/test_performance.py). + molecule_list = list(molecule) + if len(signals) == 0 or len(molecule_list) == 0: + return pd.Series(0.0, index=molecule_list) + + molecule_array = np.asarray(molecule_list) + signal_values = signals["signal"].to_numpy(dtype=float) + + starts = signals.index.values + start_order = np.argsort(starts, kind="mergesort") + sorted_starts = starts[start_order] + cum_signal_start = np.concatenate(([0.0], np.cumsum(signal_values[start_order]))) + cum_count_start = np.arange(len(sorted_starts) + 1) + + finite = ~pd.isnull(signals["t1"]).to_numpy() + end_times = signals["t1"].to_numpy()[finite] + end_order = np.argsort(end_times, kind="mergesort") + sorted_ends = end_times[end_order] + cum_signal_end = np.concatenate( + ([0.0], np.cumsum(signal_values[finite][end_order])) + ) + cum_count_end = np.arange(len(sorted_ends) + 1) + + started = np.searchsorted(sorted_starts, molecule_array, side="right") + ended = np.searchsorted(sorted_ends, molecule_array, side="right") + active_signal = cum_signal_start[started] - cum_signal_end[ended] + active_count = cum_count_start[started] - cum_count_end[ended] + + with np.errstate(invalid="ignore", divide="ignore"): + averages = np.where(active_count > 0, active_signal / active_count, 0.0) + + return pd.Series(averages, index=molecule_list) + + +def discrete_signal(signal: pd.Series, step_size: float) -> pd.Series: + """ + Discretize a signal to a specific step size, capping at +/- 1. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: SNIPPET 10.3 + + Parameters + ---------- + signal : pd.Series + The continuous signal values (e.g., from -1 to 1). + step_size : float + The step size for discretization (e.g., 0.1). + + Returns + ------- + pd.Series + The discretized signal. + """ + discretized = (signal / step_size).round() * step_size + discretized[discretized > 1] = 1.0 + discretized[discretized < -1] = -1.0 + return discretized + + +def generate_signal( + events: pd.DataFrame, + step_size: float, + probability: pd.Series, + prediction: pd.Series, + n_classes: int, + n_threads: int, +) -> pd.Series: + """ + Generate a discretized, averaged signal from model predictions. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: SNIPPET 10.1 + + Parameters + ---------- + events : pd.DataFrame + DataFrame of events, must include 't1' (end times) and + optionally 'side' (for meta-labeling). + step_size : float + The step size for final signal discretization. + probability : pd.Series + Probability of class 1 (e.g., from `predict_proba`). + prediction : pd.Series + The predicted class (e.g., 1 or -1). + n_classes : int + Number of classes in the prediction. + n_threads : int + Number of threads for `avg_active_signals`. + + Returns + ------- + pd.Series + The final, averaged, and discretized signal. + """ + if probability.shape[0] == 0: + return pd.Series() + + # 1) generate signals from multinomial classification (one-vs-rest, OvR) + # t-value of OvR + t_value = (probability - 1.0 / n_classes) / ( + (probability * (1.0 - probability)) ** 0.5 + ) + signal = prediction * (2 * norm.cdf(t_value) - 1) # signal = side * size + + if "side" in events: + signal *= events.loc[signal.index, "side"] # meta-labeling + + # 2) compute average signal among those concurrently open + signal_df = signal.to_frame("signal").join(events[["t1"]], how="left") + avg_signal = avg_active_signals(signal_df, n_threads) + + # 3) discretize signal + discretized_signal = discrete_signal(signal=avg_signal, step_size=step_size) + return discretized_signal + + +def bet_size_sigmoid(w: float, x: float) -> float: + """ + Calculate bet size using a sigmoid function. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: p.145 SNIPPET 10.4 + + Parameters + ---------- + w : float + Coefficient regulating the sigmoid width. + x : float + The divergence (e.g., forecast_price - market_price). + + Returns + ------- + float + The bet size, bounded between -1 and 1. + """ + return x / np.sqrt(w + x**2) + + +def target_position( + w: float, f: float, actual_price: float, maximum_position_size: int +) -> int: + """ + Calculate the target position size. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: p.145 SNIPPET 10.4 + + Parameters + ---------- + w : float + Coefficient regulating the sigmoid width. + f : float + Forecasted price. + actual_price : float + Actual (current) market price. + maximum_position_size : int + Maximum absolute position size. + + Returns + ------- + int + The target position size (integer). + """ + return int(bet_size_sigmoid(w, f - actual_price) * maximum_position_size) + + +def inverse_price(f: float, w: float, m: float) -> float: + """ + Calculates the inverse price given a bet size. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: p.145 SNIPPET 10.4 + + Parameters + ---------- + f : float + Forecasted price. + w : float + Coefficient regulating the sigmoid width. + m : float + Bet size (as a fraction of max size, -1 to 1). + + Returns + ------- + float + The implied price for bet size `m`. + """ + if m == 1.0 or m == -1.0: + return f # Avoid division by zero + return f - m * np.sqrt(w / (1 - m**2)) + + +def limit_price( + target_position_size: int, + current_position: int, + f: float, + w: float, + maximum_position_size: int, +) -> float: + """ + Calculate the limit price for adjusting position. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: p.145 SNIPPET 10.4 + + Parameters + ---------- + target_position_size : int + The target position size. + current_position : int + The current position size. + f : float + Forecasted price. + w : float + Coefficient regulating the sigmoid width. + maximum_position_size : int + Maximum absolute position size. + + Returns + ------- + float + The average limit price. + """ + if target_position_size == current_position: + return f # No change + + sgn = np.sign(target_position_size - current_position) + limit = 0.0 + + # Average price from current to target position + for i in range(abs(current_position + sgn), abs(target_position_size + sgn)): + limit += inverse_price(f, w, i / float(maximum_position_size)) + + limit /= abs(target_position_size - current_position) + return limit + + +def compute_sigmoid_width(x: float, m: float) -> float: + """ + Get the 'w' coefficient implied by a divergence and bet size. + + Reference: De Prado, M. (2018) Advances in financial machine learning. + Methodology: p.145 SNIPPET 10.4 + + Parameters + ---------- + x : float + Divergence (forecast - market). + m : float + Bet size (fraction, -1 to 1, not 0, 1, or -1). + + Returns + ------- + float + The implied 'w' coefficient. + """ + if m == 0.0 or m == 1.0 or m == -1.0: + return np.inf # w is undefined + return x**2 * ((1 / m**2) - 1) + + +# --------------------------------------------------------------------------- # +# Deprecated camelCase aliases (the historical AFML-style names). Each keeps +# working and emits a DeprecationWarning; scheduled for removal in 4.0.0. +# See NAMING_CANON_2.0.0.md. +# --------------------------------------------------------------------------- # +avgActiveSignals = deprecated_alias( + avg_active_signals, "avgActiveSignals", removed_in="4.0.0" +) +mpAvgActiveSignals = deprecated_alias( + mp_avg_active_signals, "mpAvgActiveSignals", removed_in="4.0.0" +) +discreteSignal = deprecated_alias(discrete_signal, "discreteSignal", removed_in="4.0.0") +Signal = deprecated_alias(generate_signal, "Signal", removed_in="4.0.0") +betSize = deprecated_alias(bet_size_sigmoid, "betSize", removed_in="4.0.0") +TPos = deprecated_alias(target_position, "TPos", removed_in="4.0.0") +inversePrice = deprecated_alias(inverse_price, "inversePrice", removed_in="4.0.0") +limitPrice = deprecated_alias(limit_price, "limitPrice", removed_in="4.0.0") +getW = deprecated_alias(compute_sigmoid_width, "getW", removed_in="4.0.0") diff --git a/src/RiskLabAI/backtest/multiple_testing.py b/src/RiskLabAI/backtest/multiple_testing.py new file mode 100644 index 0000000..9968f8d --- /dev/null +++ b/src/RiskLabAI/backtest/multiple_testing.py @@ -0,0 +1,193 @@ +r""" +Multiple-testing haircuts for the Sharpe ratio (Holm FWER, Benjamini-Hochberg-Yekutieli FDR). + +When a researcher screens many strategies or factors and reports the significant ones, the selection +inflates the apparent significance. These corrections discount it by controlling, across the whole +family of trials, either the family-wise error rate (the chance of *any* false discovery) or the +false-discovery rate (the expected *fraction* of false discoveries). They complement the Deflated +Sharpe Ratio (`probabilistic_sharpe_ratio` / `expected_max_sharpe_ratio`): the DSR asks whether the +single best strategy survives its trial count, while these screen a whole family with a stated +error-control target. + +Admitted in Appraisal 07 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tag, verbatim from the verdict: + + To judge a family of screened strategies or factors (not just the single best), prefer Holm when + you must control the chance of any false positive across the family (FWER), and BHY when you want + to bound the expected fraction of false discoveries (FDR) and can accept lower power. The Deflated + Sharpe remains the tool when the question is whether one specific best strategy survives its trial + count. Bonferroni is dominated by Holm; the t>3 hurdle over-rejects; the double-bootstrap does not + control its stated error as implemented and is not admitted. + +Evidence and caveats: appraisals/07_verdict.md. + +References +---------- +Holm, S. (1979) A simple sequentially rejective multiple test procedure. + Scandinavian Journal of Statistics, 6(2), 65-70. +Benjamini, Y. and Yekutieli, D. (2001) The control of the false discovery rate in multiple testing + under dependency. Annals of Statistics, 29(4), 1165-1188. +Harvey, C. R. and Liu, Y. (2015) Backtesting. Journal of Portfolio Management, 42(1), 13-28. +""" + +from __future__ import annotations + +import numpy as np +from scipy import stats as ss + + +def sharpe_ratio_p_values( + sharpe_ratios: np.ndarray, number_of_returns: int +) -> np.ndarray: + r""" + One-sided p-values for a positive edge (H1: SR > 0) from per-period Sharpe ratios. + + Under the i.i.d. null the Sharpe t-statistic is :math:`t = \hat{SR}\sqrt{T}`, so the one-sided + p-value is :math:`\Phi(-t)`. + + Parameters + ---------- + sharpe_ratios : np.ndarray + Per-period Sharpe ratios of the screened strategies. + number_of_returns : int + The number of return observations T each Sharpe is estimated from. + + Returns + ------- + np.ndarray + One-sided p-values, one per strategy. + """ + t_stats = np.asarray(sharpe_ratios, dtype=float) * np.sqrt(number_of_returns) + return ss.norm.sf(t_stats) + + +def holm_adjusted_p_values(p_values: np.ndarray) -> np.ndarray: + r""" + Holm (1979) step-down family-wise-error-rate adjusted p-values. + + Sorting the p-values ascending, the adjusted value at rank i is + :math:`\max_{j \le i} (M - j + 1) p_{(j)}`, clipped to 1, then mapped back to the input order. A + strategy is a discovery at level alpha when its adjusted p-value is below alpha. Holm controls the + FWER under any dependence and uniformly dominates Bonferroni (same size, at least as powerful). + + Parameters + ---------- + p_values : np.ndarray + Raw per-strategy p-values. + + Returns + ------- + np.ndarray + Holm-adjusted p-values, in the original order. + """ + p = np.asarray(p_values, dtype=float) + m = p.size + if m == 0: + return p.copy() + order = np.argsort(p) + sorted_p = p[order] + adjusted_sorted = np.maximum.accumulate((m - np.arange(m)) * sorted_p) + adjusted_sorted = np.clip(adjusted_sorted, 0.0, 1.0) + adjusted = np.empty(m) + adjusted[order] = adjusted_sorted + return adjusted + + +def benjamini_hochberg_yekutieli_adjusted_p_values(p_values: np.ndarray) -> np.ndarray: + r""" + Benjamini-Hochberg-Yekutieli (BHY, 2001) step-up false-discovery-rate adjusted p-values. + + Uses the Yekutieli dependence constant :math:`c(M) = \sum_{i=1}^{M} 1/i`, valid under arbitrary + dependence. Sorting the p-values ascending, the adjusted value at rank i is the running minimum + (from the largest rank down) of :math:`c(M)\, M\, p_{(i)} / i`, clipped to 1. A strategy is a + discovery at level alpha when its adjusted p-value is below alpha. BHY controls the FDR (the + expected fraction of false discoveries) under any dependence. + + Parameters + ---------- + p_values : np.ndarray + Raw per-strategy p-values. + + Returns + ------- + np.ndarray + BHY-adjusted p-values, in the original order. + """ + p = np.asarray(p_values, dtype=float) + m = p.size + if m == 0: + return p.copy() + order = np.argsort(p) + sorted_p = p[order] + c_m = np.sum(1.0 / np.arange(1, m + 1)) + ranks = np.arange(1, m + 1) + factor = c_m * m / ranks + # Step-up: enforce monotonicity from the largest p-value downward. + adjusted_sorted = np.minimum.accumulate((factor * sorted_p)[::-1])[::-1] + adjusted_sorted = np.clip(adjusted_sorted, 0.0, 1.0) + adjusted = np.empty(m) + adjusted[order] = adjusted_sorted + return adjusted + + +def haircut_sharpe_ratios( + sharpe_ratios: np.ndarray, + number_of_returns: int, + method: str = "holm", + significance_level: float = 0.05, +) -> dict[str, np.ndarray]: + r""" + Apply a multiple-testing haircut to a family of screened Sharpe ratios. + + Converts each Sharpe to its one-sided p-value, adjusts the p-values for multiple testing (Holm for + FWER or BHY for FDR), flags the surviving discoveries at ``significance_level``, and reports the + haircut Sharpe - the Sharpe implied by the adjusted p-value, :math:`\hat{SR}_{adj} = + \Phi^{-1}(1 - p_{adj}) / \sqrt{T}` - so the discount from selection is explicit. + + Prefer Holm when you must control the chance of any false positive across the family (FWER), and + BHY when you want to bound the expected fraction of false discoveries (FDR) and can accept lower + power. The Deflated Sharpe remains the tool when the question is whether one specific best strategy + survives its trial count. See the module docstring for the full regime tag and + appraisals/07_verdict.md. + + Parameters + ---------- + sharpe_ratios : np.ndarray + Per-period Sharpe ratios of the screened strategies (the trial set). + number_of_returns : int + The number of return observations T. + method : str, default "holm" + ``"holm"`` (FWER) or ``"bhy"`` (FDR). + significance_level : float, default 0.05 + The error-control level alpha. + + Returns + ------- + dict + ``p_values`` (raw), ``adjusted_p_values``, ``significant`` (boolean discoveries at alpha), and + ``haircut_sharpe_ratios`` (the Sharpe implied by each adjusted p-value). + + Examples + -------- + >>> import numpy as np + >>> from RiskLabAI.backtest import haircut_sharpe_ratios + >>> sharpes = np.array([0.25, 0.18, 0.05, 0.02]) # per-period Sharpes from a 4-strategy search + >>> result = haircut_sharpe_ratios(sharpes, number_of_returns=120, method="holm") + >>> bool(result["significant"][0]) + True + """ + p_values = sharpe_ratio_p_values(sharpe_ratios, number_of_returns) + if method == "holm": + adjusted = holm_adjusted_p_values(p_values) + elif method == "bhy": + adjusted = benjamini_hochberg_yekutieli_adjusted_p_values(p_values) + else: + raise ValueError(f"unknown method {method!r}; use 'holm' or 'bhy'.") + significant = adjusted < significance_level + haircut_t = ss.norm.isf(np.clip(adjusted, 1e-300, 1.0)) + haircut_sr = haircut_t / np.sqrt(number_of_returns) + return { + "p_values": p_values, + "adjusted_p_values": adjusted, + "significant": significant, + "haircut_sharpe_ratios": haircut_sr, + } diff --git a/src/RiskLabAI/backtest/ou_trading_rules.py b/src/RiskLabAI/backtest/ou_trading_rules.py new file mode 100644 index 0000000..7e25287 --- /dev/null +++ b/src/RiskLabAI/backtest/ou_trading_rules.py @@ -0,0 +1,311 @@ +r""" +Closed-form optimal Ornstein-Uhlenbeck trading rules (Lipton-Lopez de Prado 2020). + +de Prado finds optimal profit-take / stop-loss thresholds for a mean-reverting (OU) process by a +Monte-Carlo grid search (AFML ch.13): simulate many OU paths and, for each (profit-take, stop-loss) pair +on a grid, score the realized trade outcomes and pick the grid argmax. That grid carries simulation +noise, costs compute, and is limited by the grid resolution. The Lipton-Lopez de Prado closed form gives +the exact optimum for the OU model from first-passage theory, with no simulation noise and negligible +compute. + +For an OU deviation ``Y`` with ``dY = -theta Y dt + sigma dW`` entered at deviation ``y0 = -entry_gap``, +with an upper barrier ``b = y0 + profit_take`` (gain ``+profit_take``) and a lower barrier +``a = y0 - stop_loss`` (gain ``-stop_loss``), the probability of hitting the upper barrier first is +``u(y0) = [S(y0) - S(a)] / [S(b) - S(a)]`` with scale function ``S(x) = int_0^x exp(theta s^2 / sigma^2) ds`` +(proportional to the imaginary error function), and the mean first-exit time follows the Karlin-Taylor +Green's-function form. The per-trade gain is two-valued, so ``E[gain] = profit_take * u - stop_loss * (1-u) +- cost``, and the objective is the expected net return per unit time ``E[gain] / E[tau]`` (the well-posed +optimal-trading-rule objective; the raw per-trade Sharpe is degenerate, maximized by profit_take -> 0). + +Admitted in Appraisal 23 (CONTRIBUTIONS_LEDGER 2026-06-27). Scope tag, verbatim from the verdict: + + prefer the closed-form OU rule over the Monte-Carlo PT/SL grid whenever the OU model is used - it + gives the exact optimum with no simulation noise at a fraction of the compute; both are only as good + as the OU assumption, so check the OU fit at decision time. + +Held-out confirmed (appraisals/23_results, HELDOUT.md): on the sealed OU corner the closed form agrees +with the Monte-Carlo grid optimum within grid resolution (the grid pick's regret on the exact surface +stays ~0.0002-0.0008 and tightens as the grid refines) at a compute speed-up that grows with resolution +(3.4x -> 16x), and it degrades in step with the grid off-model (no false out-of-model win). The advantage +is exactness and speed under the OU model, not robustness to misspecification, so check the OU fit at +decision time. The de Prado Monte-Carlo PT/SL grid is left unchanged. Evidence: appraisals/23_verdict.md. + +References +---------- +Lipton, A. and Lopez de Prado, M. (2020) A closed-form solution for optimal mean-reverting trading + strategies. SSRN / Risk. +Lopez de Prado, M. (2018) Advances in Financial Machine Learning, ch. 13 (the Monte-Carlo PT/SL grid). +""" + +from __future__ import annotations + +import numpy as np +from scipy.optimize import minimize +from scipy.special import erfi + + +def theta_from_half_life(half_life: float) -> float: + """Continuous mean-reversion speed ``theta`` from a half-life in time units.""" + return float(np.log(2.0) / float(half_life)) + + +def stationary_std(theta: float, sigma: float) -> float: + """Stationary standard deviation of the OU deviation: ``sigma / sqrt(2 theta)``.""" + return float(sigma) / np.sqrt(2.0 * float(theta)) + + +def _scale_relative(x: float, a: float, b: float, c: float) -> float: + """``exp(-cK)(S(x) - S(a))`` with ``K = max(a^2, b^2)`` (overflow-safe), S the OU scale function.""" + big = max(a * a, b * b) + rc = np.sqrt(c) + val = ( + (np.sqrt(np.pi) / (2.0 * rc)) * (erfi(rc * x) - erfi(rc * a)) * np.exp(-c * big) + ) + if np.isfinite(val): + return float(val) + s = np.linspace(a, x, 2000) + return float(np.trapezoid(np.exp(c * (s * s - big)), s)) + + +def hit_upper_probability( + entry_gap: float, profit_take: float, stop_loss: float, theta: float, sigma: float +) -> float: + r""" + Probability the OU deviation hits the upper barrier (profit-take) before the lower (stop-loss). + + Entered at ``y0 = -entry_gap``, barriers ``a = y0 - stop_loss`` and ``b = y0 + profit_take``; + ``u = [S(y0) - S(a)] / [S(b) - S(a)]`` in an overflow-safe max-subtracted form. + """ + c = theta / (sigma * sigma) + y0 = -float(entry_gap) + a, b = y0 - stop_loss, y0 + profit_take + if b <= a: + return float("nan") + num = _scale_relative(y0, a, b, c) + den = _scale_relative(b, a, b, c) + if den <= 0: + return float("nan") + return float(np.clip(num / den, 0.0, 1.0)) + + +def mean_exit_time( + entry_gap: float, profit_take: float, stop_loss: float, theta: float, sigma: float +) -> float: + r""" + Closed-form OU mean first-exit time ``E[tau]`` from the two-barrier interval, via the Karlin-Taylor + Green's function ``v(x) = int_a^b G(x, y) m(y) dy`` (speed density ``m``), evaluated by quadrature. + """ + c = theta / (sigma * sigma) + y0 = -float(entry_gap) + a, b = y0 - stop_loss, y0 + profit_take + if b <= a: + return float("nan") + rc = np.sqrt(c) + pref = np.sqrt(np.pi) / (2.0 * rc) + ys = np.linspace(a, b, 400) + erfi_y, erfi_a, erfi_b, erfi_y0 = ( + erfi(rc * ys), + float(erfi(rc * a)), + float(erfi(rc * b)), + float(erfi(rc * y0)), + ) + if not (np.all(np.isfinite(erfi_y)) and np.isfinite(erfi_b)): + return _mean_exit_time_quadrature( + entry_gap, profit_take, stop_loss, theta, sigma + ) + s_ya = pref * (erfi_y - erfi_a) + s_by = pref * (erfi_b - erfi_y) + s_ba = pref * (erfi_b - erfi_a) + s_y0a = pref * (erfi_y0 - erfi_a) + s_by0 = pref * (erfi_b - erfi_y0) + green = np.where(ys <= y0, s_ya * s_by0, s_y0a * s_by) / s_ba + speed = (2.0 / (sigma * sigma)) * np.exp(-c * ys * ys) + return float(np.trapezoid(green * speed, ys)) + + +def _mean_exit_time_quadrature( + entry_gap: float, profit_take: float, stop_loss: float, theta: float, sigma: float +) -> float: + """Overflow-safe ``E[tau]`` for very wide barriers (max-subtracted scale-function quadrature).""" + c = theta / (sigma * sigma) + y0 = -float(entry_gap) + a, b = y0 - stop_loss, y0 + profit_take + big = max(a * a, b * b) + + def s_hat(x): + s = np.linspace(a, x, 3000) + return np.trapezoid(np.exp(c * (s * s - big)), s) + + s_hat_b, s_hat_y0 = s_hat(b), s_hat(y0) + ys = np.linspace(a, b, 3000) + s_ya = np.array([s_hat(y) for y in ys]) + s_by = s_hat_b - s_ya + green = np.where(ys <= y0, s_ya * (s_hat_b - s_hat_y0), s_hat_y0 * s_by) / s_hat_b + speed = (2.0 / (sigma * sigma)) * np.exp(c * (big - ys * ys)) + return float(np.trapezoid(green * speed, ys)) + + +def ou_rule_metrics( + profit_take: float, + stop_loss: float, + theta: float, + sigma: float, + entry_gap: float, + cost: float = 0.0, +) -> dict: + r""" + Exact OU per-trade metrics for a rule entered ``entry_gap`` from the mean (all thresholds in the same + units as the OU deviation; time in model time units). + + Returns ``hit_probability`` (``u``), ``expected_gain`` (net mean per-trade gain), ``std_gain``, + ``expected_holding_time`` (``E[tau]``), ``return_rate`` (``expected_gain / E[tau]``, the optimization + objective), and ``time_scaled_sharpe`` (``expected_gain / (std_gain sqrt(E[tau]))``, a secondary + metric that is degenerate as ``u -> 1``). + """ + bad = dict( + hit_probability=np.nan, + expected_gain=-np.inf, + std_gain=np.nan, + expected_holding_time=np.nan, + return_rate=-np.inf, + time_scaled_sharpe=-np.inf, + ) + if profit_take <= 0 or stop_loss <= 0: + return bad + u = hit_upper_probability(entry_gap, profit_take, stop_loss, theta, sigma) + if not np.isfinite(u): + return bad + q = 1.0 - u + expected_gain = profit_take * u - stop_loss * q - cost + std_gain = (profit_take + stop_loss) * np.sqrt(max(u * q, 0.0)) + e_tau = mean_exit_time(entry_gap, profit_take, stop_loss, theta, sigma) + if not np.isfinite(e_tau) or e_tau <= 0: + return bad + return dict( + hit_probability=float(u), + expected_gain=float(expected_gain), + std_gain=float(std_gain), + expected_holding_time=float(e_tau), + return_rate=float(expected_gain / e_tau), + time_scaled_sharpe=( + float(expected_gain / (std_gain * np.sqrt(e_tau))) + if std_gain > 0 + else -np.inf + ), + ) + + +def optimal_ou_trading_rule( + theta: float, + sigma: float, + entry_gap: float, + cost: float = 0.0, + bounds: tuple = (0.25, 4.0), +) -> dict: + r""" + Exact optimal OU profit-take / stop-loss by maximizing the closed-form expected net return per unit + time (no simulation). + + Multi-start L-BFGS-B on the negative ``return_rate`` objective over the ``(profit_take, stop_loss)`` + box. Prefer this over the Monte-Carlo PT/SL grid whenever the OU model is used: it gives the exact + optimum with no simulation noise at a fraction of the compute. Both are only as good as the OU + assumption, so check the OU fit at decision time (see :func:`fit_ornstein_uhlenbeck` and the module + docstring for the verbatim scope tag and appraisals/23_verdict.md). + + Parameters + ---------- + theta : float + OU mean-reversion speed. + sigma : float + OU instantaneous volatility. + entry_gap : float + Deviation from the mean at entry (the trade is opened ``entry_gap`` below the mean). + cost : float, default 0.0 + Per-trade transaction cost in the same units as the thresholds. + bounds : tuple, default (0.25, 4.0) + ``(low, high)`` box for both the profit-take and the stop-loss. + + Returns + ------- + dict + ``profit_take``, ``stop_loss``, ``return_rate`` (the maximized objective), and the full + :func:`ou_rule_metrics` of the optimum. + + Examples + -------- + >>> rule = optimal_ou_trading_rule(theta=0.1, sigma=0.1, entry_gap=1.0) + >>> rule["profit_take"] > 0 and rule["stop_loss"] > 0 + True + """ + lo, hi = bounds + + def negative_objective(x): + v = ou_rule_metrics(x[0], x[1], theta, sigma, entry_gap, cost)["return_rate"] + return -v if np.isfinite(v) else 1e9 + + span = hi - lo + starts = [ + (lo + 0.15 * span, lo + 0.2 * span), + (lo + 0.1 * span, lo + 0.6 * span), + (lo + 0.3 * span, lo + 0.4 * span), + ] + best = None + for start in starts: + res = minimize( + negative_objective, x0=start, method="L-BFGS-B", bounds=[(lo, hi), (lo, hi)] + ) + if best is None or res.fun < best.fun: + best = res + profit_take, stop_loss = float(best.x[0]), float(best.x[1]) + metrics = ou_rule_metrics(profit_take, stop_loss, theta, sigma, entry_gap, cost) + return { + "profit_take": profit_take, + "stop_loss": stop_loss, + "return_rate": -float(best.fun), + **metrics, + } + + +def fit_ornstein_uhlenbeck(series: np.ndarray, dt: float = 1.0) -> dict: + r""" + Fit an OU process by AR(1) regression ``x_t = a + b x_{t-1} + e`` (the OU goodness-of-fit check). + + Returns ``theta`` (mean-reversion speed), ``sigma`` (per-unit-time noise), ``mu`` (long-run mean), + ``rho`` (AR(1) coefficient), ``half_life``, ``stationary_std``, and ``r2`` (the AR(1) fit quality, a + simple OU goodness-of-fit proxy). Check ``r2`` / the half-life before trusting the closed-form rule: + both the closed form and the Monte-Carlo grid are only as good as the OU assumption. + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> x = np.zeros(2000) + >>> for t in range(1, 2000): + ... x[t] = 0.9 * x[t - 1] + rng.standard_normal() * 0.1 + >>> fit = fit_ornstein_uhlenbeck(x) + >>> fit["theta"] > 0 and 0.0 <= fit["r2"] <= 1.0 + True + """ + x = np.asarray(series, dtype=float) + x = x[np.isfinite(x)] + x0, x1 = x[:-1], x[1:] + design = np.vstack([np.ones_like(x0), x0]).T + coef, *_ = np.linalg.lstsq(design, x1, rcond=None) + a, b = float(coef[0]), float(coef[1]) + residual = x1 - (a + b * x0) + b = min(max(b, 1e-6), 0.999999) + theta = -np.log(b) / dt + mu = a / (1.0 - b) + var_eps = float(np.var(residual, ddof=2)) + sigma = np.sqrt(var_eps * 2.0 * theta / (1.0 - b * b)) + ss_tot = float(np.var(x1, ddof=1)) * (len(x1) - 1) + r2 = 1.0 - float(np.sum(residual**2)) / ss_tot if ss_tot > 0 else 0.0 + return dict( + theta=float(theta), + sigma=float(sigma), + mu=float(mu), + rho=float(b), + half_life=float(np.log(2.0) / theta), + stationary_std=stationary_std(theta, sigma), + r2=float(r2), + ) diff --git a/src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py b/src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py new file mode 100644 index 0000000..6f644a1 --- /dev/null +++ b/src/RiskLabAI/backtest/probabilistic_sharpe_ratio.py @@ -0,0 +1,144 @@ +""" +Implements the Probabilistic Sharpe Ratio (PSR) and related metrics +as described by Marcos Lopez de Prado. + +## TODO: +- [ ] Add a `compute_psr_curve` helper function (as seen in the + original notebook) that iterates `probabilistic_sharpe_ratio` + over a range of `observed_sharpe_ratio` values to + easily plot the PSR curve. +""" + +import numpy as np +from scipy import stats as ss + + +def probabilistic_sharpe_ratio( + observed_sharpe_ratio: float, + benchmark_sharpe_ratio: float, + number_of_returns: int, + skewness_of_returns: float = 0.0, + kurtosis_of_returns: float = 3.0, + return_test_statistic: bool = False, +) -> float: + r""" + Calculate the Probabilistic Sharpe Ratio (PSR). + + The PSR estimates the probability that the observed Sharpe ratio (SR) + is greater than a benchmark SR (e.g., the expected maximum SR from + multiple trials) given the track record length and return moments. + + The test statistic is: + .. math:: + Z = \frac{(\hat{SR} - SR^*) \sqrt{T-1}} + {\sqrt{1 - S \hat{SR} + \frac{K-1}{4} \hat{SR}^2}} + + PSR = N(Z) + + Where: + - \(\hat{SR}\) is the observed Sharpe ratio + - \(SR^*\) is the benchmark Sharpe ratio + - \(T\) is the number of returns + - \(S\) is the skewness of returns + - \(K\) is the kurtosis of returns + + Parameters + ---------- + observed_sharpe_ratio : float + The observed Sharpe ratio of the strategy. + benchmark_sharpe_ratio : float + The benchmark Sharpe ratio (e.g., E[max(SR)]). + number_of_returns : int + The number of return observations (T). + skewness_of_returns : float, default=0.0 + The skewness of the returns (S). + kurtosis_of_returns : float, default=3.0 + The kurtosis of the returns (K). Assumes 3 for normal. + return_test_statistic : bool, default=False + If True, return the Z-statistic instead of the CDF value (PSR). + + Returns + ------- + float + The Probabilistic Sharpe Ratio (if `return_test_statistic` is False) + or the Z test statistic (if True). + + Example + ------- + >>> psr = probabilistic_sharpe_ratio( + ... observed_sharpe_ratio=2.5, + ... benchmark_sharpe_ratio=1.0, + ... number_of_returns=252, + ... skewness_of_returns=-0.5, + ... kurtosis_of_returns=4.0 + ... ) + >>> print(f"PSR: {psr:.2f}") + PSR: 1.00 + """ + denominator = ( + 1 + - skewness_of_returns * observed_sharpe_ratio + + (kurtosis_of_returns - 1) / 4 * observed_sharpe_ratio**2 + ) + + # Handle cases where denominator is non-positive due to extreme inputs + if denominator <= 0: + return 0.0 if not return_test_statistic else -np.inf + + test_statistic = ( + (observed_sharpe_ratio - benchmark_sharpe_ratio) + * np.sqrt(number_of_returns - 1) + ) / np.sqrt(denominator) + + if return_test_statistic: + return test_statistic + + return ss.norm.cdf(test_statistic) + + +def benchmark_sharpe_ratio(sharpe_ratio_estimates: list[float]) -> float: + r""" + Calculate the expected maximum Sharpe Ratio (Benchmark SR). + + This is used as the benchmark SR* in the PSR calculation. It represents + the expected maximum SR one would observe from N independent trials. + + .. math:: + SR^* = \sigma_{SR} \left[ (1 - \gamma) \Phi^{-1}(1 - \frac{1}{N}) + + \gamma \Phi^{-1}(1 - \frac{1}{N} e^{-1}) \right] + + Where: + - \(\sigma_{SR}\) is the standard deviation of SR estimates + - \(\gamma\) is the Euler-Mascheroni constant + - \(\Phi^{-1}\) is the inverse CDF of a standard normal distribution + - \(N\) is the number of SR estimates (trials) + + Parameters + ---------- + sharpe_ratio_estimates : List[float] + A list or array of Sharpe ratio estimates from N different trials. + + Returns + ------- + float + The Benchmark Sharpe Ratio (E[max(SR)]). + + Example + ------- + >>> sr_list = [0.5, 1.2, -0.3, 0.8, 1.5, 0.9] + >>> bsr = benchmark_sharpe_ratio(sr_list) + >>> print(f"Benchmark SR: {bsr:.2f}") + Benchmark SR: 1.03 + """ + n_estimates = len(sharpe_ratio_estimates) + if n_estimates <= 1: + return np.mean(sharpe_ratio_estimates) if n_estimates == 1 else 0.0 + + standard_deviation = np.array(sharpe_ratio_estimates).std() + + term1 = (1 - np.euler_gamma) * ss.norm.ppf(1 - 1 / n_estimates) + term2 = np.euler_gamma * ss.norm.ppf(1 - 1 / (n_estimates * np.e)) + + benchmark_value = standard_deviation * (term1 + term2) + + return benchmark_value diff --git a/src/RiskLabAI/backtest/probability_of_backtest_overfitting.py b/src/RiskLabAI/backtest/probability_of_backtest_overfitting.py new file mode 100644 index 0000000..9ccb6d4 --- /dev/null +++ b/src/RiskLabAI/backtest/probability_of_backtest_overfitting.py @@ -0,0 +1,156 @@ +""" +Implements the Probability of Backtest Overfitting (PBO) calculation. + +## TODO: +- [ ] Add a `get_pbo` wrapper function (as seen in the + original notebook) that simplifies the call to + `probability_of_backtest_overfitting` and returns + only the PBO value. +- [ ] Add a `pbo_overfitting_plot` helper function to + visualize the logit distribution (as seen in the + original notebook). +""" + +from itertools import combinations +from typing import Callable, Optional + +import numpy as np +from joblib import Parallel, delayed + +from .backtest_statistics import sharpe_ratio + + +def performance_evaluation( + train_partition: np.ndarray, + test_partition: np.ndarray, + n_strategies: int, + metric: Callable[[np.ndarray, float], float], + risk_free_return: float, +) -> tuple[bool, float]: + """ + Evaluate strategy performance on train/test splits. + + This function finds the best strategy on the training partition and + then ranks its performance on the test partition. + + Parameters + ---------- + train_partition : np.ndarray + Training data partition (T_train x N_strategies). + test_partition : np.ndarray + Testing data partition (T_test x N_strategies). + n_strategies : int + Number of strategies (N). + metric : Callable + Metric function (e.g., `sharpe_ratio`). + risk_free_return : float + Risk-free return for the metric. + + Returns + ------- + Tuple[bool, float] + - is_overfit: True if the best in-sample strategy is not + in the top half of out-of-sample strategies. + - logit_value: The logit-transformed relative rank. + """ + # 1. Find best strategy on training data + evaluate_train = [ + metric(train_partition[:, i], risk_free_return) for i in range(n_strategies) + ] + best_strategy_idx = np.argmax(evaluate_train) + + # 2. Evaluate all strategies on test data + evaluate_test = [ + metric(test_partition[:, i], risk_free_return) for i in range(n_strategies) + ] + + # 3. Find rank of the best_strategy in the test set + # Sort test evaluations and find where best_strategy_idx lands + ranks = np.argsort(np.argsort(evaluate_test)) + rank_of_best_is_strategy = ranks[best_strategy_idx] + 1 # 1-based rank + + # 4. Calculate relative rank (omega_bar) and logit + w_bar = rank_of_best_is_strategy / (n_strategies + 1) + logit_value = np.log(w_bar / (1 - w_bar)) + + # Overfit if logit is <= 0 (i.e., rank is in bottom half) + return logit_value <= 0.0, logit_value + + +def probability_of_backtest_overfitting( + performances: np.ndarray, + n_partitions: int = 16, + risk_free_return: float = 0.0, + metric: Optional[Callable[[np.ndarray, float], float]] = None, + n_jobs: int = 1, +) -> tuple[float, np.ndarray]: + r""" + Compute the Probability of Backtest Overfitting (PBO). + + PBO is the frequency with which the best in-sample (IS) strategy + underperforms relative to the median out-of-sample (OOS) performance. + + It uses combinatorial splits of the data into train/test partitions. + Total combinations = C(S, S/2), where S = n_partitions. + + .. math:: + \left(\begin{array}{c} + S \\ + S / 2 + \end{array}\right) = \prod_{i=0}^{S / 2^{-1}} \frac{S-i}{S / 2-i} + + Parameters + ---------- + performances : np.ndarray + Matrix of T observations x N strategies. + n_partitions : int, default=16 + Number of partitions (S). Must be even. + risk_free_return : float, default=0.0 + Risk-free return for calculating the metric (e.g., Sharpe ratio). + metric : Callable, optional + Metric function (e.g., `sharpe_ratio`). Defaults to the + Numba-jitted `sharpe_ratio` in this module. + n_jobs : int, default=1 + Number of parallel jobs for `joblib`. + + Returns + ------- + Tuple[float, np.ndarray] + - pbo: The Probability of Backtest Overfitting (0 to 1). + - logit_values: Array of logit values from all combinations. + """ + if n_partitions % 2 != 0: + raise ValueError("Number of partitions must be even.") + + if metric is None: + metric = sharpe_ratio + + _, n_strategies = performances.shape + partitions = np.array_split(performances, n_partitions) + partition_indices = list(range(n_partitions)) + + # Get all combinations of training partition indices + partition_combinations_indices = list( + combinations(partition_indices, n_partitions // 2) + ) + + results = Parallel(n_jobs=n_jobs)( + delayed(performance_evaluation)( + np.concatenate([partitions[i] for i in train_indices], axis=0), + np.concatenate( + [partitions[i] for i in partition_indices if i not in train_indices], + axis=0, + ), + n_strategies, + metric, + risk_free_return, + ) + for train_indices in partition_combinations_indices + ) + + results_arr = np.array(results) + + pbo = results_arr[:, 0].mean() + logit_values = results_arr[:, 1] + + return pbo, logit_values diff --git a/src/RiskLabAI/backtest/robust_statistics.py b/src/RiskLabAI/backtest/robust_statistics.py new file mode 100644 index 0000000..89c224b --- /dev/null +++ b/src/RiskLabAI/backtest/robust_statistics.py @@ -0,0 +1,289 @@ +r""" +Robust backtest statistics: Conditional Expected Drawdown (CED) and the Ledoit-Wolf Sharpe-difference test. + +de Prado reports maximum drawdown and time-under-water and compares Sharpe ratios directly +(`backtest_statistics.py`). Maximum drawdown is a single extreme order statistic - high variance, not +coherent, not attributable - and the naive Sharpe-difference comparison assumes i.i.d. returns, so it +over-rejects under the serial dependence real returns carry. Two admitted upgrades sit alongside those +baselines: + +- **Conditional Expected Drawdown (CED, Goldberg-Mahmoud 2017)**: the tail mean (CVaR) of the + maximum-drawdown distribution at a fixed horizon, a lower-variance, coherent, factor-attributable + drawdown-risk measure; and +- **the Ledoit-Wolf (2008) bootstrap Sharpe-difference test**: a HAC-studentized statistic calibrated by + a studentized circular block bootstrap, which holds nominal size under serial dependence where the + naive test inflates. + +Admitted in Appraisal 22 (CONTRIBUTIONS_LEDGER 2026-06-27). Scope tags, verbatim from the verdict: + + CED: prefer CED over max-drawdown as a drawdown-risk statistic: lower estimator variance and better + ranking of true drawdown risk, most of all on short/heavy-tailed tracks; converges to max-DD on + benign returns; factor-attributable. + + Ledoit-Wolf Sharpe-difference test: prefer it over the naive test when comparing two Sharpes under + serial dependence / heavy tails (it holds nominal size where the naive test inflates ~3x); converges + on i.i.d. returns. + +Held-out confirmed (appraisals/22_results, HELDOUT.md): on the sealed short-sample x heavy-tail corner CED +has lower estimator variance (CV 0.30-0.32 vs max-drawdown 0.40-0.42) and better rank-recovery (Spearman +0.42-0.78 vs 0.31-0.69); the LW test holds nominal size (0.056) under AR(0.3) where the naive test rejects +0.161 (~3x), and converges on i.i.d. returns. CDaR@0.95 was NOT admitted (no variance/rank advantage over +max-drawdown as a descriptive statistic). Evidence and caveats: appraisals/22_verdict.md. + +References +---------- +Goldberg, L. R. and Mahmoud, O. (2017) Drawdown: from practice to theory and back again. + Quantitative Finance, 17(5), 745-761. (Conditional Expected Drawdown.) +Ledoit, O. and Wolf, M. (2008) Robust performance hypothesis testing with the Sharpe ratio. + Journal of Empirical Finance, 15(5), 850-859. +""" + +from __future__ import annotations + +from math import erfc, sqrt + +import numpy as np + + +# -------------------------------------------------------------------------------------------------- +# Conditional Expected Drawdown +# -------------------------------------------------------------------------------------------------- +def _wealth_from_returns(returns: np.ndarray) -> np.ndarray: + return np.cumprod(1.0 + np.asarray(returns, dtype=float)) + + +def _drawdown_series(wealth: np.ndarray) -> np.ndarray: + w = np.asarray(wealth, dtype=float) + return 1.0 - w / np.maximum.accumulate(w) + + +def _cvar_upper(values: np.ndarray, alpha: float) -> float: + """Empirical upper-tail CVaR: the mean of the worst ``1 - alpha`` fraction of ``values``.""" + v = np.sort(np.asarray(values, dtype=float)) + n = v.size + if n == 0: + return 0.0 + k = (1.0 - alpha) * n + if k <= 0: + return float(v[-1]) + n_full = int(np.floor(k)) + total = v[n - n_full :].sum() if n_full > 0 else 0.0 + frac = k - n_full + if frac > 0 and n - n_full - 1 >= 0: + total += frac * v[n - n_full - 1] + return float(total / k) + + +def _rolling_max_drawdowns(wealth: np.ndarray, horizon: int) -> np.ndarray: + w = np.asarray(wealth, dtype=float) + n = w.size + if n < horizon or horizon < 2: + d = _drawdown_series(w) + return np.array([float(d.max()) if d.size else 0.0], dtype=float) + windows = np.lib.stride_tricks.sliding_window_view(w, horizon) + high_water = np.maximum.accumulate(windows, axis=1) + return (1.0 - windows / high_water).max(axis=1) + + +def conditional_expected_drawdown( + returns: np.ndarray, + horizon: int, + alpha: float = 0.90, +) -> float: + r""" + Conditional Expected Drawdown (CED; Goldberg-Mahmoud 2017): the CVaR of the maximum-drawdown + distribution at a fixed horizon. + + From a single return track it is estimated by the upper-tail (CVaR at level ``alpha``) mean of the + maximum drawdowns within overlapping windows of length ``horizon``. Because it is a tail MEAN of the + max-drawdown distribution rather than the single most-extreme realization, it has lower estimator + variance and ranks true drawdown risk better than max-drawdown, especially on short / heavy-tailed + tracks, and it converges to agree with max-drawdown on benign returns. It is coherent and + factor-attributable. + + Prefer CED over max-drawdown as a drawdown-risk statistic (see the module docstring for the full + verbatim scope tag and appraisals/22_verdict.md). The de Prado max-drawdown / time-under-water + statistics are unchanged. + + Parameters + ---------- + returns : np.ndarray + Per-period simple returns of the track. + horizon : int + The rolling-window length (the drawdown-measurement horizon, in periods). + alpha : float, default 0.90 + The CVaR tail level (the mean is taken over the worst ``1 - alpha`` fraction of window maxima). + + Returns + ------- + float + The conditional expected drawdown in ``[0, 1)``. + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> r = rng.standard_normal(2000) * 0.01 + >>> ced = conditional_expected_drawdown(r, horizon=60) + >>> 0.0 <= ced < 1.0 + True + """ + if not 0.0 < alpha < 1.0: + raise ValueError("alpha must be in (0, 1)") + wealth = _wealth_from_returns(returns) + return _cvar_upper(_rolling_max_drawdowns(wealth, int(horizon)), alpha) + + +# -------------------------------------------------------------------------------------------------- +# Sharpe-difference tests +# -------------------------------------------------------------------------------------------------- +def _sharpe_influence(r_a: np.ndarray, r_b: np.ndarray): + """Influence-function series of the Sharpe difference SR_A - SR_B and the point estimate.""" + a = np.asarray(r_a, dtype=float) + b = np.asarray(r_b, dtype=float) + mu_a, mu_b = a.mean(), b.mean() + g_a, g_b = (a * a).mean(), (b * b).mean() + s_a = np.sqrt(max(g_a - mu_a * mu_a, 1e-300)) + s_b = np.sqrt(max(g_b - mu_b * mu_b, 1e-300)) + delta = mu_a / s_a - mu_b / s_b + ga0, ga1 = g_a / s_a**3, -mu_a / (2.0 * s_a**3) + gb0, gb1 = g_b / s_b**3, -mu_b / (2.0 * s_b**3) + influence = ( + ga0 * (a - mu_a) + + ga1 * (a * a - g_a) + - (gb0 * (b - mu_b) + gb1 * (b * b - g_b)) + ) + return float(delta), influence + + +def _bartlett_hac_var(influence: np.ndarray, bandwidth: int) -> float: + """Bartlett-kernel HAC variance of the mean of the influence series (= Var(delta_hat)).""" + x = np.asarray(influence, dtype=float) + x = x - x.mean() + t = x.size + if t < 2: + return float("nan") + s = np.dot(x, x) / t + h = max(0, int(bandwidth)) + for j in range(1, h + 1): + if j >= t: + break + s += 2.0 * (1.0 - j / (h + 1.0)) * (np.dot(x[j:], x[:-j]) / t) + return float(max(s, 1e-300) / t) + + +def _iid_var(influence: np.ndarray) -> float: + x = np.asarray(influence, dtype=float) + t = x.size + if t < 2: + return float("nan") + return float(max(x.var(), 1e-300) / t) + + +def _circular_block_indices( + t: int, block_len: int, rng: np.random.Generator +) -> np.ndarray: + n_blocks = int(np.ceil(t / block_len)) + starts = rng.integers(0, t, size=n_blocks) + offsets = np.arange(block_len) + return ((starts[:, None] + offsets[None, :]) % t).reshape(-1)[:t] + + +def sharpe_difference_test( + returns_a: np.ndarray, + returns_b: np.ndarray, + method: str = "ledoit_wolf", + block_length: int | None = None, + n_boot: int = 1000, + bandwidth: int | None = None, + random_state: int | None = 0, +) -> dict: + r""" + Test H0: SR_A = SR_B (equal Sharpe ratios), naive or Ledoit-Wolf robust. + + ``method="naive"`` is the influence-function delta-method z-test using the i.i.d. (lag-0) variance of + the Sharpe-difference influence function and the normal cutoff - it ignores serial dependence and + over-rejects under it. ``method="ledoit_wolf"`` (Ledoit-Wolf 2008) studentizes the same Sharpe + difference by a Bartlett-kernel HAC standard error and calibrates the two-sided p-value by a + studentized circular block bootstrap that preserves serial dependence, so it holds nominal size where + the naive test inflates (~3x under AR), and converges to the naive test on i.i.d. returns. + + Prefer the Ledoit-Wolf test when comparing two Sharpes under serial dependence / heavy tails (see the + module docstring for the full verbatim scope tag and appraisals/22_verdict.md). The Sharpe ratio uses + the population standard deviation, matching ``backtest_statistics.sharpe_ratio``. + + Parameters + ---------- + returns_a, returns_b : np.ndarray + The two equal-length per-period return series being compared. + method : {"ledoit_wolf", "naive"}, default "ledoit_wolf" + The test variant. + block_length : int, optional + Circular-block length for the LW bootstrap (also the default HAC bandwidth). Defaults to + ``ceil(T ** (1/3))``. Ignored for ``method="naive"``. + n_boot : int, default 1000 + Bootstrap resamples for the LW p-value. Ignored for ``method="naive"``. + bandwidth : int, optional + HAC (Bartlett) bandwidth; defaults to ``block_length``. Ignored for ``method="naive"``. + random_state : int, optional + Seed for the LW bootstrap. + + Returns + ------- + dict + ``delta`` (SR_A - SR_B), ``se``, ``stat``, ``pvalue``, and ``reject`` (at the 5% level). + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> a = rng.standard_normal(1000) * 0.01 + 0.001 + >>> b = rng.standard_normal(1000) * 0.01 + >>> res = sharpe_difference_test(a, b, method="ledoit_wolf", n_boot=200) + >>> set(res) == {"delta", "se", "stat", "pvalue", "reject"} + True + """ + if method not in ("ledoit_wolf", "naive"): + raise ValueError("method must be 'ledoit_wolf' or 'naive'") + a = np.asarray(returns_a, dtype=float) + b = np.asarray(returns_b, dtype=float) + if a.size != b.size: + raise ValueError("returns_a and returns_b must have the same length") + delta, influence = _sharpe_influence(a, b) + + if method == "naive": + se = np.sqrt(_iid_var(influence)) + z = delta / se if se > 0 else 0.0 + pvalue = float(erfc(abs(z) / sqrt(2.0))) + return { + "delta": delta, + "se": float(se), + "stat": float(z), + "pvalue": pvalue, + "reject": bool(pvalue < 0.05), + } + + t = a.size + if block_length is None: + block_length = max(1, int(np.ceil(t ** (1.0 / 3.0)))) + bw = block_length if bandwidth is None else bandwidth + se_hat = np.sqrt(_bartlett_hac_var(influence, bw)) + z_hat = abs(delta / se_hat) if se_hat > 0 else 0.0 + rng = np.random.default_rng(random_state) + count = 0 + for _ in range(n_boot): + idx = _circular_block_indices(t, block_length, rng) + delta_b, influence_b = _sharpe_influence(a[idx], b[idx]) + se_b = np.sqrt(_bartlett_hac_var(influence_b, bw)) + if se_b <= 0: + continue + if abs((delta_b - delta) / se_b) >= z_hat: + count += 1 + pvalue = (count + 1.0) / (n_boot + 1.0) + return { + "delta": delta, + "se": float(se_hat), + "stat": float(z_hat), + "pvalue": float(pvalue), + "reject": bool(pvalue < 0.05), + } diff --git a/src/RiskLabAI/backtest/sharpe_inference.py b/src/RiskLabAI/backtest/sharpe_inference.py new file mode 100644 index 0000000..cdbdbb5 --- /dev/null +++ b/src/RiskLabAI/backtest/sharpe_inference.py @@ -0,0 +1,200 @@ +r""" +Sharpe-ratio inference under non-normal, autocorrelated returns (Lopez de Prado-Lipton-Zoonekynd 2025). + +The sampling variance of the Sharpe estimator depends on the higher moments AND the serial correlation +of returns. de Prado's Probabilistic Sharpe Ratio (`probabilistic_sharpe_ratio`) corrects the higher +moments but assumes serial independence, so under autocorrelation its confidence interval is too narrow +and its test over-rejects. The LPLZ (2025) method takes the heteroskedasticity-and-autocorrelation- +consistent (HAC, Newey-West) long-run variance of the Sharpe influence function, correcting for both at +once. + +The Sharpe influence function is + +.. math:: + + \mathrm{IF}_t = z_t - \tfrac{1}{2}\,\widehat{SR}\,(z_t^2 - 1), \qquad z_t = (r_t - \mu)/\sigma, + +and the asymptotic variance of :math:`\sqrt{T}(\widehat{SR} - SR)` is the long-run variance of +:math:`\mathrm{IF}_t`. Under i.i.d. returns this long-run variance equals +:math:`1 - S\,\widehat{SR} + \tfrac{K-1}{4}\widehat{SR}^2` (exactly the PSR denominator), so the method +converges to the PSR when there is no serial correlation. + +Admitted in Appraisal 08 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tag, verbatim from the verdict: + + Prefer the LPLZ (2025) Sharpe inference - a HAC of the Sharpe influence function - when returns + show material serial correlation and/or non-normality (estimable from the sample): it restores + near-nominal CI coverage and test size where the PSR under-covers and over-rejects (PSR size ~ 0.20 + vs nominal 0.05 under AR(1)). It converges to the PSR on near-normal i.i.d. returns with no + over-coverage. Caveats: mild small-sample over-rejection at short tracks, coverage restored toward + (not fully to) nominal at strong autocorrelation, and wider CIs - the honest power cost. Lo (2002) + is the autocorrelation-only intermediate, dominated by LPLZ under non-normality. + +Evidence and caveats: appraisals/08_verdict.md. + +References +---------- +Lopez de Prado, M., Lipton, A. and Zoonekynd, V. (2025) Sharpe-ratio inference under non-normal, + serially-correlated returns. (Open code at github.com/zoonek/2025-sharpe-ratio - NOT consulted or + vendored; this is reimplemented clean-room from the influence-function / HAC math, GOVERNANCE 3.) +Newey, W. K. and West, K. D. (1987) A simple, positive semi-definite, heteroskedasticity and + autocorrelation consistent covariance matrix. Econometrica, 55(3), 703-708. +Lo, A. W. (2002) The statistics of Sharpe ratios. Financial Analysts Journal, 58(4), 36-52. +""" + +from __future__ import annotations + +from typing import Optional + +import numpy as np +from scipy import stats as ss + + +def _sharpe(returns: np.ndarray) -> float: + r = np.asarray(returns, dtype=float) + sd = r.std(ddof=1) + return float(r.mean() / sd) if sd > 0 else 0.0 + + +def sharpe_ratio_influence_function(returns: np.ndarray) -> np.ndarray: + r""" + The Sharpe-ratio influence function :math:`\mathrm{IF}_t = z_t - \tfrac{1}{2}SR(z_t^2 - 1)`. + + Parameters + ---------- + returns : np.ndarray + The return series. + + Returns + ------- + np.ndarray + The per-observation influence function (mean ~ 0). + """ + r = np.asarray(returns, dtype=float) + mu = r.mean() + sigma = r.std(ddof=1) + if sigma == 0: + return np.zeros_like(r) + z = (r - mu) / sigma + sr = mu / sigma + return z - 0.5 * sr * (z**2 - 1.0) + + +def newey_west_long_run_variance(series: np.ndarray, lag: int) -> float: + r""" + Newey-West (Bartlett-kernel) long-run variance of a series. + + .. math:: + \widehat{\Omega} = \gamma_0 + 2 \sum_{k=1}^{L} \left(1 - \frac{k}{L+1}\right) \gamma_k, + + where :math:`\gamma_k` is the lag-k autocovariance. With ``lag == 0`` this is the sample variance. + + Parameters + ---------- + series : np.ndarray + The series whose long-run variance is estimated. + lag : int + The HAC bandwidth L (number of Bartlett lags). + + Returns + ------- + float + The estimated long-run variance (floored at a small positive number). + """ + x = np.asarray(series, dtype=float) + x = x - x.mean() + t = x.size + if t == 0: + return 1e-12 + total = float(x @ x) / t + for k in range(1, lag + 1): + weight = 1.0 - k / (lag + 1.0) + total += 2.0 * weight * float(x[k:] @ x[:-k]) / t + return max(total, 1e-12) + + +def newey_west_automatic_lag(number_of_returns: int) -> int: + r"""Newey-West automatic bandwidth :math:`\lfloor 4 (T/100)^{2/9} \rfloor` (at least 1).""" + return max(int(np.floor(4.0 * (number_of_returns / 100.0) ** (2.0 / 9.0))), 1) + + +def lplz_sharpe_inference( + returns: np.ndarray, + confidence_level: float = 0.95, + lag: Optional[int] = None, + null_sharpe_ratio: float = 0.0, +) -> dict[str, float]: + r""" + Lopez de Prado-Lipton-Zoonekynd (2025) Sharpe-ratio inference (HAC of the influence function). + + Returns the per-period Sharpe ratio, its autocorrelation-and-non-normality-robust standard error, + a confidence interval, and the test of ``null_sharpe_ratio``. The standard error is + :math:`\sqrt{\widehat{\Omega}/T}` where :math:`\widehat{\Omega}` is the Newey-West long-run variance + of the Sharpe influence function. + + Prefer this over the PSR when returns show material serial correlation and/or non-normality + (estimable from the sample): it restores near-nominal CI coverage and test size where the PSR + under-covers and over-rejects, and converges to the PSR on near-normal i.i.d. returns with no + over-coverage. The honest cost is a wider CI under autocorrelation. See the module docstring for the + full regime tag and appraisals/08_verdict.md. + + Parameters + ---------- + returns : np.ndarray + The strategy return series. + confidence_level : float, default 0.95 + The CI confidence level. + lag : int, optional + The HAC (Newey-West) bandwidth. Defaults to :func:`newey_west_automatic_lag`. + null_sharpe_ratio : float, default 0.0 + The Sharpe under the null hypothesis tested. + + Returns + ------- + dict + ``sharpe_ratio``, ``standard_error``, ``confidence_interval`` (a (lo, hi) tuple), + ``test_statistic``, ``p_value`` (two-sided), ``significant`` (reject the null at the level), + and ``lag`` (the bandwidth used). + + Examples + -------- + >>> import numpy as np + >>> from RiskLabAI.backtest import lplz_sharpe_inference + >>> rng = np.random.default_rng(0) + >>> result = lplz_sharpe_inference(0.1 + rng.standard_normal(240)) + >>> result["standard_error"] > 0 + True + """ + r = np.asarray(returns, dtype=float) + t = r.size + sr = _sharpe(r) + if t < 3 or r.std(ddof=1) == 0: + return { + "sharpe_ratio": sr, + "standard_error": float("nan"), + "confidence_interval": (float("nan"), float("nan")), + "test_statistic": float("nan"), + "p_value": float("nan"), + "significant": False, + "lag": 0, + } + if lag is None: + lag = newey_west_automatic_lag(t) + lrv = newey_west_long_run_variance(sharpe_ratio_influence_function(r), lag) + standard_error = float(np.sqrt(lrv / t)) + z = ss.norm.ppf(0.5 + confidence_level / 2.0) + ci = (sr - z * standard_error, sr + z * standard_error) + test_statistic = ( + (sr - null_sharpe_ratio) / standard_error + if standard_error > 0 + else float("nan") + ) + p_value = float(2.0 * ss.norm.sf(abs(test_statistic))) + return { + "sharpe_ratio": sr, + "standard_error": standard_error, + "confidence_interval": ci, + "test_statistic": float(test_statistic), + "p_value": p_value, + "significant": bool(p_value < (1.0 - confidence_level)), + "lag": int(lag), + } diff --git a/src/RiskLabAI/backtest/strategy_risk.py b/src/RiskLabAI/backtest/strategy_risk.py new file mode 100644 index 0000000..15e5a36 --- /dev/null +++ b/src/RiskLabAI/backtest/strategy_risk.py @@ -0,0 +1,379 @@ +""" +Functions for calculating strategy risk metrics, such as implied +precision, binomial Sharpe ratio, and probability of failure. +""" + +import logging +from typing import TYPE_CHECKING + +import numpy as np +import scipy.stats as ss + +if TYPE_CHECKING: + import sympy + +logger = logging.getLogger(__name__) + + +def sharpe_ratio_trials(p: float, n_run: int) -> tuple[float, float, float]: + r""" + Simulate binomial trials to estimate mean, std dev, and Sharpe ratio. + + Each trial is a binomial outcome (1 or -1) with probability `p`. + + .. math:: + \text{outcome} = \begin{cases} 1 & \text{with probability } p \\ + -1 & \text{with probability } 1-p + \end{cases} + SR = \frac{\mu}{\sigma} + + Parameters + ---------- + p : float + Probability of success (outcome 1). + n_run : int + Number of simulation runs (trials). + + Returns + ------- + Tuple[float, float, float] + (mean_outcome, std_outcome, sharpe_ratio) + """ + outcomes = np.random.binomial(n=1, p=p, size=n_run) * 2 - 1 + mean_outcome = np.mean(outcomes) + std_outcome = np.std(outcomes) + sharpe_ratio = mean_outcome / std_outcome if std_outcome > 0 else 0.0 + + return mean_outcome, std_outcome, sharpe_ratio + + +def target_sharpe_ratio_symbolic() -> "sympy.Expr": + r""" + Calculate the variance of a binomial trial symbolically. + + The variance is: + .. math:: + Var = E[X^2] - (E[X])^2 + Var = (p u^2 + (1 - p) d^2) - (p u + (1 - p) d)^2 + + This function simplifies this expression using SymPy. + + Returns + ------- + sympy.Expr + The symbolic expression for variance: `p*(1 - p)*(d - u)**2` + """ + try: + from sympy import factor, symbols + except ImportError as exc: + raise ImportError( + "Symbolic strategy-risk analysis requires the optional 'sympy' dependency." + ) from exc + + p, u, d = symbols("p u d") + + m2 = p * u**2 + (1 - p) * d**2 + m1 = p * u + (1 - p) * d + v = m2 - m1**2 + + return factor(v) + + +def implied_precision( + stop_loss: float, + profit_taking: float, + frequency: float, + target_sharpe_ratio: float, +) -> float: + r""" + Calculate the implied precision (probability) to achieve a target SR. + + Solves the quadratic equation for `p` from the binomial SR formula: + .. math:: + a = (f + S^2) (pt - sl)^2 + b = (2 f sl - S^2 (pt - sl)) (pt - sl) + c = f sl^2 + p = \frac{-b + \sqrt{b^2 - 4ac}}{2a} + + Where: + - \(f\) is the frequency (bets per year) + - \(S\) is the target_sharpe_ratio + - \(pt\) is the profit_taking + - \(sl\) is the stop_loss (as a positive value) + + Parameters + ---------- + stop_loss : float + Stop-loss threshold (positive value, e.g., 0.02 for 2%). + profit_taking : float + Profit-taking threshold (positive value, e.g., 0.04 for 4%). + frequency : float + Number of bets per year. + target_sharpe_ratio : float + Target annual Sharpe ratio. + + Returns + ------- + float + The required precision (probability of success). + """ + a = (frequency + target_sharpe_ratio**2) * (profit_taking - stop_loss) ** 2 + b = ( + 2 * frequency * stop_loss - target_sharpe_ratio**2 * (profit_taking - stop_loss) + ) * (profit_taking - stop_loss) + c = frequency * stop_loss**2 + + discriminant = b**2 - 4 * a * c + if discriminant < 0: + return np.nan # No real solution + + precision = (-b + np.sqrt(discriminant)) / (2.0 * a) + return precision + + +def bin_frequency( + stop_loss: float, + profit_taking: float, + precision: float, + target_sharpe_ratio: float, +) -> float: + r""" + Calculate the frequency needed to achieve a target SR. + + .. math:: + f = \frac{S^2 (pt - sl)^2 p (1 - p)}{((pt - sl) p - sl)^2} + + Note: The original formula had `((pt - sl) * p + sl)` in the denominator, + which corresponds to `sl` being negative (a loss). This + implementation assumes `stop_loss` is a positive value, so the + expected return is `(pt - sl) * p - sl * (1-p) = p*pt - p*sl - sl + p*sl = p*pt - sl`. + Wait, the original `binomial_sharpe_ratio` uses `(pt - sl) * p + sl`. + This implies `sl` is treated as a negative number. + Let's assume `stop_loss` is a positive threshold, so `d = -stop_loss`. + E[R] = p*pt + (1-p)*(-sl) = p*(pt+sl) - sl + Stdev = (pt - (-sl)) * sqrt(p(1-p)) = (pt+sl)*sqrt(p(1-p)) + SR = (p*(pt+sl) - sl) / ((pt+sl)*sqrt(p(1-p))) + Let's stick to the formula provided in `binomial_sharpe_ratio`. + + .. math:: + f = \frac{S^2 (pt - sl)^2 p (1-p)}{((pt - sl) p + sl)^2} + + Parameters + ---------- + stop_loss : float + Stop-loss threshold (e.g., -0.02 for 2% loss). + profit_taking : float + Profit-taking threshold (e.g., 0.04 for 4% gain). + precision : float + The precision (probability of a profitable trade). + target_sharpe_ratio : float + The target annual Sharpe ratio. + + Returns + ------- + float + The required number of bets per year (frequency). + """ + if precision <= 0 or precision >= 1: + return np.inf + + numerator = ( + (target_sharpe_ratio * (profit_taking - stop_loss)) ** 2 + * precision + * (1 - precision) + ) + denominator = ((profit_taking - stop_loss) * precision + stop_loss) ** 2 + + if denominator == 0: + return np.inf + + return numerator / denominator + + +def binomial_sharpe_ratio( + stop_loss: float, + profit_taking: float, + frequency: float, + probability: float, +) -> float: + r""" + Calculate the Sharpe Ratio for a binary outcome strategy. + + .. math:: + E[R] = p \cdot pt + (1-p) \cdot sl + \sigma[R] = (pt - sl) \sqrt{p(1-p)} + SR_{trade} = \frac{E[R]}{\sigma[R]} + SR_{annual} = SR_{trade} \times \sqrt{f} + + Parameters + ---------- + stop_loss : float + Stop-loss outcome (negative value, e.g., -0.02). + profit_taking : float + Profit-taking outcome (positive value, e.g., 0.04). + frequency : float + Number of bets per year. + probability : float + Probability of success (profit_taking). + + Returns + ------- + float + The annualized Sharpe Ratio. + """ + expected_return = (profit_taking * probability) + (stop_loss * (1 - probability)) + + p = probability + stdev_return = (profit_taking - stop_loss) * np.sqrt(p * (1 - p)) + + if stdev_return == 0: + return 0.0 if expected_return == 0 else np.inf * np.sign(expected_return) + + sr_trade = expected_return / stdev_return + sr_annual = sr_trade * np.sqrt(frequency) + + return sr_annual + + +def mix_gaussians( + mu1: float, + mu2: float, + sigma1: float, + sigma2: float, + probability: float, + n_obs: int, +) -> np.ndarray: + """ + Generate a mixture of two Gaussian-distributed outcomes. + + Parameters + ---------- + mu1 : float + Mean of the first Gaussian (e.g., winning trades). + mu2 : float + Mean of the second Gaussian (e.g., losing trades). + sigma1 : float + Standard deviation of the first Gaussian. + sigma2 : float + Standard deviation of the second Gaussian. + probability : float + Probability of drawing from the first Gaussian (e.g., prob of win). + n_obs : int + Total number of observations (trades) to generate. + + Returns + ------- + np.ndarray + Array of generated outcomes, shuffled. + """ + n_obs1 = int(n_obs * probability) + n_obs2 = n_obs - n_obs1 + + returns1 = np.random.normal(mu1, sigma1, size=n_obs1) + returns2 = np.random.normal(mu2, sigma2, size=n_obs2) + + returns = np.append(returns1, returns2, axis=0) + np.random.shuffle(returns) + + return returns + + +def failure_probability( + returns: np.ndarray, frequency: float, target_sharpe_ratio: float +) -> float: + """ + Calculate the probability that the strategy may fail. + + This compares the observed precision (from returns) with the + implied precision required to meet the target SR. + + Parameters + ---------- + returns : np.ndarray + Array of returns from a strategy. + frequency : float + Number of bets per year (observed). + target_sharpe_ratio : float + The target annual Sharpe ratio. + + Returns + ------- + float + Calculated failure probability (a Z-score CDF). + """ + positive_returns = returns[returns > 0] + negative_returns = returns[returns <= 0] + + if len(positive_returns) == 0 or len(negative_returns) == 0: + return 0.0 # Cannot calculate + + profit_taking = positive_returns.mean() + stop_loss = negative_returns.mean() # This will be negative + + # Observed precision + observed_precision = positive_returns.shape[0] / float(returns.shape[0]) + + # Required precision + required_precision = implied_precision( + abs(stop_loss), profit_taking, frequency, target_sharpe_ratio + ) + + if np.isnan(required_precision): + return 1.0 # Cannot achieve target SR + + # Probability that observed_precision < required_precision + # This is a test on a proportion + p_var = observed_precision * (1 - observed_precision) + if p_var == 0: + return 0.0 if observed_precision >= required_precision else 1.0 + + p_std = np.sqrt(p_var / returns.shape[0]) # Std dev of the proportion + + z_score = (observed_precision - required_precision) / p_std + risk = ss.norm.cdf(z_score) # Prob of being <= required_precision + + return risk + + +def calculate_strategy_risk( + mu1: float, + mu2: float, + sigma1: float, + sigma2: float, + probability: float, + n_obs: int, + frequency: float, + target_sharpe_ratio: float, +) -> float: + """ + Run a simulation to calculate the strategy risk. + + Parameters + ---------- + mu1 : float + Mean of winning trades. + mu2 : float + Mean of losing trades. + sigma1 : float + Standard deviation of winning trades. + sigma2 : float + Standard deviation of losing trades. + probability : float + Probability of a winning trade. + n_obs : int + Number of observations (trades) in the simulation. + frequency : float + Number of bets per year. + target_sharpe_ratio : float + Target annual Sharpe ratio. + + Returns + ------- + float + Calculated probability of strategy failure. + """ + returns = mix_gaussians(mu1, mu2, sigma1, sigma2, probability, n_obs) + probability_fail = failure_probability(returns, frequency, target_sharpe_ratio) + + logger.info("Probability that strategy will fail: %.2f%%", probability_fail * 100) + return probability_fail diff --git a/src/RiskLabAI/backtest/test_set_overfitting.py b/src/RiskLabAI/backtest/test_set_overfitting.py new file mode 100644 index 0000000..3c8afaf --- /dev/null +++ b/src/RiskLabAI/backtest/test_set_overfitting.py @@ -0,0 +1,320 @@ +""" +Functions for calculating metrics related to test-set overfitting, +including the expected maximum Sharpe Ratio and Type I/II errors. +""" + +import numpy as np +import pandas as pd +import scipy.stats as ss +from scipy.stats import norm + + +def expected_max_sharpe_ratio( + n_trials: int, mean_sharpe_ratio: float, std_sharpe_ratio: float +) -> float: + r""" + Calculate the expected maximum Sharpe Ratio from N trials. + + Uses the formula for the expected maximum of N standard normal variables: + .. math:: + E[\max(SR)] = \mu + \sigma \left[ (1 - \gamma) \Phi^{-1}(1 - \frac{1}{N}) + + \gamma \Phi^{-1}(1 - (N e)^{-1}) \right] + + Where: + - \(\mu\) is the mean_sharpe_ratio + - \(\sigma\) is the std_sharpe_ratio + - \(\gamma\) is the Euler-Mascheroni constant + - \(\Phi^{-1}\) is the inverse CDF of the standard normal distribution + - \(N\) is n_trials + + Parameters + ---------- + n_trials : int + Number of trials (e.g., number of strategies tested). + mean_sharpe_ratio : float + The mean Sharpe Ratio across all trials. + std_sharpe_ratio : float + The standard deviation of Sharpe Ratios across all trials. + + Returns + ------- + float + The expected maximum Sharpe Ratio. + """ + if n_trials == 0: + return 0.0 + if n_trials == 1: + return mean_sharpe_ratio + + euler_gamma = 0.5772156649 + + term1 = (1 - euler_gamma) * norm.ppf(1.0 - 1.0 / n_trials) + term2 = euler_gamma * norm.ppf(1.0 - (n_trials * np.e) ** -1) + + expected_max_sr = mean_sharpe_ratio + std_sharpe_ratio * (term1 + term2) + + return expected_max_sr + + +def generate_max_sharpe_ratios( + n_sims: int, + n_trials_list: list[int], + std_sharpe_ratio: float, + mean_sharpe_ratio: float, +) -> pd.DataFrame: + """ + Generate a DataFrame of maximum Sharpe Ratios from simulations. + + Parameters + ---------- + n_sims : int + Number of simulations to run for each `n_trials`. + n_trials_list : List[int] + A list of N-trials (e.g., [10, 50, 100] strategies). + std_sharpe_ratio : float + The standard deviation of Sharpe Ratios. + mean_sharpe_ratio : float + The mean of Sharpe Ratios. + + Returns + ------- + pd.DataFrame + A long-format DataFrame with columns ['max_SR', 'n_trials']. + """ + rng = np.random.default_rng() + output_list = [] + + for n_trials in n_trials_list: + # Generate all simulations for this n_trials + sr_sims = rng.normal(loc=0.0, scale=1.0, size=(n_sims, n_trials)) + + # Normalize (z-score) each simulation row + sr_sims = (sr_sims - sr_sims.mean(axis=1, keepdims=True)) / sr_sims.std( + axis=1, keepdims=True + ) + + # Scale by target mean and std + sr_sims = mean_sharpe_ratio + sr_sims * std_sharpe_ratio + + # Get max SR for each simulation + max_sr = sr_sims.max(axis=1) + + output_temp = pd.DataFrame({"max_SR": max_sr, "n_trials": n_trials}) + output_list.append(output_temp) + + return pd.concat(output_list, ignore_index=True) + + +def mean_std_error( + n_sims0: int, + n_sims1: int, + n_trials: list[int], + std_sharpe_ratio: float = 1.0, + mean_sharpe_ratio: float = 0.0, +) -> pd.DataFrame: + """ + Calculate the mean and standard deviation of the estimation error. + + This function compares the analytical E[max{SR}] with the simulated + average max{SR} over `n_sims1` repetitions. + + Parameters + ---------- + n_sims0 : int + Number of max{SR} simulations used to estimate E[max{SR}] (inner loop). + n_sims1 : int + Number of errors on which to compute the std dev (outer loop). + n_trials : List[int] + List of numbers of trials (e.g., [10, 20, 50]). + std_sharpe_ratio : float, default=1.0 + Standard deviation of Sharpe Ratios. + mean_sharpe_ratio : float, default=0.0 + Mean of Sharpe Ratios. + + Returns + ------- + pd.DataFrame + DataFrame indexed by 'nTrials' with columns ['meanErr', 'stdErr']. + """ + # 1. Analytical E[max{SR}] + expected_sr = pd.Series( + { + n: expected_max_sharpe_ratio(n, mean_sharpe_ratio, std_sharpe_ratio) + for n in n_trials + }, + name="E[max{SR}]", + ) + expected_sr.index.name = "nTrials" + + error_list = [] + + # 2. Run n_sims1 experiments + for _ in range(int(n_sims1)): + # 3. Generate simulated max SRs + simulated_sr = generate_max_sharpe_ratios( + n_sims=n_sims0, + n_trials_list=n_trials, + mean_sharpe_ratio=mean_sharpe_ratio, + std_sharpe_ratio=std_sharpe_ratio, + ) + # 4. Average simulated max SRs + avg_simulated_sr = simulated_sr.groupby("n_trials")["max_SR"].mean() + + # 5. Calculate error + error = avg_simulated_sr / expected_sr - 1.0 + error_list.append(error.rename("error")) + + all_errors = pd.concat(error_list, axis=1).T + + # 6. Compute mean and std of errors + output = pd.DataFrame( + { + "meanErr": all_errors.mean(), + "stdErr": all_errors.std(), + } + ) + + return output + + +def estimated_sharpe_ratio_z_statistics( + sharpe_ratio: float, + t: int, + true_sharpe_ratio: float = 0.0, + skew: float = 0.0, + kurt: int = 3, +) -> float: + r""" + Calculate the Z-statistic for an estimated Sharpe Ratio. + + This is the test statistic used in the Probabilistic Sharpe Ratio. + + .. math:: + Z = \frac{(\hat{SR} - SR_0) \sqrt{T - 1}} + {\sqrt{1 - S \hat{SR} + \frac{K - 1}{4} \hat{SR}^2}} + + Parameters + ---------- + sharpe_ratio : float + Estimated Sharpe Ratio (\(\hat{SR}\)). + t : int + Number of observations (T). + true_sharpe_ratio : float, default=0.0 + The null hypothesis Sharpe Ratio (\(SR_0\)). + skew : float, default=0.0 + Skewness of returns (S). + kurt : int, default=3 + Kurtosis of returns (K). + + Returns + ------- + float + The calculated Z-statistic. + """ + denominator = 1 - skew * sharpe_ratio + (kurt - 1) / 4.0 * sharpe_ratio**2 + if denominator <= 0: + return np.nan + + z = (sharpe_ratio - true_sharpe_ratio) * np.sqrt(t - 1) + z /= np.sqrt(denominator) + + return z + + +def strategy_type1_error_probability(z: float, k: int = 1) -> float: + r""" + Calculate the Type I error probability (alpha) for multiple tests. + + This is the probability of at least one false positive (rejecting + a true null) when conducting `k` independent tests. + + .. math:: + \alpha_k = 1 - (1 - \alpha)^k + + Where \(\alpha = N(-z)\) is the Type I error for a single test. + + Parameters + ---------- + z : float + Z-statistic for the significance threshold (e.g., 1.96). + k : int, default=1 + Number of independent tests. + + Returns + ------- + float + The family-wise Type I error rate. + """ + alpha_single_test = ss.norm.cdf(-z) + alpha_k = 1 - (1 - alpha_single_test) ** k + return alpha_k + + +def theta_for_type2_error( + sharpe_ratio: float, + t: int, + true_sharpe_ratio: float, + skew: float = 0.0, + kurt: int = 3, +) -> float: + r""" + Calculate the \(\theta\) parameter for Type II error probability. + + .. math:: + \theta = \frac{SR_{True} \sqrt{T - 1}} + {\sqrt{1 - S \hat{SR} + \frac{K - 1}{4} \hat{SR}^2}} + + Parameters + ---------- + sharpe_ratio : float + The estimated Sharpe Ratio (\(\hat{SR}\)). + t : int + Number of observations (T). + true_sharpe_ratio : float + The true Sharpe Ratio (\(SR_{True}\)) (the alternative hypothesis). + skew : float, default=0.0 + Skewness of returns (S). + kurt : int, default=3 + Kurtosis of returns (K). + + Returns + ------- + float + The \(\theta\) parameter. + """ + denominator = 1 - skew * sharpe_ratio + (kurt - 1) / 4.0 * sharpe_ratio**2 + if denominator <= 0: + return np.nan + + theta = true_sharpe_ratio * np.sqrt(t - 1) + theta /= np.sqrt(denominator) + return theta + + +def strategy_type2_error_probability(alpha_k: float, k: int, theta: float) -> float: + r""" + Calculate the Type II error probability (beta) for multiple tests. + + This is the probability of failing to reject a false null hypothesis. + + .. math:: + Z_{\alpha} = \Phi^{-1}((1 - \alpha_k)^{1/k}) + \beta = N(Z_{\alpha} - \theta) + + Parameters + ---------- + alpha_k : float + The family-wise Type I error rate. + k : int + Number of independent tests. + theta : float + The \(\theta\) parameter from `theta_for_type2_error`. + + Returns + ------- + float + The Type II error probability (\(\beta\)). + """ + z_alpha = ss.norm.ppf((1 - alpha_k) ** (1.0 / k)) + beta = ss.norm.cdf(z_alpha - theta) + return beta diff --git a/src/RiskLabAI/backtest/validation/__init__.py b/src/RiskLabAI/backtest/validation/__init__.py new file mode 100644 index 0000000..385b4a4 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/__init__.py @@ -0,0 +1,55 @@ +""" +RiskLabAI Cross-Validation Module + +Implements advanced financial cross-validation techniques from +"Advances in Financial Machine Learning" by de Prado, including: +- Purged K-Fold (for removing group leakage) +- Walk-Forward +- Combinatorial Purged K-Fold (CPSC-V) + +This module is built on an AbstractCrossValidator interface and +a CrossValidatorFactory for easy instantiation. +""" + +from .adaptive_combinatorial_purged import AdaptiveCombinatorialPurged +from .bagged_combinatorial_purged import BaggedCombinatorialPurged +from .combinatorial_purged import CombinatorialPurged +from .cross_validator_controller import CrossValidatorController +from .cross_validator_factory import CrossValidatorFactory +from .cross_validator_interface import CrossValidator +from .kfold import KFold +from .leakage_aware_hpo import deflated_sharpe_gate, leakage_aware_hpo +from .path_adaptive_cpcv import ( + adaptive_probability_of_backtest_overfitting, + estimate_volatility_regimes, +) +from .path_bagged_cpcv import ( + bagged_probability_of_backtest_overfitting, + moving_block_bootstrap_indices, +) +from .purged_kfold import PurgedKFold +from .walk_forward import WalkForward + +__all__ = [ + # Interface + "CrossValidator", + # Validators + "KFold", + "PurgedKFold", + "WalkForward", # <-- Fix + "CombinatorialPurged", # <-- Fix + "BaggedCombinatorialPurged", # <-- Fix + "AdaptiveCombinatorialPurged", # <-- Fix + # Path-level bagged CPCV PBO (Appraisal 09) + "bagged_probability_of_backtest_overfitting", + "moving_block_bootstrap_indices", + # Path-level adaptive (regime-weighted) CPCV PBO (Appraisal 09b) + "adaptive_probability_of_backtest_overfitting", + "estimate_volatility_regimes", + # Leakage-aware HPO methodology (Appraisal 20) + "leakage_aware_hpo", + "deflated_sharpe_gate", + # Utilities + "CrossValidatorFactory", + "CrossValidatorController", +] diff --git a/src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py b/src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py new file mode 100644 index 0000000..7e81bc0 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/adaptive_combinatorial_purged.py @@ -0,0 +1,472 @@ +""" +Implements an Adaptive Combinatorial Purged Cross-Validation (A-CPCV), +which adjusts split boundaries based on an external feature. +""" + +import warnings +from collections import ChainMap +from collections.abc import Generator +from itertools import combinations +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd +from joblib import Parallel, delayed +from sklearn.base import clone +from sklearn.exceptions import ConvergenceWarning + +from .combinatorial_purged import CombinatorialPurged + +# For type hinting sklearn-like estimators +Estimator = Any + + +class AdaptiveCombinatorialPurged(CombinatorialPurged): + """ + Adaptive Combinatorial Purged Cross-Validation (A-CPCV). + + This method extends CPCV by making the group splits "adaptive". + Instead of fixed-size groups, it adjusts the group boundaries + based on the quantiles of an external feature (e.g., volatility, + market stress indicator). + + Parameters + ---------- + n_splits : int + Number of splits/groups to partition the data into. + n_test_groups : int + Size of the testing set in terms of groups. + times : pd.Series or Dict[str, pd.Series] + The timestamp series associated with the labels. + embargo : float, default=0 + The embargo rate for purging. + n_subsplits : int, default=3 + Number of subsplits within each group to check for boundary + adjustment. Higher numbers allow for finer adjustments. + external_feature : pd.Series or Dict[str, pd.Series] + The external feature used to adjust split boundaries. + Must share the same index as `data` and `times`. + lower_quantile : float, default=0.25 + The lower quantile threshold. If a boundary point's + feature value is below this, the boundary is shifted. + upper_quantile : float, default=0.75 + The upper quantile threshold. If a boundary point's + feature value is above this, the boundary is shifted. + subtract_border_adjustments : bool, default=True + If True, shifts boundaries left (subtracts) for low + quantile values and right (adds) for high. If False, + does the opposite. + """ + + def __init__( + self, + n_splits: int, + n_test_groups: int, + times: Union[pd.Series, dict[str, pd.Series]], + embargo: float = 0, + n_subsplits: int = 3, + external_feature: Union[pd.Series, dict[str, pd.Series]] = None, + lower_quantile: float = 0.25, + upper_quantile: float = 0.75, + subtract_border_adjustments: bool = True, + ): + """ + Initialize the AdaptiveCombinatorialPurged class. + """ + super().__init__(n_splits, n_test_groups, times, embargo) + + if external_feature is None: + raise ValueError("external_feature must be provided for A-CPCV") + + self.n_subsplits = n_subsplits + self.external_feature = external_feature + self.lower_quantile = lower_quantile + self.upper_quantile = upper_quantile + self.subtract_border_adjustments = subtract_border_adjustments + + # Check for multiple datasets consistency + if self.is_multiple_datasets and not isinstance(external_feature, dict): + raise ValueError( + "If 'times' is a dict, 'external_feature' must also be a dict." + ) + if not self.is_multiple_datasets and not isinstance( + external_feature, pd.Series + ): + raise ValueError( + "If 'times' is a Series, 'external_feature' must be a Series." + ) + + def _validate_input( + self, + single_times: pd.Series, + single_data: pd.DataFrame, + single_external_feature: pd.Series, + ) -> None: + """ + Validate that data, times, and external feature share the same index. + """ + # Call parent validation + super()._validate_input(single_times, single_data) + + if not single_data.index.equals(single_external_feature.index): + raise ValueError("Data and external_feature must have the same index") + + def _single_adaptive_split_segments( + self, indices: np.ndarray, single_external_feature: pd.Series + ) -> list[np.ndarray]: + """ + Adaptively split data indices based on the external feature. + + Parameters + ---------- + indices : np.ndarray + Array of data indices to be split. + single_external_feature : pd.Series + The external feature for the entire dataset. + + Returns + ------- + List[np.ndarray] + List of adaptively split data indices (the `n_splits` groups). + """ + # Get feature values for the relevant indices + external_values = single_external_feature.iloc[indices].values + + # Calculate quantile thresholds + lower_threshold = np.percentile(external_values, self.lower_quantile * 100) + upper_threshold = np.percentile(external_values, self.upper_quantile * 100) + + # Partition indices into fine-grained subsplits + n_total_subsplits = self.n_splits * self.n_subsplits + subsplits = np.array_split(indices, n_total_subsplits) + + # Get the start index of each subsplit + subsplit_starts_loc = np.array( + [split[0] for split in subsplits if len(split) > 0] + ) + + # Get the feature values at the start of each subsplit + # We need to map iloc back to the feature series's index + subsplit_start_indices = single_external_feature.index[subsplit_starts_loc] + subsplit_start_features = single_external_feature[subsplit_start_indices].values + + # Determine the initial (non-adaptive) borders + # These are the indices of the *subsplits* that start a new group + borders = np.arange(self.n_subsplits, n_total_subsplits, self.n_subsplits) + + # Vectorized comparison of external feature values at borders + border_feature_values = subsplit_start_features[borders] + + # Determine adjustments + border_adjustments = np.zeros_like(borders) + border_adjustments[border_feature_values < lower_threshold] = -1 + border_adjustments[border_feature_values > upper_threshold] = 1 + + if self.subtract_border_adjustments: + adjusted_borders = borders - border_adjustments + else: + adjusted_borders = borders + border_adjustments + + # Ensure borders are within valid range + adjusted_borders = np.clip(adjusted_borders, 1, len(subsplit_starts_loc) - 1) + + # Get the integer-location split points + split_points = subsplit_starts_loc[adjusted_borders] + + # Create the final N-split segments + # We split the original `indices` array at the calculated `split_points` + # `np.searchsorted` maps the split point values back to their locations in `indices` + split_point_locs = np.searchsorted(indices, split_points) + split_segments = np.split(indices, split_point_locs) + + return split_segments + + def _get_split_segments( + self, + single_data: pd.DataFrame, + single_external_feature: Optional[pd.Series] = None, + ) -> list[np.ndarray]: + """Override to use adaptive splitting.""" + if single_external_feature is None: + raise ValueError("_get_split_segments requires external_feature") + + indices = np.arange(single_data.shape[0]) + return self._single_adaptive_split_segments(indices, single_external_feature) + + def _single_split( + self, + single_times: pd.Series, + single_data: pd.DataFrame, + single_external_feature: Optional[pd.Series] = None, + ) -> Generator[tuple[np.ndarray, np.ndarray], None, None]: + """ + Split a single dataset into C(n, k) adaptively purged indices. + """ + if single_external_feature is None: + raise ValueError("_single_split requires single_external_feature") + + self._validate_input(single_times, single_data, single_external_feature) + + split_segments = self._get_split_segments(single_data, single_external_feature) + combinations_list = list(combinations(range(self.n_splits), self.n_test_groups)) + + all_combinatorial_splits = self._combinatorial_splits( + combinations_list, split_segments + ) + + for test_indices in all_combinatorial_splits: + train_indices = self._get_train_indices( + test_indices, single_times, continous_test_times=False + ) + yield train_indices, test_indices + + def split( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> Union[ + Generator[tuple[np.ndarray, np.ndarray], None, None], + Generator[tuple[str, tuple[np.ndarray, np.ndarray]], None, None], + ]: + """ + Split data (or dictionary of data) into adaptively purged indices. + """ + if self.is_multiple_datasets: + if not isinstance(data, dict): + raise ValueError("If 'times' is a dict, 'data' must be a dict.") + for key in self.times: + for train_indices, test_indices in self._single_split( + self.times[key], data[key], self.external_feature[key] + ): + yield key, (train_indices, test_indices) + else: + if not isinstance(data, pd.DataFrame): + raise ValueError("If 'times' is a Series, 'data' must be a DataFrame.") + for train_indices, test_indices in self._single_split( + self.times, data, self.external_feature + ): + yield train_indices, test_indices + + def _combinations_and_path_locations_and_split_segments( + self, data: pd.DataFrame, single_external_feature: Optional[pd.Series] = None + ) -> tuple[ + list[tuple[int, ...]], dict[int, list[tuple[int, int]]], list[np.ndarray] + ]: + """Helper to compute all components, now including adaptive splits.""" + if single_external_feature is None: + raise ValueError("Method requires single_external_feature") + + combinations_list = list(combinations(range(self.n_splits), self.n_test_groups)) + locations = self._path_locations(self.n_splits, combinations_list) + split_segments = self._get_split_segments(data, single_external_feature) + + return combinations_list, locations, split_segments + + def _single_backtest_paths( + self, + single_times: pd.Series, + single_data: pd.DataFrame, + single_external_feature: Optional[pd.Series] = None, + ) -> dict[str, list[dict[str, np.ndarray]]]: + """ + Generate all adaptive combinatorial backtest paths. + """ + if single_external_feature is None: + raise ValueError("Method requires single_external_feature") + + self._validate_input(single_times, single_data, single_external_feature) + + paths = {} + combinations_list, locations, split_segments = ( + self._combinations_and_path_locations_and_split_segments( + single_data, single_external_feature + ) + ) + + all_combinatorial_splits = list( + self._combinatorial_splits(combinations_list, split_segments) + ) + + for path_num, locs in locations.items(): + path_data = [] + for group_idx, split_idx in locs: + combinatorial_test_indices = all_combinatorial_splits[split_idx] + train_indices = self._get_train_indices( + combinatorial_test_indices, single_times, continous_test_times=False + ) + test_indices_segment = split_segments[group_idx] + + path_data.append( + { + "Train": train_indices, + "Test": test_indices_segment, + } + ) + + paths[f"Path {path_num}"] = path_data + return paths + + def backtest_paths( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + ) -> Union[ + dict[str, list[dict[str, np.ndarray]]], + dict[str, dict[str, list[dict[str, np.ndarray]]]], + ]: + """ + Generate adaptive backtest paths for data or a dictionary of data. + """ + if self.is_multiple_datasets: + if not isinstance(data, dict): + raise ValueError("If 'times' is a dict, 'data' must be a dict.") + multiple_paths = {} + for key in self.times: + multiple_paths[key] = self._single_backtest_paths( + self.times[key], data[key], self.external_feature[key] + ) + return multiple_paths + + if not isinstance(data, pd.DataFrame): + raise ValueError("If 'times' is a Series, 'data' must be a DataFrame.") + return self._single_backtest_paths(self.times, data, self.external_feature) + + def _single_backtest_predictions( + self, + single_estimator: Estimator, + single_times: pd.Series, + single_data: pd.DataFrame, + single_labels: pd.Series, + single_weights: Optional[np.ndarray] = None, + single_external_feature: Optional[pd.Series] = None, # New arg + predict_probability: bool = False, + n_jobs: int = 1, + ) -> dict[str, np.ndarray]: + """ + Obtain backtest predictions for all A-CPCV paths. + """ + if single_external_feature is None: + raise ValueError("Method requires single_external_feature") + + self._validate_input(single_times, single_data, single_external_feature) + + if single_weights is None: + single_weights = np.ones(len(single_data)) + + combinations_list, locations, split_segments = ( + self._combinations_and_path_locations_and_split_segments( + single_data, single_external_feature + ) + ) + + def train_single_estimator( + estimator_: Estimator, combinatorial_test_indices: np.ndarray + ) -> Estimator: + """Train one estimator for one C(n,k) split.""" + train_indices = self._get_train_indices( + combinatorial_test_indices, single_times, continous_test_times=False + ) + X_train = single_data.iloc[train_indices] + y_train = single_labels.iloc[train_indices] + weights_train = single_weights[train_indices] + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ConvergenceWarning) + try: + estimator_.fit(X_train, y_train, sample_weight=weights_train) + except (TypeError, ValueError): + estimator_.fit(X_train, y_train) + + return estimator_ + + # 1. Train all C(n, k) estimators in parallel + combinatorial_trained_estimators = Parallel(n_jobs=n_jobs)( + delayed(train_single_estimator)(clone(single_estimator), test_indices) + for test_indices in self._combinatorial_splits( + combinations_list, split_segments + ) + ) + + def get_path_data( + path_num: int, locs: list[tuple[int, int]] + ) -> dict[str, np.ndarray]: + """Assemble predictions for one path.""" + path_predictions = [] + for group_idx, split_idx in locs: + test_indices_segment = split_segments[group_idx] + X_test = single_data.iloc[test_indices_segment] + estimator = combinatorial_trained_estimators[split_idx] + + if predict_probability: + preds = estimator.predict_proba(X_test) + else: + preds = estimator.predict(X_test) + + path_predictions.append(preds) + + return {f"Path {path_num}": np.concatenate(path_predictions)} + + # 2. Assemble predictions for all paths + path_results = Parallel(n_jobs=n_jobs)( + delayed(get_path_data)(path_num, locs) + for path_num, locs in locations.items() + ) + + paths_predictions = dict(ChainMap(*reversed(path_results))) + return paths_predictions + + def backtest_predictions( + self, + estimator: Union[Estimator, dict[str, Estimator]], + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Union[pd.Series, dict[str, pd.Series]], + sample_weights: Optional[Union[np.ndarray, dict[str, np.ndarray]]] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> Union[dict[str, np.ndarray], dict[str, dict[str, np.ndarray]]]: + """ + Generate adaptive backtest predictions. + """ + if self.is_multiple_datasets: + if not ( + isinstance(data, dict) + and isinstance(estimator, dict) + and isinstance(labels, dict) + ): + raise ValueError( + "If 'times' is a dict, 'data', 'estimator', and 'labels' " + "must also be dicts." + ) + + multiple_paths_predictions = {} + for key in self.times: + s_weights = sample_weights[key] if sample_weights else None + multiple_paths_predictions[key] = self._single_backtest_predictions( + estimator[key], + self.times[key], + data[key], + labels[key], + s_weights, + self.external_feature[key], + predict_probability, + n_jobs, + ) + return multiple_paths_predictions + + # Handle single dataset case + if not (isinstance(data, pd.DataFrame) and isinstance(labels, pd.Series)): + raise ValueError( + "If 'times' is a Series, 'data' must be a DataFrame " + "and 'labels' must be a Series." + ) + + return self._single_backtest_predictions( + estimator, + self.times, + data, + labels, + sample_weights, + self.external_feature, + predict_probability, + n_jobs, + ) diff --git a/src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py b/src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py new file mode 100644 index 0000000..27d8d45 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/bagged_combinatorial_purged.py @@ -0,0 +1,247 @@ +""" +Implements Bagged Combinatorial Purged Cross-Validation (B-CPCV). +""" + +import warnings +from collections import ChainMap +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd +from joblib import Parallel, delayed +from sklearn.base import clone +from sklearn.ensemble import BaggingClassifier, BaggingRegressor +from sklearn.exceptions import ConvergenceWarning + +from .combinatorial_purged import CombinatorialPurged + +# For type hinting sklearn-like estimators +Estimator = Any + + +class BaggedCombinatorialPurged(CombinatorialPurged): + """ + Bagged Combinatorial Purged Cross-Validation (B-CPCV). + + This class extends CPCV by applying a bagging (Bootstrap Aggregating) + wrapper to the estimator for each combinatorial split. This helps + to reduce variance and improve model stability. + + Parameters + ---------- + n_splits : int + Number of splits/groups to partition the data into. + n_test_groups : int + Size of the testing set in terms of groups. + times : pd.Series or Dict[str, pd.Series] + The timestamp series associated with the labels. + embargo : float, default=0 + The embargo rate for purging. + classifier : bool, default=True + If True, use `sklearn.ensemble.BaggingClassifier`. + If False, use `sklearn.ensemble.BaggingRegressor`. + n_estimators : int, default=10 + The number of base estimators in the ensemble. + max_samples : float, default=1.0 + The number of samples to draw to train each base estimator. + max_features : float, default=1.0 + The number of features to draw to train each base estimator. + bootstrap : bool, default=True + Whether samples are drawn with replacement. + bootstrap_features : bool, default=False + Whether features are drawn with replacement. + random_state : int, optional + Controls the randomness of the bootstrapping. + """ + + def __init__( + self, + n_splits: int, + n_test_groups: int, + times: Union[pd.Series, dict[str, pd.Series]], + embargo: float = 0, + classifier: bool = True, + n_estimators: int = 10, + max_samples: float = 1.0, + max_features: float = 1.0, + bootstrap: bool = True, + bootstrap_features: bool = False, + random_state: int = None, + ): + """ + Initialize the BaggedCombinatorialPurged class. + """ + super().__init__(n_splits, n_test_groups, times, embargo) + self.classifier = classifier + self.n_estimators = n_estimators + self.max_samples = max_samples + self.max_features = max_features + self.bootstrap = bootstrap + self.bootstrap_features = bootstrap_features + self.random_state = random_state + + def _single_backtest_predictions( + self, + single_estimator: Estimator, + single_times: pd.Series, + single_data: pd.DataFrame, + single_labels: pd.Series, + single_weights: Optional[np.ndarray] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> dict[str, np.ndarray]: + """ + Obtain backtest predictions for all B-CPCV paths. + + This method overrides the parent by wrapping the `single_estimator` + in a `BaggingClassifier` or `BaggingRegressor` before training. + + Parameters + ---------- + single_estimator : Estimator + The *base* estimator to be used inside the bagging ensemble. + single_times : pd.Series + The 'times' (info range) Series for this dataset. + single_data : pd.DataFrame + Data for the single dataset. + single_labels : pd.Series + Labels for the single dataset. + single_weights : np.ndarray, optional + Sample weights. Defaults to equal weights. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + Only valid if `classifier=True`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Dict[str, np.ndarray] + A dictionary where keys are "Path {id}" and values are + the contiguous arrays of out-of-sample predictions for each path. + """ + self._validate_input(single_times, single_data) + + if single_weights is None: + single_weights = np.ones(len(single_data)) + + if predict_probability and not self.classifier: + raise ValueError( + "Cannot use predict_probability=True when classifier=False" + ) + + combinations_list, locations, split_segments = ( + self._combinations_and_path_locations_and_split_segments(single_data) + ) + + def train_single_bagging_estimator( + base_estimator_: Estimator, combinatorial_test_indices: np.ndarray + ) -> Estimator: + """Train one Bagging estimator for one C(n,k) split.""" + train_indices = self._get_train_indices( + combinatorial_test_indices, single_times, continous_test_times=False + ) + + X_train = single_data.iloc[train_indices] + y_train = single_labels.iloc[train_indices] + weights_train = single_weights[train_indices] + + # --- Bagging Wrapper --- + if self.classifier: + bagging_estimator = BaggingClassifier( + estimator=base_estimator_, + n_estimators=self.n_estimators, + max_samples=self.max_samples, + max_features=self.max_features, + bootstrap=self.bootstrap, + bootstrap_features=self.bootstrap_features, + random_state=self.random_state, + n_jobs=n_jobs, # Parallelize bagging itself + ) + else: + bagging_estimator = BaggingRegressor( + estimator=base_estimator_, + n_estimators=self.n_estimators, + max_samples=self.max_samples, + max_features=self.max_features, + bootstrap=self.bootstrap, + bootstrap_features=self.bootstrap_features, + random_state=self.random_state, + n_jobs=n_jobs, # Parallelize bagging itself + ) + # --- End Bagging Wrapper --- + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ConvergenceWarning) + try: + bagging_estimator.fit(X_train, y_train, sample_weight=weights_train) + except (TypeError, ValueError): + # Fallback for models without sample_weight + bagging_estimator.fit(X_train, y_train) + + return bagging_estimator + + # 1. Train all C(n, k) *bagging* estimators in parallel + # Note: We set n_jobs=1 for the *outer* parallel loop + # and let the *inner* bagging estimator use `n_jobs`. + # This avoids nested parallelization issues. + + # Determine parallelization strategy + if n_jobs > 1 or n_jobs == -1: + outer_n_jobs = n_jobs + inner_n_jobs = 1 + + # Update bagging params to use inner_n_jobs + if self.classifier: + BaggingClassifier.__init__ = lambda self, **kwargs: super( + BaggingClassifier, self + ).__init__(**kwargs, n_jobs=inner_n_jobs) + else: + BaggingRegressor.__init__ = lambda self, **kwargs: super( + BaggingRegressor, self + ).__init__(**kwargs, n_jobs=inner_n_jobs) + else: + # Let bagging use all cores if outer loop is serial + outer_n_jobs = 1 + inner_n_jobs = -1 # Use all + + combinatorial_trained_estimators = Parallel(n_jobs=outer_n_jobs)( + delayed(train_single_bagging_estimator)( + clone(single_estimator), test_indices + ) + for test_indices in self._combinatorial_splits( + combinations_list, split_segments + ) + ) + + # Restore default constructors + BaggingClassifier.__init__ = BaggingClassifier.__init__ + BaggingRegressor.__init__ = BaggingRegressor.__init__ + + # 2. Assemble predictions (this is fast, can be serial or parallel) + def get_path_data( + path_num: int, locs: list[tuple[int, int]] + ) -> dict[str, np.ndarray]: + """Assemble predictions for one path.""" + path_predictions = [] + for group_idx, split_idx in locs: + test_indices_segment = split_segments[group_idx] + X_test = single_data.iloc[test_indices_segment] + estimator = combinatorial_trained_estimators[split_idx] + + if predict_probability: + preds = estimator.predict_proba(X_test) + else: + preds = estimator.predict(X_test) + + path_predictions.append(preds) + + return {f"Path {path_num}": np.concatenate(path_predictions)} + + path_results = Parallel(n_jobs=outer_n_jobs)( + delayed(get_path_data)(path_num, locs) + for path_num, locs in locations.items() + ) + + paths_predictions = dict(ChainMap(*reversed(path_results))) + return paths_predictions diff --git a/src/RiskLabAI/backtest/validation/combinatorial_purged.py b/src/RiskLabAI/backtest/validation/combinatorial_purged.py new file mode 100644 index 0000000..e6a0a25 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/combinatorial_purged.py @@ -0,0 +1,413 @@ +""" +Implements Combinatorial Purged Cross-Validation (CPCV) as described by +Marcos Lopez de Prado. +""" + +import warnings +from collections import ChainMap, defaultdict +from collections.abc import Generator +from copy import deepcopy +from itertools import combinations +from math import comb +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd +from joblib import Parallel, delayed +from sklearn.exceptions import ConvergenceWarning + +from .purged_kfold import PurgedKFold + +# For type hinting sklearn-like estimators +Estimator = Any + + +class CombinatorialPurged(PurgedKFold): + """ + Combinatorial Purged Cross-Validation (CPCV). + + This method generates multiple backtest paths by creating all possible + combinations of train/test splits, given `n_splits` groups and + `n_test_groups` for the test set size. + + It inherits from `PurgedKFold` to ensure that all generated training + sets are properly purged and embargoed against their corresponding + test set. + + Parameters + ---------- + n_splits : int + Total number of groups to partition the data into. + n_test_groups : int + The number of groups to use for testing in each combination. + e.g., n_splits=8, n_test_groups=2 -> C(8, 2) = 28 splits. + times : pd.Series or Dict[str, pd.Series] + A Series where the index is the observation start time and + the value is the observation end time. + embargo : float, default=0 + The embargo fraction (e.g., 0.01 for 1%). + """ + + @staticmethod + def _path_locations( + n_splits: int, combinations_list: list[tuple[int, ...]] + ) -> dict[int, list[tuple[int, int]]]: + """ + Generate a labeled path matrix to map splits to backtest paths. + + Parameters + ---------- + n_splits : int + Total number of groups. + combinations_list : List[Tuple[int, ...]] + The list of all C(n_splits, n_test_groups) combinations. + + Returns + ------- + Dict[int, List[Tuple[int, int]]] + A dictionary mapping each path ID (int) to a list of its + coordinates (group, split_num) in the path matrix. + """ + n_combinations = len(combinations_list) + + # Initialize a zero matrix + matrix = np.zeros((n_splits, n_combinations), dtype=int) + + # Populate matrix: 1 if group `i` is in combination `j` + for j, indices in enumerate(combinations_list): + matrix[indices, j] = 1 + + # Label path numbers + def label_path_row(row: np.ndarray) -> np.ndarray: + """Assigns a path number to each test split in a group.""" + path_counter = 1 + labeled_row = np.zeros_like(row) + for i, val in enumerate(row): + if val == 1: + labeled_row[i] = path_counter + path_counter += 1 + return labeled_row + + path_numbers = np.apply_along_axis(label_path_row, 1, matrix) + + # Extract path locations + locations = defaultdict(list) + for group_idx in range(n_splits): + for split_idx in range(n_combinations): + path_id = path_numbers[group_idx, split_idx] + if path_id != 0: + locations[path_id].append((group_idx, split_idx)) + + return dict(locations) + + @staticmethod + def _combinatorial_splits( + combinations_list: list[tuple[int, ...]], split_segments: list[np.ndarray] + ) -> Generator[np.ndarray, None, None]: + """ + Generate combinatorial test sets. + + Parameters + ---------- + combinations_list : List[Tuple[int, ...]] + List of all C(n_splits, n_test_groups) combinations. + split_segments : List[np.ndarray] + List of index arrays, one for each of the `n_splits` groups. + + Yields + ------- + Generator[np.ndarray, None, None] + A generator yielding the concatenated test set indices for + each combination. + """ + for test_groups in combinations_list: + test_sets = [ + split for i, split in enumerate(split_segments) if i in test_groups + ] + yield np.concatenate(test_sets) + + def __init__( + self, + n_splits: int, + n_test_groups: int, + times: Union[pd.Series, dict[str, pd.Series]], + embargo: float = 0, + ) -> None: + """ + Initialize the CombinatorialPurged class. + + Parameters + ---------- + n_splits : int + Number of splits/groups to partition the data into. + n_test_groups : int + Size of the testing set in terms of groups. + times : pd.Series or Dict[str, pd.Series] + The timestamp series associated with the labels. + embargo : float, default=0 + The embargo rate for purging. + """ + super().__init__(n_splits, times, embargo) + if n_test_groups >= n_splits: + raise ValueError("n_test_groups must be strictly less than n_splits") + self.n_test_groups = n_test_groups + + def get_n_splits( + self, + data: Optional[Union[pd.DataFrame, dict[str, pd.DataFrame]]] = None, + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> int: + """ + Return the total number of combinatorial splits. + + This is C(n_splits, n_test_groups). + + Parameters + ---------- + data : pd.DataFrame or dict, optional + Dataset or dictionary of datasets. + labels : pd.Series or dict, optional + Labels or dictionary of labels. + groups : np.ndarray, optional + Group labels for the samples. + + Returns + ------- + int + The total number of splits, C(n_splits, n_test_groups). + """ + return comb(self.n_splits, self.n_test_groups) + + def _get_split_segments(self, single_data: pd.DataFrame) -> list[np.ndarray]: + """Helper to get the base K-Fold segments.""" + indices = np.arange(single_data.shape[0]) + return np.array_split(indices, self.n_splits) + + def _single_split( + self, + single_times: pd.Series, + single_data: pd.DataFrame, + ) -> Generator[tuple[np.ndarray, np.ndarray], None, None]: + """ + Split a single dataset into C(n, k) purged train-test indices. + + Parameters + ---------- + single_times : pd.Series + The 'times' (info range) Series for this dataset. + single_data : pd.DataFrame + Input dataset. + + Yields + ------- + Generator[Tuple[np.ndarray, np.ndarray], None, None] + A generator of (train_indices, test_indices) for all + C(n_splits, n_test_groups) combinations. + """ + self._validate_input(single_times, single_data) + + split_segments = self._get_split_segments(single_data) + combinations_list = list(combinations(range(self.n_splits), self.n_test_groups)) + + all_combinatorial_splits = self._combinatorial_splits( + combinations_list, split_segments + ) + + for test_indices in all_combinatorial_splits: + # Purge against the *non-contiguous* test set + train_indices = self._get_train_indices( + test_indices, single_times, continous_test_times=False + ) + yield train_indices, test_indices + + def _combinations_and_path_locations_and_split_segments( + self, data: pd.DataFrame + ) -> tuple[ + list[tuple[int, ...]], dict[int, list[tuple[int, int]]], list[np.ndarray] + ]: + """ + Helper to compute all necessary components for CPCV. + + Parameters + ---------- + data : pd.DataFrame + Input dataset. + + Returns + ------- + Tuple[List, Dict, List] + - combinations_list: List of all C(n, k) combinations. + - locations: The path location dictionary from `_path_locations`. + - split_segments: List of index arrays for each of the `n_splits` groups. + """ + combinations_list = list(combinations(range(self.n_splits), self.n_test_groups)) + locations = self._path_locations(self.n_splits, combinations_list) + split_segments = self._get_split_segments(data) + + return combinations_list, locations, split_segments + + def _single_backtest_paths( + self, + single_times: pd.Series, + single_data: pd.DataFrame, + ) -> dict[str, list[dict[str, np.ndarray]]]: + """ + Generate all combinatorial backtest paths for a single dataset. + + Parameters + ---------- + single_times : pd.Series + The 'times' (info range) Series for this dataset. + single_data : pd.DataFrame + Input dataset. + + Returns + ------- + Dict[str, List[Dict[str, np.ndarray]]] + A dictionary where each key is a "Path {id}" and the value + is the list of (Train, Test) splits for that path. + """ + self._validate_input(single_times, single_data) + + paths = {} + combinations_list, locations, split_segments = ( + self._combinations_and_path_locations_and_split_segments(single_data) + ) + + all_combinatorial_splits = list( + self._combinatorial_splits(combinations_list, split_segments) + ) + + for path_num, locs in locations.items(): + path_data = [] + for group_idx, split_idx in locs: + # Get the full test set for this *combination* + combinatorial_test_indices = all_combinatorial_splits[split_idx] + + # Get the train set purged against this *combination* + train_indices = self._get_train_indices( + combinatorial_test_indices, single_times, continous_test_times=False + ) + + # The test set for this *path segment* is just one group + test_indices_segment = split_segments[group_idx] + + path_data.append( + { + "Train": train_indices, + "Test": test_indices_segment, + } + ) + + # This path is complete + paths[f"Path {path_num}"] = path_data + + return paths + + def _single_backtest_predictions( + self, + single_estimator: Estimator, + single_times: pd.Series, + single_data: pd.DataFrame, + single_labels: pd.Series, + single_weights: Optional[np.ndarray] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> dict[str, np.ndarray]: + """ + Obtain backtest predictions for all CPCV paths. + + Parameters + ---------- + single_estimator : Estimator + A scikit-learn-like estimator. + single_times : pd.Series + The 'times' (info range) Series for this dataset. + single_data : pd.DataFrame + Data for the single dataset. + single_labels : pd.Series + Labels for the single dataset. + single_weights : np.ndarray, optional + Sample weights. Defaults to equal weights. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Dict[str, np.ndarray] + A dictionary where keys are "Path {id}" and values are + the contiguous arrays of out-of-sample predictions for each path. + """ + self._validate_input(single_times, single_data) + + if single_weights is None: + single_weights = np.ones(len(single_data)) + + combinations_list, locations, split_segments = ( + self._combinations_and_path_locations_and_split_segments(single_data) + ) + + def train_single_estimator( + estimator_: Estimator, combinatorial_test_indices: np.ndarray + ) -> Estimator: + """Train one estimator for one C(n,k) split.""" + train_indices = self._get_train_indices( + combinatorial_test_indices, single_times, continous_test_times=False + ) + + X_train = single_data.iloc[train_indices] + y_train = single_labels.iloc[train_indices] + weights_train = single_weights[train_indices] + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ConvergenceWarning) + try: + estimator_.fit(X_train, y_train, sample_weight=weights_train) + except TypeError: + estimator_.fit(X_train, y_train) + + return estimator_ + + # 1. Train all C(n, k) estimators in parallel + combinatorial_trained_estimators = Parallel(n_jobs=n_jobs)( + delayed(train_single_estimator)(deepcopy(single_estimator), test_indices) + for test_indices in self._combinatorial_splits( + combinations_list, split_segments + ) + ) + + def get_path_data( + path_num: int, locs: list[tuple[int, int]] + ) -> dict[str, np.ndarray]: + """Assemble predictions for one path.""" + path_predictions = [] + + for group_idx, split_idx in locs: + # Get the test segment for this path + test_indices_segment = split_segments[group_idx] + X_test = single_data.iloc[test_indices_segment] + + # Get the estimator trained for this split + estimator = combinatorial_trained_estimators[split_idx] + + if predict_probability: + preds = estimator.predict_proba(X_test) + else: + preds = estimator.predict(X_test) + + path_predictions.append(preds) + + return {f"Path {path_num}": np.concatenate(path_predictions)} + + # 2. Assemble predictions for all paths in parallel + path_results = Parallel(n_jobs=n_jobs)( + delayed(get_path_data)(path_num, locs) + for path_num, locs in locations.items() + ) + + # Combine list of dicts into one dict + paths_predictions = dict(ChainMap(*reversed(path_results))) + return paths_predictions diff --git a/src/RiskLabAI/backtest/validation/cross_validator_controller.py b/src/RiskLabAI/backtest/validation/cross_validator_controller.py new file mode 100644 index 0000000..56820a5 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/cross_validator_controller.py @@ -0,0 +1,38 @@ +""" +Deprecated controller for creating and holding a cross-validator. + +Use the core component registry (:data:`RiskLabAI.core.CROSS_VALIDATORS`) +directly instead. This controller is retained for backward compatibility and +delegates to that registry; it is scheduled for removal in 4.0.0. +""" + +import warnings +from typing import Any + +from .cross_validator_interface import CrossValidator + + +class CrossValidatorController: + """ + Deprecated. Use ``RiskLabAI.core.CROSS_VALIDATORS.create(...)`` instead. + + Scheduled for removal in 4.0.0. + """ + + def __init__(self, validator_type: str, **kwargs: Any): + warnings.warn( + "CrossValidatorController is deprecated and will be removed in " + "4.0.0; use RiskLabAI.core.CROSS_VALIDATORS.create(validator_type, " + "...) instead.", + DeprecationWarning, + stacklevel=2, + ) + from RiskLabAI.core import CROSS_VALIDATORS + + self.cross_validator: CrossValidator = CROSS_VALIDATORS.create( + validator_type, filter_unknown_kwargs=True, **kwargs + ) + + def get_validator(self) -> CrossValidator: + """Return the created cross-validator instance.""" + return self.cross_validator diff --git a/src/RiskLabAI/backtest/validation/cross_validator_factory.py b/src/RiskLabAI/backtest/validation/cross_validator_factory.py new file mode 100644 index 0000000..ee34e0a --- /dev/null +++ b/src/RiskLabAI/backtest/validation/cross_validator_factory.py @@ -0,0 +1,74 @@ +""" +Deprecated factory for creating cross-validator instances. + +The core component registry (:data:`RiskLabAI.core.CROSS_VALIDATORS`) is now the +single source of truth for creating cross-validators. This factory is retained +for backward compatibility and simply delegates to that registry; it is removed +in 4.0.0. +""" + +import warnings +from typing import Any + +from .adaptive_combinatorial_purged import AdaptiveCombinatorialPurged +from .bagged_combinatorial_purged import BaggedCombinatorialPurged +from .combinatorial_purged import CombinatorialPurged +from .cross_validator_interface import CrossValidator +from .kfold import KFold +from .purged_kfold import PurgedKFold +from .walk_forward import WalkForward + + +class CrossValidatorFactory: + """ + Deprecated. Use ``RiskLabAI.core.CROSS_VALIDATORS.create(...)`` instead. + + Retained for backward compatibility (scheduled for removal in 4.0.0). ``VALIDATORS`` is + kept so existing introspection keeps working; ``create_cross_validator`` + now delegates to the core registry. + """ + + VALIDATORS = { + "kfold": KFold, + "walkforward": WalkForward, + "purgedkfold": PurgedKFold, + "combinatorialpurged": CombinatorialPurged, + "baggedcombinatorialpurged": BaggedCombinatorialPurged, + "adaptivecombinatorialpurged": AdaptiveCombinatorialPurged, + } + + @staticmethod + def create_cross_validator(validator_type: str, **kwargs: Any) -> CrossValidator: + """ + Deprecated. Create a cross-validator via the core registry. + + Parameters + ---------- + validator_type : str + One of the keys in :attr:`VALIDATORS` (case-insensitive). + **kwargs : Any + Forwarded to the validator's constructor; arguments it does not + accept are dropped (matching the historical behaviour). + + Raises + ------ + ValueError + If ``validator_type`` is not a known validator. + """ + warnings.warn( + "CrossValidatorFactory is deprecated and will be removed in 4.0.0; " + "use RiskLabAI.core.CROSS_VALIDATORS.create(validator_type, ...) " + "instead.", + DeprecationWarning, + stacklevel=2, + ) + from RiskLabAI.core import CROSS_VALIDATORS + + if validator_type.lower() not in CROSS_VALIDATORS: + raise ValueError( + f"Invalid validator_type: {validator_type}. " + f"Valid types are: {list(CrossValidatorFactory.VALIDATORS.keys())}" + ) + return CROSS_VALIDATORS.create( + validator_type, filter_unknown_kwargs=True, **kwargs + ) diff --git a/src/RiskLabAI/backtest/validation/cross_validator_interface.py b/src/RiskLabAI/backtest/validation/cross_validator_interface.py new file mode 100644 index 0000000..b4cb587 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/cross_validator_interface.py @@ -0,0 +1,235 @@ +""" +Defines the abstract base class (ABC) for all cross-validation strategies +in the RiskLabAI library. +""" + +from abc import ABC, abstractmethod +from collections.abc import Generator +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd + +# For type hinting sklearn-like estimators +Estimator = Any + + +class CrossValidator(ABC): + """ + Abstract Base Class (ABC) for cross-validation strategies. + + This interface defines the common methods required for all cross-validators, + ensuring they can handle both single and multiple datasets (passed as dicts) + and provide methods for splitting, generating backtest paths, and + running backtest predictions. + """ + + @abstractmethod + def get_n_splits( + self, + data: Optional[Union[pd.DataFrame, dict[str, pd.DataFrame]]] = None, + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> int: + """ + Return the total number of splits. + + Parameters + ---------- + data : pd.DataFrame or dict, optional + Dataset or dictionary of datasets. + labels : pd.Series or dict, optional + Labels or dictionary of labels. + groups : np.ndarray, optional + Group labels for the samples. + + Returns + ------- + int + The total number of splits. + """ + raise NotImplementedError + + @abstractmethod + def _single_split( + self, + single_data: pd.DataFrame, + ) -> Generator[tuple[np.ndarray, np.ndarray], None, None]: + """ + Split a single dataset into train-test indices. + + This function provides train-test indices to split a single dataset + into train/test sets. + + Parameters + ---------- + single_data : pd.DataFrame + Input dataset. + + Yields + ------- + Generator[Tuple[np.ndarray, np.ndarray], None, None] + A generator where each item is a tuple of (train_indices, test_indices). + """ + raise NotImplementedError + + @abstractmethod + def split( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> Union[ + Generator[tuple[np.ndarray, np.ndarray], None, None], + Generator[tuple[str, tuple[np.ndarray, np.ndarray]], None, None], + ]: + """ + Split data (or dictionary of data) into train-test indices. + + This function returns a generator that yields train-test indices. If a + dictionary of data is provided, the generator yields a key followed + by the (train_indices, test_indices) tuple. + + Parameters + ---------- + data : pd.DataFrame or dict + Dataset or dictionary of datasets. + labels : pd.Series or dict, optional + Labels or dictionary of labels. + groups : np.ndarray, optional + Group labels for the samples. + + Yields + ------- + Generator + - If data is a pd.DataFrame: (train_indices, test_indices) + - If data is a dict: (key, (train_indices, test_indices)) + """ + raise NotImplementedError + + @abstractmethod + def _single_backtest_paths( + self, single_data: pd.DataFrame + ) -> dict[str, list[dict[str, np.ndarray]]]: + """ + Generate backtest paths for a single dataset. + + A "path" is a specific sequence of train/test folds. For simple + K-Fold, there is only one path. For combinatorial methods, there + can be multiple. + + Parameters + ---------- + single_data : pd.DataFrame + Input dataset. + + Returns + ------- + Dict[str, List[Dict[str, np.ndarray]]] + A dictionary where keys are path identifiers (e.g., "Path 1") + and values are lists of splits. Each split is a dictionary + with "Train" and "Test" keys holding their respective indices. + """ + raise NotImplementedError + + @abstractmethod + def backtest_paths( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + ) -> Union[ + dict[str, list[dict[str, np.ndarray]]], + dict[str, dict[str, list[dict[str, np.ndarray]]]], + ]: + """ + Generate backtest paths for data or a dictionary of data. + + Parameters + ---------- + data : pd.DataFrame or dict + Dataset or dictionary of datasets. + + Returns + ------- + Union[Dict, Dict[str, Dict]] + - If data is a pd.DataFrame: A dictionary of backtest paths. + - If data is a dict: A nested dictionary, where the first-level + key is the dataset key, and the value is its + dictionary of backtest paths. + """ + raise NotImplementedError + + @abstractmethod + def _single_backtest_predictions( + self, + single_estimator: Estimator, + single_data: pd.DataFrame, + single_labels: pd.Series, + single_weights: Optional[np.ndarray] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> dict[str, np.ndarray]: + """ + Obtain backtest predictions for a single dataset. + + Parameters + ---------- + single_estimator : Estimator + A scikit-learn-like estimator to be trained and used for predictions. + single_data : pd.DataFrame + Data for the single dataset. + single_labels : pd.Series + Labels for the single dataset. + single_weights : np.ndarray, optional + Sample weights for the observations. Defaults to equal weights. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Dict[str, np.ndarray] + A dictionary where keys are path identifiers (e.g., "Path 1") + and values are the contiguous arrays of out-of-sample predictions. + """ + raise NotImplementedError + + @abstractmethod + def backtest_predictions( + self, + estimator: Union[Estimator, dict[str, Estimator]], + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Union[pd.Series, dict[str, pd.Series]], + sample_weights: Optional[Union[np.ndarray, dict[str, np.ndarray]]] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> Union[dict[str, np.ndarray], dict[str, dict[str, np.ndarray]]]: + """ + Generate backtest predictions for single or multiple datasets. + + Parameters + ---------- + estimator : Estimator or dict + Model(s) to be trained. Can be a single estimator or a + dictionary of estimators for multiple datasets. + data : pd.DataFrame or dict + Input data. Can be a single dataset or a dictionary of datasets. + labels : pd.Series or dict + Target labels. Can be a single series or a dictionary of series. + sample_weights : np.ndarray or dict, optional + Sample weights. Can be a single array or a dictionary of arrays. + Defaults to None (equal weights). + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Union[Dict, Dict[str, Dict]] + - If data is a pd.DataFrame: A dictionary of predictions. + - If data is a dict: A nested dictionary, where the first-level + key is the dataset key, and the value is its + dictionary of predictions. + """ + raise NotImplementedError diff --git a/src/RiskLabAI/backtest/validation/kfold.py b/src/RiskLabAI/backtest/validation/kfold.py new file mode 100644 index 0000000..9dbb7b4 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/kfold.py @@ -0,0 +1,359 @@ +""" +Implements a standard K-Fold cross-validator. +""" + +import warnings +from collections.abc import Generator +from copy import deepcopy +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd +from joblib import Parallel, delayed +from sklearn.exceptions import ConvergenceWarning + +from .cross_validator_interface import CrossValidator + +# For type hinting sklearn-like estimators +Estimator = Any + + +class KFold(CrossValidator): + """ + K-Fold cross-validator. + + This class implements the standard K-Fold cross-validation strategy, where + the dataset is divided into `k` consecutive folds. Each fold is then + used once as a validation set while the `k - 1` remaining folds + form the training set. + + This implementation allows for optional shuffling. + """ + + def __init__( + self, n_splits: int, shuffle: bool = False, random_seed: int = None + ) -> None: + """ + Initialize the K-Fold cross-validator. + + Parameters + ---------- + n_splits : int + Number of splits or folds for the cross-validation. + The dataset will be divided into `n_splits` consecutive parts. + shuffle : bool, default=False + Whether to shuffle the data before splitting it into folds. + random_seed : int, optional + Seed used for random shuffling. Only used when `shuffle` is True. + """ + self.n_splits = n_splits + self.shuffle = shuffle + self.random_seed = random_seed + self.rng_ = np.random.default_rng(random_seed) + + def get_n_splits( + self, + data: Optional[Union[pd.DataFrame, dict[str, pd.DataFrame]]] = None, + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> int: + """ + Return the number of splits. + + Parameters + ---------- + data : pd.DataFrame or dict, optional + Dataset or dictionary of datasets. + labels : pd.Series or dict, optional + Labels or dictionary of labels. + groups : np.ndarray, optional + Group labels for the samples. + + Returns + ------- + int + The number of splits (`n_splits`). + """ + return self.n_splits + + def _get_shuffled_indices(self, n_samples: int) -> np.ndarray: + """Helper to get shuffled or sequential indices.""" + indices = np.arange(n_samples) + if self.shuffle: + self.rng_.shuffle(indices) + return indices + + def _single_split( + self, + single_data: pd.DataFrame, + ) -> Generator[tuple[np.ndarray, np.ndarray], None, None]: + """ + Split a single dataset into train-test indices. + + Parameters + ---------- + single_data : pd.DataFrame + Input dataset. + + Yields + ------- + Generator[Tuple[np.ndarray, np.ndarray], None, None] + A generator where each item is a tuple of (train_indices, test_indices). + """ + indices = self._get_shuffled_indices(single_data.shape[0]) + + for test_indices in np.array_split(indices, self.n_splits): + train_indices = np.setdiff1d(indices, test_indices) + yield train_indices, test_indices + + def split( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> Union[ + Generator[tuple[np.ndarray, np.ndarray], None, None], + Generator[tuple[str, tuple[np.ndarray, np.ndarray]], None, None], + ]: + """ + Split data (or dictionary of data) into train-test indices. + + Parameters + ---------- + data : pd.DataFrame or dict + Dataset or dictionary of datasets. + labels : pd.Series or dict, optional + Labels or dictionary of labels. + groups : np.ndarray, optional + Group labels for the samples. + + Yields + ------- + Generator + - If data is a pd.DataFrame: (train_indices, test_indices) + - If data is a dict: (key, (train_indices, test_indices)) + """ + if isinstance(data, dict): + for key in data: + for train_indices, test_indices in self._single_split(data[key]): + yield key, (train_indices, test_indices) + else: + for train_indices, test_indices in self._single_split(data): + yield train_indices, test_indices + + def _single_backtest_paths( + self, single_data: pd.DataFrame + ) -> dict[str, list[dict[str, np.ndarray]]]: + """ + Generate backtest paths for a single dataset. + For K-Fold, there is only one "path". + + Parameters + ---------- + single_data : pd.DataFrame + Input dataset. + + Returns + ------- + Dict[str, List[Dict[str, np.ndarray]]] + A dictionary with one key "Path 1", containing a list of all + (Train, Test) index dictionaries. + """ + path_data = [] + paths = {} + + for train_indices, test_indices in self._single_split(single_data): + path_data.append( + { + "Train": train_indices, + "Test": test_indices, + } + ) + + paths["Path 1"] = path_data + return paths + + def backtest_paths( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + ) -> Union[ + dict[str, list[dict[str, np.ndarray]]], + dict[str, dict[str, list[dict[str, np.ndarray]]]], + ]: + """ + Generate backtest paths for data or a dictionary of data. + + Parameters + ---------- + data : pd.DataFrame or dict + Dataset or dictionary of datasets. + + Returns + ------- + Union[Dict, Dict[str, Dict]] + - If data is a pd.DataFrame: A dictionary of backtest paths. + - If data is a dict: A nested dictionary, where the first-level + key is the dataset key, and the value is its + dictionary of backtest paths. + """ + if isinstance(data, dict): + multiple_paths = {} + for key in data: + multiple_paths[key] = self._single_backtest_paths(data[key]) + return multiple_paths + + return self._single_backtest_paths(data) + + def _single_backtest_predictions( + self, + single_estimator: Estimator, + single_data: pd.DataFrame, + single_labels: pd.Series, + single_weights: Optional[np.ndarray] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> dict[str, np.ndarray]: + """ + Obtain backtest predictions for a single dataset. + + Parameters + ---------- + single_estimator : Estimator + A scikit-learn-like estimator to be trained and used for predictions. + single_data : pd.DataFrame + Data for the single dataset. + single_labels : pd.Series + Labels for the single dataset. + single_weights : np.ndarray, optional + Sample weights for the observations. Defaults to equal weights. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Dict[str, np.ndarray] + A dictionary with one key "Path 1", containing the contiguous + array of out-of-sample predictions, ordered by the original + dataset index. + """ + if single_weights is None: + single_weights = np.ones(len(single_data)) + + def train_test_single_estimator( + estimator_: Estimator, train_indices: np.ndarray, test_indices: np.ndarray + ) -> tuple[np.ndarray, np.ndarray]: + """Train model and return (predictions, test_indices).""" + X_train = single_data.iloc[train_indices] + y_train = single_labels.iloc[train_indices] + weights_train = single_weights[train_indices] + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ConvergenceWarning) + try: + estimator_.fit(X_train, y_train, sample_weight=weights_train) + except TypeError: + # Fallback for estimators without sample_weight + estimator_.fit(X_train, y_train) + + X_test = single_data.iloc[test_indices] + + if predict_probability: + preds = estimator_.predict_proba(X_test) + else: + preds = estimator_.predict(X_test) + + return preds, test_indices + + # Run all folds in parallel + results = Parallel(n_jobs=n_jobs)( + delayed(train_test_single_estimator)( + deepcopy(single_estimator), train_indices, test_indices + ) + for train_indices, test_indices in self._single_split(single_data) + ) + + # Unpack predictions and their corresponding original indices + all_preds = [res[0] for res in results] + all_indices = [res[1] for res in results] + + # Concatenate results + path_data = np.concatenate(all_preds) + original_indices = np.concatenate(all_indices) + + # Re-order predictions to match the original dataset order + # This is critical if shuffle=True + if self.shuffle: + reorder_indices = np.argsort(original_indices) + path_data = path_data[reorder_indices] + + paths_predictions = {"Path 1": path_data} + return paths_predictions + + def backtest_predictions( + self, + estimator: Union[Estimator, dict[str, Estimator]], + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Union[pd.Series, dict[str, pd.Series]], + sample_weights: Optional[Union[np.ndarray, dict[str, np.ndarray]]] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> Union[dict[str, np.ndarray], dict[str, dict[str, np.ndarray]]]: + """ + Generate backtest predictions for single or multiple datasets. + + Parameters + ---------- + estimator : Estimator or dict + Model(s) to be trained. + data : pd.DataFrame or dict + Input data. + labels : pd.Series or dict + Target labels. + sample_weights : np.ndarray or dict, optional + Sample weights. Defaults to None. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Union[Dict, Dict[str, Dict]] + - If data is a pd.DataFrame: A dictionary of predictions. + - If data is a dict: A nested dictionary of predictions. + """ + if isinstance(data, dict): + if ( + not isinstance(estimator, dict) + or not isinstance(labels, dict) + or (sample_weights and not isinstance(sample_weights, dict)) + ): + raise ValueError( + "If data is a dict, estimator, labels, " + "and sample_weights (if provided) must also be dicts." + ) + + multiple_paths_predictions = {} + for key in data: + multiple_paths_predictions[key] = self._single_backtest_predictions( + estimator[key], + data[key], + labels[key], + sample_weights[key] if sample_weights else None, + predict_probability, + n_jobs, + ) + return multiple_paths_predictions + + # Handle single dataset case + if not (isinstance(data, pd.DataFrame) and isinstance(labels, pd.Series)): + raise ValueError( + "If data is a DataFrame, estimator must be a single estimator " + "and labels must be a single Series." + ) + + return self._single_backtest_predictions( + estimator, data, labels, sample_weights, predict_probability, n_jobs + ) diff --git a/src/RiskLabAI/backtest/validation/leakage_aware_hpo.py b/src/RiskLabAI/backtest/validation/leakage_aware_hpo.py new file mode 100644 index 0000000..e299bd6 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/leakage_aware_hpo.py @@ -0,0 +1,240 @@ +r""" +Leakage-aware hyper-parameter optimization: Optuna wired through purged CV, gated by the Deflated Sharpe. + +de Prado tunes via grid / random search under purged cross-validation. Principled Bayesian / evolutionary +search (Optuna's TPE / CMA-ES) reaches good configurations in far fewer trials, but two hazards make naive +HPO dangerous on financial data: leakage-prone CV (a shuffled k-fold) massively inflates the apparent +score under overlapping labels, and every trial is a configuration, so the search itself overfits and the +in-sample HPO score must be deflated by the trial count before it can be trusted. + +This module wires Optuna through the repo's :class:`PurgedKFold` / :class:`CombinatorialPurged` (so the +per-trial score is leakage-controlled) and provides a Deflated-Sharpe gate that deflates the selected +model's out-of-sample Sharpe by the HPO-inclusive trial count. + +Admitted in Appraisal 20 (CONTRIBUTIONS_LEDGER 2026-06-27) as a methodology / infrastructure admit, NOT a +performance claim. Scope tag, verbatim from the verdict: + + Optuna (TPE/CMA-ES) wired through PurgedKFold/CPCV with PBO/DSR gating, preferred over grid/random for + search efficiency and over naive-CV tuning for leakage-safety, and carrying the honest, characterized + finding that tuning yields no OOS edge after deflation - so it must be gated by the Deflated Sharpe at + the trial count, never trusted on the in-sample HPO score. + +**Plainly: this improves search efficiency and leakage-safety, NOT out-of-sample performance. Tuning does +not create edge.** In the appraisal the principled-HPO real-leg gain did not survive the Deflated Sharpe at +the HPO-inclusive trial count (DSR 0.44 / 0.34 < 0.5, PBO 0.67 / 0.69, OOS at the base rate); the value is +that the search reaches the optimum in fewer trials and that purged CV removes the leakage a naive k-fold +inflates. Always gate the selected model by :func:`deflated_sharpe_gate` at the HPO trial count; never +trust the in-sample HPO score. Evidence and caveats: appraisals/20_verdict.md. + +References +---------- +Akiba, T. et al. (2019) Optuna: a next-generation hyperparameter optimization framework. KDD. +Lopez de Prado, M. (2018) Advances in Financial Machine Learning (PurgedKFold/CPCV, PBO, the Deflated + Sharpe Ratio). +""" + +from __future__ import annotations + +from typing import Callable + +import numpy as np + +from RiskLabAI.backtest.probabilistic_sharpe_ratio import probabilistic_sharpe_ratio +from RiskLabAI.backtest.test_set_overfitting import expected_max_sharpe_ratio + +from .purged_kfold import PurgedKFold + + +def _purged_cv_score( + estimator_factory, params, x, y, times, n_splits, embargo, scoring, seed +): + """Mean leakage-controlled CV score of one configuration under PurgedKFold.""" + purged = PurgedKFold(n_splits=n_splits, times=times, embargo=embargo) + x_values = x.values if hasattr(x, "values") else np.asarray(x) + y_values = y.values if hasattr(y, "values") else np.asarray(y) + scores = [] + for train, test in purged.split(x): + if len(train) < 25 or len(test) < 5 or len(np.unique(y_values[train])) < 2: + continue + model = estimator_factory(params) + model.fit(x_values[train], y_values[train]) + if scoring is None: + scores.append( + float((model.predict(x_values[test]) == y_values[test]).mean()) + ) + else: + scores.append(float(scoring(model, x_values[test], y_values[test]))) + return float(np.mean(scores)) if scores else float("-inf") + + +def leakage_aware_hpo( + suggest_params: Callable, + estimator_factory: Callable, + x, + y, + times, + n_trials: int = 50, + sampler: str = "tpe", + n_splits: int = 5, + embargo: float = 0.0, + scoring: Callable | None = None, + random_state: int = 0, +) -> dict: + r""" + Optuna hyper-parameter search with every trial scored under :class:`PurgedKFold` (leakage-controlled). + + This improves SEARCH EFFICIENCY (TPE / CMA-ES reach the optimum in far fewer trials than grid / random) + and LEAKAGE-SAFETY (the per-trial score is purged, not a leaky shuffled k-fold). It does NOT improve + out-of-sample performance - tuning does not create edge - so the selected model must be gated by + :func:`deflated_sharpe_gate` at the returned ``n_trials``; never trust ``best_score`` directly. See the + module docstring for the verbatim scope tag and appraisals/20_verdict.md. + + Parameters + ---------- + suggest_params : callable + ``suggest_params(trial) -> dict``: maps an Optuna trial to a hyper-parameter dict. + estimator_factory : callable + ``estimator_factory(params) -> estimator``: builds a fresh scikit-learn-style estimator + (``fit`` / ``predict``). + x, y : array-like or pandas + Design matrix and labels. ``x`` must support ``PurgedKFold.split(x)`` (a DataFrame indexed by the + event start times). + times : pd.Series + Event start -> end times for purging / embargo (passed to :class:`PurgedKFold`). + n_trials : int, default 50 + Optuna trial budget (this is the HPO trial count to deflate by). + sampler : {"tpe", "cmaes"}, default "tpe" + The Optuna sampler. + n_splits : int, default 5 + Purged CV folds. + embargo : float, default 0.0 + PurgedKFold embargo fraction. + scoring : callable, optional + ``scoring(estimator, x_test, y_test) -> float`` (higher is better). Defaults to accuracy. + random_state : int, default 0 + Sampler seed. + + Returns + ------- + dict + ``best_params``, ``best_score`` (purged CV), ``n_trials``, ``trial_scores``, ``mean_trial_score``, + ``std_trial_score``. + + Raises + ------ + ImportError + If ``optuna`` is not installed (it is an optional dependency). + """ + try: + import optuna + except ImportError as exc: # pragma: no cover - optional dependency + raise ImportError( + "leakage_aware_hpo requires optuna (pip install optuna)" + ) from exc + + if sampler == "tpe": + optuna_sampler = optuna.samplers.TPESampler(seed=random_state) + elif sampler == "cmaes": + optuna_sampler = optuna.samplers.CmaEsSampler(seed=random_state) + else: + raise ValueError("sampler must be 'tpe' or 'cmaes'") + + optuna.logging.set_verbosity(optuna.logging.WARNING) + + def objective(trial): + params = suggest_params(trial) + return _purged_cv_score( + estimator_factory, + params, + x, + y, + times, + n_splits, + embargo, + scoring, + random_state, + ) + + study = optuna.create_study(direction="maximize", sampler=optuna_sampler) + study.optimize(objective, n_trials=n_trials, show_progress_bar=False) + trial_scores = [ + t.value for t in study.trials if t.value is not None and np.isfinite(t.value) + ] + return { + "best_params": study.best_params, + "best_score": float(study.best_value), + "n_trials": int(n_trials), + "trial_scores": trial_scores, + "mean_trial_score": ( + float(np.mean(trial_scores)) if trial_scores else float("nan") + ), + "std_trial_score": ( + float(np.std(trial_scores)) if trial_scores else float("nan") + ), + } + + +def deflated_sharpe_gate( + out_of_sample_returns: np.ndarray, + n_trials: int, + trial_sharpe_std: float, + threshold: float = 0.5, +) -> dict: + r""" + Gate a selected model's out-of-sample Sharpe by the Deflated Sharpe at the HPO trial count. + + The benchmark is the expected maximum Sharpe over ``n_trials`` independent trials (with the observed + cross-trial Sharpe dispersion ``trial_sharpe_std``); the Deflated Sharpe is the probability that the + realized OOS Sharpe exceeds that benchmark. A selection passes only if the Deflated Sharpe exceeds + ``threshold`` (0.5). This is the decisive control: in the appraisal the principled-HPO gain did not + pass it - tuning yielded no out-of-sample edge after deflation. See the module docstring for the + verbatim scope tag and appraisals/20_verdict.md. + + Parameters + ---------- + out_of_sample_returns : np.ndarray + The selected model's out-of-sample (e.g. CPCV) per-period return series. + n_trials : int + The HPO-inclusive trial count to deflate by (every configuration tried). + trial_sharpe_std : float + The standard deviation of the per-trial (per-period) Sharpe ratios across the search. + threshold : float, default 0.5 + The Deflated-Sharpe pass threshold. + + Returns + ------- + dict + ``observed_sharpe``, ``benchmark_sharpe`` (E[max SR] at ``n_trials``), ``n_trials``, + ``deflated_sharpe``, and ``passes``. + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> r = rng.standard_normal(500) * 0.01 + >>> gate = deflated_sharpe_gate(r, n_trials=100, trial_sharpe_std=0.05) + >>> set(gate) >= {"deflated_sharpe", "passes", "benchmark_sharpe"} + True + """ + from scipy.stats import kurtosis, skew + + r = np.asarray(out_of_sample_returns, dtype=float) + sd = r.std() + observed_sharpe = float(r.mean() / sd) if sd > 0 else 0.0 + benchmark_sharpe = float(expected_max_sharpe_ratio(n_trials, 0.0, trial_sharpe_std)) + deflated = float( + probabilistic_sharpe_ratio( + observed_sharpe, + benchmark_sharpe, + r.size, + skewness_of_returns=float(skew(r)) if r.size > 2 else 0.0, + kurtosis_of_returns=float(kurtosis(r, fisher=False)) if r.size > 3 else 3.0, + ) + ) + return { + "observed_sharpe": observed_sharpe, + "benchmark_sharpe": benchmark_sharpe, + "n_trials": int(n_trials), + "deflated_sharpe": deflated, + "passes": bool(deflated > threshold), + } diff --git a/src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py b/src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py new file mode 100644 index 0000000..bdf8460 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/path_adaptive_cpcv.py @@ -0,0 +1,219 @@ +""" +Path-level Adaptive Combinatorial Purged Cross-Validation PBO (Arian-Norouzi-Seco 2024). + +de Prado's CPCV / CSCV weights every combinatorial backtest path equally when it estimates the +Probability of Backtest Overfitting (PBO) and when it selects a configuration. When the train/test +regime shifts, that equal weighting biases the selection toward the wrong regime. Adaptive CPCV +weights each path by how much its TEST block resembles the FORWARD regime, where the regime is read +from an observable, point-in-time volatility signal (the cross-configuration dispersion). Down-weighting +paths that test on a non-representative regime lowers selection error when the regime is detectable and +selection is signal-bearing, and the weights collapse to uniform in a stationary single regime, so the +estimate converges to plain CPCV with no over-adaptation. + +This is the **path-level** mechanism from the Appraisal 09 pre-registration. It is distinct from, and +does NOT touch, the existing :class:`AdaptiveCombinatorialPurged`, which adapts the split *boundaries* +of an ML backtest (a different mechanism). The plain-CPCV baseline here is the repo's CSCV PBO: the +per-path overfit indicator is taken from +:func:`~RiskLabAI.backtest.probability_of_backtest_overfitting.performance_evaluation`, so with uniform +weights this function reproduces +:func:`~RiskLabAI.backtest.probability_of_backtest_overfitting.probability_of_backtest_overfitting` +exactly. + +Admitted in Appraisal 09b (CONTRIBUTIONS_LEDGER 2026-06-27; in-house method under a conflict of +interest, held to the identical bar, admitted only after a re-designed signal-bearing held-out). +Regime tag, verbatim from the 09b verdict: + + prefer Adaptive CPCV over plain CPCV for model selection when the train/test regime may shift, the + shift is identifiable at decision time from observable volatility, and there is adequate data for + selection (it does not help where selection is noise-dominated by too little data); it converges to + plain CPCV in a stationary regime with no over-adaptation. + +The advantage is, as of admission, Monte-Carlo evidence where the regime is observable through the +adaptive feature; a real-data regime-shift confirmation on a genuine crisis/volatility period is a +tracked obligation (``REAL_DATA_FOLLOWUPS.md``). Evidence and caveats: appraisals/09_verdict.md (the +09b re-appraisal). + +References +---------- +Arian, H., Norouzi M., L. and Seco, L. (2024) Bagged and Adaptive Combinatorial Purged + Cross-Validation. (Clean-room from the path-level mechanism; the verdict harness validated plain + CPCV against probability_of_backtest_overfitting exactly.) +Bailey, D. H., Borwein, J., Lopez de Prado, M. and Zhu, Q. J. (2017) The probability of backtest + overfitting. Journal of Computational Finance, 20(4), 39-69. +""" + +from __future__ import annotations + +from itertools import combinations +from typing import Callable + +import numpy as np + +from ..backtest_statistics import sharpe_ratio +from ..probability_of_backtest_overfitting import performance_evaluation + + +def estimate_volatility_regimes( + performances: np.ndarray, n_regimes: int = 2, window: int | None = None +) -> np.ndarray: + """ + Observable regime label per period from the cross-configuration volatility. + + The signal is the per-period dispersion across configurations (a market-stress proxy), smoothed by a + **trailing** moving average so the volatility feature at period ``t`` uses only returns up to ``t``. + Periods are then labelled high- vs low-volatility relative to the median of the observed signal (an + evaluation-time normalisation over the in-sample window). The estimate keys on volatility alone: it + is invariant to the sign of the returns and never uses the forward performance the method selects, so + it carries no look-ahead into the quantity being predicted. + + Parameters + ---------- + performances : np.ndarray + A T-observations x N-configurations matrix of per-period returns. + n_regimes : int, default 2 + The number of regimes (>= 2); 2 gives a low/high-volatility split at the observed median. + window : int, optional + The trailing smoothing window. Defaults to ``max(T // 20, 3)``. + + Returns + ------- + np.ndarray + An integer regime label in ``[0, n_regimes)`` for each of the T periods. + """ + performances = np.asarray(performances, dtype=float) + t_len = performances.shape[0] + dispersion = performances.std(axis=1) + if window is None: + window = max(t_len // 20, 3) + window = max(int(window), 1) + # trailing (causal) moving average: cumulative-sum difference over a back-window + cumulative = np.concatenate([[0.0], np.cumsum(dispersion)]) + counts = np.minimum(np.arange(1, t_len + 1), window) + starts = np.maximum(np.arange(1, t_len + 1) - window, 0) + smooth = (cumulative[1:] - cumulative[starts]) / counts + if n_regimes <= 2: + return (smooth > np.median(smooth)).astype(int) + edges = np.quantile(smooth, np.linspace(0.0, 1.0, n_regimes + 1)[1:-1]) + return np.digitize(smooth, edges) + + +def _cscv_path_stats( + performances: np.ndarray, + n_partitions: int, + metric: Callable[[np.ndarray, float], float], + risk_free_return: float, +): + """ + Per CSCV path: ``(is_overfit, test_config_metrics, test_partition_ids)``. + + The overfit indicator comes from the repo's + :func:`~RiskLabAI.backtest.probability_of_backtest_overfitting.performance_evaluation`, over the same + symmetric train/test combinations the plain PBO uses, so the unweighted mean reproduces the repo PBO + exactly. The per-configuration test metrics are recomputed with the same metric for the (weighted) + selection and weighting. + """ + _, n_strategies = performances.shape + partitions = np.array_split(performances, n_partitions) + all_ids = list(range(n_partitions)) + stats = [] + for train_ids in combinations(all_ids, n_partitions // 2): + test_ids = [i for i in all_ids if i not in train_ids] + train = np.concatenate([partitions[i] for i in train_ids], axis=0) + test = np.concatenate([partitions[i] for i in test_ids], axis=0) + is_overfit, _logit = performance_evaluation( + train, test, n_strategies, metric, risk_free_return + ) + test_metrics = np.array( + [metric(test[:, j], risk_free_return) for j in range(n_strategies)] + ) + stats.append((1.0 if is_overfit else 0.0, test_metrics, test_ids)) + return stats + + +def adaptive_probability_of_backtest_overfitting( + performances: np.ndarray, + n_partitions: int = 16, + target_fraction: float = 0.25, + risk_free_return: float = 0.0, + metric: Callable[[np.ndarray, float], float] | None = None, + n_regimes: int = 2, +) -> tuple[float, int]: + r""" + Adaptive (path-level) Probability of Backtest Overfitting and regime-aware configuration selection. + + Each CSCV path is weighted by how much its test block resembles the forward regime (the regime of + the most recent ``target_fraction`` of the sample, read from the observable volatility signal of + :func:`estimate_volatility_regimes`). The regime-weighted PBO and the configuration with the highest + regime-weighted out-of-sample metric are returned. In a stationary regime the weights are uniform, + so the PBO converges to plain CPCV and the selection matches it; see the module docstring for the + full regime tag and appraisals/09_verdict.md. + + Parameters + ---------- + performances : np.ndarray + A T-observations x N-configurations matrix of per-period returns. + n_partitions : int, default 16 + The number of CSCV partitions S (must be even). + target_fraction : float, default 0.25 + The trailing fraction of the sample whose regime defines the forward target (in ``(0, 1]``). + risk_free_return : float, default 0.0 + Passed through to the metric. + metric : Callable, optional + The performance metric (defaults to the PBO module's Sharpe). + n_regimes : int, default 2 + The number of volatility regimes to estimate. + + Returns + ------- + (float, int) + The regime-weighted PBO and the index of the selected configuration. + + Examples + -------- + >>> import numpy as np + >>> from RiskLabAI.backtest.validation import ( + ... adaptive_probability_of_backtest_overfitting, + ... ) + >>> rng = np.random.default_rng(0) + >>> # a regime shift: config 0 leads early, config 9 leads in the volatile forward block + >>> perf = rng.standard_normal((480, 10)) * 0.05 + >>> perf[:360, 0] += 0.02 + >>> perf[360:, 9] += 0.02 + rng.standard_normal((120, 1))[:, 0] * 0.05 + >>> pbo, selected = adaptive_probability_of_backtest_overfitting(perf, n_partitions=8) + >>> 0.0 <= pbo <= 1.0 and 0 <= selected < 10 + True + """ + performances = np.asarray(performances, dtype=float) + if n_partitions % 2 != 0: + raise ValueError("Number of partitions must be even.") + if performances.ndim != 2: + raise ValueError("performances must be a 2-D (T x N) matrix.") + t_len, _ = performances.shape + if t_len < n_partitions: + raise ValueError( + f"Too few observations ({t_len}) for {n_partitions} partitions." + ) + if not 0.0 < target_fraction <= 1.0: + raise ValueError("target_fraction must be in (0, 1].") + if metric is None: + metric = sharpe_ratio + + regimes = estimate_volatility_regimes(performances, n_regimes=n_regimes) + forward = regimes[int((1.0 - target_fraction) * t_len) :] + # majority regime label over the forward block (the target the selection is steered toward) + forward_label = int(np.bincount(forward).argmax()) if forward.size else 0 + + regime_partitions = np.array_split(regimes, n_partitions) + stats = _cscv_path_stats(performances, n_partitions, metric, risk_free_return) + + overfit = np.array([s[0] for s in stats]) + test_metrics = np.array([s[1] for s in stats]) # (n_paths, n_configs) + weights = np.empty(len(stats)) + for k, (_overfit, _metrics, test_ids) in enumerate(stats): + test_regime = np.concatenate([regime_partitions[i] for i in test_ids]) + # similarity = share of the test block in the forward regime; epsilon avoids all-zero weights + weights[k] = float(np.mean(test_regime == forward_label)) + 1e-3 + + pbo = float(np.average(overfit, weights=weights)) + selected = int(np.argmax(np.average(test_metrics, axis=0, weights=weights))) + return pbo, selected diff --git a/src/RiskLabAI/backtest/validation/path_bagged_cpcv.py b/src/RiskLabAI/backtest/validation/path_bagged_cpcv.py new file mode 100644 index 0000000..ab15b2c --- /dev/null +++ b/src/RiskLabAI/backtest/validation/path_bagged_cpcv.py @@ -0,0 +1,137 @@ +""" +Path-level Bagged Combinatorial Purged Cross-Validation PBO (Arian-Norouzi-Seco 2024). + +de Prado's CPCV / CSCV estimates the Probability of Backtest Overfitting (PBO) from a finite set of +combinatorial backtest paths, so the estimate carries estimation variance. Bagging the PATHS - taking +a moving-block bootstrap of the performance series, recomputing the CSCV PBO on each resample, and +averaging - reduces that variance, giving a more accurate PBO estimate whenever the path set is small +or noisy, and converging to plain CPCV in the data-rich limit. + +This is the **path-level** mechanism from the Appraisal 09 pre-registration. It is distinct from, and +does NOT touch, the existing :class:`BaggedCombinatorialPurged`, which bags a sklearn *estimator* on an +ML backtest (a different mechanism). The plain-CPCV baseline here is the repo's CSCV PBO +(:func:`~RiskLabAI.backtest.probability_of_backtest_overfitting.probability_of_backtest_overfitting`), +which this function calls on each bootstrap resample. + +Admitted in Appraisal 09 (CONTRIBUTIONS_LEDGER 2026-06-27; in-house method, held to the identical bar). +Regime tag, verbatim from the verdict: + + For a more accurate, lower-variance overfitting (PBO) estimate whenever the CPCV path set is small + or noisy, converging to plain CPCV in the data-rich limit; neutral on model selection. Point-in-time + and mechanism-backed. + +Evidence and caveats: appraisals/09_verdict.md. + +References +---------- +Arian, H., Norouzi M., L. and Seco, L. (2024) Bagged and Adaptive Combinatorial Purged + Cross-Validation. (Clean-room from the path-level mechanism; the verdict harness validated plain + CPCV against probability_of_backtest_overfitting exactly.) +Bailey, D. H., Borwein, J., Lopez de Prado, M. and Zhu, Q. J. (2017) The probability of backtest + overfitting. Journal of Computational Finance, 20(4), 39-69. +""" + +from __future__ import annotations + +from typing import Callable, Optional + +import numpy as np + +from ..probability_of_backtest_overfitting import probability_of_backtest_overfitting + + +def moving_block_bootstrap_indices( + n_observations: int, block_size: int, rng: np.random.Generator +) -> np.ndarray: + """ + Moving-block bootstrap row indices for a length-n series (blocks preserve serial structure). + + Random block start points are tiled, each contributing ``block_size`` consecutive (wrap-around) + rows, truncated to ``n_observations``. + + Parameters + ---------- + n_observations : int + The series length n. + block_size : int + The block length. + rng : np.random.Generator + The random generator (seed it for reproducibility). + + Returns + ------- + np.ndarray + An index array of length ``n_observations``. + """ + block_size = max(int(block_size), 1) + n_blocks = int(np.ceil(n_observations / block_size)) + starts = rng.integers(0, n_observations, size=n_blocks) + offsets = np.arange(block_size) + return (starts[:, None] + offsets[None, :]).ravel()[ + :n_observations + ] % n_observations + + +def bagged_probability_of_backtest_overfitting( + performances: np.ndarray, + n_partitions: int = 16, + n_bag: int = 30, + block_size: Optional[int] = None, + risk_free_return: float = 0.0, + metric: Optional[Callable[[np.ndarray, float], float]] = None, + seed: int = 0, +) -> tuple[float, np.ndarray]: + r""" + Bagged (path-level) Probability of Backtest Overfitting. + + Takes ``n_bag`` moving-block bootstrap resamples of the performance matrix, computes the CSCV PBO + (:func:`probability_of_backtest_overfitting`) on each, and returns the average - a lower-variance + estimate than a single CPCV PBO. Prefer it whenever the CPCV path set is small or noisy; it + converges to plain CPCV in the data-rich limit and is neutral on model selection. See the module + docstring for the full regime tag and appraisals/09_verdict.md. + + Parameters + ---------- + performances : np.ndarray + A T-observations x N-strategies matrix of per-period returns. + n_partitions : int, default 16 + The number of CSCV partitions S (must be even); passed through to the plain PBO. + n_bag : int, default 30 + The number of moving-block bootstrap resamples to aggregate. + block_size : int, optional + The bootstrap block length. Defaults to ``max(T // 20, 5)``. + risk_free_return : float, default 0.0 + Passed through to the metric. + metric : Callable, optional + The performance metric (defaults to the PBO module's Sharpe). + seed : int, default 0 + The bootstrap seed (for reproducibility). + + Returns + ------- + (float, np.ndarray) + The bagged PBO (the mean over resamples) and the array of per-resample PBOs. + + Examples + -------- + >>> import numpy as np + >>> from RiskLabAI.backtest.validation import bagged_probability_of_backtest_overfitting + >>> rng = np.random.default_rng(0) + >>> perf = rng.standard_normal((480, 10)) * 0.05 + np.linspace(0, 0.01, 10) + >>> pbo, per_resample = bagged_probability_of_backtest_overfitting(perf, n_partitions=8, n_bag=20) + >>> 0.0 <= pbo <= 1.0 + True + """ + performances = np.asarray(performances, dtype=float) + t_len = performances.shape[0] + if block_size is None: + block_size = max(t_len // 20, 5) + rng = np.random.default_rng(seed) + pbos = np.empty(n_bag) + for b in range(n_bag): + rows = moving_block_bootstrap_indices(t_len, block_size, rng) + pbo, _ = probability_of_backtest_overfitting( + performances[rows], n_partitions, risk_free_return, metric + ) + pbos[b] = pbo + return float(pbos.mean()), pbos diff --git a/src/RiskLabAI/backtest/validation/purged_kfold.py b/src/RiskLabAI/backtest/validation/purged_kfold.py new file mode 100644 index 0000000..9872834 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/purged_kfold.py @@ -0,0 +1,543 @@ +""" +Implements a Purged K-Fold cross-validator to prevent information leakage +in financial time-series data. +""" + +import warnings +from collections.abc import Generator +from copy import deepcopy +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd +from joblib import Parallel, delayed +from sklearn.exceptions import ConvergenceWarning + +from .cross_validator_interface import CrossValidator + +# For type hinting sklearn-like estimators +Estimator = Any + + +class PurgedKFold(CrossValidator): + """ + Purged K-Fold cross-validator. + + Implements a K-Fold strategy where each fold is purged of observations + that overlap in time with the test set. An "embargo" period can also be + added after the test set to prevent information leakage from the training + set immediately following the test set. + + This is essential for financial data where labels are defined over a + future period (e.g., T+k returns) and simple K-Fold would leak + information. + + Parameters + ---------- + n_splits : int + Number of splits or folds. + times : pd.Series or Dict[str, pd.Series] + A Series where the index is the time the observation starts and + the values are the time the observation ends. This defines the + "information range" of each observation. + embargo : float, default=0 + The embargo fraction. A fraction of the dataset length (e.g., 0.01) + to be removed *after* the end of the test set information range. + """ + + @staticmethod + def filtered_training_indices_with_embargo( + data_info_range: pd.Series, + test_time_range: pd.Series, + embargo_fraction: float = 0, + continous_test_times: bool = False, + ) -> pd.Series: + r""" + Purge observations in the training set with embargo. + + Finds training set indices that do not overlap with the test set's + information range. + + .. math:: + \text{embargo\_length} = \text{len(data\_info_range)} \times \text{embargo\_fraction} + + Parameters + ---------- + data_info_range : pd.Series + Series detailing the information range for *all* data. + - Index: Time when the information extraction started. + - Value: Time when the information extraction ended. + test_time_range : pd.Series + Series detailing the information range for *test set* data. + - Index: Time when the information extraction started. + - Value: Time when the information extraction ended. + embargo_fraction : float, default=0 + Fraction of the dataset trailing the test observations to + exclude from training. + continous_test_times : bool, default=False + If True, considers the test time range as one continuous block + from the earliest start to the latest end. + + Returns + ------- + pd.Series + A view of `data_info_range` containing only the filtered + training data (i.e., observations that do not overlap with + test + embargo). + """ + indices_to_drop: set[int] = set() + embargo_length = int(len(data_info_range) * embargo_fraction) + + if test_time_range.empty: + return data_info_range + + sorted_test_time_range = test_time_range.sort_index().copy() + + if not continous_test_times: + # Create blocks of contiguous test ranges + test_ranges = pd.DataFrame( + { + "start": sorted_test_time_range.index, + "end": sorted_test_time_range.values, + } + ) + gaps = test_ranges["start"] > test_ranges["end"].shift(1) + blocks = gaps.cumsum() + effective_test_time_range = test_ranges.groupby(blocks).agg( + {"start": "min", "end": "max"} + ) + effective_test_time_range = pd.Series( + effective_test_time_range["end"].values, + index=effective_test_time_range["start"], + ) + else: + effective_test_time_range = pd.Series( + sorted_test_time_range.values[-1], + index=[sorted_test_time_range.index[0]], + ) + + if embargo_length == 0: + embargoed_ranges = effective_test_time_range + else: + embargoed_values = [] + for end_val in effective_test_time_range.values: + end_iloc = data_info_range.index.searchsorted(end_val, side="left") + + if end_iloc >= len(data_info_range): + embargoed_values.append(end_val) + else: + embargoed_iloc = min( + end_iloc + embargo_length, len(data_info_range) - 1 + ) + embargoed_values.append(data_info_range.index[embargoed_iloc]) + + embargoed_ranges = pd.Series( + embargoed_values, index=effective_test_time_range.index + ) + # === END OF FIX === + + # Purge + for test_start, test_end_embargoed in embargoed_ranges.items(): + # 1. Overlap: train starts during test/embargo + cond1 = (data_info_range.index >= test_start) & ( + data_info_range.index <= test_end_embargoed + ) + # 2. Overlap: train ends during test/embargo + cond2 = (data_info_range.values >= test_start) & ( + data_info_range.values <= test_end_embargoed + ) + # 3. Overlap: train envelops test/embargo + cond3 = (data_info_range.index <= test_start) & ( + data_info_range.values >= test_end_embargoed + ) + + indices_to_drop.update(data_info_range[cond1 | cond2 | cond3].index) + + return data_info_range.drop(indices_to_drop) + + def __init__( + self, + n_splits: int, + times: Union[pd.Series, dict[str, pd.Series]], + embargo: float = 0, + ) -> None: + """ + Initialize the PurgedKFold cross-validator. + + Parameters + ---------- + n_splits : int + Number of splits or folds. + times : pd.Series or Dict[str, pd.Series] + A Series (or dict of Series) where the index is the observation + start time and the value is the observation end time. + embargo : float, default=0 + The embargo fraction (e.g., 0.01 for 1%). + """ + self.n_splits = n_splits + self.times = times + self.embargo = embargo + self.is_multiple_datasets = isinstance(times, dict) + + def get_n_splits( + self, + data: Optional[Union[pd.DataFrame, dict[str, pd.DataFrame]]] = None, + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> int: + """ + Return the number of splits. + + Parameters + ---------- + data : pd.DataFrame or dict, optional + Dataset or dictionary of datasets. + labels : pd.Series or dict, optional + Labels or dictionary of labels. + groups : np.ndarray, optional + Group labels for the samples. + + Returns + ------- + int + The number of splits (`n_splits`). + """ + return self.n_splits + + def _validate_input( + self, single_times: pd.Series, single_data: pd.DataFrame + ) -> None: + """Validate that data and times indices match.""" + if not single_data.index.equals(single_times.index): + raise ValueError("Data and 'times' (info range) must have the same index") + + def _get_train_indices( + self, + test_indices: np.ndarray, + single_times: pd.Series, + continous_test_times: bool = False, + ) -> np.ndarray: + """ + Get training indices after purging and embargo. + + Parameters + ---------- + test_indices : np.ndarray + The integer-location indices of the test set. + single_times : pd.Series + The 'times' (info range) Series for *all* data. + continous_test_times : bool, default=False + Passed to `filtered_training_indices_with_embargo`. + + Returns + ------- + np.ndarray + The integer-location indices of the training set. + """ + if len(test_indices) == 0: + return np.arange(len(single_times)) + + test_time_range = single_times.iloc[test_indices] + + train_times = self.filtered_training_indices_with_embargo( + single_times, test_time_range, self.embargo, continous_test_times + ) + + # Convert index labels back to integer locations + train_indices = single_times.index.get_indexer(train_times.index) + return train_indices + + def _single_split( + self, + single_times: pd.Series, + single_data: pd.DataFrame, + ) -> Generator[tuple[np.ndarray, np.ndarray], None, None]: + """ + Split a single dataset into purged train-test indices. + + Parameters + ---------- + single_times : pd.Series + The 'times' (info range) Series for this dataset. + single_data : pd.DataFrame + Input dataset. + + Yields + ------- + Generator[Tuple[np.ndarray, np.ndarray], None, None] + A generator of (train_indices, test_indices). + """ + self._validate_input(single_times, single_data) + + indices = np.arange(len(single_data)) + + for test_indices in np.array_split(indices, self.n_splits): + train_indices = self._get_train_indices(test_indices, single_times, True) + yield train_indices, test_indices + + def split( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Optional[Union[pd.Series, dict[str, pd.Series]]] = None, + groups: Optional[np.ndarray] = None, + ) -> Union[ + Generator[tuple[np.ndarray, np.ndarray], None, None], + Generator[tuple[str, tuple[np.ndarray, np.ndarray]], None, None], + ]: + """ + Split data (or dictionary of data) into purged train-test indices. + + Parameters + ---------- + data : pd.DataFrame or dict + Dataset or dictionary of datasets. + labels : pd.Series or dict, optional + Labels or dictionary of labels. + groups : np.ndarray, optional + Group labels for the samples. + + Yields + ------- + Generator + - If data is a pd.DataFrame: (train_indices, test_indices) + - If data is a dict: (key, (train_indices, test_indices)) + """ + if self.is_multiple_datasets: + if not isinstance(data, dict): + raise ValueError("If 'times' is a dict, 'data' must also be a dict.") + for key in self.times: + for train_indices, test_indices in self._single_split( + self.times[key], data[key] + ): + yield key, (train_indices, test_indices) + else: + if not isinstance(data, pd.DataFrame): + raise ValueError("If 'times' is a Series, 'data' must be a DataFrame.") + for train_indices, test_indices in self._single_split(self.times, data): + yield train_indices, test_indices + + def _single_backtest_paths( + self, + single_times: pd.Series, + single_data: pd.DataFrame, + ) -> dict[str, list[dict[str, np.ndarray]]]: + """ + Generate backtest paths for a single dataset. + For PurgedKFold, there is only one "path". + + Parameters + ---------- + single_times : pd.Series + The 'times' (info range) Series for this dataset. + single_data : pd.DataFrame + Input dataset. + + Returns + ------- + Dict[str, List[Dict[str, np.ndarray]]] + A dictionary with one key "Path 1", containing a list of all + (Train, Test) index dictionaries. + """ + self._validate_input(single_times, single_data) + + path_data = [] + paths = {} + + for train_indices, test_indices in self._single_split( + single_times, single_data + ): + path_data.append( + { + "Train": train_indices, + "Test": test_indices, + } + ) + + paths["Path 1"] = path_data + return paths + + def backtest_paths( + self, + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + ) -> Union[ + dict[str, list[dict[str, np.ndarray]]], + dict[str, dict[str, list[dict[str, np.ndarray]]]], + ]: + """ + Generate backtest paths for data or a dictionary of data. + + Parameters + ---------- + data : pd.DataFrame or dict + Dataset or dictionary of datasets. + + Returns + ------- + Union[Dict, Dict[str, Dict]] + - If data is a pd.DataFrame: A dictionary of backtest paths. + - If data is a dict: A nested dictionary of backtest paths. + """ + if self.is_multiple_datasets: + if not isinstance(data, dict): + raise ValueError("If 'times' is a dict, 'data' must also be a dict.") + multiple_paths = {} + for key in self.times: + multiple_paths[key] = self._single_backtest_paths( + self.times[key], data[key] + ) + return multiple_paths + + if not isinstance(data, pd.DataFrame): + raise ValueError("If 'times' is a Series, 'data' must be a DataFrame.") + return self._single_backtest_paths(self.times, data) + + def _single_backtest_predictions( + self, + single_estimator: Estimator, + single_times: pd.Series, + single_data: pd.DataFrame, + single_labels: pd.Series, + single_weights: Optional[np.ndarray] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> dict[str, np.ndarray]: + """ + Obtain backtest predictions for a single dataset. + + Parameters + ---------- + single_estimator : Estimator + A scikit-learn-like estimator. + single_times : pd.Series + The 'times' (info range) Series for this dataset. + single_data : pd.DataFrame + Data for the single dataset. + single_labels : pd.Series + Labels for the single dataset. + single_weights : np.ndarray, optional + Sample weights. Defaults to equal weights. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Dict[str, np.ndarray] + A dictionary with one key "Path 1", containing the contiguous + array of out-of-sample predictions. + """ + self._validate_input(single_times, single_data) + + if single_weights is None: + single_weights = np.ones(len(single_data)) + + def train_test_single_estimator( + estimator_: Estimator, train_indices: np.ndarray, test_indices: np.ndarray + ) -> np.ndarray: + """Train model and return predictions.""" + X_train = single_data.iloc[train_indices] + y_train = single_labels.iloc[train_indices] + weights_train = single_weights[train_indices] + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ConvergenceWarning) + try: + estimator_.fit(X_train, y_train, sample_weight=weights_train) + except TypeError: + estimator_.fit(X_train, y_train) + + X_test = single_data.iloc[test_indices] + + if predict_probability: + return estimator_.predict_proba(X_test) + + return estimator_.predict(X_test) + + path_data = Parallel(n_jobs=n_jobs)( + delayed(train_test_single_estimator)( + deepcopy(single_estimator), train_indices, test_indices + ) + for train_indices, test_indices in self._single_split( + single_times, single_data + ) + ) + + # Since PurgedKFold is not shuffled, we can just concatenate + paths_predictions = {"Path 1": np.concatenate(path_data)} + return paths_predictions + + def backtest_predictions( + self, + estimator: Union[Estimator, dict[str, Estimator]], + data: Union[pd.DataFrame, dict[str, pd.DataFrame]], + labels: Union[pd.Series, dict[str, pd.Series]], + sample_weights: Optional[Union[np.ndarray, dict[str, np.ndarray]]] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> Union[dict[str, np.ndarray], dict[str, dict[str, np.ndarray]]]: + """ + Generate backtest predictions for single or multiple datasets. + + Parameters + ---------- + estimator : Estimator or dict + Model(s) to be trained. + data : pd.DataFrame or dict + Input data. + labels : pd.Series or dict + Target labels. + sample_weights : np.ndarray or dict, optional + Sample weights. Defaults to None. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Union[Dict, Dict[str, Dict]] + - If data is a pd.DataFrame: A dictionary of predictions. + - If data is a dict: A nested dictionary of predictions. + """ + if self.is_multiple_datasets: + if not ( + isinstance(data, dict) + and isinstance(estimator, dict) + and isinstance(labels, dict) + ): + raise ValueError( + "If 'times' is a dict, 'data', 'estimator', and 'labels' " + "must also be dicts." + ) + + multiple_paths_predictions = {} + for key in self.times: + s_weights = sample_weights[key] if sample_weights else None + multiple_paths_predictions[key] = self._single_backtest_predictions( + estimator[key], + self.times[key], + data[key], + labels[key], + s_weights, + predict_probability, + n_jobs, + ) + return multiple_paths_predictions + + # Handle single dataset case + if not (isinstance(data, pd.DataFrame) and isinstance(labels, pd.Series)): + raise ValueError( + "If 'times' is a Series, 'data' must be a DataFrame " + "and 'labels' must be a Series." + ) + + return self._single_backtest_predictions( + estimator, + self.times, + data, + labels, + sample_weights, + predict_probability, + n_jobs, + ) diff --git a/src/RiskLabAI/backtest/validation/walk_forward.py b/src/RiskLabAI/backtest/validation/walk_forward.py new file mode 100644 index 0000000..8b64ab0 --- /dev/null +++ b/src/RiskLabAI/backtest/validation/walk_forward.py @@ -0,0 +1,183 @@ +""" +Implements a Walk-Forward cross-validator for time-series data. +""" + +import warnings +from collections.abc import Generator +from copy import deepcopy +from typing import Any, Optional + +import numpy as np +import pandas as pd +from joblib import Parallel, delayed +from sklearn.exceptions import ConvergenceWarning + +from .kfold import KFold + +# For type hinting sklearn-like estimators +Estimator = Any + + +class WalkForward(KFold): + """ + Walk-Forward Cross-Validator for Time Series Data. + + This cross-validator splits time-series data in a "walk-forward" + manner. In each split, the training set progressively grows (up to + `max_train_size`) while the test set "walks" forward. + + This method respects the temporal order of data, ensuring the model is + always trained on past data and validated on future data. + + Parameters + ---------- + n_splits : int, default=5 + Number of splits/folds. Must be at least 2. + max_train_size : int, optional + Maximum number of observations allowed in the training dataset. + If provided, the most recent `max_train_size` observations are + used for training. + gap : int, default=0 + Number of observations to skip between the end of the training + data and the start of the test data. + """ + + def __init__( + self, n_splits: int = 5, max_train_size: Optional[int] = None, gap: int = 0 + ) -> None: + """ + Initialize the WalkForward cross-validator. + + Parameters + ---------- + n_splits : int, default=5 + Number of splits/folds. Must be at least 2. + max_train_size : int, optional + Maximum number of observations allowed in the training dataset. + gap : int, default=0 + Number of observations to skip between the end of the training + data and the start of the test data. + """ + # WalkForward cannot be shuffled + super().__init__(n_splits=n_splits, shuffle=False) + self.max_train_size = max_train_size + self.gap = gap + + def _single_split( + self, + single_data: pd.DataFrame, + ) -> Generator[tuple[np.ndarray, np.ndarray], None, None]: + """ + Split a single dataset into walk-forward train-test indices. + + Parameters + ---------- + single_data : pd.DataFrame + Input dataset. + + Yields + ------- + Generator[Tuple[np.ndarray, np.ndarray], None, None] + A generator where each item is a tuple of (train_indices, test_indices). + """ + indices = np.arange(single_data.shape[0]) + + # np.array_split handles non-divisible splits + for test_indices in np.array_split(indices, self.n_splits): + # The first test index + first_test_idx_loc = test_indices[0] + + # Train indices end `gap` samples before the test set + train_end_loc = first_test_idx_loc - self.gap + + if train_end_loc < 0: + # No training data possible + train_indices = np.array([], dtype=int) + else: + if self.max_train_size and train_end_loc > self.max_train_size: + train_start_loc = train_end_loc - self.max_train_size + train_indices = indices[train_start_loc:train_end_loc] + else: + train_indices = indices[:train_end_loc] + + yield train_indices, test_indices + + def _single_backtest_predictions( + self, + single_estimator: Estimator, + single_data: pd.DataFrame, + single_labels: pd.Series, + single_weights: Optional[np.ndarray] = None, + predict_probability: bool = False, + n_jobs: int = 1, + ) -> dict[str, np.ndarray]: + """ + Obtain backtest predictions for a single dataset. + + Parameters + ---------- + single_estimator : Estimator + A scikit-learn-like estimator to be trained and used for predictions. + single_data : pd.DataFrame + Data for the single dataset. + single_labels : pd.Series + Labels for the single dataset. + single_weights : np.ndarray, optional + Sample weights for the observations. Defaults to equal weights. + predict_probability : bool, default=False + If True, call `predict_proba()` instead of `predict()`. + n_jobs : int, default=1 + The number of jobs to run in parallel. + + Returns + ------- + Dict[str, np.ndarray] + A dictionary with one key "Path 1", containing the contiguous + array of out-of-sample predictions. + """ + if single_weights is None: + single_weights = np.ones(len(single_data)) + + def train_test_single_estimator( + estimator_: Estimator, train_indices: np.ndarray, test_indices: np.ndarray + ) -> np.ndarray: + """Train model and return predictions.""" + + if len(train_indices) == 0: + # No training data, return NaNs + n_classes = len(np.unique(single_labels)) if predict_probability else 1 + shape = ( + (len(test_indices), n_classes) + if predict_probability + else (len(test_indices),) + ) + return np.full(shape, np.nan) + + X_train = single_data.iloc[train_indices] + y_train = single_labels.iloc[train_indices] + weights_train = single_weights[train_indices] + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ConvergenceWarning) + try: + estimator_.fit(X_train, y_train, sample_weight=weights_train) + except TypeError: + estimator_.fit(X_train, y_train) + + X_test = single_data.iloc[test_indices] + + if predict_probability: + return estimator_.predict_proba(X_test) + + return estimator_.predict(X_test) + + path_data = Parallel(n_jobs=n_jobs)( + delayed(train_test_single_estimator)( + deepcopy(single_estimator), train_indices, test_indices + ) + for train_indices, test_indices in self._single_split(single_data) + ) + + # Since shuffle=False, we can just concatenate + paths_predictions = {"Path 1": np.concatenate(path_data)} + return paths_predictions diff --git a/src/RiskLabAI/causal_factor_analysis/__init__.py b/src/RiskLabAI/causal_factor_analysis/__init__.py new file mode 100644 index 0000000..e2b800e --- /dev/null +++ b/src/RiskLabAI/causal_factor_analysis/__init__.py @@ -0,0 +1,135 @@ +"""Causal-factor portfolio, graph, specification-error, and protocol methods. + +This package implements externally published Causal Factor Analysis equations +and graphical identification criteria together with the seven-stage research +protocol, without importing the broader causal or portfolio-optimization +namespaces. +""" + +from .factor_mirage import ( + ColliderCoefficients, + ColliderDiagnostics, + StrategyPerformance, + collider_factor_return, + collider_forecast_return, + collider_model_diagnostics, + collider_overcontrolled_coefficients, + confounder_factor_return, + confounder_forecast_return, + confounder_undercontrolled_coefficient, +) +from .graph_identification import ( + BackdoorAdjustmentEvidence, + CausalDAG, + DSeparationEvidence, + FrontdoorAdjustmentEvidence, + InstrumentEvidence, + NodeRoleEvidence, + PathEvidence, + causal_role_evidence, + check_backdoor_adjustment_set, + check_frontdoor_adjustment_set, + check_instrument, + d_separation, + minimal_backdoor_adjustment_sets, + minimal_frontdoor_adjustment_sets, +) +from .optimizer import minimum_variance_factor_weights +from .protocol import ( + BacktestStage, + CausalAdjustmentSetStage, + CausalDiscoveryStage, + CausalExplanatoryAndPredictivePowerStage, + CausalFactorProtocolReport, + CausalPortfolioConstructionStage, + EventHorizon, + FoldEvidence, + MultipleTestingAdjustmentsStage, + ValidationEvidence, + VariableSelectionStage, + validate_causal_factor_protocol, +) +from .specification_diagnostics import ( + PopulationRegressionDiagnostics, + RegressionDiagnostics, + SpecificationExperimentResult, + SpecificationPopulationResult, + collider_population_diagnostics, + collider_specification_experiment, + confounded_mediator_population_diagnostics, + confounded_mediator_specification_experiment, + fork_population_diagnostics, + fork_specification_experiment, +) +from .treatment_effects import ( + DifferenceInDifferencesEstimate, + TreatmentEffectDecomposition, + average_treatment_effect, + backdoor_adjusted_average_treatment_effect, + backdoor_adjusted_expectation, + difference_in_differences, + frontdoor_adjusted_probability, + linear_instrumental_variable_effect, + randomized_mean_difference, + treatment_effect_decomposition, +) + +__all__ = [ + "BackdoorAdjustmentEvidence", + "BacktestStage", + "CausalDAG", + "CausalAdjustmentSetStage", + "ColliderCoefficients", + "ColliderDiagnostics", + "CausalDiscoveryStage", + "CausalExplanatoryAndPredictivePowerStage", + "CausalFactorProtocolReport", + "CausalPortfolioConstructionStage", + "DSeparationEvidence", + "DifferenceInDifferencesEstimate", + "EventHorizon", + "FoldEvidence", + "FrontdoorAdjustmentEvidence", + "InstrumentEvidence", + "MultipleTestingAdjustmentsStage", + "NodeRoleEvidence", + "PathEvidence", + "PopulationRegressionDiagnostics", + "RegressionDiagnostics", + "SpecificationExperimentResult", + "SpecificationPopulationResult", + "StrategyPerformance", + "TreatmentEffectDecomposition", + "ValidationEvidence", + "VariableSelectionStage", + "average_treatment_effect", + "backdoor_adjusted_average_treatment_effect", + "backdoor_adjusted_expectation", + "causal_role_evidence", + "check_backdoor_adjustment_set", + "check_frontdoor_adjustment_set", + "check_instrument", + "collider_factor_return", + "collider_forecast_return", + "collider_model_diagnostics", + "collider_overcontrolled_coefficients", + "collider_population_diagnostics", + "collider_specification_experiment", + "confounded_mediator_population_diagnostics", + "confounded_mediator_specification_experiment", + "confounder_factor_return", + "confounder_forecast_return", + "confounder_undercontrolled_coefficient", + "d_separation", + "difference_in_differences", + "fork_population_diagnostics", + "fork_specification_experiment", + "frontdoor_adjusted_probability", + "linear_instrumental_variable_effect", + "minimal_backdoor_adjustment_sets", + "minimal_frontdoor_adjustment_sets", + "minimum_variance_factor_weights", + "randomized_mean_difference", + "treatment_effect_decomposition", + "validate_causal_factor_protocol", +] diff --git a/src/RiskLabAI/causal_factor_analysis/factor_mirage.py b/src/RiskLabAI/causal_factor_analysis/factor_mirage.py new file mode 100644 index 0000000..08ddade --- /dev/null +++ b/src/RiskLabAI/causal_factor_analysis/factor_mirage.py @@ -0,0 +1,399 @@ +"""Closed-form analytics for specification errors in factor investing. + +The functions implement the standardized linear-Gaussian results in Lopez de +Prado and Zoonekynd (2026), Equations 2, 5-19 and Appendices D-E. Factor +returns condition on the supplied exposures. Forecast returns integrate over +the exposure distribution assumed by the article. + +Appendix F is not exposed as a separate parameter-shift model because its +equations contain only the training-regime parameters. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from decimal import Decimal, localcontext +from fractions import Fraction +from numbers import Integral, Real +from typing import Optional + +__all__ = [ + "ColliderCoefficients", + "ColliderDiagnostics", + "StrategyPerformance", + "collider_factor_return", + "collider_forecast_return", + "collider_model_diagnostics", + "collider_overcontrolled_coefficients", + "confounder_factor_return", + "confounder_forecast_return", + "confounder_undercontrolled_coefficient", +] + + +@dataclass(frozen=True) +class StrategyPerformance: + """Expected returns from the correct and misspecified strategies.""" + + correct: float + misspecified: float + + +@dataclass(frozen=True) +class ColliderCoefficients: + """Population coefficients after conditioning on the collider.""" + + beta_hat: float + theta_hat: float + + +@dataclass(frozen=True) +class ColliderDiagnostics: + """Population model-selection diagnostics for collider conditioning. + + The t-statistic fields are the expected signals in Appendix D. They are + not p-values. In particular, the article assigns different degrees of + freedom to the correct and overcontrolled regressions. + """ + + residual_variance: float + outcome_variance: float + correct_r_squared: float + overcontrolled_r_squared: float + correct_adjusted_r_squared: float + overcontrolled_adjusted_r_squared: float + correct_beta_variance: float + overcontrolled_beta_variance: float + collider_coefficient_variance: float + correct_beta_t_statistic: float + overcontrolled_beta_t_statistic: float + collider_t_statistic: float + adjusted_r_squared_prefers_overcontrolled: bool + absolute_beta_t_prefers_overcontrolled: bool + + +def _as_finite_real(name: str, value: Real) -> float: + if isinstance(value, bool) or not isinstance(value, Real): + raise ValueError(f"{name} must be a real scalar") + try: + result = float(value) + except (TypeError, ValueError, OverflowError) as error: + raise ValueError(f"{name} must be a representable real scalar") from error + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def _as_sample_size(value: Integral) -> int: + if isinstance(value, bool) or not isinstance(value, Integral): + raise ValueError("n_observations must be an integer") + result = int(value) + if result <= 3: + raise ValueError("n_observations must be greater than 3") + return result + + +def _fraction(value: float) -> Fraction: + return Fraction.from_float(value) + + +def _as_result(name: str, value: Fraction) -> float: + try: + result = float(value) + except (OverflowError, ValueError) as error: + raise ValueError(f"{name} cannot be represented as a finite float") from error + if not math.isfinite(result): + raise ValueError(f"{name} cannot be represented as a finite float") + return result + + +def _as_decimal(value: Fraction) -> Decimal: + return Decimal(value.numerator) / Decimal(value.denominator) + + +def _coefficient_times_square_root( + name: str, coefficient: Fraction, radicand: Fraction +) -> float: + if coefficient == 0: + return 0.0 + try: + with localcontext() as context: + context.prec = 80 + value = _as_decimal(coefficient) * _as_decimal(radicand).sqrt() + result = float(value) + except (ArithmeticError, OverflowError, ValueError) as error: + raise ValueError(f"{name} cannot be represented as a finite float") from error + if not math.isfinite(result): + raise ValueError(f"{name} cannot be represented as a finite float") + return result + + +def _confounder_coefficient( + beta: Fraction, gamma: Fraction, delta: Fraction +) -> Fraction: + return beta + gamma * delta / (1 + delta * delta) + + +def _collider_coefficients( + beta: Fraction, gamma: Fraction, delta: Fraction +) -> tuple[Fraction, Fraction]: + denominator = 1 + gamma * gamma + return (beta - delta * gamma) / denominator, gamma / denominator + + +def confounder_undercontrolled_coefficient( + beta: Real, gamma: Real, delta: Real +) -> float: + r"""Return the coefficient obtained when the confounder is omitted. + + For ``X = delta * Z + v`` and ``Y = beta * X + gamma * Z + u``, the + undercontrolled regression coefficient is + ``beta + gamma * delta / (1 + delta**2)`` (Equation 2). + """ + + beta_value = _fraction(_as_finite_real("beta", beta)) + gamma_value = _fraction(_as_finite_real("gamma", gamma)) + delta_value = _fraction(_as_finite_real("delta", delta)) + return _as_result( + "undercontrolled coefficient", + _confounder_coefficient(beta_value, gamma_value, delta_value), + ) + + +def confounder_factor_return( + x: Real, + z: Real, + beta: Real, + gamma: Real, + delta_estimated: Real, + *, + delta_realized: Optional[Real] = None, # noqa: UP045, Python 3.9 API +) -> StrategyPerformance: + r"""Return correct and undercontrolled factor-strategy performance. + + With one correlation regime, omit ``delta_realized`` to evaluate Equations + 5 and 7. Supply it to evaluate the train-to-execution shift in Appendix E, + where ``delta_estimated`` is the training value and ``delta_realized`` is + the execution value. + """ + + x_value = _fraction(_as_finite_real("x", x)) + z_value = _fraction(_as_finite_real("z", z)) + beta_value = _fraction(_as_finite_real("beta", beta)) + gamma_value = _fraction(_as_finite_real("gamma", gamma)) + estimated = _fraction(_as_finite_real("delta_estimated", delta_estimated)) + realized = ( + estimated + if delta_realized is None + else _fraction(_as_finite_real("delta_realized", delta_realized)) + ) + + correct_signal = x_value * beta_value + z_value * gamma_value + estimated_coefficient = _confounder_coefficient(beta_value, gamma_value, estimated) + realized_coefficient = _confounder_coefficient(beta_value, gamma_value, realized) + correct = correct_signal * correct_signal + misspecified = x_value * x_value * estimated_coefficient * realized_coefficient + return StrategyPerformance( + correct=_as_result("correct factor return", correct), + misspecified=_as_result("undercontrolled factor return", misspecified), + ) + + +def confounder_forecast_return( + beta: Real, + gamma: Real, + delta_estimated: Real, + *, + delta_realized: Optional[Real] = None, # noqa: UP045, Python 3.9 API +) -> StrategyPerformance: + r"""Return correct and undercontrolled forecasting performance. + + Omitting ``delta_realized`` evaluates Equations 8 and 9. Supplying it + evaluates Equations E-3 and E-4 after a change in the confounder link. + """ + + beta_value = _fraction(_as_finite_real("beta", beta)) + gamma_value = _fraction(_as_finite_real("gamma", gamma)) + estimated = _fraction(_as_finite_real("delta_estimated", delta_estimated)) + realized = ( + estimated + if delta_realized is None + else _fraction(_as_finite_real("delta_realized", delta_realized)) + ) + + estimated_coefficient = _confounder_coefficient(beta_value, gamma_value, estimated) + realized_coefficient = _confounder_coefficient(beta_value, gamma_value, realized) + second_signal = beta_value * realized + gamma_value + correct = beta_value * beta_value + second_signal * second_signal + misspecified = ( + (1 + realized * realized) * estimated_coefficient * realized_coefficient + ) + return StrategyPerformance( + correct=_as_result("correct forecast return", correct), + misspecified=_as_result("undercontrolled forecast return", misspecified), + ) + + +def collider_overcontrolled_coefficients( + beta: Real, gamma: Real, delta: Real +) -> ColliderCoefficients: + r"""Return the two coefficients after conditioning on the collider. + + The coefficient on ``X`` is ``(beta - delta*gamma)/(1 + gamma**2)`` and + the coefficient on ``Z`` is ``gamma/(1 + gamma**2)`` (Equations 11-12). + """ + + beta_value = _fraction(_as_finite_real("beta", beta)) + gamma_value = _fraction(_as_finite_real("gamma", gamma)) + delta_value = _fraction(_as_finite_real("delta", delta)) + beta_hat, theta_hat = _collider_coefficients(beta_value, gamma_value, delta_value) + return ColliderCoefficients( + beta_hat=_as_result("overcontrolled beta coefficient", beta_hat), + theta_hat=_as_result("collider coefficient", theta_hat), + ) + + +def collider_factor_return( + x: Real, + collider_proxy: Real, + beta: Real, + gamma: Real, + delta: Real, +) -> StrategyPerformance: + r"""Return correct and overcontrolled factor-strategy performance. + + ``collider_proxy`` is the article's pre-outcome proxy ``Z tilde``. It is + not the contemporaneous collider, which contains the outcome. The return + maps to Equations 16 and 17. + """ + + x_value = _fraction(_as_finite_real("x", x)) + proxy_value = _fraction(_as_finite_real("collider_proxy", collider_proxy)) + beta_value = _fraction(_as_finite_real("beta", beta)) + gamma_value = _fraction(_as_finite_real("gamma", gamma)) + delta_value = _fraction(_as_finite_real("delta", delta)) + + beta_x = beta_value * x_value + denominator = 1 + gamma_value * gamma_value + overcontrolled_signal = ( + beta_value - delta_value * gamma_value + ) * x_value + gamma_value * proxy_value + correct = beta_x * beta_x + misspecified = beta_x * overcontrolled_signal / denominator + return StrategyPerformance( + correct=_as_result("correct collider factor return", correct), + misspecified=_as_result("overcontrolled factor return", misspecified), + ) + + +def collider_forecast_return( + beta: Real, gamma: Real, delta: Real +) -> StrategyPerformance: + r"""Return correct and overcontrolled forecasting performance. + + The two values are ``beta**2`` and + ``beta * (beta - gamma*delta) / (1 + gamma**2)`` (Equations 18-19). + """ + + beta_value = _fraction(_as_finite_real("beta", beta)) + gamma_value = _fraction(_as_finite_real("gamma", gamma)) + delta_value = _fraction(_as_finite_real("delta", delta)) + denominator = 1 + gamma_value * gamma_value + correct = beta_value * beta_value + misspecified = beta_value * (beta_value - gamma_value * delta_value) / denominator + return StrategyPerformance( + correct=_as_result("correct collider forecast return", correct), + misspecified=_as_result("overcontrolled forecast return", misspecified), + ) + + +def collider_model_diagnostics( + beta: Real, + gamma: Real, + delta: Real, + n_observations: Integral, +) -> ColliderDiagnostics: + r"""Return the Appendix D diagnostics for conditioning on a collider. + + ``n_observations`` must be an integer greater than three. The returned + booleans implement the strict adjusted-R-squared and absolute expected + beta t-statistic comparisons. The latter is not an exact p-value test. + """ + + beta_value = _fraction(_as_finite_real("beta", beta)) + gamma_value = _fraction(_as_finite_real("gamma", gamma)) + delta_value = _fraction(_as_finite_real("delta", delta)) + n_integer = _as_sample_size(n_observations) + n_value = Fraction(n_integer) + + gamma_denominator = 1 + gamma_value * gamma_value + outcome_variance = 1 + beta_value * beta_value + residual_variance = 1 / gamma_denominator + correct_r_squared = 1 - 1 / outcome_variance + overcontrolled_r_squared = 1 - 1 / (gamma_denominator * outcome_variance) + correct_adjusted_r_squared = 1 - ( + Fraction(n_integer - 1, n_integer - 2) / outcome_variance + ) + overcontrolled_adjusted_r_squared = 1 - ( + Fraction(n_integer - 1, n_integer - 3) / (gamma_denominator * outcome_variance) + ) + + diagnostic_sum = ( + (beta_value * gamma_value + delta_value) ** 2 + gamma_value * gamma_value + 1 + ) + correct_beta_variance = 1 / n_value + overcontrolled_beta_variance = diagnostic_sum / ( + n_value * gamma_denominator * gamma_denominator + ) + collider_coefficient_variance = 1 / ( + n_value * gamma_denominator * gamma_denominator + ) + + overcontrolled_numerator = beta_value - delta_value * gamma_value + correct_beta_t_statistic = _coefficient_times_square_root( + "correct beta t-statistic", beta_value, n_value + ) + overcontrolled_beta_t_statistic = _coefficient_times_square_root( + "overcontrolled beta t-statistic", + overcontrolled_numerator, + n_value / diagnostic_sum, + ) + collider_t_statistic = _coefficient_times_square_root( + "collider t-statistic", gamma_value, n_value + ) + + return ColliderDiagnostics( + residual_variance=_as_result("residual variance", residual_variance), + outcome_variance=_as_result("outcome variance", outcome_variance), + correct_r_squared=_as_result("correct R-squared", correct_r_squared), + overcontrolled_r_squared=_as_result( + "overcontrolled R-squared", overcontrolled_r_squared + ), + correct_adjusted_r_squared=_as_result( + "correct adjusted R-squared", correct_adjusted_r_squared + ), + overcontrolled_adjusted_r_squared=_as_result( + "overcontrolled adjusted R-squared", + overcontrolled_adjusted_r_squared, + ), + correct_beta_variance=_as_result( + "correct beta variance", correct_beta_variance + ), + overcontrolled_beta_variance=_as_result( + "overcontrolled beta variance", overcontrolled_beta_variance + ), + collider_coefficient_variance=_as_result( + "collider coefficient variance", collider_coefficient_variance + ), + correct_beta_t_statistic=correct_beta_t_statistic, + overcontrolled_beta_t_statistic=overcontrolled_beta_t_statistic, + collider_t_statistic=collider_t_statistic, + adjusted_r_squared_prefers_overcontrolled=( + gamma_value * gamma_value * (n_integer - 3) > 1 + ), + absolute_beta_t_prefers_overcontrolled=( + overcontrolled_numerator * overcontrolled_numerator + > beta_value * beta_value * diagnostic_sum + ), + ) diff --git a/src/RiskLabAI/causal_factor_analysis/graph_identification.py b/src/RiskLabAI/causal_factor_analysis/graph_identification.py new file mode 100644 index 0000000..02ec948 --- /dev/null +++ b/src/RiskLabAI/causal_factor_analysis/graph_identification.py @@ -0,0 +1,1051 @@ +# ruff: noqa: UP045 +"""Graphical identification evidence for causal-factor research. + +The routines in this module operate on an explicit, immutable directed acyclic +graph. They implement path activation, d-separation, the back-door and +front-door criteria, and two distinct graphical instrument profiles. The +results describe implications of an accepted graph; they do not discover a +graph, verify its causal assumptions, establish positivity, or estimate an +effect. +""" + +from __future__ import annotations + +from collections import namedtuple +from collections.abc import Callable, Iterable, Sequence +from typing import NamedTuple, Optional + +__all__ = [ + "BackdoorAdjustmentEvidence", + "CausalDAG", + "DSeparationEvidence", + "FrontdoorAdjustmentEvidence", + "InstrumentEvidence", + "NodeRoleEvidence", + "PathEvidence", + "causal_role_evidence", + "check_backdoor_adjustment_set", + "check_frontdoor_adjustment_set", + "check_instrument", + "d_separation", + "minimal_backdoor_adjustment_sets", + "minimal_frontdoor_adjustment_sets", +] + + +NodeTuple = tuple[str, ...] +EdgeTuple = tuple[tuple[str, str], ...] +PathTuple = tuple[str, ...] +_DEFAULT_MAX_PATHS = 10_000 +_DEFAULT_MAX_PATH_STATES = 100_000 +_DEFAULT_MAX_CANDIDATES = 65_536 + + +def _materialize_collection(value, subject: str) -> tuple: + if isinstance(value, (str, bytes)) or not isinstance(value, Iterable): + raise TypeError(f"{subject} must be a collection, not a scalar string.") + try: + return tuple(value) + except TypeError as exc: + raise TypeError(f"{subject} must be an iterable collection.") from exc + + +def _require_node(value, subject: str) -> str: + if not isinstance(value, str): + raise TypeError(f"{subject} must be a string node identifier.") + if not value.strip(): + raise ValueError(f"{subject} must be nonblank.") + return value + + +def _normalize_nodes(value, subject: str, *, allow_empty: bool) -> NodeTuple: + items = _materialize_collection(value, subject) + normalized = tuple(_require_node(item, f"{subject} member") for item in items) + if not allow_empty and not normalized: + raise ValueError(f"{subject} must be nonempty.") + if len(set(normalized)) != len(normalized): + raise ValueError(f"{subject} must not contain duplicate nodes.") + return tuple(sorted(normalized)) + + +def _normalize_edges(value) -> EdgeTuple: + items = _materialize_collection(value, "directed_edges") + normalized = [] + for raw_edge in items: + if isinstance(raw_edge, (str, bytes)) or not isinstance(raw_edge, Sequence): + raise TypeError("Each directed edge must be an ordered two-node sequence.") + edge = _materialize_collection(raw_edge, "each directed edge") + if len(edge) != 2: + raise ValueError("Each directed edge must contain exactly two nodes.") + source = _require_node(edge[0], "directed-edge source") + target = _require_node(edge[1], "directed-edge target") + normalized.append((source, target)) + result = tuple(normalized) + if len(set(result)) != len(result): + raise ValueError("directed_edges must not contain duplicate edges.") + return tuple(sorted(result)) + + +def _children(nodes: NodeTuple, edges: EdgeTuple): + result = {node: set() for node in nodes} + for source, target in edges: + result[source].add(target) + return result + + +def _parents(nodes: NodeTuple, edges: EdgeTuple): + result = {node: set() for node in nodes} + for source, target in edges: + result[target].add(source) + return result + + +def _is_acyclic(nodes: NodeTuple, edges: EdgeTuple) -> bool: + children = _children(nodes, edges) + indegree = {node: 0 for node in nodes} + for _, target in edges: + indegree[target] += 1 + ready = sorted(node for node, degree in indegree.items() if degree == 0) + visited = 0 + while ready: + node = ready.pop() + visited += 1 + for child in sorted(children[node], reverse=True): + indegree[child] -= 1 + if indegree[child] == 0: + ready.append(child) + return visited == len(nodes) + + +_CausalDAGBase = namedtuple("CausalDAG", ("nodes", "directed_edges", "observed_nodes")) + + +class CausalDAG(_CausalDAGBase): + """An immutable DAG with explicit observed and latent nodes. + + Nodes absent from ``observed_nodes`` remain in every graphical calculation + but are never eligible for an adjustment or mediator set. + """ + + __slots__ = () + + def __new__(cls, nodes, directed_edges, observed_nodes): + nodes = _normalize_nodes(nodes, "nodes", allow_empty=False) + edges = _normalize_edges(directed_edges) + observed = _normalize_nodes(observed_nodes, "observed_nodes", allow_empty=True) + node_set = set(nodes) + unknown_observed = set(observed) - node_set + if unknown_observed: + raise ValueError("observed_nodes must be a subset of nodes.") + for source, target in edges: + if source not in node_set or target not in node_set: + raise ValueError("Every directed edge endpoint must be a graph node.") + if source == target: + raise ValueError("Directed self-loops are not admissible in a DAG.") + if not _is_acyclic(nodes, edges): + raise ValueError("directed_edges must define an acyclic graph.") + return super().__new__(cls, nodes, edges, observed) + + @classmethod + def _make(cls, iterable): + values = tuple(iterable) + if len(values) != 3: + raise TypeError("CausalDAG._make requires exactly three values.") + return cls(*values) + + def _replace(self, **changes): + unknown = set(changes) - set(self._fields) + if unknown: + names = ", ".join(sorted(unknown)) + raise TypeError(f"Unexpected CausalDAG field names: {names}.") + return type(self)( + changes.get("nodes", self.nodes), + changes.get("directed_edges", self.directed_edges), + changes.get("observed_nodes", self.observed_nodes), + ) + + +class PathEvidence(NamedTuple): + """Path-relative collider, blocking, and direction evidence.""" + + nodes: PathTuple + colliders: NodeTuple + noncolliders: NodeTuple + conditioned_noncolliders: NodeTuple + activated_colliders: NodeTuple + blocking_colliders: NodeTuple + is_open: bool + is_directed_from_left: bool + is_backdoor_from_left: bool + + +class DSeparationEvidence(NamedTuple): + """Complete simple-path evidence for one d-separation query.""" + + left: NodeTuple + right: NodeTuple + conditioned: NodeTuple + separated: bool + paths: tuple[PathEvidence, ...] + + +class BackdoorAdjustmentEvidence(NamedTuple): + """Evidence for the ordinary total-effect back-door criterion.""" + + treatment: str + outcome: str + adjustment_set: NodeTuple + all_observed: bool + endpoints_in_set: NodeTuple + descendants_in_set: NodeTuple + open_backdoor_paths: tuple[PathEvidence, ...] + admissible: bool + + +class FrontdoorAdjustmentEvidence(NamedTuple): + """Graphical front-door evidence; positivity remains an external premise.""" + + treatment: str + outcome: str + mediator_set: NodeTuple + all_observed: bool + endpoints_in_set: NodeTuple + directed_path_exists: bool + unintercepted_directed_paths: tuple[PathEvidence, ...] + open_treatment_mediator_backdoor_paths: tuple[PathEvidence, ...] + open_mediator_outcome_backdoor_paths_given_treatment: tuple[PathEvidence, ...] + graphically_admissible: bool + positivity_required: bool + + +class InstrumentEvidence(NamedTuple): + """Separate Pearl-graphical and simple CFI instrument premises.""" + + instrument: str + treatment: str + outcome: str + conditioned: NodeTuple + all_observed: bool + conditioning_disjoint: bool + conditioning_unaffected_by_treatment: bool + pearl_relevance: bool + pearl_exclusion_exogeneity: bool + pearl_graphical: bool + cfi_direct_relevance: bool + cfi_full_mediation: bool + cfi_exogeneity: bool + cfi_simple: bool + + +class NodeRoleEvidence(NamedTuple): + """Non-exclusive, path-relative evidence for one internal graph node.""" + + node: str + treatment: str + outcome: str + collider_on_paths: tuple[PathTuple, ...] + noncollider_on_paths: tuple[PathTuple, ...] + mediator_on_directed_paths: tuple[PathTuple, ...] + backdoor_noncollider_on_paths: tuple[PathTuple, ...] + common_cause_paths: tuple[PathTuple, ...] + descendant_of_treatment: bool + ancestor_of_outcome: bool + + +def _validate_dag(dag) -> CausalDAG: + if not isinstance(dag, CausalDAG): + raise TypeError("dag must be a CausalDAG.") + return dag + + +def _known_node_set(dag: CausalDAG, value, subject: str, *, allow_empty: bool): + nodes = _normalize_nodes(value, subject, allow_empty=allow_empty) + if not set(nodes).issubset(dag.nodes): + raise ValueError(f"{subject} must contain only known graph nodes.") + return nodes + + +def _validate_endpoints(dag: CausalDAG, treatment, outcome) -> tuple[str, str]: + treatment = _require_node(treatment, "treatment") + outcome = _require_node(outcome, "outcome") + if treatment == outcome: + raise ValueError("treatment and outcome must be distinct nodes.") + if treatment not in dag.nodes or outcome not in dag.nodes: + raise ValueError("treatment and outcome must be known graph nodes.") + observed = set(dag.observed_nodes) + if treatment not in observed or outcome not in observed: + raise ValueError("treatment and outcome must both be observed nodes.") + return treatment, outcome + + +def _descendants(dag: CausalDAG, node: str, edges: Optional[EdgeTuple] = None): + active_edges = dag.directed_edges if edges is None else edges + children = _children(dag.nodes, active_edges) + visited = set() + pending = list(children[node]) + while pending: + current = pending.pop() + if current in visited: + continue + visited.add(current) + pending.extend(children[current]) + return visited + + +def _ancestors(dag: CausalDAG, node: str): + parents = _parents(dag.nodes, dag.directed_edges) + visited = set() + pending = list(parents[node]) + while pending: + current = pending.pop() + if current in visited: + continue + visited.add(current) + pending.extend(parents[current]) + return visited + + +def _normalize_limit(value, subject: str) -> Optional[int]: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{subject} must be a positive integer or None.") + if value < 1: + raise ValueError(f"{subject} must be positive.") + return value + + +def _has_directed_path( + dag: CausalDAG, + left: str, + right: str, + edges: Optional[EdgeTuple] = None, + blocked: Iterable[str] = (), +) -> bool: + active_edges = dag.directed_edges if edges is None else edges + blocked_nodes = set(blocked) + if left in blocked_nodes or right in blocked_nodes: + return False + children = _children(dag.nodes, active_edges) + pending = [left] + visited = set() + while pending: + node = pending.pop() + if node == right: + return True + if node in visited: + continue + visited.add(node) + pending.extend( + child + for child in children[node] + if child not in visited and child not in blocked_nodes + ) + return False + + +def _is_d_separated( + dag: CausalDAG, + left: NodeTuple, + right: NodeTuple, + conditioned: NodeTuple, + edges: Optional[EdgeTuple] = None, +) -> bool: + """Test d-separation through ancestral-subgraph moralization.""" + + active_edges = dag.directed_edges if edges is None else edges + parents = _parents(dag.nodes, active_edges) + relevant = set(left).union(right, conditioned) + pending = list(relevant) + while pending: + node = pending.pop() + for parent in parents[node]: + if parent not in relevant: + relevant.add(parent) + pending.append(parent) + + moral = {node: set() for node in relevant} + for source, target in active_edges: + if source in relevant and target in relevant: + moral[source].add(target) + moral[target].add(source) + for child in relevant: + child_parents = sorted( + parent for parent in parents[child] if parent in relevant + ) + for index, first in enumerate(child_parents): + for second in child_parents[index + 1 :]: + moral[first].add(second) + moral[second].add(first) + + blocked = set(conditioned) + targets = set(right) + reachable = [node for node in left if node not in blocked] + visited = set() + while reachable: + node = reachable.pop() + if node in targets: + return False + if node in visited: + continue + visited.add(node) + reachable.extend( + neighbor + for neighbor in moral[node] + if neighbor not in blocked and neighbor not in visited + ) + return True + + +def _without_outgoing(edges: EdgeTuple, nodes) -> EdgeTuple: + removed = set(nodes) + return tuple(edge for edge in edges if edge[0] not in removed) + + +def _without_incoming(edges: EdgeTuple, nodes) -> EdgeTuple: + removed = set(nodes) + return tuple(edge for edge in edges if edge[1] not in removed) + + +def _simple_paths( + dag: CausalDAG, + left: NodeTuple, + right: NodeTuple, + edges: Optional[EdgeTuple] = None, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> tuple[PathTuple, ...]: + active_edges = dag.directed_edges if edges is None else edges + adjacency = {node: set() for node in dag.nodes} + for source, target in active_edges: + adjacency[source].add(target) + adjacency[target].add(source) + paths = [] + states = 0 + for start in left: + for finish in right: + stack = [(start, (start,), frozenset((start,)))] + while stack: + states += 1 + if max_path_states is not None and states > max_path_states: + raise ValueError( + "Complete path search exceeds max_path_states; " + "increase the explicit budget or simplify the query." + ) + node, path, visited = stack.pop() + if node == finish: + paths.append(path) + if max_paths is not None and len(paths) > max_paths: + raise ValueError( + "Complete path evidence exceeds max_paths; " + "increase the explicit budget or simplify the query." + ) + continue + for neighbor in sorted(adjacency[node], reverse=True): + if neighbor not in visited: + stack.append( + (neighbor, path + (neighbor,), visited | {neighbor}) + ) + return tuple(sorted(paths, key=lambda path: (len(path), path))) + + +def _path_evidence( + dag: CausalDAG, + path: PathTuple, + conditioned: NodeTuple, + edges: Optional[EdgeTuple] = None, +) -> PathEvidence: + active_edges = dag.directed_edges if edges is None else edges + edge_set = set(active_edges) + conditioned_set = set(conditioned) + colliders = [] + noncolliders = [] + for index in range(1, len(path) - 1): + previous, node, following = path[index - 1 : index + 2] + if (previous, node) in edge_set and (following, node) in edge_set: + colliders.append(node) + else: + noncolliders.append(node) + activated = [] + blocking = [] + for collider in colliders: + collider_family = {collider} | _descendants(dag, collider, active_edges) + if collider_family.intersection(conditioned_set): + activated.append(collider) + else: + blocking.append(collider) + conditioned_noncolliders = tuple( + node for node in noncolliders if node in conditioned_set + ) + is_directed = all( + (path[index], path[index + 1]) in edge_set for index in range(len(path) - 1) + ) + is_backdoor = len(path) > 1 and (path[1], path[0]) in edge_set + return PathEvidence( + nodes=path, + colliders=tuple(colliders), + noncolliders=tuple(noncolliders), + conditioned_noncolliders=conditioned_noncolliders, + activated_colliders=tuple(activated), + blocking_colliders=tuple(blocking), + is_open=not conditioned_noncolliders and not blocking, + is_directed_from_left=is_directed, + is_backdoor_from_left=is_backdoor, + ) + + +def _d_separation_from_normalized( + dag: CausalDAG, + left: NodeTuple, + right: NodeTuple, + conditioned: NodeTuple, + edges: Optional[EdgeTuple] = None, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> DSeparationEvidence: + paths = tuple( + _path_evidence(dag, path, conditioned, edges) + for path in _simple_paths( + dag, + left, + right, + edges, + max_paths, + max_path_states, + ) + ) + return DSeparationEvidence( + left=left, + right=right, + conditioned=conditioned, + separated=all(not path.is_open for path in paths), + paths=paths, + ) + + +def d_separation( + dag: CausalDAG, + left: Iterable[str], + right: Iterable[str], + conditioned: Iterable[str] = (), + *, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> DSeparationEvidence: + """Return complete simple-path evidence for a d-separation query. + + The three node sets must be pairwise disjoint. ``separated`` describes a + graphical implication for distributions compatible with ``dag``; its + negation is not a sample-level dependence test. Complete path evidence is + output-sensitive. ``max_paths`` bounds returned witnesses and + ``max_path_states`` bounds search expansions. ``None`` explicitly opts + out of the respective bound. Evidence is never truncated. + """ + + dag = _validate_dag(dag) + left_nodes = _known_node_set(dag, left, "left", allow_empty=False) + right_nodes = _known_node_set(dag, right, "right", allow_empty=False) + conditioned_nodes = _known_node_set( + dag, conditioned, "conditioned", allow_empty=True + ) + max_paths = _normalize_limit(max_paths, "max_paths") + max_path_states = _normalize_limit(max_path_states, "max_path_states") + if ( + set(left_nodes).intersection(right_nodes) + or set(left_nodes).intersection(conditioned_nodes) + or set(right_nodes).intersection(conditioned_nodes) + ): + raise ValueError("left, right, and conditioned must be pairwise disjoint.") + return _d_separation_from_normalized( + dag, + left_nodes, + right_nodes, + conditioned_nodes, + max_paths=max_paths, + max_path_states=max_path_states, + ) + + +def check_backdoor_adjustment_set( + dag: CausalDAG, + treatment: str, + outcome: str, + adjustment_set: Iterable[str] = (), + *, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> BackdoorAdjustmentEvidence: + """Check one observed set against the ordinary back-door criterion. + + ``max_paths`` bounds returned witnesses and ``max_path_states`` bounds + search expansions. Evidence is never truncated. + """ + + dag = _validate_dag(dag) + treatment, outcome = _validate_endpoints(dag, treatment, outcome) + adjustment = _known_node_set( + dag, adjustment_set, "adjustment_set", allow_empty=True + ) + max_paths = _normalize_limit(max_paths, "max_paths") + max_path_states = _normalize_limit(max_path_states, "max_path_states") + observed = set(dag.observed_nodes) + endpoints = tuple(sorted(set(adjustment).intersection((treatment, outcome)))) + descendants = tuple( + sorted(set(adjustment).intersection(_descendants(dag, treatment))) + ) + backdoor_edges = _without_outgoing(dag.directed_edges, (treatment,)) + separation = _d_separation_from_normalized( + dag, + (treatment,), + (outcome,), + adjustment, + backdoor_edges, + max_paths, + max_path_states, + ) + open_paths = tuple(path for path in separation.paths if path.is_open) + all_observed = set(adjustment).issubset(observed) + admissible = all_observed and not endpoints and not descendants and not open_paths + return BackdoorAdjustmentEvidence( + treatment=treatment, + outcome=outcome, + adjustment_set=adjustment, + all_observed=all_observed, + endpoints_in_set=endpoints, + descendants_in_set=descendants, + open_backdoor_paths=open_paths, + admissible=admissible, + ) + + +def _inclusion_minimal_sets(valid_sets): + ordered = sorted(valid_sets, key=lambda item: (len(item), item)) + result = [] + for candidate in ordered: + candidate_set = set(candidate) + if not any(set(existing).issubset(candidate_set) for existing in result): + result.append(candidate) + return tuple(result) + + +def _search_inclusion_minimal_sets( + candidates: NodeTuple, + is_valid: Callable[[NodeTuple], bool], + max_candidates: Optional[int], +) -> tuple[NodeTuple, ...]: + valid_sets = [] + selected = [] + frames = [[0, False]] + work = 0 + while frames: + next_index, evaluated = frames[-1] + candidate = tuple(selected) + if not evaluated: + work += 1 + len(candidate) + if max_candidates is not None and work > max_candidates: + raise ValueError( + "Exact minimal-set search exceeds max_candidates; " + "increase the explicit budget or simplify the graph." + ) + candidate_set = set(candidate) + if any(set(existing).issubset(candidate_set) for existing in valid_sets): + frames.pop() + if selected: + selected.pop() + continue + if is_valid(candidate): + if not candidate: + return ((),) + valid_sets.append(candidate) + frames.pop() + selected.pop() + continue + frames[-1][1] = True + continue + if next_index >= len(candidates): + frames.pop() + if selected: + selected.pop() + continue + frames[-1][0] += 1 + selected.append(candidates[next_index]) + frames.append([next_index + 1, False]) + return _inclusion_minimal_sets(valid_sets) + + +def minimal_backdoor_adjustment_sets( + dag: CausalDAG, + treatment: str, + outcome: str, + *, + max_candidates: Optional[int] = _DEFAULT_MAX_CANDIDATES, +) -> tuple[NodeTuple, ...]: + """Return all inclusion-minimal observed back-door adjustment sets. + + ``((),)`` means that the empty set is admissible. ``()`` means that no + admissible set exists among the observed, non-descendant graph nodes. + Search is exact and stops with ``ValueError`` before exceeding the explicit + ``max_candidates`` budget; no partial result is returned. + """ + + dag = _validate_dag(dag) + treatment, outcome = _validate_endpoints(dag, treatment, outcome) + max_candidates = _normalize_limit(max_candidates, "max_candidates") + descendants = _descendants(dag, treatment) + candidates = tuple( + node + for node in dag.observed_nodes + if node not in {treatment, outcome} and node not in descendants + ) + backdoor_edges = _without_outgoing(dag.directed_edges, (treatment,)) + + def is_valid(candidate: NodeTuple) -> bool: + return _is_d_separated(dag, (treatment,), (outcome,), candidate, backdoor_edges) + + return _search_inclusion_minimal_sets(candidates, is_valid, max_candidates) + + +def _directed_path_tuples( + dag: CausalDAG, + left: str, + right: str, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> tuple[PathTuple, ...]: + children = _children(dag.nodes, dag.directed_edges) + parents = _parents(dag.nodes, dag.directed_edges) + can_reach_right = {right} + pending = [right] + while pending: + node = pending.pop() + for parent in parents[node]: + if parent not in can_reach_right: + can_reach_right.add(parent) + pending.append(parent) + paths = [] + stack = [(left, (left,))] + states = 0 + while stack: + states += 1 + if max_path_states is not None and states > max_path_states: + raise ValueError( + "Complete directed-path search exceeds max_path_states; " + "increase the explicit budget or simplify the query." + ) + node, path = stack.pop() + if node == right: + paths.append(path) + if max_paths is not None and len(paths) > max_paths: + raise ValueError( + "Complete directed-path evidence exceeds max_paths; " + "increase the explicit budget or simplify the query." + ) + continue + for child in sorted(children[node], reverse=True): + if child in can_reach_right: + stack.append((child, path + (child,))) + return tuple(sorted(paths, key=lambda path: (len(path), path))) + + +def _directed_paths( + dag: CausalDAG, + left: str, + right: str, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> tuple[PathEvidence, ...]: + return tuple( + _path_evidence(dag, path, ()) + for path in _directed_path_tuples(dag, left, right, max_paths, max_path_states) + ) + + +def check_frontdoor_adjustment_set( + dag: CausalDAG, + treatment: str, + outcome: str, + mediator_set: Iterable[str], + *, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> FrontdoorAdjustmentEvidence: + """Check Pearl's front-door conditions under PROTOCOL-01 eligibility. + + The policy layer requires a nonempty observed mediator set and at least one + directed treatment-to-outcome path, avoiding vacuous eligibility when no + causal path exists. A positive result is deliberately called + ``graphically_admissible``. Positivity is required by the identification + theorem but cannot be checked from a DAG. ``max_paths`` bounds returned + witnesses and ``max_path_states`` bounds search expansions. Evidence is + never truncated. + """ + + dag = _validate_dag(dag) + treatment, outcome = _validate_endpoints(dag, treatment, outcome) + mediators = _known_node_set(dag, mediator_set, "mediator_set", allow_empty=True) + max_paths = _normalize_limit(max_paths, "max_paths") + max_path_states = _normalize_limit(max_path_states, "max_path_states") + observed = set(dag.observed_nodes) + endpoints = tuple(sorted(set(mediators).intersection((treatment, outcome)))) + directed_paths = _directed_paths( + dag, treatment, outcome, max_paths, max_path_states + ) + unintercepted = tuple( + path + for path in directed_paths + if not set(path.nodes[1:-1]).intersection(mediators) + ) + treatment_edges = _without_outgoing(dag.directed_edges, (treatment,)) + treatment_mediator = ( + _d_separation_from_normalized( + dag, + (treatment,), + mediators, + (), + treatment_edges, + max_paths, + max_path_states, + ) + if mediators and not endpoints + else None + ) + mediator_edges = _without_outgoing(dag.directed_edges, mediators) + mediator_outcome = ( + _d_separation_from_normalized( + dag, + mediators, + (outcome,), + (treatment,), + mediator_edges, + max_paths, + max_path_states, + ) + if mediators and not endpoints + else None + ) + open_treatment_mediator = ( + tuple(path for path in treatment_mediator.paths if path.is_open) + if treatment_mediator is not None + else () + ) + open_mediator_outcome = ( + tuple(path for path in mediator_outcome.paths if path.is_open) + if mediator_outcome is not None + else () + ) + all_observed = set(mediators).issubset(observed) + directed_path_exists = bool(directed_paths) + admissible = ( + bool(mediators) + and all_observed + and not endpoints + and directed_path_exists + and not unintercepted + and not open_treatment_mediator + and not open_mediator_outcome + ) + return FrontdoorAdjustmentEvidence( + treatment=treatment, + outcome=outcome, + mediator_set=mediators, + all_observed=all_observed, + endpoints_in_set=endpoints, + directed_path_exists=directed_path_exists, + unintercepted_directed_paths=unintercepted, + open_treatment_mediator_backdoor_paths=open_treatment_mediator, + open_mediator_outcome_backdoor_paths_given_treatment=open_mediator_outcome, + graphically_admissible=admissible, + positivity_required=True, + ) + + +def minimal_frontdoor_adjustment_sets( + dag: CausalDAG, + treatment: str, + outcome: str, + *, + max_candidates: Optional[int] = _DEFAULT_MAX_CANDIDATES, +) -> tuple[NodeTuple, ...]: + """Return all inclusion-minimal observed PROTOCOL-eligible front-door sets. + + Search is exact and stops with ``ValueError`` before exceeding the explicit + ``max_candidates`` budget; no partial result is returned. + """ + + dag = _validate_dag(dag) + treatment, outcome = _validate_endpoints(dag, treatment, outcome) + max_candidates = _normalize_limit(max_candidates, "max_candidates") + if not _has_directed_path(dag, treatment, outcome): + return () + candidates = tuple( + node for node in dag.observed_nodes if node not in {treatment, outcome} + ) + treatment_edges = _without_outgoing(dag.directed_edges, (treatment,)) + + def is_valid(candidate: NodeTuple) -> bool: + if not candidate: + return False + if _has_directed_path(dag, treatment, outcome, blocked=candidate): + return False + if not _is_d_separated(dag, (treatment,), candidate, (), treatment_edges): + return False + mediator_edges = _without_outgoing(dag.directed_edges, candidate) + return _is_d_separated(dag, candidate, (outcome,), (treatment,), mediator_edges) + + return _search_inclusion_minimal_sets(candidates, is_valid, max_candidates) + + +def check_instrument( + dag: CausalDAG, + instrument: str, + treatment: str, + outcome: str, + conditioned: Iterable[str] = (), +) -> InstrumentEvidence: + """Check Pearl's conditional and CFI's simple instrument profiles. + + ``pearl_graphical`` uses d-connected relevance and a conditioning set. + ``conditioned`` applies only to that Pearl profile. Every ``cfi_*`` field + evaluates the stricter, unconditional simple CFI profile on the same DAG + and can remain true when ``conditioned`` is nonempty. Neither Boolean is + an effect-estimation or nonparametric-identification claim. + """ + + dag = _validate_dag(dag) + treatment, outcome = _validate_endpoints(dag, treatment, outcome) + instrument = _require_node(instrument, "instrument") + if instrument not in dag.nodes: + raise ValueError("instrument must be a known graph node.") + if instrument in {treatment, outcome}: + raise ValueError("instrument, treatment, and outcome must be distinct.") + controls = _known_node_set(dag, conditioned, "conditioned", allow_empty=True) + forbidden = {instrument, treatment, outcome} + conditioning_disjoint = not set(controls).intersection(forbidden) + observed = set(dag.observed_nodes) + all_observed = {instrument, treatment, outcome}.union(controls).issubset(observed) + unaffected = not set(controls).intersection(_descendants(dag, treatment)) + if conditioning_disjoint: + treatment_edges = _without_incoming(dag.directed_edges, (treatment,)) + pearl_relevance = not _is_d_separated( + dag, (instrument,), (treatment,), controls + ) + pearl_exclusion = _is_d_separated( + dag, (instrument,), (outcome,), controls, treatment_edges + ) + else: + pearl_relevance = False + pearl_exclusion = False + pearl_graphical = ( + all_observed + and conditioning_disjoint + and unaffected + and pearl_relevance + and pearl_exclusion + ) + + edge_set = set(dag.directed_edges) + cfi_direct = (instrument, treatment) in edge_set + instrument_outcome_path = _has_directed_path(dag, instrument, outcome) + bypasses_treatment = _has_directed_path( + dag, instrument, outcome, blocked=(treatment,) + ) + cfi_full_mediation = instrument_outcome_path and not bypasses_treatment + instrument_edges = _without_outgoing(dag.directed_edges, (instrument,)) + cfi_exogeneity = _is_d_separated( + dag, (instrument,), (outcome,), (), instrument_edges + ) + cfi_simple = ( + instrument in observed and cfi_direct and cfi_full_mediation and cfi_exogeneity + ) + return InstrumentEvidence( + instrument=instrument, + treatment=treatment, + outcome=outcome, + conditioned=controls, + all_observed=all_observed, + conditioning_disjoint=conditioning_disjoint, + conditioning_unaffected_by_treatment=unaffected, + pearl_relevance=pearl_relevance, + pearl_exclusion_exogeneity=pearl_exclusion, + pearl_graphical=pearl_graphical, + cfi_direct_relevance=cfi_direct, + cfi_full_mediation=cfi_full_mediation, + cfi_exogeneity=cfi_exogeneity, + cfi_simple=cfi_simple, + ) + + +def causal_role_evidence( + dag: CausalDAG, + treatment: str, + outcome: str, + node: str, + *, + max_paths: Optional[int] = _DEFAULT_MAX_PATHS, + max_path_states: Optional[int] = _DEFAULT_MAX_PATH_STATES, +) -> NodeRoleEvidence: + """Return non-exclusive path witnesses for one internal graph node. + + ``max_paths`` bounds returned witnesses and ``max_path_states`` bounds + search expansions. Evidence is never truncated. + """ + + dag = _validate_dag(dag) + treatment, outcome = _validate_endpoints(dag, treatment, outcome) + node = _require_node(node, "node") + if node not in dag.nodes: + raise ValueError("node must be a known graph node.") + if node in {treatment, outcome}: + raise ValueError("node must be distinct from treatment and outcome.") + max_paths = _normalize_limit(max_paths, "max_paths") + max_path_states = _normalize_limit(max_path_states, "max_path_states") + paths = tuple( + _path_evidence(dag, path, ()) + for path in _simple_paths( + dag, + (treatment,), + (outcome,), + max_paths=max_paths, + max_path_states=max_path_states, + ) + ) + collider_paths = tuple(path.nodes for path in paths if node in path.colliders) + noncollider_paths = tuple(path.nodes for path in paths if node in path.noncolliders) + mediator_paths = tuple( + path.nodes + for path in paths + if path.is_directed_from_left and node in path.nodes[1:-1] + ) + backdoor_noncollider_paths = tuple( + path.nodes + for path in paths + if path.is_backdoor_from_left and node in path.noncolliders + ) + edge_set = set(dag.directed_edges) + common_cause_paths = [] + for path in paths: + if node not in path.noncolliders: + continue + index = path.nodes.index(node) + directed_to_treatment = all( + (path.nodes[position], path.nodes[position - 1]) in edge_set + for position in range(index, 0, -1) + ) + directed_to_outcome = all( + (path.nodes[position], path.nodes[position + 1]) in edge_set + for position in range(index, len(path.nodes) - 1) + ) + if directed_to_treatment and directed_to_outcome: + common_cause_paths.append(path.nodes) + return NodeRoleEvidence( + node=node, + treatment=treatment, + outcome=outcome, + collider_on_paths=collider_paths, + noncollider_on_paths=noncollider_paths, + mediator_on_directed_paths=mediator_paths, + backdoor_noncollider_on_paths=backdoor_noncollider_paths, + common_cause_paths=tuple(common_cause_paths), + descendant_of_treatment=node in _descendants(dag, treatment), + ancestor_of_outcome=node in _ancestors(dag, outcome), + ) diff --git a/src/RiskLabAI/causal_factor_analysis/optimizer.py b/src/RiskLabAI/causal_factor_analysis/optimizer.py new file mode 100644 index 0000000..c39a124 --- /dev/null +++ b/src/RiskLabAI/causal_factor_analysis/optimizer.py @@ -0,0 +1,212 @@ +r"""Minimum-variance allocation subject to target factor exposures. + +The optimization problem is + +.. math:: + + \min_{\omega}\; \omega^\mathsf{T} V \omega + \quad\text{subject to}\quad + B^\mathsf{T}\omega = c, + +where ``V`` is an asset covariance matrix, ``B`` contains asset-by-factor +exposures, and ``c`` is the requested factor-exposure vector. Equations (3)-(6) +and (A-1)-(A-7) of López de Prado, Lipton, and Zoonekynd (2025) give + +.. math:: + + \omega^* = V^{-1}B(B^\mathsf{T}V^{-1}B)^{-1}c. + +This implementation uses linear solves rather than forming either inverse. It +implements only the published allocation problem; factor-exposure estimation +and causal-model selection are separate responsibilities. + +References +---------- +López de Prado, M., Lipton, A., and Zoonekynd, V. (2025). "Causal Factor +Analysis Is a Necessary Condition for Investment Efficiency." Equations +(3)-(6) and Appendix A. SSRN 5131050. +""" + +from __future__ import annotations + +import numpy as np +from numpy.typing import ArrayLike, NDArray + +__all__ = ["minimum_variance_factor_weights"] + +_SYMMETRY_RTOL = 1e-10 +_SYMMETRY_ATOL = 1e-12 +_FEASIBILITY_RTOL = 1e-10 +_FEASIBILITY_ATOL = 1e-12 + + +def _as_real_array(values: ArrayLike, name: str) -> NDArray[np.float64]: + """Convert an input to a real ``float64`` array with a stable error.""" + try: + raw = np.asarray(values) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError(f"{name} must be a numeric array") from exc + if np.iscomplexobj(raw): + raise ValueError(f"{name} must be real-valued") + try: + return np.asarray(values, dtype=np.float64) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError(f"{name} must be a numeric array") from exc + + +def minimum_variance_factor_weights( + covariance: ArrayLike, + factor_exposures: ArrayLike, + target_exposures: ArrayLike, +) -> NDArray[np.float64]: + r"""Compute minimum-variance weights with prescribed factor exposures. + + Parameters + ---------- + covariance : array-like of shape (n_assets, n_assets) + Symmetric positive-definite asset covariance matrix :math:`V`. + factor_exposures : array-like of shape (n_assets, n_factors) + Full-column-rank asset-by-factor exposure matrix :math:`B`. + target_exposures : array-like of shape (n_factors,) + Requested factor exposures :math:`c`. + + Returns + ------- + numpy.ndarray + One-dimensional ``float64`` vector of optimal asset weights. + + Raises + ------ + ValueError + If an input has the wrong shape, contains a non-finite or complex value, + if ``covariance`` is not symmetric positive definite, or if + ``factor_exposures`` is not full column rank or its constraints cannot + be resolved to numerical precision. + + Notes + ----- + The published inverse formula is evaluated by Cholesky-whitening the + exposures and solving the resulting minimum-norm problem with reduced QR. + This avoids both explicit inverses and the condition-number squaring of a + normal-equation Gram matrix. Covariance and constraint units are normalized + by positive scalars without changing the optimizer, and the requested + exposures are verified before return. No diagonal jitter or pseudoinverse + is applied: singular covariance matrices and redundant factor constraints + are outside the stated assumptions of the model. + + Examples + -------- + >>> covariance = np.diag([1.0, 2.0, 4.0]) + >>> exposures = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]) + >>> targets = np.array([0.0, 1.0]) + >>> minimum_variance_factor_weights(covariance, exposures, targets) + array([-0.28571429, 0.71428571, 0.28571429]) + """ + covariance_array = _as_real_array(covariance, "covariance") + exposure_array = _as_real_array(factor_exposures, "factor_exposures") + target_array = _as_real_array(target_exposures, "target_exposures") + + if covariance_array.ndim != 2: + raise ValueError("covariance must be a 2-D square matrix") + n_assets, covariance_columns = covariance_array.shape + if n_assets == 0 or covariance_columns != n_assets: + raise ValueError("covariance must be a non-empty square matrix") + + if exposure_array.ndim != 2: + raise ValueError("factor_exposures must be a 2-D matrix") + if exposure_array.shape[0] != n_assets: + raise ValueError( + "factor_exposures must have the same number of rows as covariance" + ) + n_factors = exposure_array.shape[1] + if n_factors == 0: + raise ValueError("factor_exposures must contain at least one factor column") + if n_factors > n_assets: + raise ValueError("factor_exposures cannot have more columns than rows") + + if target_array.ndim != 1: + raise ValueError("target_exposures must be a 1-D vector") + if target_array.shape[0] != n_factors: + raise ValueError( + "target_exposures length must equal the number of factor columns" + ) + + if not np.all(np.isfinite(covariance_array)): + raise ValueError("covariance must contain only finite values") + if not np.all(np.isfinite(exposure_array)): + raise ValueError("factor_exposures must contain only finite values") + if not np.all(np.isfinite(target_array)): + raise ValueError("target_exposures must contain only finite values") + + covariance_scale = float(np.max(np.abs(covariance_array))) + if covariance_scale == 0.0: + raise ValueError("covariance must be positive definite") + scaled_covariance = covariance_array / covariance_scale + if not np.allclose( + scaled_covariance, + scaled_covariance.T, + rtol=_SYMMETRY_RTOL, + atol=_SYMMETRY_ATOL, + ): + raise ValueError("covariance must be symmetric") + scaled_covariance = 0.5 * scaled_covariance + 0.5 * scaled_covariance.T + + try: + covariance_cholesky = np.linalg.cholesky(scaled_covariance) + except np.linalg.LinAlgError as exc: + raise ValueError("covariance must be positive definite") from exc + + maximum_exposures = np.max(np.abs(exposure_array), axis=0) + if np.any(maximum_exposures == 0.0): + raise ValueError("factor_exposures must have full column rank") + target_scales = np.abs(target_array) / (np.finfo(np.float64).max / 2.0) + constraint_scales = np.maximum(maximum_exposures, target_scales) + with np.errstate(over="ignore", invalid="ignore", divide="ignore"): + scaled_exposures = exposure_array / constraint_scales + scaled_targets = target_array / constraint_scales + target_underflow = (target_array != 0.0) & (scaled_targets == 0.0) + if ( + np.any(target_underflow) + or not np.all(np.isfinite(scaled_exposures)) + or not np.all(np.isfinite(scaled_targets)) + ): + raise ValueError( + "factor constraints could not be represented at numerical precision" + ) + + try: + whitened_exposures = np.linalg.solve(covariance_cholesky, scaled_exposures) + if not np.all(np.isfinite(whitened_exposures)): + raise ValueError("factor_exposures produce non-finite whitened values") + if np.linalg.matrix_rank(whitened_exposures) != n_factors: + raise ValueError("factor_exposures must have full column rank") + + orthonormal_basis, triangular_factor = np.linalg.qr( + whitened_exposures, mode="reduced" + ) + transformed_targets = np.linalg.solve(triangular_factor.T, scaled_targets) + whitened_weights = orthonormal_basis @ transformed_targets + weights = np.linalg.solve(covariance_cholesky.T, whitened_weights) + except np.linalg.LinAlgError as exc: + raise ValueError("factor_exposures must have full column rank") from exc + + weights = np.asarray(weights, dtype=np.float64).reshape(n_assets) + if not np.all(np.isfinite(weights)): + raise ValueError("the allocation produced non-finite weights") + + with np.errstate(over="ignore", invalid="ignore"): + achieved_exposures = scaled_exposures.T @ weights + absolute_residuals = np.abs(achieved_exposures - scaled_targets) + feasibility_tolerances = np.where( + scaled_targets != 0.0, + _FEASIBILITY_RTOL * np.abs(scaled_targets), + _FEASIBILITY_ATOL, + ) + if not np.all(np.isfinite(achieved_exposures)) or np.any( + absolute_residuals > feasibility_tolerances + ): + raise ValueError( + "factor constraints could not be resolved to numerical precision" + ) + + return weights.copy() diff --git a/src/RiskLabAI/causal_factor_analysis/protocol.py b/src/RiskLabAI/causal_factor_analysis/protocol.py new file mode 100644 index 0000000..281afe2 --- /dev/null +++ b/src/RiskLabAI/causal_factor_analysis/protocol.py @@ -0,0 +1,1717 @@ +# ruff: noqa: UP045 +"""Immutable evidence contract for the causal-factor research protocol. + +The module represents the seven stages in Lopez de Prado and Zoonekynd +(2026), PDF pages 18-22 (journal pages 27-31) and Exhibit 10. It validates a +completed research record; it does not execute discovery, estimation, +portfolio construction, backtesting, or multiple-testing algorithms. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from numbers import Integral, Real +from typing import Optional + +__all__ = [ + "BacktestStage", + "CausalAdjustmentSetStage", + "CausalDiscoveryStage", + "CausalExplanatoryAndPredictivePowerStage", + "CausalFactorProtocolReport", + "CausalPortfolioConstructionStage", + "EventHorizon", + "FoldEvidence", + "MultipleTestingAdjustmentsStage", + "ValidationEvidence", + "VariableSelectionStage", + "validate_causal_factor_protocol", +] + + +_STAGE_NAMES = ( + "Variable Selection", + "Causal Discovery", + "Causal Adjustment Set", + "Causal Explanatory and Predictive Power", + "Causal Portfolio Construction", + "Backtest", + "Multiple Testing Adjustments", +) + +_VARIABLE_SELECTION_METHODS = { + "MUTUAL_INFORMATION", + "SHAPLEY_VALUES", + "MEAN_DECREASE_IMPURITY", + "PERMUTATION_FEATURE_IMPORTANCE", +} +_RESEARCH_PURPOSES = {"CAUSAL_ATTRIBUTION", "RISK_PREMIA_HARVESTING"} +_DISCOVERY_METHODS = { + "PC", + "LINGAM", + "ECONOMIC_REASONING", + "EX_ANTE_VIEWS", + "OBSERVED_OUTCOMES", + "PEER_REVIEWED_ASSUMPTIONS", + "DOMAIN_EXPERTISE", +} +_ADJUSTMENT_METHODS = { + "BACKDOOR_ADJUSTMENT", + "FRONT_DOOR_ADJUSTMENT", + "INSTRUMENTAL_VARIABLES", +} +_VALIDATION_METHODS = { + "PURGED_CROSS_VALIDATION", + "WALK_FORWARD", + "RESAMPLING", + "COMBINATORIAL_PURGED_CROSS_VALIDATION", +} +_PURGED_VALIDATION_METHODS = { + "PURGED_CROSS_VALIDATION", + "COMBINATORIAL_PURGED_CROSS_VALIDATION", +} +_METRICS_BY_TASK = { + "PROBABILITY": {"LOG_LOSS", "BRIER_SCORE"}, + "RANKING": { + "ROC_CURVE", + "PRECISION_RECALL_CURVE", + "MEAN_RECIPROCAL_RANK", + "CLASSIFICATION_ACCURACY", + }, + "RETURN_SIZE": { + "MEAN_SQUARED_ERROR", + "R_SQUARED", + "SPEARMAN_CORRELATION", + }, +} +_PORTFOLIO_METHODS = { + "POSITION_SIZING", + "EXPOSURE_CONTROL", + "ECONOMIC_RATIONALE", + "FRAGILITY_STRESS_TEST", + "TRANSACTION_COST_OPTIMIZATION", + "TRANSFER_COEFFICIENT", +} +_BACKTEST_METHODS = { + "WALK_FORWARD", + "RESAMPLING", + "COMBINATORIAL_PURGED_CROSS_VALIDATION", + "MONTE_CARLO", +} +_MULTIPLE_TESTING_METHODS = { + "HOLM", + "HOCHBERG", + "BENJAMINI_HOCHBERG", + "DEFLATED_SHARPE_RATIO", +} +_P_VALUE_METHODS = {"HOLM", "HOCHBERG", "BENJAMINI_HOCHBERG"} +_REFIT_COMPONENTS = { + "VARIABLE_SELECTION", + "CAUSAL_ESTIMATION", + "PORTFOLIO_CONSTRUCTION", +} + + +def _freeze_tuple(value): + if isinstance(value, list): + return tuple(value) + return value + + +def _freeze_nested_tuple(value): + outer = _freeze_tuple(value) + if isinstance(outer, tuple): + return tuple(_freeze_tuple(item) for item in outer) + return outer + + +def _freeze_partitions(value): + outer = _freeze_tuple(value) + if not isinstance(outer, tuple): + return outer + result = [] + for item in outer: + pair = _freeze_tuple(item) + if isinstance(pair, tuple) and len(pair) == 2: + pair = (pair[0], _freeze_tuple(pair[1])) + result.append(pair) + return tuple(result) + + +def _store_normalized_field(instance, field_name, value): + """Store a normalized value while a frozen record is being constructed.""" + + instance.__dict__[field_name] = value + + +@dataclass(frozen=True) +class EventHorizon: + """Observation identifier and its half-open return interval.""" + + observation_id: str + start: Real + end: Real + + +@dataclass(frozen=True) +class FoldEvidence: + """Membership and training-only pipeline components for one fold.""" + + fold_id: str + path_id: str + train_ids: tuple[str, ...] + test_ids: tuple[str, ...] + fit_ids: tuple[str, ...] + prediction_ids: tuple[str, ...] + refit_id: str + refit_components: tuple[str, ...] + + def __post_init__(self): + for name in ( + "train_ids", + "test_ids", + "fit_ids", + "prediction_ids", + "refit_components", + ): + _store_normalized_field(self, name, _freeze_tuple(getattr(self, name))) + + +@dataclass(frozen=True) +class ValidationEvidence: + """Temporal validation evidence shared by source stages that require it.""" + + method_labels: tuple[str, ...] + event_horizons: tuple[EventHorizon, ...] + folds: tuple[FoldEvidence, ...] + embargo: Real = 0.0 + + def __post_init__(self): + _store_normalized_field(self, "method_labels", _freeze_tuple(self.method_labels)) + _store_normalized_field(self, "event_horizons", _freeze_tuple(self.event_horizons)) + _store_normalized_field(self, "folds", _freeze_tuple(self.folds)) + + +@dataclass(frozen=True) +class VariableSelectionStage: + """Stage 1 evidence for candidate-variable selection and leakage controls.""" + + purpose: str + selected_variables: tuple[str, ...] + method_labels: tuple[str, ...] + overlapping_returns: bool + strong_time_dependence: bool + validation: Optional[ValidationEvidence] + + def __post_init__(self): + _store_normalized_field( + self, "selected_variables", _freeze_tuple(self.selected_variables) + ) + _store_normalized_field(self, "method_labels", _freeze_tuple(self.method_labels)) + + +@dataclass(frozen=True) +class CausalDiscoveryStage: + """Stage 2 evidence for an explicit, resolved causal graph.""" + + method_labels: tuple[str, ...] + graph_id: str + graph_nodes: tuple[str, ...] + directed_edges: tuple[tuple[str, str], ...] + graph_kind: str + ambiguous_edges: tuple[tuple[str, str], ...] + assumptions: tuple[str, ...] + + def __post_init__(self): + _store_normalized_field(self, "method_labels", _freeze_tuple(self.method_labels)) + _store_normalized_field(self, "graph_nodes", _freeze_tuple(self.graph_nodes)) + _store_normalized_field( + self, "directed_edges", _freeze_nested_tuple(self.directed_edges) + ) + _store_normalized_field( + self, "ambiguous_edges", _freeze_nested_tuple(self.ambiguous_edges) + ) + _store_normalized_field(self, "assumptions", _freeze_tuple(self.assumptions)) + + +@dataclass(frozen=True) +class CausalAdjustmentSetStage: + """Stage 3 evidence for identification and admissible ordinary controls.""" + + treatment: str + outcome: str + method_label: str + identified: bool + admissible_adjustment_sets: tuple[tuple[str, ...], ...] + selected_adjustment_set: tuple[str, ...] + confounders: tuple[str, ...] + descendants: tuple[str, ...] + mediators: tuple[str, ...] + colliders: tuple[str, ...] + instruments: tuple[str, ...] + control_justifications: tuple[tuple[str, str], ...] + open_backdoor_paths: tuple[str, ...] + frontdoor_criteria_satisfied: Optional[bool] = None + instrument_relevance_satisfied: Optional[bool] = None + instrument_exclusion_satisfied: Optional[bool] = None + instrument_exogeneity_satisfied: Optional[bool] = None + + def __post_init__(self): + _store_normalized_field( + self, + "admissible_adjustment_sets", + _freeze_nested_tuple(self.admissible_adjustment_sets), + ) + for name in ( + "selected_adjustment_set", + "confounders", + "descendants", + "mediators", + "colliders", + "instruments", + "open_backdoor_paths", + ): + _store_normalized_field(self, name, _freeze_tuple(getattr(self, name))) + _store_normalized_field( + self, + "control_justifications", + _freeze_nested_tuple(self.control_justifications), + ) + + +@dataclass(frozen=True) +class CausalExplanatoryAndPredictivePowerStage: + """Stage 4 generalization evidence, separated by research purpose.""" + + task_types: tuple[str, ...] + estimator_label: str + explanatory_metric_labels: tuple[str, ...] + predictive_metric_labels: tuple[str, ...] + explanatory_evidence_id: Optional[str] + predictive_evidence_id: Optional[str] + naive_benchmark_id: str + validation: ValidationEvidence + multiclass_encoding: Optional[str] = None + averaging_method: Optional[str] = None + + def __post_init__(self): + _store_normalized_field(self, "task_types", _freeze_tuple(self.task_types)) + _store_normalized_field( + self, + "explanatory_metric_labels", + _freeze_tuple(self.explanatory_metric_labels), + ) + _store_normalized_field( + self, + "predictive_metric_labels", + _freeze_tuple(self.predictive_metric_labels), + ) + + +@dataclass(frozen=True) +class CausalPortfolioConstructionStage: + """Stage 5 evidence for a causally motivated implemented portfolio.""" + + method_labels: tuple[str, ...] + causal_exposures: tuple[str, ...] + controlled_unintended_exposures: tuple[str, ...] + cost_model_id: str + constraint_set_id: str + economic_rationale: str + fragility_scenarios: tuple[str, ...] + transfer_coefficient: Real + + def __post_init__(self): + for name in ( + "method_labels", + "causal_exposures", + "controlled_unintended_exposures", + "fragility_scenarios", + ): + _store_normalized_field(self, name, _freeze_tuple(getattr(self, name))) + + +@dataclass(frozen=True) +class BacktestStage: + """Stage 6 evidence for one or more source-listed backtest types.""" + + method_labels: tuple[str, ...] + trial_family_id: str + declared_trial_ids: tuple[str, ...] + validation: Optional[ValidationEvidence] = None + monte_carlo_dgp: Optional[str] = None + + def __post_init__(self): + _store_normalized_field(self, "method_labels", _freeze_tuple(self.method_labels)) + _store_normalized_field( + self, "declared_trial_ids", _freeze_tuple(self.declared_trial_ids) + ) + + +@dataclass(frozen=True) +class MultipleTestingAdjustmentsStage: + """Stage 7 evidence for the complete declared family of trials.""" + + method_labels: tuple[str, ...] + trial_family_id: str + declared_trial_ids: tuple[str, ...] + family_partitions: tuple[tuple[str, tuple[str, ...]], ...] + alpha: Real + backtests_independent: bool + p_value_estimator: Optional[str] = None + time_dependence_model: Optional[str] = None + p_value_inputs_assume_independence: Optional[bool] = None + sharpe_variance: Optional[Real] = None + effective_trials: Optional[Real] = None + sample_length: Optional[Integral] = None + skewness: Optional[Real] = None + kurtosis: Optional[Real] = None + selection_bias_evidence_id: Optional[str] = None + + def __post_init__(self): + _store_normalized_field(self, "method_labels", _freeze_tuple(self.method_labels)) + _store_normalized_field( + self, "declared_trial_ids", _freeze_tuple(self.declared_trial_ids) + ) + _store_normalized_field( + self, "family_partitions", _freeze_partitions(self.family_partitions) + ) + + +@dataclass(frozen=True) +class CausalFactorProtocolReport: + """Complete immutable record of the seven published protocol stages.""" + + trial_family_id: str + declared_trial_ids: tuple[str, ...] + variable_selection: VariableSelectionStage + causal_discovery: CausalDiscoveryStage + causal_adjustment_set: CausalAdjustmentSetStage + causal_explanatory_and_predictive_power: CausalExplanatoryAndPredictivePowerStage + causal_portfolio_construction: CausalPortfolioConstructionStage + backtest: BacktestStage + multiple_testing_adjustments: MultipleTestingAdjustmentsStage + source_oracle: str = field(default="PROTOCOL-01", init=False) + source_locator: str = field( + default="PDF pages 18-22 (journal pages 27-31), Exhibit 10", + init=False, + ) + + def __post_init__(self): + _store_normalized_field( + self, "declared_trial_ids", _freeze_tuple(self.declared_trial_ids) + ) + + @property + def stages(self): + """Return the seven named stage records in the source order.""" + + return ( + (_STAGE_NAMES[0], self.variable_selection), + (_STAGE_NAMES[1], self.causal_discovery), + (_STAGE_NAMES[2], self.causal_adjustment_set), + (_STAGE_NAMES[3], self.causal_explanatory_and_predictive_power), + (_STAGE_NAMES[4], self.causal_portfolio_construction), + (_STAGE_NAMES[5], self.backtest), + (_STAGE_NAMES[6], self.multiple_testing_adjustments), + ) + + @property + def stage_names(self): + """Return the exact seven source stage names.""" + + return _STAGE_NAMES + + +def _add_error(errors: list[str], message: str) -> None: + if message not in errors: + errors.append(message) + + +def _nonblank(value) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _safe_label_set(value) -> set[str]: + if not isinstance(value, tuple) or not all( + isinstance(item, str) for item in value + ): + return set() + return set(value) + + +def _finite_real(value) -> bool: + if isinstance(value, bool) or not isinstance(value, Real): + return False + try: + return math.isfinite(float(value)) + except (TypeError, ValueError, OverflowError): + return False + + +def _validate_string_tuple( + value, + stage: str, + subject: str, + errors: list[str], + *, + allow_empty: bool = False, +) -> bool: + if not isinstance(value, tuple): + _add_error(errors, f"{stage}: {subject} must be a tuple.") + return False + if not allow_empty and not value: + _add_error(errors, f"{stage}: {subject} must not be empty.") + return False + if not all(_nonblank(item) for item in value): + _add_error(errors, f"{stage}: {subject} contains an invalid identifier.") + return False + if len(set(value)) != len(value): + _add_error(errors, f"{stage}: {subject} contains duplicate identifiers.") + return False + return True + + +def _validate_labels( + value, + allowed: set[str], + stage: str, + subject: str, + errors: list[str], + *, + allow_empty: bool = False, +) -> bool: + valid = _validate_string_tuple( + value, stage, subject, errors, allow_empty=allow_empty + ) + if valid and not set(value).issubset(allowed): + _add_error(errors, f"{stage}: {subject} contains a non-source label.") + return False + return valid + + +def _validate_optional_identifier( + value, stage: str, subject: str, errors: list[str] +) -> bool: + if value is None: + return True + if not _nonblank(value): + _add_error(errors, f"{stage}: {subject} is invalid.") + return False + return True + + +def _validate_event_horizons( + evidence: ValidationEvidence, stage: str, errors: list[str] +) -> dict[str, tuple[float, float]]: + horizons = evidence.event_horizons + if not isinstance(horizons, tuple) or not horizons: + _add_error(errors, f"{stage}: event horizons must be a nonempty tuple.") + return {} + + result: dict[str, tuple[float, float]] = {} + for horizon in horizons: + if not isinstance(horizon, EventHorizon): + _add_error(errors, f"{stage}: an event-horizon record has an invalid type.") + continue + if not _nonblank(horizon.observation_id): + _add_error(errors, f"{stage}: an event horizon has an invalid identifier.") + continue + if horizon.observation_id in result: + _add_error(errors, f"{stage}: event-horizon identifiers must be unique.") + continue + if not _finite_real(horizon.start) or not _finite_real(horizon.end): + _add_error(errors, f"{stage}: event-horizon bounds must be finite numbers.") + continue + start = float(horizon.start) + end = float(horizon.end) + if not start < end: + _add_error(errors, f"{stage}: every event horizon must have positive length.") + continue + result[horizon.observation_id] = (start, end) + return result + + +def _intervals_overlap( + left: tuple[float, float], right: tuple[float, float] +) -> bool: + return left[0] < right[1] and right[0] < left[1] + + +def _has_any_overlap(horizons: dict[str, tuple[float, float]]) -> bool: + intervals = list(horizons.values()) + for left_index, left in enumerate(intervals): + for right in intervals[left_index + 1 :]: + if _intervals_overlap(left, right): + return True + return False + + +def _validate_fold_evidence( + evidence: ValidationEvidence, + stage: str, + errors: list[str], + *, + required_refit_components: set[str], +) -> dict[str, tuple[float, float]]: + _validate_labels( + evidence.method_labels, + _VALIDATION_METHODS, + stage, + "validation method labels", + errors, + ) + horizon_map = _validate_event_horizons(evidence, stage, errors) + if not _finite_real(evidence.embargo) or float(evidence.embargo) < 0.0: + _add_error(errors, f"{stage}: embargo must be a finite nonnegative number.") + embargo = 0.0 + else: + embargo = float(evidence.embargo) + + folds = evidence.folds + if not isinstance(folds, tuple) or not folds: + _add_error(errors, f"{stage}: validation folds must be a nonempty tuple.") + return horizon_map + + fold_ids: set[str] = set() + refit_training_sets: dict[str, frozenset] = {} + path_predictions: set[tuple[str, str]] = set() + path_test_ids: dict[str, set[str]] = {} + path_partitions: dict[str, list[tuple[str, ...]]] = {} + nonresampled_test_ids: set[str] = set() + known_ids = set(horizon_map) + validation_methods = _safe_label_set(evidence.method_labels) + + for fold in folds: + if not isinstance(fold, FoldEvidence): + _add_error(errors, f"{stage}: a fold record has an invalid type.") + continue + if not _nonblank(fold.fold_id) or not _nonblank(fold.path_id): + _add_error(errors, f"{stage}: fold and path identifiers must be nonblank.") + elif fold.fold_id in fold_ids: + _add_error(errors, f"{stage}: fold identifiers must be unique.") + else: + fold_ids.add(fold.fold_id) + if not _nonblank(fold.refit_id): + _add_error(errors, f"{stage}: every fold must record a refit identifier.") + components_valid = _validate_labels( + fold.refit_components, + _REFIT_COMPONENTS, + stage, + "refit components", + errors, + ) + if components_valid and not required_refit_components.issubset( + set(fold.refit_components) + ): + _add_error( + errors, + f"{stage}: every fold must refit all required pipeline components.", + ) + + train_valid = _validate_string_tuple( + fold.train_ids, stage, "training identifiers", errors + ) + test_valid = _validate_string_tuple( + fold.test_ids, stage, "test identifiers", errors + ) + fit_valid = _validate_string_tuple( + fold.fit_ids, stage, "fit identifiers", errors + ) + prediction_valid = _validate_string_tuple( + fold.prediction_ids, stage, "prediction identifiers", errors + ) + if not (train_valid and test_valid and fit_valid and prediction_valid): + continue + + train_set = set(fold.train_ids) + test_set = set(fold.test_ids) + fit_set = set(fold.fit_ids) + prediction_set = set(fold.prediction_ids) + if train_set.intersection(test_set): + _add_error(errors, f"{stage}: training and test membership must be disjoint.") + if fit_set != train_set or fit_set.intersection(test_set): + _add_error( + errors, + f"{stage}: every fold must refit on its complete training membership only.", + ) + if prediction_set != test_set: + _add_error(errors, f"{stage}: predictions must match the test membership.") + if not (train_set | test_set | fit_set | prediction_set).issubset(known_ids): + _add_error(errors, f"{stage}: every fold identifier must have an event horizon.") + if validation_methods.intersection( + {"PURGED_CROSS_VALIDATION", "WALK_FORWARD"} + ): + if nonresampled_test_ids.intersection(test_set): + _add_error( + errors, + f"{stage}: ordinary temporal validation cannot duplicate test predictions.", + ) + nonresampled_test_ids.update(test_set.intersection(known_ids)) + + if _nonblank(fold.refit_id): + training_signature = frozenset(fold.train_ids) + previous = refit_training_sets.get(fold.refit_id) + if previous is not None and previous != training_signature: + _add_error( + errors, + f"{stage}: a refit identifier cannot represent different training sets.", + ) + refit_training_sets[fold.refit_id] = training_signature + + if _nonblank(fold.path_id): + path_test_ids.setdefault(fold.path_id, set()) + path_partitions.setdefault(fold.path_id, []) + path_partitions[fold.path_id].append(tuple(sorted(test_set))) + for observation_id in fold.test_ids: + contribution = (fold.path_id, observation_id) + if contribution in path_predictions: + _add_error( + errors, + f"{stage}: a path cannot contain duplicate test contributions.", + ) + path_predictions.add(contribution) + path_test_ids[fold.path_id].add(observation_id) + + if not (train_set | test_set).issubset(known_ids): + continue + for train_id in fold.train_ids: + train_horizon = horizon_map[train_id] + for test_id in fold.test_ids: + test_horizon = horizon_map[test_id] + if _intervals_overlap(train_horizon, test_horizon): + _add_error( + errors, + f"{stage}: train and test event horizons must be purged.", + ) + if ( + embargo > 0.0 + and train_horizon[0] >= test_horizon[1] + and train_horizon[0] < test_horizon[1] + embargo + ): + _add_error( + errors, + f"{stage}: post-test embargo observations must be excluded.", + ) + if "WALK_FORWARD" in validation_methods and train_set and test_set: + latest_train_end = max(horizon_map[item][1] for item in train_set) + earliest_test_start = min(horizon_map[item][0] for item in test_set) + if latest_train_end > earliest_test_start: + _add_error( + errors, + f"{stage}: walk-forward training must precede the test interval.", + ) + + if "COMBINATORIAL_PURGED_CROSS_VALIDATION" in validation_methods: + if len(path_test_ids) < 2: + _add_error(errors, f"{stage}: CPCV requires multiple complete paths.") + if known_ids and any(ids != known_ids for ids in path_test_ids.values()): + _add_error( + errors, + f"{stage}: every CPCV path must predict each declared observation once.", + ) + partition_signatures = [ + tuple(sorted(partition)) for partition in path_partitions.values() + ] + if len(set(partition_signatures)) != len(partition_signatures): + _add_error(errors, f"{stage}: CPCV paths must have distinct test partitions.") + if ( + "PURGED_CROSS_VALIDATION" in validation_methods + and known_ids + and nonresampled_test_ids != known_ids + ): + _add_error( + errors, + f"{stage}: purged cross-validation must predict every observation once.", + ) + return horizon_map + + +def _valid_edge_tuple(value, stage: str, subject: str, errors: list[str]) -> bool: + if not isinstance(value, tuple): + _add_error(errors, f"{stage}: {subject} must be a tuple.") + return False + valid = True + seen: set[tuple[str, str]] = set() + for edge in value: + if ( + not isinstance(edge, tuple) + or len(edge) != 2 + or not _nonblank(edge[0]) + or not _nonblank(edge[1]) + ): + _add_error(errors, f"{stage}: {subject} contains an invalid edge.") + valid = False + continue + if edge in seen: + _add_error(errors, f"{stage}: {subject} contains duplicate edges.") + valid = False + seen.add(edge) + return valid + + +def _adjacency(nodes: set[str], edges: tuple[tuple[str, str], ...]): + result = {node: set() for node in nodes} + for source, target in edges: + if source in result and target in result: + result[source].add(target) + return result + + +def _is_acyclic(nodes: set[str], edges: tuple[tuple[str, str], ...]) -> bool: + indegree = {node: 0 for node in nodes} + adjacency = _adjacency(nodes, edges) + for targets in adjacency.values(): + for target in targets: + indegree[target] += 1 + ready = [node for node, degree in indegree.items() if degree == 0] + visited = 0 + while ready: + node = ready.pop() + visited += 1 + for target in adjacency[node]: + indegree[target] -= 1 + if indegree[target] == 0: + ready.append(target) + return visited == len(nodes) + + +def _reachable(adjacency, start: str) -> set[str]: + visited: set[str] = set() + pending = list(adjacency.get(start, ())) + while pending: + node = pending.pop() + if node in visited: + continue + visited.add(node) + pending.extend(adjacency.get(node, ())) + return visited + + +def _reachable_avoiding(adjacency, start: str, blocked: set[str]) -> set[str]: + if start in blocked: + return set() + visited: set[str] = set() + pending = [node for node in adjacency.get(start, ()) if node not in blocked] + while pending: + node = pending.pop() + if node in visited or node in blocked: + continue + visited.add(node) + pending.extend( + target for target in adjacency.get(node, ()) if target not in blocked + ) + return visited + + +def _d_separated( + edges: tuple[tuple[str, str], ...], + left: str, + right: str, + conditioned: set[str], +) -> bool: + nodes = {left, right} | conditioned + for source, target in edges: + nodes.add(source) + nodes.add(target) + parents = {node: set() for node in nodes} + for source, target in edges: + parents[target].add(source) + + ancestors = {left, right} | conditioned + pending = list(ancestors) + while pending: + node = pending.pop() + for parent in parents.get(node, ()): + if parent not in ancestors: + ancestors.add(parent) + pending.append(parent) + + moral = {node: set() for node in ancestors} + for source, target in edges: + if source in ancestors and target in ancestors: + moral[source].add(target) + moral[target].add(source) + for child in ancestors: + child_parents = [parent for parent in parents[child] if parent in ancestors] + for index, first in enumerate(child_parents): + for second in child_parents[index + 1 :]: + moral[first].add(second) + moral[second].add(first) + + if left in conditioned or right in conditioned: + return True + visited = set(conditioned) + pending = [left] + while pending: + node = pending.pop() + if node == right: + return False + if node in visited: + continue + visited.add(node) + pending.extend(neighbor for neighbor in moral[node] if neighbor not in visited) + return True + + +def _backdoor_d_separated( + edges: tuple[tuple[str, str], ...], + exposure: str, + outcome: str, + conditioned: set[str], +) -> bool: + backdoor_edges = tuple(edge for edge in edges if edge[0] != exposure) + return _d_separated(backdoor_edges, exposure, outcome, conditioned) + + +def _validate_discovery( + stage: CausalDiscoveryStage, + selected_variables: set[str], + outcome: str, + errors: list[str], +): + label = "Stage 2" + _validate_labels( + stage.method_labels, + _DISCOVERY_METHODS, + label, + "causal-discovery method labels", + errors, + ) + if not _nonblank(stage.graph_id): + _add_error(errors, f"{label}: graph identifier must be nonblank.") + nodes_valid = _validate_string_tuple( + stage.graph_nodes, label, "graph nodes", errors + ) + nodes = set(stage.graph_nodes) if nodes_valid else set() + if nodes_valid and nodes != selected_variables | {outcome}: + _add_error( + errors, + f"{label}: graph nodes must match the selected variables and outcome.", + ) + edges_valid = _valid_edge_tuple( + stage.directed_edges, label, "directed edges", errors + ) + if edges_valid: + for source, target in stage.directed_edges: + if source not in nodes or target not in nodes: + _add_error(errors, f"{label}: every directed edge must use graph nodes.") + if source == target: + _add_error(errors, f"{label}: directed self-loops are not admissible.") + if nodes and not _is_acyclic(nodes, stage.directed_edges): + _add_error(errors, f"{label}: the resolved graph must be acyclic.") + if stage.graph_kind != "DAG": + _add_error(errors, f"{label}: the accepted graph must be a resolved DAG.") + if not isinstance(stage.ambiguous_edges, tuple): + _add_error(errors, f"{label}: ambiguous edges must be a tuple.") + elif stage.ambiguous_edges: + _valid_edge_tuple(stage.ambiguous_edges, label, "ambiguous edges", errors) + _add_error(errors, f"{label}: identification-relevant ambiguity must be resolved.") + _validate_string_tuple(stage.assumptions, label, "graph assumptions", errors) + safe_edges = stage.directed_edges if edges_valid else () + return nodes, _adjacency(nodes, safe_edges), safe_edges + + +def _validate_adjustment_sets( + value, stage: str, errors: list[str] +) -> tuple[set[frozenset], bool]: + if not isinstance(value, tuple) or not value: + _add_error(errors, f"{stage}: admissible adjustment sets must be nonempty.") + return set(), False + result: set[frozenset] = set() + valid = True + for adjustment_set in value: + if not _validate_string_tuple( + adjustment_set, + stage, + "an admissible adjustment set", + errors, + allow_empty=True, + ): + valid = False + continue + frozen = frozenset(adjustment_set) + if frozen in result: + _add_error(errors, f"{stage}: admissible adjustment sets must be distinct.") + valid = False + result.add(frozen) + return result, valid + + +def _validate_justifications(value, stage: str, errors: list[str]): + if not isinstance(value, tuple): + _add_error(errors, f"{stage}: control justifications must be a tuple.") + return {} + result = {} + for item in value: + if ( + not isinstance(item, tuple) + or len(item) != 2 + or not _nonblank(item[0]) + or not _nonblank(item[1]) + ): + _add_error(errors, f"{stage}: a control justification is invalid.") + continue + if item[0] in result: + _add_error(errors, f"{stage}: every control must have one justification.") + result[item[0]] = item[1] + return result + + +def _validate_adjustment( + stage: CausalAdjustmentSetStage, + graph_nodes: set[str], + adjacency, + directed_edges: tuple[tuple[str, str], ...], + errors: list[str], +) -> None: + label = "Stage 3" + treatment = stage.treatment if _nonblank(stage.treatment) else "" + outcome = stage.outcome if _nonblank(stage.outcome) else "" + if not _nonblank(stage.treatment) or not _nonblank(stage.outcome): + _add_error(errors, f"{label}: treatment and outcome must be nonblank.") + elif treatment == outcome: + _add_error(errors, f"{label}: treatment and outcome must be distinct.") + if treatment not in graph_nodes or outcome not in graph_nodes: + _add_error(errors, f"{label}: treatment and outcome must be graph nodes.") + if not _nonblank(stage.method_label) or stage.method_label not in _ADJUSTMENT_METHODS: + _add_error(errors, f"{label}: adjustment method is not source-listed.") + if not isinstance(stage.identified, bool) or not stage.identified: + _add_error(errors, f"{label}: the causal effect must be identified.") + + selected_valid = _validate_string_tuple( + stage.selected_adjustment_set, + label, + "selected adjustment set", + errors, + allow_empty=True, + ) + admissible_sets, _ = _validate_adjustment_sets( + stage.admissible_adjustment_sets, label, errors + ) + selected = set(stage.selected_adjustment_set) if selected_valid else set() + if selected_valid and frozenset(selected) not in admissible_sets: + _add_error( + errors, + f"{label}: the selected adjustment set must be declared admissible.", + ) + + role_names = {} + for field_name, subject in ( + ("confounders", "confounders"), + ("descendants", "descendants"), + ("mediators", "mediators"), + ("colliders", "colliders"), + ("instruments", "instruments"), + ): + value = getattr(stage, field_name) + valid = _validate_string_tuple( + value, label, subject, errors, allow_empty=True + ) + role_names[field_name] = set(value) if valid else set() + if valid and not set(value).issubset(graph_nodes): + _add_error(errors, f"{label}: every declared causal role must be a graph node.") + if valid and {treatment, outcome}.intersection(value): + _add_error(errors, f"{label}: treatment and outcome cannot be role variables.") + + computed_descendants = _reachable(adjacency, treatment) + computed_descendants.discard(outcome) + if role_names["descendants"] != computed_descendants: + _add_error( + errors, + f"{label}: declared descendants must exactly match the graph.", + ) + for mediator in role_names["mediators"]: + if mediator not in computed_descendants or outcome not in _reachable( + adjacency, mediator + ): + _add_error(errors, f"{label}: declared mediators must lie on a causal path.") + incoming = {node: 0 for node in graph_nodes} + for targets in adjacency.values(): + for target in targets: + incoming[target] += 1 + if any(incoming.get(collider, 0) < 2 for collider in role_names["colliders"]): + _add_error(errors, f"{label}: declared colliders must have converging arrows.") + if outcome not in _reachable(adjacency, treatment): + _add_error(errors, f"{label}: the graph must retain a treatment-to-outcome path.") + + graph_confounders = { + node + for node in graph_nodes - {treatment, outcome} + if treatment in _reachable(adjacency, node) + and outcome in _reachable_avoiding(adjacency, node, {treatment}) + } + if role_names["confounders"] != graph_confounders: + _add_error( + errors, + f"{label}: declared confounders must match graph-implied common causes.", + ) + graph_colliders = { + node + for node, degree in incoming.items() + if degree >= 2 and node not in {treatment, outcome} + } + if role_names["colliders"] != graph_colliders: + _add_error( + errors, + f"{label}: declared colliders must match graph-implied converging nodes.", + ) + graph_mediators = { + node + for node in graph_nodes - {treatment, outcome} + if node in _reachable(adjacency, treatment) + and outcome in _reachable(adjacency, node) + } + if role_names["mediators"] != graph_mediators: + _add_error( + errors, + f"{label}: declared mediators must match graph-implied causal-path nodes.", + ) + if role_names["confounders"].intersection(role_names["instruments"]): + _add_error( + errors, + f"{label}: a variable cannot be both a confounder and an instrument.", + ) + instrument_edges = tuple( + edge for edge in directed_edges if edge[0] != treatment + ) + for instrument in role_names["instruments"]: + if treatment not in _reachable_avoiding(adjacency, instrument, selected): + _add_error( + errors, + f"{label}: every declared instrument must be relevant in the graph.", + ) + if outcome in _reachable_avoiding(adjacency, instrument, {treatment}): + _add_error( + errors, + f"{label}: every declared instrument must satisfy graph-level exclusion.", + ) + if not _d_separated(instrument_edges, instrument, outcome, selected): + _add_error( + errors, + f"{label}: every declared instrument must be graph-separated from outcome shocks.", + ) + + forbidden_controls = ( + computed_descendants + | role_names["descendants"] + | role_names["mediators"] + | role_names["colliders"] + | role_names["instruments"] + | {treatment, outcome} + ) + if selected.intersection(forbidden_controls): + _add_error( + errors, + f"{label}: ordinary controls cannot include descendants, mediators, colliders, or instruments.", + ) + for admissible in admissible_sets: + candidate = set(admissible) + if not candidate.issubset(graph_nodes): + _add_error( + errors, + f"{label}: every declared admissible set must use graph nodes.", + ) + if candidate.intersection(forbidden_controls): + _add_error( + errors, + f"{label}: no declared admissible set may contain a descendant, mediator, collider, or instrument.", + ) + if stage.method_label == "BACKDOOR_ADJUSTMENT" and not _backdoor_d_separated( + directed_edges, + treatment, + outcome, + candidate, + ): + _add_error( + errors, + f"{label}: every declared admissible set must block graph-implied backdoor paths.", + ) + if not selected.issubset(graph_nodes): + _add_error(errors, f"{label}: every selected control must be a graph node.") + justifications = _validate_justifications( + stage.control_justifications, label, errors + ) + if set(justifications) != selected: + _add_error(errors, f"{label}: every selected control needs one justification.") + if not _validate_string_tuple( + stage.open_backdoor_paths, + label, + "open backdoor paths", + errors, + allow_empty=True, + ): + pass + elif stage.open_backdoor_paths: + _add_error(errors, f"{label}: no backdoor path may remain open.") + + if ( + stage.method_label != "FRONT_DOOR_ADJUSTMENT" + and stage.frontdoor_criteria_satisfied is not None + ): + _add_error( + errors, + f"{label}: front-door evidence requires the front-door method.", + ) + instrument_premises = ( + stage.instrument_relevance_satisfied, + stage.instrument_exclusion_satisfied, + stage.instrument_exogeneity_satisfied, + ) + if stage.method_label != "INSTRUMENTAL_VARIABLES" and any( + value is not None for value in instrument_premises + ): + _add_error( + errors, + f"{label}: instrumental-variable premises require the IV method.", + ) + + if stage.method_label == "BACKDOOR_ADJUSTMENT": + if not _backdoor_d_separated( + directed_edges, + treatment, + outcome, + selected, + ): + _add_error( + errors, + f"{label}: selected controls must block every graph-implied backdoor path.", + ) + elif stage.method_label == "FRONT_DOOR_ADJUSTMENT": + if not role_names["mediators"] or stage.frontdoor_criteria_satisfied is not True: + _add_error(errors, f"{label}: front-door criteria must be explicitly satisfied.") + elif outcome in _reachable_avoiding( + adjacency, treatment, role_names["mediators"] + ): + _add_error( + errors, + f"{label}: front-door mediators must intercept every directed causal path.", + ) + for mediator in role_names["mediators"]: + if not _backdoor_d_separated( + directed_edges, treatment, mediator, selected + ) or not _backdoor_d_separated( + directed_edges, mediator, outcome, {treatment} | selected + ): + _add_error( + errors, + f"{label}: graph-implied front-door backdoor criteria must hold.", + ) + elif stage.method_label == "INSTRUMENTAL_VARIABLES": + if not role_names["instruments"] or any( + value is not True for value in instrument_premises + ): + _add_error(errors, f"{label}: instrumental-variable premises must all hold.") + + +def _same_horizons( + left: dict[str, tuple[float, float]], right: dict[str, tuple[float, float]] +) -> bool: + return left == right + + +def validate_causal_factor_protocol( + report: CausalFactorProtocolReport, +) -> CausalFactorProtocolReport: + """Validate and return a complete seven-stage protocol evidence record. + + The validation is structural and fail-closed. A successful result means + the record conforms to the published stage contract; it does not certify + that the caller's empirical evidence or causal assumptions are true. + + Raises + ------ + ValueError + If any stage is missing, has the wrong record type, uses a method label + outside the source contract, or fails a cross-stage safety invariant. + """ + + if not isinstance(report, CausalFactorProtocolReport): + raise ValueError("Protocol: report type is invalid.") + + errors: list[str] = [] + if not _nonblank(report.trial_family_id): + _add_error(errors, "Protocol: trial-family identifier must be nonblank.") + trials_valid = _validate_string_tuple( + report.declared_trial_ids, + "Protocol", + "declared trial identifiers", + errors, + ) + + stage_types = ( + ("Stage 1", report.variable_selection, VariableSelectionStage), + ("Stage 2", report.causal_discovery, CausalDiscoveryStage), + ("Stage 3", report.causal_adjustment_set, CausalAdjustmentSetStage), + ( + "Stage 4", + report.causal_explanatory_and_predictive_power, + CausalExplanatoryAndPredictivePowerStage, + ), + ( + "Stage 5", + report.causal_portfolio_construction, + CausalPortfolioConstructionStage, + ), + ("Stage 6", report.backtest, BacktestStage), + ( + "Stage 7", + report.multiple_testing_adjustments, + MultipleTestingAdjustmentsStage, + ), + ) + for label, value, expected in stage_types: + if not isinstance(value, expected): + _add_error(errors, f"{label}: stage record type is invalid.") + if any(not isinstance(value, expected) for _, value, expected in stage_types): + raise ValueError("Causal factor protocol is invalid. " + " ".join(errors)) + + selection = report.variable_selection + label = "Stage 1" + if not _nonblank(selection.purpose) or selection.purpose not in _RESEARCH_PURPOSES: + _add_error(errors, f"{label}: research purpose is not source-listed.") + selected_valid = _validate_string_tuple( + selection.selected_variables, label, "selected variables", errors + ) + selected_variables = ( + set(selection.selected_variables) if selected_valid else set() + ) + selection_methods_valid = _validate_labels( + selection.method_labels, + _VARIABLE_SELECTION_METHODS, + label, + "variable-selection method labels", + errors, + ) + if not isinstance(selection.overlapping_returns, bool): + _add_error(errors, f"{label}: overlapping-returns flag must be boolean.") + if not isinstance(selection.strong_time_dependence, bool): + _add_error(errors, f"{label}: time-dependence flag must be boolean.") + selection_methods = ( + set(selection.method_labels) if selection_methods_valid else set() + ) + if selection.validation is None: + selection_horizons = {} + if ( + selection.overlapping_returns is True + or selection.strong_time_dependence is True + or "PERMUTATION_FEATURE_IMPORTANCE" in selection_methods + ): + _add_error( + errors, + f"{label}: the declared selection design requires temporal validation.", + ) + elif not isinstance(selection.validation, ValidationEvidence): + _add_error(errors, f"{label}: temporal validation evidence has an invalid type.") + selection_horizons = {} + else: + selection_horizons = _validate_fold_evidence( + selection.validation, + label, + errors, + required_refit_components={"VARIABLE_SELECTION"}, + ) + actual_overlap = _has_any_overlap(selection_horizons) + if isinstance(selection.overlapping_returns, bool) and ( + selection.overlapping_returns != actual_overlap + ): + _add_error(errors, f"{label}: overlap declaration must match event horizons.") + validation_methods = _safe_label_set(selection.validation.method_labels) + if actual_overlap and not validation_methods.intersection( + _PURGED_VALIDATION_METHODS + ): + _add_error(errors, f"{label}: overlapping returns require purged validation.") + if selection.strong_time_dependence and ( + not _finite_real(selection.validation.embargo) + or float(selection.validation.embargo) <= 0.0 + ): + _add_error(errors, f"{label}: strong time dependence requires an embargo.") + + adjustment = report.causal_adjustment_set + safe_outcome = adjustment.outcome if _nonblank(adjustment.outcome) else "" + graph_nodes, graph_adjacency, graph_edges = _validate_discovery( + report.causal_discovery, + selected_variables, + safe_outcome, + errors, + ) + _validate_adjustment( + adjustment, + graph_nodes, + graph_adjacency, + graph_edges, + errors, + ) + + stage4 = report.causal_explanatory_and_predictive_power + label = "Stage 4" + task_types_valid = _validate_labels( + stage4.task_types, + set(_METRICS_BY_TASK), + label, + "task types", + errors, + ) + allowed_metrics = set() + if task_types_valid: + for task_type in stage4.task_types: + allowed_metrics.update(_METRICS_BY_TASK[task_type]) + if not _nonblank(stage4.estimator_label): + _add_error(errors, f"{label}: estimator label must be nonblank.") + explanatory_valid = _validate_labels( + stage4.explanatory_metric_labels, + allowed_metrics, + label, + "explanatory metric labels", + errors, + allow_empty=True, + ) + predictive_valid = _validate_labels( + stage4.predictive_metric_labels, + allowed_metrics, + label, + "predictive metric labels", + errors, + allow_empty=True, + ) + has_explanatory = explanatory_valid and bool(stage4.explanatory_metric_labels) + has_predictive = predictive_valid and bool(stage4.predictive_metric_labels) + if not has_explanatory and not has_predictive: + _add_error(errors, f"{label}: at least one performance aspect is required.") + reported_metrics = set() + if explanatory_valid: + reported_metrics.update(stage4.explanatory_metric_labels) + if predictive_valid: + reported_metrics.update(stage4.predictive_metric_labels) + if task_types_valid: + for task_type in stage4.task_types: + if not reported_metrics.intersection(_METRICS_BY_TASK[task_type]): + _add_error( + errors, + f"{label}: every declared task needs a compatible metric.", + ) + if has_explanatory and not _nonblank(stage4.explanatory_evidence_id): + _add_error(errors, f"{label}: explanatory evidence identifier is required.") + if not has_explanatory and stage4.explanatory_evidence_id is not None: + _add_error(errors, f"{label}: explanatory evidence must match reported metrics.") + if has_predictive and not _nonblank(stage4.predictive_evidence_id): + _add_error(errors, f"{label}: predictive evidence identifier is required.") + if not has_predictive and stage4.predictive_evidence_id is not None: + _add_error(errors, f"{label}: predictive evidence must match reported metrics.") + _validate_optional_identifier( + stage4.explanatory_evidence_id, + label, + "explanatory evidence identifier", + errors, + ) + _validate_optional_identifier( + stage4.predictive_evidence_id, + label, + "predictive evidence identifier", + errors, + ) + if ( + has_explanatory + and has_predictive + and stage4.explanatory_evidence_id == stage4.predictive_evidence_id + ): + _add_error(errors, f"{label}: explanatory and predictive evidence must be separate.") + if selection.purpose == "CAUSAL_ATTRIBUTION" and not has_explanatory: + _add_error(errors, f"{label}: causal attribution requires explanatory evidence.") + if selection.purpose == "RISK_PREMIA_HARVESTING" and not has_predictive: + _add_error(errors, f"{label}: risk-premia harvesting requires predictive evidence.") + if not _nonblank(stage4.naive_benchmark_id): + _add_error(errors, f"{label}: a naive benchmark identifier is required.") + multiclass_fields = (stage4.multiclass_encoding, stage4.averaging_method) + if any(value is not None for value in multiclass_fields): + if not isinstance(stage4.multiclass_encoding, str) or ( + stage4.multiclass_encoding != "ONE_VS_REST" + ): + _add_error(errors, f"{label}: multiclass encoding is not source-listed.") + if not isinstance(stage4.averaging_method, str) or ( + stage4.averaging_method + not in {"MICRO", "MACRO", "WEIGHTED", "SAMPLES"} + ): + _add_error(errors, f"{label}: multiclass averaging is not source-listed.") + if not _safe_label_set(stage4.task_types).intersection( + {"PROBABILITY", "RANKING"} + ): + _add_error(errors, f"{label}: multiclass settings require a classification task.") + if not isinstance(stage4.validation, ValidationEvidence): + _add_error(errors, f"{label}: purged validation evidence is required.") + else: + stage4_horizons = _validate_fold_evidence( + stage4.validation, + label, + errors, + required_refit_components={"VARIABLE_SELECTION", "CAUSAL_ESTIMATION"}, + ) + if not _safe_label_set(stage4.validation.method_labels).intersection( + _PURGED_VALIDATION_METHODS + ): + _add_error(errors, f"{label}: purged cross-validation is required.") + if selection_horizons and not _same_horizons( + selection_horizons, stage4_horizons + ): + _add_error(errors, f"{label}: event horizons must match Stage 1.") + if selection.strong_time_dependence and ( + not _finite_real(stage4.validation.embargo) + or float(stage4.validation.embargo) <= 0.0 + ): + _add_error(errors, f"{label}: strong time dependence requires an embargo.") + + portfolio = report.causal_portfolio_construction + label = "Stage 5" + portfolio_labels_valid = _validate_labels( + portfolio.method_labels, + _PORTFOLIO_METHODS, + label, + "portfolio-construction method labels", + errors, + ) + if portfolio_labels_valid and set(portfolio.method_labels) != _PORTFOLIO_METHODS: + _add_error(errors, f"{label}: all source portfolio considerations are required.") + causal_valid = _validate_string_tuple( + portfolio.causal_exposures, label, "causal exposures", errors + ) + neutral_valid = _validate_string_tuple( + portfolio.controlled_unintended_exposures, + label, + "controlled unintended exposures", + errors, + allow_empty=True, + ) + causal_exposures = set(portfolio.causal_exposures) if causal_valid else set() + neutral_exposures = ( + set(portfolio.controlled_unintended_exposures) if neutral_valid else set() + ) + if causal_valid and not causal_exposures.issubset(graph_nodes): + _add_error(errors, f"{label}: causal exposures must be graph variables.") + if causal_valid and any( + safe_outcome not in _reachable(graph_adjacency, exposure) + for exposure in causal_exposures + ): + _add_error( + errors, + f"{label}: every causal exposure must have a directed path to the outcome.", + ) + if neutral_valid and not neutral_exposures.issubset(graph_nodes): + _add_error( + errors, + f"{label}: controlled unintended exposures must be graph variables.", + ) + safe_treatment = adjustment.treatment if _nonblank(adjustment.treatment) else "" + if safe_treatment not in causal_exposures: + _add_error(errors, f"{label}: the target causal factor must drive position sizing.") + if causal_exposures.intersection(neutral_exposures): + _add_error(errors, f"{label}: causal and neutralized exposures must be disjoint.") + declared_colliders = _safe_label_set(adjustment.colliders) + if not declared_colliders.issubset(neutral_exposures): + _add_error(errors, f"{label}: declared colliders must be neutralized.") + for value, subject in ( + (portfolio.cost_model_id, "cost model identifier"), + (portfolio.constraint_set_id, "constraint-set identifier"), + (portfolio.economic_rationale, "economic rationale"), + ): + if not _nonblank(value): + _add_error(errors, f"{label}: {subject} must be nonblank.") + _validate_string_tuple( + portfolio.fragility_scenarios, + label, + "causal-fragility scenarios", + errors, + ) + if not _finite_real(portfolio.transfer_coefficient) or not ( + -1.0 <= float(portfolio.transfer_coefficient) <= 1.0 + ): + _add_error( + errors, + f"{label}: transfer coefficient must be finite and between minus one and one.", + ) + + backtest = report.backtest + label = "Stage 6" + backtest_labels_valid = _validate_labels( + backtest.method_labels, + _BACKTEST_METHODS, + label, + "backtest method labels", + errors, + ) + if backtest.trial_family_id != report.trial_family_id: + _add_error(errors, f"{label}: trial-family identifier must match the report.") + if backtest.declared_trial_ids != report.declared_trial_ids: + _add_error(errors, f"{label}: declared trials must match the report exactly.") + non_monte_carlo = ( + set(backtest.method_labels) - {"MONTE_CARLO"} + if backtest_labels_valid + else set() + ) + if non_monte_carlo: + if not isinstance(backtest.validation, ValidationEvidence): + _add_error(errors, f"{label}: temporal backtests require validation evidence.") + else: + _validate_fold_evidence( + backtest.validation, + label, + errors, + required_refit_components=_REFIT_COMPONENTS, + ) + if selection.strong_time_dependence and ( + not _finite_real(backtest.validation.embargo) + or float(backtest.validation.embargo) <= 0.0 + ): + _add_error( + errors, + f"{label}: strong time dependence requires a backtest embargo.", + ) + if not non_monte_carlo.issubset( + _safe_label_set(backtest.validation.method_labels) + ): + _add_error( + errors, + f"{label}: backtest and validation method labels must agree.", + ) + elif backtest.validation is not None: + _add_error( + errors, + f"{label}: Monte Carlo-only evidence cannot carry temporal folds.", + ) + if backtest_labels_valid and "MONTE_CARLO" in backtest.method_labels: + if not _nonblank(backtest.monte_carlo_dgp): + _add_error(errors, f"{label}: Monte Carlo requires an explicit DGP.") + elif backtest.monte_carlo_dgp is not None: + _add_error(errors, f"{label}: a Monte Carlo DGP requires the Monte Carlo method.") + + multiple_testing = report.multiple_testing_adjustments + label = "Stage 7" + methods_valid = _validate_labels( + multiple_testing.method_labels, + _MULTIPLE_TESTING_METHODS, + label, + "multiple-testing method labels", + errors, + ) + if multiple_testing.trial_family_id != report.trial_family_id: + _add_error(errors, f"{label}: trial-family identifier must match the report.") + if multiple_testing.declared_trial_ids != report.declared_trial_ids: + _add_error(errors, f"{label}: declared trials must match the report exactly.") + + partitions = multiple_testing.family_partitions + covered_trials: set[str] = set() + partition_ids: set[str] = set() + if not isinstance(partitions, tuple) or not partitions: + _add_error(errors, f"{label}: family partitions must be a nonempty tuple.") + else: + for partition in partitions: + if ( + not isinstance(partition, tuple) + or len(partition) != 2 + or not _nonblank(partition[0]) + ): + _add_error(errors, f"{label}: a family partition is invalid.") + continue + if partition[0] in partition_ids: + _add_error(errors, f"{label}: family partition identifiers must be unique.") + partition_ids.add(partition[0]) + if not _validate_string_tuple( + partition[1], label, "partition trial identifiers", errors + ): + continue + for trial_id in partition[1]: + if trial_id in covered_trials: + _add_error(errors, f"{label}: family partitions must be disjoint.") + covered_trials.add(trial_id) + if trials_valid and covered_trials != set(report.declared_trial_ids): + _add_error(errors, f"{label}: family partitions must cover declared trials exactly.") + + if not _finite_real(multiple_testing.alpha) or not ( + 0.0 < float(multiple_testing.alpha) < 1.0 + ): + _add_error(errors, f"{label}: alpha must be a finite number between zero and one.") + if not isinstance(multiple_testing.backtests_independent, bool): + _add_error(errors, f"{label}: backtest-dependence flag must be boolean.") + elif multiple_testing.backtests_independent: + _add_error(errors, f"{label}: backtest dependence must be represented.") + + methods = set(multiple_testing.method_labels) if methods_valid else set() + if methods.intersection(_P_VALUE_METHODS): + if not _nonblank(multiple_testing.p_value_estimator): + _add_error(errors, f"{label}: p-value estimator must be declared.") + if not _nonblank(multiple_testing.time_dependence_model): + _add_error(errors, f"{label}: time dependence treatment must be declared.") + if multiple_testing.p_value_inputs_assume_independence is not False: + _add_error( + errors, + f"{label}: p-value inputs must explicitly reject independence assumptions.", + ) + elif any( + value is not None + for value in ( + multiple_testing.p_value_estimator, + multiple_testing.time_dependence_model, + multiple_testing.p_value_inputs_assume_independence, + ) + ): + _add_error( + errors, + f"{label}: p-value inputs require a p-value correction.", + ) + if "DEFLATED_SHARPE_RATIO" in methods: + trial_count = ( + len(report.declared_trial_ids) + if isinstance(report.declared_trial_ids, tuple) + else 0 + ) + if ( + not _finite_real(multiple_testing.sharpe_variance) + or float(multiple_testing.sharpe_variance) <= 0.0 + ): + _add_error(errors, f"{label}: positive Sharpe-ratio variance is required.") + if not _finite_real(multiple_testing.effective_trials) or not ( + 1.0 + <= float(multiple_testing.effective_trials) + < trial_count + ): + _add_error(errors, f"{label}: effective trials must be below total trials.") + if ( + isinstance(multiple_testing.sample_length, bool) + or not isinstance(multiple_testing.sample_length, Integral) + or int(multiple_testing.sample_length) <= 1 + ): + _add_error(errors, f"{label}: sample length must be an integer above one.") + if not _finite_real(multiple_testing.skewness): + _add_error(errors, f"{label}: skewness must be finite.") + if not _finite_real(multiple_testing.kurtosis) or float( + multiple_testing.kurtosis + ) < 1.0: + _add_error(errors, f"{label}: Pearson kurtosis must be finite and at least one.") + elif _finite_real(multiple_testing.skewness) and abs( + float(multiple_testing.skewness) + ) > math.sqrt(float(multiple_testing.kurtosis) - 1.0): + _add_error( + errors, + f"{label}: skewness and Pearson kurtosis must satisfy the moment inequality.", + ) + if not _nonblank(multiple_testing.selection_bias_evidence_id): + _add_error(errors, f"{label}: selection-bias evidence is required.") + elif any( + value is not None + for value in ( + multiple_testing.sharpe_variance, + multiple_testing.effective_trials, + multiple_testing.sample_length, + multiple_testing.skewness, + multiple_testing.kurtosis, + multiple_testing.selection_bias_evidence_id, + ) + ): + _add_error( + errors, + f"{label}: Sharpe-correction inputs require the deflated Sharpe ratio.", + ) + + if errors: + raise ValueError("Causal factor protocol is invalid. " + " ".join(errors)) + return report diff --git a/src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py b/src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py new file mode 100644 index 0000000..3ad6407 --- /dev/null +++ b/src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py @@ -0,0 +1,267 @@ +"""Specification diagnostics from the 2023 causal-factor experiments. + +This module independently implements the three linear-Gaussian structural +experiments in Sections 7.1-7.3 and their Appendix procedures. Population +functions expose exact analytical regression targets; experiment functions +generate seeded samples and fit both specifications with NumPy only. +""" + +from __future__ import annotations + +import math +from numbers import Integral +from typing import NamedTuple, Tuple + +import numpy as np + +__all__ = [ + "PopulationRegressionDiagnostics", + "RegressionDiagnostics", + "SpecificationExperimentResult", + "SpecificationPopulationResult", + "collider_population_diagnostics", + "collider_specification_experiment", + "confounded_mediator_population_diagnostics", + "confounded_mediator_specification_experiment", + "fork_population_diagnostics", + "fork_specification_experiment", +] + + +class PopulationRegressionDiagnostics(NamedTuple): + """Exact population coefficients and explained-variance fraction.""" + + coefficient_names: Tuple[str, ...] + coefficients: Tuple[float, ...] + r_squared: float + + +class RegressionDiagnostics(NamedTuple): + """Classical least-squares diagnostics for one generated sample.""" + + coefficient_names: Tuple[str, ...] + coefficients: Tuple[float, ...] + standard_errors: Tuple[float, ...] + t_statistics: Tuple[float, ...] + r_squared: float + adjusted_r_squared: float + residual_variance: float + n_observations: int + + +class SpecificationPopulationResult(NamedTuple): + """Exact population targets for unconditioned and conditioned models.""" + + structure: str + conditioning_variable_role: str + unconditioned: PopulationRegressionDiagnostics + conditioned: PopulationRegressionDiagnostics + + +class SpecificationExperimentResult(NamedTuple): + """Sample diagnostics for unconditioned and conditioned regressions.""" + + structure: str + conditioning_variable_role: str + unconditioned: RegressionDiagnostics + conditioned: RegressionDiagnostics + + +_MAX_SAMPLE_SIZE = 1_000_000 + + +def _as_sample_size(value: Integral) -> int: + if isinstance(value, (bool, np.bool_)) or not isinstance(value, Integral): + raise ValueError("sample_size must be an integer") + result = int(value) + if result <= 3: + raise ValueError("sample_size must be greater than 3") + if result > _MAX_SAMPLE_SIZE: + raise ValueError(f"sample_size must not exceed {_MAX_SAMPLE_SIZE}") + return result + + +def _as_seed(value: Integral) -> int: + if isinstance(value, (bool, np.bool_)) or not isinstance(value, Integral): + raise ValueError("seed must be an integer") + result = int(value) + if result < 0 or result > 2**32 - 1: + raise ValueError("seed must be between 0 and 2**32 - 1") + return result + + +def _population_regression( + names: Tuple[str, ...], coefficients: Tuple[float, ...], r_squared: float +) -> PopulationRegressionDiagnostics: + return PopulationRegressionDiagnostics( + coefficient_names=names, + coefficients=coefficients, + r_squared=r_squared, + ) + + +def _ols_diagnostics( + outcome: np.ndarray, + regressors: Tuple[Tuple[str, np.ndarray], ...], +) -> RegressionDiagnostics: + n_observations = int(outcome.size) + design = np.column_stack( + (np.ones(n_observations, dtype=float), *(values for _, values in regressors)) + ) + n_parameters = int(design.shape[1]) + if n_observations <= n_parameters: + raise ValueError("the regression requires positive residual degrees of freedom") + if np.linalg.matrix_rank(design) != n_parameters: + raise ValueError("the generated regression design is rank deficient") + + coefficients, _, _, _ = np.linalg.lstsq(design, outcome, rcond=None) + fitted = design @ coefficients + residuals = outcome - fitted + residual_sum_squares = float(residuals @ residuals) + centered = outcome - float(np.mean(outcome)) + total_sum_squares = float(centered @ centered) + if not math.isfinite(total_sum_squares) or total_sum_squares <= 0.0: + raise ValueError("the generated outcome must have positive finite variance") + + residual_degrees_of_freedom = n_observations - n_parameters + residual_variance = residual_sum_squares / residual_degrees_of_freedom + gram_inverse = np.linalg.inv(design.T @ design) + coefficient_variances = np.diag(gram_inverse) * residual_variance + if np.any(coefficient_variances < -1.0e-12): + raise ValueError("the coefficient variance calculation is invalid") + standard_errors = np.sqrt(np.maximum(coefficient_variances, 0.0)) + t_statistics = np.divide( + coefficients, + standard_errors, + out=np.full(coefficients.shape, np.nan, dtype=float), + where=standard_errors > 0.0, + ) + if not np.all(np.isfinite(coefficients)) or not np.all( + np.isfinite(standard_errors) + ): + raise ValueError("the regression diagnostics must be finite") + + r_squared = 1.0 - residual_sum_squares / total_sum_squares + if r_squared < 0.0 and r_squared >= -1.0e-12: + r_squared = 0.0 + if r_squared > 1.0 and r_squared <= 1.0 + 1.0e-12: + r_squared = 1.0 + adjusted_r_squared = 1.0 - (1.0 - r_squared) * ( + (n_observations - 1) / residual_degrees_of_freedom + ) + names = ("intercept", *(name for name, _ in regressors)) + return RegressionDiagnostics( + coefficient_names=names, + coefficients=tuple(float(value) for value in coefficients), + standard_errors=tuple(float(value) for value in standard_errors), + t_statistics=tuple(float(value) for value in t_statistics), + r_squared=float(r_squared), + adjusted_r_squared=float(adjusted_r_squared), + residual_variance=float(residual_variance), + n_observations=n_observations, + ) + + +def fork_population_diagnostics() -> SpecificationPopulationResult: + """Return exact targets for Equations 20-24 (the confounder fork).""" + + return SpecificationPopulationResult( + structure="fork", + conditioning_variable_role="confounder", + unconditioned=_population_regression( + ("intercept", "X"), (0.0, 0.5), 0.25 + ), + conditioned=_population_regression( + ("intercept", "X", "Z"), (0.0, 0.0, 1.0), 0.5 + ), + ) + + +def collider_population_diagnostics() -> SpecificationPopulationResult: + """Return exact targets for Equations 27-33 (the collider).""" + + return SpecificationPopulationResult( + structure="collider", + conditioning_variable_role="collider", + unconditioned=_population_regression( + ("intercept", "X"), (0.0, 0.0), 0.0 + ), + conditioned=_population_regression( + ("intercept", "X", "Z"), (0.0, -0.5, 0.5), 0.5 + ), + ) + + +def confounded_mediator_population_diagnostics() -> SpecificationPopulationResult: + """Return exact targets for Equations 37-42 (the confounded mediator).""" + + return SpecificationPopulationResult( + structure="confounded_mediator", + conditioning_variable_role="confounded_mediator", + unconditioned=_population_regression( + ("intercept", "X"), (0.0, 1.0), 1.0 / 7.0 + ), + conditioned=_population_regression( + ("intercept", "X", "Z"), + (0.0, -0.5, 1.5), + 11.0 / 14.0, + ), + ) + + +def fork_specification_experiment( + sample_size: Integral = 5_000, + seed: Integral = 0, +) -> SpecificationExperimentResult: + """Run the seeded fork experiment from Section 7.1 and the Appendix.""" + + n_observations = _as_sample_size(sample_size) + random_state = np.random.RandomState(_as_seed(seed)) + z = random_state.standard_normal(n_observations) + x = z + random_state.standard_normal(n_observations) + y = z + random_state.standard_normal(n_observations) + return SpecificationExperimentResult( + structure="fork", + conditioning_variable_role="confounder", + unconditioned=_ols_diagnostics(y, (("X", x),)), + conditioned=_ols_diagnostics(y, (("X", x), ("Z", z))), + ) + + +def collider_specification_experiment( + sample_size: Integral = 5_000, + seed: Integral = 0, +) -> SpecificationExperimentResult: + """Run the seeded collider experiment from Section 7.2 and the Appendix.""" + + n_observations = _as_sample_size(sample_size) + random_state = np.random.RandomState(_as_seed(seed)) + x = random_state.standard_normal(n_observations) + y = random_state.standard_normal(n_observations) + z = x + y + random_state.standard_normal(n_observations) + return SpecificationExperimentResult( + structure="collider", + conditioning_variable_role="collider", + unconditioned=_ols_diagnostics(y, (("X", x),)), + conditioned=_ols_diagnostics(y, (("X", x), ("Z", z))), + ) + + +def confounded_mediator_specification_experiment( + sample_size: Integral = 5_000, + seed: Integral = 0, +) -> SpecificationExperimentResult: + """Run the seeded confounded-mediator experiment from Section 7.3.""" + + n_observations = _as_sample_size(sample_size) + random_state = np.random.RandomState(_as_seed(seed)) + x = random_state.standard_normal(n_observations) + w = random_state.standard_normal(n_observations) + z = x + w + random_state.standard_normal(n_observations) + y = z + w + random_state.standard_normal(n_observations) + return SpecificationExperimentResult( + structure="confounded_mediator", + conditioning_variable_role="confounded_mediator", + unconditioned=_ols_diagnostics(y, (("X", x),)), + conditioned=_ols_diagnostics(y, (("X", x), ("Z", z))), + ) diff --git a/src/RiskLabAI/causal_factor_analysis/treatment_effects.py b/src/RiskLabAI/causal_factor_analysis/treatment_effects.py new file mode 100644 index 0000000..8d5bab9 --- /dev/null +++ b/src/RiskLabAI/causal_factor_analysis/treatment_effects.py @@ -0,0 +1,328 @@ +"""Treatment-effect estimands from *Causal Factor Investing* (2023). + +The functions are independent implementations of Equations 2-9 and the +linear instrumental-variable ratio in Section 4.3.2.4. They operate on +already identified interventional means, conditional means, probabilities, +or covariances. They do not infer a causal graph from data and do not claim +that the supplied adjustment variables or instrument are valid. +""" + +from __future__ import annotations + +import math +from numbers import Real +from typing import NamedTuple, Sequence + +import numpy as np + +__all__ = [ + "DifferenceInDifferencesEstimate", + "TreatmentEffectDecomposition", + "average_treatment_effect", + "backdoor_adjusted_average_treatment_effect", + "backdoor_adjusted_expectation", + "difference_in_differences", + "frontdoor_adjusted_probability", + "linear_instrumental_variable_effect", + "randomized_mean_difference", + "treatment_effect_decomposition", +] + + +class TreatmentEffectDecomposition(NamedTuple): + """Observed mean difference decomposed into ATT and selection bias.""" + + observed_difference: float + average_treatment_effect_on_treated: float + sample_selection_bias: float + + +class DifferenceInDifferencesEstimate(NamedTuple): + """Two-group, two-period difference-in-differences decomposition.""" + + treated_change: float + control_change: float + estimate: float + + +_PROBABILITY_TOLERANCE = 1.0e-12 + + +def _as_finite_real(name: str, value: Real) -> float: + if isinstance(value, (bool, np.bool_)) or not isinstance(value, Real): + raise ValueError(f"{name} must be a real scalar") + try: + result = float(value) + except (TypeError, ValueError, OverflowError) as error: + raise ValueError(f"{name} must be a representable real scalar") from error + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def _as_finite_vector(name: str, values: Sequence[Real]) -> np.ndarray: + try: + raw = np.asarray(values) + except (TypeError, ValueError) as error: + raise ValueError( + f"{name} must be a one-dimensional numeric sequence" + ) from error + if raw.ndim != 1 or raw.size == 0 or raw.dtype.kind not in "iuf": + raise ValueError(f"{name} must be a nonempty one-dimensional numeric sequence") + try: + result = np.array(raw, dtype=float, copy=True) + except (TypeError, ValueError, OverflowError) as error: + raise ValueError(f"{name} must contain representable real values") from error + if not np.all(np.isfinite(result)): + raise ValueError(f"{name} must contain only finite values") + return result + + +def _as_probability_vector(name: str, values: Sequence[Real]) -> np.ndarray: + result = _as_finite_vector(name, values) + if np.any(result < 0.0) or np.any(result > 1.0): + raise ValueError(f"{name} must contain probabilities in [0, 1]") + if not math.isclose( + float(np.sum(result)), + 1.0, + rel_tol=_PROBABILITY_TOLERANCE, + abs_tol=_PROBABILITY_TOLERANCE, + ): + raise ValueError(f"{name} must sum to one") + return result + + +def average_treatment_effect( + treated_interventional_mean: Real, + control_interventional_mean: Real, +) -> float: + """Return ``E[Y|do(X=x1)] - E[Y|do(X=x0)]`` (Equation 2).""" + + treated = _as_finite_real( + "treated_interventional_mean", treated_interventional_mean + ) + control = _as_finite_real( + "control_interventional_mean", control_interventional_mean + ) + return _as_finite_real("average_treatment_effect", treated - control) + + +def treatment_effect_decomposition( + observed_treated_mean: Real, + observed_control_mean: Real, + counterfactual_control_mean_for_treated: Real, +) -> TreatmentEffectDecomposition: + """Decompose the observed difference into ATT and selection bias. + + This is Equation 3 under consistency: the treated group's observed mean + is its treated potential-outcome mean. + """ + + treated = _as_finite_real("observed_treated_mean", observed_treated_mean) + control = _as_finite_real("observed_control_mean", observed_control_mean) + counterfactual = _as_finite_real( + "counterfactual_control_mean_for_treated", + counterfactual_control_mean_for_treated, + ) + observed_difference = _as_finite_real( + "observed_difference", treated - control + ) + effect_on_treated = _as_finite_real( + "average_treatment_effect_on_treated", treated - counterfactual + ) + selection_bias = _as_finite_real( + "sample_selection_bias", counterfactual - control + ) + return TreatmentEffectDecomposition( + observed_difference=observed_difference, + average_treatment_effect_on_treated=effect_on_treated, + sample_selection_bias=selection_bias, + ) + + +def randomized_mean_difference( + treated_outcomes: Sequence[Real], + control_outcomes: Sequence[Real], +) -> float: + """Return the randomized-study difference in sample means. + + Under the random-assignment identities in Equations 4-6, this is the + finite-sample estimator of the average treatment effect. + """ + + treated = _as_finite_vector("treated_outcomes", treated_outcomes) + control = _as_finite_vector("control_outcomes", control_outcomes) + return _as_finite_real( + "randomized_mean_difference", + float(np.mean(treated) - np.mean(control)), + ) + + +def difference_in_differences( + treated_before: Real, + treated_after: Real, + control_before: Real, + control_after: Real, +) -> DifferenceInDifferencesEstimate: + """Return the two-group, two-period difference-in-differences estimate.""" + + treated_change = _as_finite_real( + "treated_change", + _as_finite_real("treated_after", treated_after) + - _as_finite_real("treated_before", treated_before), + ) + control_change = _as_finite_real( + "control_change", + _as_finite_real("control_after", control_after) + - _as_finite_real("control_before", control_before), + ) + estimate = _as_finite_real( + "difference_in_differences", treated_change - control_change + ) + return DifferenceInDifferencesEstimate( + treated_change=treated_change, + control_change=control_change, + estimate=estimate, + ) + + +def backdoor_adjusted_expectation( + conditional_outcome_means: Sequence[Real], + adjustment_probabilities: Sequence[Real], +) -> float: + """Standardize conditional outcome means over an adjustment distribution. + + For an indicator outcome, this is the discrete probability adjustment in + Equation 8. For a general outcome, it is the expectation form used by + Equation 7. + """ + + means = _as_finite_vector("conditional_outcome_means", conditional_outcome_means) + probabilities = _as_probability_vector( + "adjustment_probabilities", adjustment_probabilities + ) + if means.shape != probabilities.shape: + raise ValueError( + "conditional_outcome_means and adjustment_probabilities must have " + "the same length" + ) + return _as_finite_real( + "backdoor_adjusted_expectation", float(np.dot(means, probabilities)) + ) + + +def backdoor_adjusted_average_treatment_effect( + treated_conditional_means: Sequence[Real], + control_conditional_means: Sequence[Real], + adjustment_probabilities: Sequence[Real], +) -> float: + """Return the standardized average treatment effect in Equation 7.""" + + treated = _as_finite_vector( + "treated_conditional_means", treated_conditional_means + ) + control = _as_finite_vector( + "control_conditional_means", control_conditional_means + ) + probabilities = _as_probability_vector( + "adjustment_probabilities", adjustment_probabilities + ) + if treated.shape != control.shape or treated.shape != probabilities.shape: + raise ValueError( + "treated_conditional_means, control_conditional_means, and " + "adjustment_probabilities must have the same length" + ) + with np.errstate(over="ignore", invalid="ignore"): + contrasts = treated - control + result = float(np.dot(contrasts, probabilities)) + return _as_finite_real("backdoor_adjusted_average_treatment_effect", result) + + +def frontdoor_adjusted_probability( + mediator_probabilities_given_treatment: Sequence[Real], + outcome_probabilities_given_mediator_and_treatment: Sequence[Sequence[Real]], + treatment_probabilities: Sequence[Real], +) -> float: + """Return the discrete front-door adjustment in Equation 9. + + Rows of ``outcome_probabilities_given_mediator_and_treatment`` correspond + to mediator states; columns correspond to treatment states. Each entry is + the conditional probability of one fixed outcome event. + """ + + mediator_probabilities = _as_probability_vector( + "mediator_probabilities_given_treatment", + mediator_probabilities_given_treatment, + ) + treatment_weights = _as_probability_vector( + "treatment_probabilities", treatment_probabilities + ) + try: + raw = np.asarray(outcome_probabilities_given_mediator_and_treatment) + except (TypeError, ValueError) as error: + raise ValueError( + "outcome_probabilities_given_mediator_and_treatment must be a " + "two-dimensional numeric array" + ) from error + if raw.ndim != 2 or raw.dtype.kind not in "iuf": + raise ValueError( + "outcome_probabilities_given_mediator_and_treatment must be a " + "two-dimensional numeric array" + ) + try: + conditional = np.array(raw, dtype=float, copy=True) + except (TypeError, ValueError, OverflowError) as error: + raise ValueError( + "outcome_probabilities_given_mediator_and_treatment must contain " + "representable real values" + ) from error + if not np.all(np.isfinite(conditional)): + raise ValueError( + "outcome_probabilities_given_mediator_and_treatment must contain " + "only finite values" + ) + if np.any(conditional < 0.0) or np.any(conditional > 1.0): + raise ValueError( + "outcome_probabilities_given_mediator_and_treatment must contain " + "probabilities in [0, 1]" + ) + expected_shape = (mediator_probabilities.size, treatment_weights.size) + if conditional.shape != expected_shape: + raise ValueError( + "outcome_probabilities_given_mediator_and_treatment must have shape " + "(number of mediator states, number of treatment states)" + ) + probability = float( + np.dot(mediator_probabilities, np.dot(conditional, treatment_weights)) + ) + probability = _as_finite_real("frontdoor_adjusted_probability", probability) + if probability < 0.0 and probability >= -_PROBABILITY_TOLERANCE: + return 0.0 + if probability > 1.0 and probability <= 1.0 + _PROBABILITY_TOLERANCE: + return 1.0 + return probability + + +def linear_instrumental_variable_effect( + outcome_instrument_covariance: Real, + treatment_instrument_covariance: Real, +) -> float: + """Return the linear IV covariance ratio from Section 4.3.2.4. + + The caller must establish the graphical and substantive IV premises. A + zero treatment-instrument covariance violates the ratio's relevance + requirement and is rejected. + """ + + numerator = _as_finite_real( + "outcome_instrument_covariance", outcome_instrument_covariance + ) + denominator = _as_finite_real( + "treatment_instrument_covariance", treatment_instrument_covariance + ) + if denominator == 0.0: + raise ValueError("treatment_instrument_covariance must be nonzero") + result = numerator / denominator + if not math.isfinite(result): + raise ValueError("the instrumental-variable effect must be finite") + return result diff --git a/src/RiskLabAI/cluster/__init__.py b/src/RiskLabAI/cluster/__init__.py new file mode 100644 index 0000000..e6bf92e --- /dev/null +++ b/src/RiskLabAI/cluster/__init__.py @@ -0,0 +1,27 @@ +""" +RiskLabAI Clustering Module + +This module provides functions for financial portfolio clustering, +including implementations of the Optimized Nested Clustering (ONC) +algorithm described by de Prado. +""" + +from .clustering import ( + cluster_k_means_base, + cluster_k_means_top, + covariance_to_correlation, + make_new_outputs, + random_block_correlation, + random_block_covariance, + random_covariance_sub, +) + +__all__ = [ + "covariance_to_correlation", + "cluster_k_means_base", + "make_new_outputs", + "cluster_k_means_top", + "random_covariance_sub", + "random_block_covariance", + "random_block_correlation", +] diff --git a/src/RiskLabAI/cluster/clustering.py b/src/RiskLabAI/cluster/clustering.py new file mode 100644 index 0000000..6fac684 --- /dev/null +++ b/src/RiskLabAI/cluster/clustering.py @@ -0,0 +1,462 @@ +""" +Implements clustering algorithms, including the Optimized Nested Clustering (ONC) +method from Marcos Lopez de Prado. + +Reference: + De Prado, M. (2020) Advances in financial machine learning. John Wiley & Sons. +""" + +from typing import Optional + +import numpy as np +import pandas as pd +from scipy.linalg import block_diag +from sklearn.cluster import KMeans +from sklearn.metrics import silhouette_samples +from sklearn.utils import check_random_state + + +def covariance_to_correlation(covariance: np.ndarray) -> np.ndarray: + r""" + Derive the correlation matrix from a covariance matrix. + + .. math:: + \text{corr}_{ij} = \frac{\text{cov}_{ij}} + {\sqrt{\text{cov}_{ii} \text{cov}_{jj}}} + + Reference: + Snippet 2.3, Page 27. + + Parameters + ---------- + covariance : np.ndarray + A square covariance matrix. + + Returns + ------- + np.ndarray + The corresponding correlation matrix. + """ + # Single source of truth: RiskLabAI.data.denoise.cov_to_corr implements the + # same conversion (Snippet 2.3) with added zero-std and diagonal safeguards. + # Output is identical to floating-point precision for valid covariance + # matrices. Imported locally to avoid any import cycle at module load. + from RiskLabAI.data.denoise.denoising import cov_to_corr + + return cov_to_corr(covariance) + + +def cluster_k_means_base( + correlation: pd.DataFrame, + max_clusters: int = 10, + iterations: int = 10, + random_state: Optional[int] = None, +) -> tuple[pd.DataFrame, dict[int, list[str]], pd.Series]: + """ + Perform the base K-Means clustering step. + + This function iterates through different numbers of clusters (from 2 to + `max_clusters`) and multiple initializations (`iterations`), selecting the + clustering result that yields the highest average silhouette score + t-statistic (mean / std). + + Reference: + Snippet 4.1, Page 56. + + Parameters + ---------- + correlation : pd.DataFrame + The correlation matrix to be clustered. + max_clusters : int, default=10 + The maximum number of clusters to try. + iterations : int, default=10 + Number of K-Means initializations for each cluster count. + random_state : int, optional + Random state for K-Means reproducibility. + + Returns + ------- + Tuple[pd.DataFrame, Dict[int, List[str]], pd.Series] + - correlation_sorted: The correlation matrix, reordered by cluster. + - clusters: A dictionary mapping cluster ID to a list of item names. + - silhouette_scores: A Series of silhouette scores for each item. + """ + # Calculate distance matrix + distance = ((1 - correlation.fillna(0)) / 2.0) ** 0.5 + + best_kmeans = None + best_silhouette_scores = None + best_score = -np.inf + + rng = check_random_state(random_state) + + for _ in range(iterations): + for n_clusters in range(2, max_clusters + 1): + # Use a different random_state for each K-Means fit + iter_seed = rng.randint(0, np.iinfo(np.int32).max) + + kmeans = KMeans( + n_clusters=n_clusters, + n_init=1, # We handle iterations externally + random_state=iter_seed, + ) + kmeans.fit(distance) + + silhouette_scores_ = silhouette_samples(distance, kmeans.labels_) + + # Use silhouette score t-statistic (mean/std) as the quality metric + stat_mean = silhouette_scores_.mean() + stat_std = silhouette_scores_.std() + + if stat_std == 0: + # Avoid division by zero if all silhouette scores are identical + score = np.sign(stat_mean) * np.inf + else: + score = stat_mean / stat_std + + if score > best_score: + best_score = score + best_kmeans = kmeans + best_silhouette_scores = silhouette_scores_ + + if best_kmeans is None: + raise ValueError("Clustering failed to find a valid solution.") + + # Sort items by cluster label + index_sorted = np.argsort(best_kmeans.labels_) + correlation_sorted = correlation.iloc[index_sorted, index_sorted] + + # Create cluster dictionary + clusters = { + i: correlation.columns[np.where(best_kmeans.labels_ == i)[0]].tolist() + for i in np.unique(best_kmeans.labels_) + } + + silhouette_series = pd.Series(best_silhouette_scores, index=distance.index) + + return correlation_sorted, clusters, silhouette_series + + +def make_new_outputs( + correlation: pd.DataFrame, + clusters_1: dict[int, list[str]], + clusters_2: dict[int, list[str]], +) -> tuple[pd.DataFrame, dict[int, list[str]], pd.Series]: + """ + Merge two disjoint sets of clusters and re-calculate metrics. + + Reference: + Snippet 4.2, Page 58. + + Parameters + ---------- + correlation : pd.DataFrame + The *original* correlation matrix. + clusters_1 : Dict[int, List[str]] + The first set of clusters (e.g., the "good" clusters). + clusters_2 : Dict[int, List[str]] + The second set of clusters (e.g., the re-clustered "bad" ones). + + Returns + ------- + Tuple[pd.DataFrame, Dict[int, List[str]], pd.Series] + - correlation_new: The correlation matrix, reordered by new clusters. + - clusters_new: The merged dictionary of all clusters. + - silhouette_new: New silhouette scores based on the merged clustering. + """ + clusters_new = {} + for i in clusters_1: + clusters_new[len(clusters_new)] = list(clusters_1[i]) + for i in clusters_2: + clusters_new[len(clusters_new)] = list(clusters_2[i]) + + # Get the new full, sorted index + index_new = [ + item for cluster_items in clusters_new.values() for item in cluster_items + ] + correlation_new = correlation.loc[index_new, index_new] + + # Calculate new silhouette scores based on the *original* distance + distance = ((1 - correlation.fillna(0)) / 2.0) ** 0.5 + labels_kmeans = np.zeros(len(distance.columns)) + + # Create the label array for silhouette_samples + for i, items in clusters_new.items(): + item_indices = [distance.index.get_loc(k) for k in items] + labels_kmeans[item_indices] = i + + silhouette_new = pd.Series( + silhouette_samples(distance, labels_kmeans), index=distance.index + ) + return correlation_new, clusters_new, silhouette_new + + +def cluster_k_means_top( + correlation: pd.DataFrame, + max_clusters: Optional[int] = None, + iterations: int = 10, + random_state: Optional[int] = None, +) -> tuple[pd.DataFrame, dict[int, list[str]], pd.Series]: + """ + Perform Optimized Nested Clustering (ONC). + + This method recursively re-clusters unstable clusters (those with a + silhouette t-statistic below the average). + + Reference: + Snippet 4.2, Page 58. + + Parameters + ---------- + correlation : pd.DataFrame + The correlation matrix to be clustered. + max_clusters : int, optional + Maximum number of clusters. Defaults to `N-1`. + iterations : int, default=10 + Number of K-Means initializations. + random_state : int, optional + Random state for reproducibility. + + Returns + ------- + Tuple[pd.DataFrame, Dict[int, List[str]], pd.Series] + - correlation_sorted: The final sorted correlation matrix. + - clusters: The final dictionary of clusters. + - silhouette_scores: The final silhouette scores for each item. + """ + n_cols = correlation.shape[1] + if max_clusters is None: + max_clusters = n_cols - 1 + + max_clusters = min(max_clusters, n_cols - 1) + if max_clusters < 2: + return ( + correlation, + {0: correlation.columns.tolist()}, + pd.Series(dtype="float64"), + ) + + # 1. Run base clustering + corr_sorted, clusters, silhouette = cluster_k_means_base( + correlation, + max_clusters=max_clusters, + iterations=iterations, + random_state=random_state, + ) + + # 2. Calculate t-statistic for each cluster + cluster_t_stats = { + i: silhouette[clusters[i]].mean() / silhouette[clusters[i]].std() + for i in clusters + if silhouette[clusters[i]].std() > 0 + } + + if not cluster_t_stats: + return corr_sorted, clusters, silhouette # No valid clusters found + + t_stat_mean = np.mean(list(cluster_t_stats.values())) + + # 3. Identify clusters to re-cluster + redo_clusters = [i for i, t_stat in cluster_t_stats.items() if t_stat < t_stat_mean] + + if len(redo_clusters) <= 1: + # Base case: All clusters are stable, or only one is unstable + return corr_sorted, clusters, silhouette + else: + # 4. Recurse on unstable clusters + keys_redo = [item for i in redo_clusters for item in clusters[i]] + corr_temp = correlation.loc[keys_redo, keys_redo] + + # Keep track of mean t-stat for comparison + t_stat_mean_redo = np.mean([cluster_t_stats[i] for i in redo_clusters]) + + # Calculate remaining clusters for recursive call + n_clusters_good = len(clusters) - len(redo_clusters) + remained_n_clusters = max_clusters - n_clusters_good + + # Recursive call + corr_sorted_2, clusters_2, silh_2 = cluster_k_means_top( + corr_temp, + max_clusters=min(remained_n_clusters, corr_temp.shape[1] - 1), + iterations=iterations, + random_state=random_state, + ) + + # 5. Merge results + clusters_1 = {i: clusters[i] for i in clusters if i not in redo_clusters} + corr_new, clusters_new, silh_new = make_new_outputs( + correlation, clusters_1, clusters_2 + ) + + # 6. Decide whether to keep the re-clustered result + new_t_stats = [ + silh_new[clusters_new[i]].mean() / silh_new[clusters_new[i]].std() + for i in clusters_new + if silh_new[clusters_new[i]].std() > 0 + ] + + if not new_t_stats: + return corr_sorted, clusters, silhouette # Re-clustering failed + + new_t_stat_mean = np.mean(new_t_stats) + + if new_t_stat_mean <= t_stat_mean_redo: + # Re-clustering did not improve, return original + return corr_sorted, clusters, silhouette + else: + # Re-clustering improved, return new results + return corr_new, clusters_new, silh_new + + +def random_covariance_sub( + n_observations: int, + n_columns: int, + sigma: float, + random_state: Optional[int] = None, +) -> np.ndarray: + """ + Generate a random covariance matrix for a single block. + + Reference: + Snippet 4.3, Page 61. + + Parameters + ---------- + n_observations : int + Number of observations to simulate. + n_columns : int + Number of columns (assets) in this block. + sigma : float + Sigma for the idiosyncratic noise. + random_state : int, optional + Random state for reproducibility. + + Returns + ------- + np.ndarray + The (n_columns x n_columns) covariance matrix. + """ + rng = check_random_state(random_state) + + if n_columns == 1: + return np.ones((1, 1)) + + # Common factor + data = rng.normal(size=(n_observations, 1)) + data = np.repeat(data, n_columns, axis=1) + + # Idiosyncratic noise + data += rng.normal(scale=sigma, size=data.shape) + + covariance = np.cov(data, rowvar=False) + return covariance + + +def random_block_covariance( + n_columns: int, + n_blocks: int, + block_size_min: int = 1, + sigma: float = 1.0, + random_state: Optional[int] = None, +) -> np.ndarray: + """ + Generate a random block-diagonal covariance matrix. + + Reference: + Snippet 4.3, Page 61. + + Parameters + ---------- + n_columns : int + Total number of columns (assets). + n_blocks : int + Number of blocks to create. + block_size_min : int, default=1 + The minimum size for each block. + sigma : float, default=1.0 + Sigma for idiosyncratic noise. + random_state : int, optional + Random state for reproducibility. + + Returns + ------- + np.ndarray + The (n_columns x n_columns) block-diagonal covariance matrix. + """ + rng = check_random_state(random_state) + + # Generate block sizes + parts = rng.choice( + range(1, n_columns - (block_size_min - 1) * n_blocks), + n_blocks - 1, + replace=False, + ) + parts.sort() + parts = np.append(parts, n_columns - (block_size_min - 1) * n_blocks) + parts = np.append(parts[0], np.diff(parts)) - 1 + block_size_min + + cov_list = [] + for col_size in parts: + # Number of observations must be > number of columns + n_obs = int(max(col_size * (col_size + 1) / 2.0, 100)) + + this_covariance = random_covariance_sub( + n_obs, col_size, sigma, random_state=rng + ) + cov_list.append(this_covariance) + + return block_diag(*cov_list) + + +def random_block_correlation( + n_columns: int, + n_blocks: int, + random_state: Optional[int] = None, + block_size_min: int = 1, +) -> pd.DataFrame: + """ + Generate a random block-diagonal correlation matrix with noise. + + This adds a "market" component (a single block covariance) + to the block-diagonal structure. + + Reference: + Snippet 4.3, Page 61. + + Parameters + ---------- + n_columns : int + Total number of columns (assets). + n_blocks : int + Number of blocks to create. + random_state : int, optional + Random state for reproducibility. + block_size_min : int, default=1 + The minimum size for each block. + + Returns + ------- + pd.DataFrame + The (n_columns x n_columns) correlation matrix. + """ + rng = check_random_state(random_state) + + # Block-diagonal structure + covariance1 = random_block_covariance( + n_columns, + n_blocks, + block_size_min=block_size_min, + sigma=0.5, + random_state=rng, + ) + + # Market component (noise) + covariance2 = random_block_covariance( + n_columns, 1, block_size_min=n_columns, sigma=1.0, random_state=rng + ) + + covariance = covariance1 + covariance2 + correlation = covariance_to_correlation(covariance) + correlation = pd.DataFrame(correlation) + + return correlation diff --git a/src/RiskLabAI/controller/__init__.py b/src/RiskLabAI/controller/__init__.py new file mode 100644 index 0000000..11fb6fd --- /dev/null +++ b/src/RiskLabAI/controller/__init__.py @@ -0,0 +1,2 @@ +from .bars_initializer import * +from .data_structure_controller import * diff --git a/src/RiskLabAI/controller/bars_initializer.py b/src/RiskLabAI/controller/bars_initializer.py new file mode 100644 index 0000000..cfd2996 --- /dev/null +++ b/src/RiskLabAI/controller/bars_initializer.py @@ -0,0 +1,299 @@ +""" +Controller class to act as a factory for initializing +various bar types (Standard, Time, Imbalance, Run). +""" + +from typing import Any, Callable, Optional, Union + +import pandas as pd + +from RiskLabAI.data.structures.abstract_bars import AbstractBars + +# Import bar types +from RiskLabAI.data.structures.imbalance_bars import ( + ExpectedImbalanceBars, + FixedImbalanceBars, +) +from RiskLabAI.data.structures.run_bars import ExpectedRunBars, FixedRunBars +from RiskLabAI.data.structures.standard_bars import StandardBars +from RiskLabAI.data.structures.time_bars import TimeBars + +# Import constants +from RiskLabAI.utils.constants import ( + CUMULATIVE_DOLLAR, + CUMULATIVE_TICKS, + CUMULATIVE_VOLUME, +) + + +class BarsInitializerController: + """ + Controller for initializing various types of bars. + + This class acts as a factory, providing static methods to + construct different bar objects with sensible default parameters. + """ + + def __init__(self): + """ + Initializes the controller and maps method names to methods. + """ + self.method_name_to_method: dict[str, Callable[..., AbstractBars]] = { + "expected_dollar_imbalance_bars": self.initialize_expected_dollar_imbalance_bars, + "expected_volume_imbalance_bars": self.initialize_expected_volume_imbalance_bars, + "expected_tick_imbalance_bars": self.initialize_expected_tick_imbalance_bars, + "fixed_dollar_imbalance_bars": self.initialize_fixed_dollar_imbalance_bars, + "fixed_volume_imbalance_bars": self.initialize_fixed_volume_imbalance_bars, + "fixed_tick_imbalance_bars": self.initialize_fixed_tick_imbalance_bars, + "expected_dollar_run_bars": self.initialize_expected_dollar_run_bars, + "expected_volume_run_bars": self.initialize_expected_volume_run_bars, + "expected_tick_run_bars": self.initialize_expected_tick_run_bars, + "fixed_dollar_run_bars": self.initialize_fixed_dollar_run_bars, + "fixed_volume_run_bars": self.initialize_fixed_volume_run_bars, + "fixed_tick_run_bars": self.initialize_fixed_tick_run_bars, + "dollar_standard_bars": self.initialize_dollar_standard_bars, + "volume_standard_bars": self.initialize_volume_standard_bars, + "tick_standard_bars": self.initialize_tick_standard_bars, + "time_bars": self.initialize_time_bars, + } + + @staticmethod + def initialize_expected_dollar_imbalance_bars( + window_size_for_expected_n_ticks_estimation: int = 10000, + window_size_for_expected_imbalance_estimation: int = 10000, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + **kwargs: Any, # Accept extra kwargs but don't use them + ) -> ExpectedImbalanceBars: + """ + Initialize expected dollar imbalance bars. + """ + return ExpectedImbalanceBars( + bar_type="dollar_imbalance", + window_size_for_expected_n_ticks_estimation=window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + expected_ticks_number_bounds=expected_ticks_number_bounds, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_expected_volume_imbalance_bars( + window_size_for_expected_n_ticks_estimation: int = 10000, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> ExpectedImbalanceBars: + """Initialize expected volume imbalance bars.""" + return ExpectedImbalanceBars( + bar_type="volume_imbalance", + window_size_for_expected_n_ticks_estimation=window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + expected_ticks_number_bounds=expected_ticks_number_bounds, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_expected_tick_imbalance_bars( + window_size_for_expected_n_ticks_estimation: int = 10000, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> ExpectedImbalanceBars: + """Initialize expected tick imbalance bars.""" + return ExpectedImbalanceBars( + bar_type="tick_imbalance", + window_size_for_expected_n_ticks_estimation=window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + expected_ticks_number_bounds=expected_ticks_number_bounds, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_fixed_dollar_imbalance_bars( + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> FixedImbalanceBars: + """Initialize fixed dollar imbalance bars.""" + return FixedImbalanceBars( + bar_type="dollar_imbalance", + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_fixed_volume_imbalance_bars( + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> FixedImbalanceBars: + """Initialize fixed volume imbalance bars.""" + return FixedImbalanceBars( + bar_type="volume_imbalance", + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_fixed_tick_imbalance_bars( + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> FixedImbalanceBars: + """Initialize fixed tick imbalance bars.""" + return FixedImbalanceBars( + bar_type="tick_imbalance", + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_expected_dollar_run_bars( + window_size_for_expected_n_ticks_estimation: int = 10000, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> ExpectedRunBars: + """Initialize expected dollar run bars.""" + return ExpectedRunBars( + bar_type="dollar_run", + window_size_for_expected_n_ticks_estimation=window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + expected_ticks_number_bounds=expected_ticks_number_bounds, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_expected_volume_run_bars( + window_size_for_expected_n_ticks_estimation: int = 10000, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> ExpectedRunBars: + """Initialize expected volume run bars.""" + return ExpectedRunBars( + bar_type="volume_run", + window_size_for_expected_n_ticks_estimation=window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + expected_ticks_number_bounds=expected_ticks_number_bounds, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_expected_tick_run_bars( + window_size_for_expected_n_ticks_estimation: int = 10000, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> ExpectedRunBars: + """Initialize expected tick run bars.""" + return ExpectedRunBars( + bar_type="tick_run", + window_size_for_expected_n_ticks_estimation=window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + expected_ticks_number_bounds=expected_ticks_number_bounds, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_fixed_dollar_run_bars( + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> FixedRunBars: + """Initialize fixed dollar run bars.""" + return FixedRunBars( + bar_type="dollar_run", + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_fixed_volume_run_bars( + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> FixedRunBars: + """Initialize fixed volume run bars.""" + return FixedRunBars( + bar_type="volume_run", + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_fixed_tick_run_bars( + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + analyse_thresholds: bool = False, + **kwargs: Any, + ) -> FixedRunBars: + """Initialize fixed tick run bars.""" + return FixedRunBars( + bar_type="tick_run", + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + analyse_thresholds=analyse_thresholds, + ) + + @staticmethod + def initialize_dollar_standard_bars( + threshold: Union[float, pd.Series] = 70000000, + **kwargs: Any, + ) -> StandardBars: + """Initialize dollar standard bars.""" + return StandardBars(bar_type=CUMULATIVE_DOLLAR, threshold=threshold) + + @staticmethod + def initialize_volume_standard_bars( + threshold: Union[float, pd.Series] = 30000, + **kwargs: Any, + ) -> StandardBars: + """Initialize volume standard bars.""" + return StandardBars(bar_type=CUMULATIVE_VOLUME, threshold=threshold) + + @staticmethod + def initialize_tick_standard_bars( + threshold: Union[float, pd.Series] = 6000, + **kwargs: Any, + ) -> StandardBars: + """Initialize tick standard bars.""" + return StandardBars(bar_type=CUMULATIVE_TICKS, threshold=threshold) + + @staticmethod + def initialize_time_bars( + resolution_type: str = "D", + resolution_units: int = 1, + **kwargs: Any, + ) -> TimeBars: + """Initialize time bars.""" + return TimeBars( + resolution_type=resolution_type, resolution_units=resolution_units + ) diff --git a/src/RiskLabAI/controller/data_structure_controller.py b/src/RiskLabAI/controller/data_structure_controller.py new file mode 100644 index 0000000..5b23082 --- /dev/null +++ b/src/RiskLabAI/controller/data_structure_controller.py @@ -0,0 +1,188 @@ +""" +Main controller for processing tick data into bars. + +This class reads data in batches from CSVs or DataFrames and +uses the BarsInitializerController to construct bars based on +a specified method. +""" + +import logging +from collections.abc import Generator +from typing import Any, Optional, Union + +import pandas as pd + +from RiskLabAI.controller.bars_initializer import BarsInitializerController +from RiskLabAI.data.structures.abstract_bars import AbstractBars +from RiskLabAI.utils.constants import ( + CLOSE_PRICE, + CUMULATIVE_BUY_VOLUME, + CUMULATIVE_DOLLAR, + CUMULATIVE_SELL_VOLUME, + CUMULATIVE_TICKS, + CUMULATIVE_VOLUME, + DATE_TIME, + HIGH_PRICE, + LOW_PRICE, + OPEN_PRICE, + THRESHOLD, + TICK_NUMBER, +) + +logger = logging.getLogger(__name__) + +# Define the bar column schema +BAR_COLUMNS = [ + DATE_TIME, + TICK_NUMBER, + OPEN_PRICE, + HIGH_PRICE, + LOW_PRICE, + CLOSE_PRICE, + CUMULATIVE_VOLUME, + CUMULATIVE_BUY_VOLUME, + CUMULATIVE_SELL_VOLUME, + CUMULATIVE_TICKS, + CUMULATIVE_DOLLAR, + THRESHOLD, +] + + +class Controller: + """ + Controller for initializing and processing bars from data sources. + """ + + def __init__(self) -> None: + """ + Initializes the Controller and its internal BarsInitializerController. + """ + self.bars_initializer = BarsInitializerController() + + def handle_input_command( + self, + method_name: str, + method_arguments: dict[str, Any], + input_data: Union[str, pd.DataFrame], + output_path: Optional[str] = None, + batch_size: int = 1_000_000, + ) -> pd.DataFrame: + """ + Handles the command to initialize a bar generator and process data. + (Parameters same as original) + """ + # 1. Initialize the bar generator + try: + initializer_method = self.bars_initializer.method_name_to_method[ + method_name + ] + except KeyError: + valid_methods = list(self.bars_initializer.method_name_to_method.keys()) + logger.error( + "Bar method '%s' not found. Valid methods are: %s", + method_name, + valid_methods, + ) + return pd.DataFrame(columns=BAR_COLUMNS) + + try: + bar_generator: AbstractBars = initializer_method(**method_arguments) + except TypeError as e: + logger.error( + "Error initializing bar method '%s' with arguments %s: %s", + method_name, + method_arguments, + e, + ) + return pd.DataFrame(columns=BAR_COLUMNS) + + # 2. Get the correct batch generator + if isinstance(input_data, str): + data_generator = self.read_batches_from_string(input_data, batch_size) + elif isinstance(input_data, pd.DataFrame): + data_generator = self.read_batches_from_dataframe(input_data, batch_size) + else: + raise TypeError("input_data must be a string (path) or pd.DataFrame") + + all_bars: list[list[Any]] = [] + + # 3. Process data in batches + logger.info("Processing data in batches...") + try: + for data_batch in data_generator: + # We assume data is [datetime, price, volume] + # .values returns a NumPy array, which is an efficient iterable + bars = bar_generator.construct_bars_from_data(data=data_batch.values) + all_bars.extend(bars) + except Exception as e: + logger.warning( + "Error during bar construction: %s. Returning DataFrame with " + "bars constructed so far.", + e, + ) + # Continue to return whatever was processed + + logger.info("Done. Constructed %d bars.", len(all_bars)) + + # 4. Create final DataFrame + bars_df = pd.DataFrame(all_bars, columns=BAR_COLUMNS) + + if output_path: + logger.info("Saving bars to %s...", output_path) + try: + bars_df.to_csv(output_path, index=False) + logger.info("Save complete.") + except Exception as e: + logger.error("Error saving file to %s: %s", output_path, e) + + return bars_df + + @staticmethod + def read_batches_from_string( + input_path: str, batch_size: int + ) -> Generator[pd.DataFrame, None, None]: + """ + Reads data in batches from a CSV file efficiently. + + Parameters + ---------- + input_path : str + File path to read from. + batch_size : int + Size of each batch. + + Yields + ------- + Generator[pd.DataFrame, None, None] + A generator yielding batches of data. + """ + try: + # Use a generator to read the file in chunks + # This is more memory-efficient and avoids reading the file twice. + yield from pd.read_csv(input_path, chunksize=batch_size, parse_dates=[0]) + except FileNotFoundError: + logger.error("File not found at %s", input_path) + return + except pd.errors.ParserError as e: + logger.error("Error parsing CSV file %s: %s", input_path, e) + return + except Exception as e: + logger.error( + "An unexpected error occurred while reading %s: %s", input_path, e + ) + return + + @staticmethod + def read_batches_from_dataframe( + input_data: pd.DataFrame, batch_size: int + ) -> Generator[pd.DataFrame, None, None]: + """ + Reads data in batches from a DataFrame. + """ + n_rows = input_data.shape[0] + if n_rows == 0: + return + + for start_row in range(0, n_rows, batch_size): + end_row = min(start_row + batch_size, n_rows) + yield input_data.iloc[start_row:end_row] diff --git a/src/RiskLabAI/core/__init__.py b/src/RiskLabAI/core/__init__.py new file mode 100644 index 0000000..06fd69c --- /dev/null +++ b/src/RiskLabAI/core/__init__.py @@ -0,0 +1,67 @@ +""" +RiskLabAI.core — the extension layer. + +This package holds the small set of abstractions that make RiskLabAI *modular, +extendable and maintainable*: a reusable component :class:`Registry`, the base +interfaces every model family targets, and a built-in catalogue of the +components the library already ships. + +It is the recommended entry point for adding a **new** model (for example, an +extension or alternative to a method from López de Prado's books): implement the +relevant base interface, register it in the matching registry, and it becomes +discoverable and constructible by name — without editing any central file. See +``EXTENDING.md`` at the repository root for a step-by-step guide. + +Importing this package is cheap: every component is registered lazily, so heavy +dependencies are only imported when a component is actually created. + +Quick tour +---------- +>>> from RiskLabAI.core import list_components, CROSS_VALIDATORS +>>> sorted(list_components()) # doctest: +SKIP +['bars', 'bet_sizers', 'cross_validators', ...] +>>> "purgedkfold" in CROSS_VALIDATORS +True +>>> cv = CROSS_VALIDATORS.create("purgedkfold", n_splits=5) # doctest: +SKIP +""" + +from __future__ import annotations + +from ._builtins import ( + BARS, + BET_SIZERS, + CROSS_VALIDATORS, + FEATURE_IMPORTANCE, + LABELERS, + PORTFOLIO_OPTIMIZERS, + REGISTRIES, + get_registry, + list_components, +) +from .base import ( + BaseBetSizer, + BaseLabeler, + BasePortfolioOptimizer, + Estimator, +) +from .registry import Registry + +__all__ = [ + # Registry machinery + "Registry", + "REGISTRIES", + "get_registry", + "list_components", + # Per-family registries + "BARS", + "CROSS_VALIDATORS", + "FEATURE_IMPORTANCE", + "LABELERS", + "BET_SIZERS", + "PORTFOLIO_OPTIMIZERS", + # Base contracts + "Estimator", + "BaseLabeler", + "BaseBetSizer", + "BasePortfolioOptimizer", +] diff --git a/src/RiskLabAI/core/_builtins.py b/src/RiskLabAI/core/_builtins.py new file mode 100644 index 0000000..7cb533a --- /dev/null +++ b/src/RiskLabAI/core/_builtins.py @@ -0,0 +1,195 @@ +""" +The built-in component catalogue. + +Defines one :class:`~RiskLabAI.core.registry.Registry` per model family and +populates it with the components the library already ships. Registration is +**lazy** (by import path), so importing this module — and therefore +``RiskLabAI.core`` — does not import pandas, numba, scikit-learn or torch. The +heavy import only happens when a component is actually created. + +Families currently exposed only as free functions (labeling, bet sizing, +portfolio optimization) get an empty registry here: it is the extension point +where new class-based models register themselves (see ``EXTENDING.md``). + +The registries are deliberately a *superset* of the existing hand-written +factories (``CrossValidatorFactory``, ``FeatureImportanceFactory``, +``BarsInitializerController``), which remain unchanged for backward +compatibility. ``test/core/test_builtin_parity.py`` asserts the registries stay +in sync with those factories so the two cannot silently drift apart. +""" + +from __future__ import annotations + +from .registry import Registry + +__all__ = [ + "BARS", + "CROSS_VALIDATORS", + "FEATURE_IMPORTANCE", + "LABELERS", + "BET_SIZERS", + "PORTFOLIO_OPTIMIZERS", + "REGISTRIES", + "get_registry", + "list_components", +] + +# --------------------------------------------------------------------------- # +# Bar structures. The extension point for a new bar type is a new AbstractBars +# subclass; register it here (or via @BARS.register in your own code). +# --------------------------------------------------------------------------- # +BARS = Registry("bars") +BARS.register_lazy( + "standard_bars", + "RiskLabAI.data.structures.standard_bars:StandardBars", + metadata={"family": "bars", "afml_chapter": 2}, +) +BARS.register_lazy( + "time_bars", + "RiskLabAI.data.structures.time_bars:TimeBars", + metadata={"family": "bars", "afml_chapter": 2}, +) +BARS.register_lazy( + "expected_imbalance_bars", + "RiskLabAI.data.structures.imbalance_bars:ExpectedImbalanceBars", + metadata={"family": "bars", "afml_chapter": 2}, +) +BARS.register_lazy( + "fixed_imbalance_bars", + "RiskLabAI.data.structures.imbalance_bars:FixedImbalanceBars", + metadata={"family": "bars", "afml_chapter": 2}, +) +BARS.register_lazy( + "expected_run_bars", + "RiskLabAI.data.structures.run_bars:ExpectedRunBars", + metadata={"family": "bars", "afml_chapter": 2}, +) +BARS.register_lazy( + "fixed_run_bars", + "RiskLabAI.data.structures.run_bars:FixedRunBars", + metadata={"family": "bars", "afml_chapter": 2}, +) + +# --------------------------------------------------------------------------- # +# Cross-validators. Keys mirror CrossValidatorFactory.VALIDATORS exactly. +# --------------------------------------------------------------------------- # +CROSS_VALIDATORS = Registry("cross_validators") +CROSS_VALIDATORS.register_lazy( + "kfold", + "RiskLabAI.backtest.validation.kfold:KFold", + metadata={"family": "cross_validator"}, +) +CROSS_VALIDATORS.register_lazy( + "walkforward", + "RiskLabAI.backtest.validation.walk_forward:WalkForward", + metadata={"family": "cross_validator"}, +) +CROSS_VALIDATORS.register_lazy( + "purgedkfold", + "RiskLabAI.backtest.validation.purged_kfold:PurgedKFold", + metadata={"family": "cross_validator"}, +) +CROSS_VALIDATORS.register_lazy( + "combinatorialpurged", + "RiskLabAI.backtest.validation.combinatorial_purged:CombinatorialPurged", + metadata={"family": "cross_validator"}, +) +CROSS_VALIDATORS.register_lazy( + "baggedcombinatorialpurged", + "RiskLabAI.backtest.validation.bagged_combinatorial_purged:BaggedCombinatorialPurged", + metadata={"family": "cross_validator"}, +) +CROSS_VALIDATORS.register_lazy( + "adaptivecombinatorialpurged", + "RiskLabAI.backtest.validation.adaptive_combinatorial_purged:AdaptiveCombinatorialPurged", + metadata={"family": "cross_validator"}, +) + +# --------------------------------------------------------------------------- # +# Feature-importance strategies. Keys mirror FeatureImportanceFactory exactly. +# --------------------------------------------------------------------------- # +FEATURE_IMPORTANCE = Registry("feature_importance") +FEATURE_IMPORTANCE.register_lazy( + "MDI", + "RiskLabAI.features.feature_importance.feature_importance_mdi:FeatureImportanceMDI", + metadata={"family": "feature_importance"}, +) +FEATURE_IMPORTANCE.register_lazy( + "ClusteredMDI", + "RiskLabAI.features.feature_importance.clustered_feature_importance_mdi:ClusteredFeatureImportanceMDI", + metadata={"family": "feature_importance"}, +) +FEATURE_IMPORTANCE.register_lazy( + "MDA", + "RiskLabAI.features.feature_importance.feature_importance_mda:FeatureImportanceMDA", + metadata={"family": "feature_importance"}, +) +FEATURE_IMPORTANCE.register_lazy( + "ClusteredMDA", + "RiskLabAI.features.feature_importance.clustered_feature_importance_mda:ClusteredFeatureImportanceMDA", + metadata={"family": "feature_importance"}, +) +FEATURE_IMPORTANCE.register_lazy( + "SFI", + "RiskLabAI.features.feature_importance.feature_importance_sfi:FeatureImportanceSFI", + metadata={"family": "feature_importance"}, +) + +# --------------------------------------------------------------------------- # +# Free-function families: empty registries, ready for new class-based models. +# These correspond to the BaseLabeler / BaseBetSizer / BasePortfolioOptimizer +# contracts in RiskLabAI.core.base. +# --------------------------------------------------------------------------- # +LABELERS = Registry("labelers") +BET_SIZERS = Registry("bet_sizers") +PORTFOLIO_OPTIMIZERS = Registry("portfolio_optimizers") + +# --------------------------------------------------------------------------- # +# Family name -> registry, for discovery and a unified catalogue. +# --------------------------------------------------------------------------- # +REGISTRIES: dict[str, Registry] = { + "bars": BARS, + "cross_validators": CROSS_VALIDATORS, + "feature_importance": FEATURE_IMPORTANCE, + "labelers": LABELERS, + "bet_sizers": BET_SIZERS, + "portfolio_optimizers": PORTFOLIO_OPTIMIZERS, +} + + +def get_registry(family: str) -> Registry: + """ + Return the registry for a model family. + + Parameters + ---------- + family : str + One of the keys of :data:`REGISTRIES` (e.g. ``"bars"``, + ``"cross_validators"``). + + Raises + ------ + KeyError + If the family name is unknown (the message lists valid families). + """ + try: + return REGISTRIES[family] + except KeyError: + valid = ", ".join(repr(k) for k in sorted(REGISTRIES)) + raise KeyError( + f"{family!r} is not a known model family. Valid families: {valid}." + ) from None + + +def list_components() -> dict[str, list[str]]: + """ + Return a catalogue mapping each family name to its available component keys. + + Useful for discovery and for documentation that should not drift:: + + >>> from RiskLabAI.core import list_components + >>> catalogue = list_components() + >>> catalogue["cross_validators"] # doctest: +SKIP + ['adaptivecombinatorialpurged', 'baggedcombinatorialpurged', ...] + """ + return {family: reg.available() for family, reg in REGISTRIES.items()} diff --git a/src/RiskLabAI/core/base.py b/src/RiskLabAI/core/base.py new file mode 100644 index 0000000..6e0c709 --- /dev/null +++ b/src/RiskLabAI/core/base.py @@ -0,0 +1,193 @@ +""" +Base interfaces (contracts) for RiskLabAI model families. + +This module gives every model family a single, documented contract that new +implementations can target. Two kinds of contract live here: + +1. **Canonical interfaces that already exist** elsewhere in the library are + re-exported from here so there is one obvious place to look: + :class:`AbstractBars`, :class:`CrossValidator`, and + :class:`FeatureImportanceStrategy`. These are re-exported *lazily* (PEP 562) + so that ``import RiskLabAI.core`` does not eagerly pull in pandas/numba and + the rest of the data/backtest/features sub-packages. + +2. **New, optional abstract base classes** for families that are currently + exposed only as free functions (labeling, bet sizing, portfolio + optimization). They are purely additive: existing functions keep working + unchanged. A *new* model can subclass one of these to advertise a uniform + interface (and to be registered and discovered through + :mod:`RiskLabAI.core.registry`). + +Plus a structural :class:`Estimator` protocol describing the scikit-learn-style +``fit``/``predict`` objects the cross-validation and feature-importance code +already expects. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any, Protocol, runtime_checkable + +import pandas as pd + +__all__ = [ + # Structural protocol + "Estimator", + # New optional contracts for free-function families + "BaseLabeler", + "BaseBetSizer", + "BasePortfolioOptimizer", + # Re-exported canonical interfaces (lazy) + "AbstractBars", + "BarBuilder", + "CrossValidator", + "FeatureImportanceStrategy", +] + + +@runtime_checkable +class Estimator(Protocol): + """ + Structural type for a scikit-learn-style estimator. + + Anything with ``fit`` and ``predict`` satisfies this protocol; no + inheritance is required (it is ``runtime_checkable``, so + ``isinstance(model, Estimator)`` works for duck-typed models). This is the + object the cross-validation and feature-importance machinery consumes. + """ + + def fit(self, X: Any, y: Any = None, **kwargs: Any) -> Any: ... + + def predict(self, X: Any) -> Any: ... + + +class BaseLabeler(ABC): + """ + Optional contract for a labeling method (e.g. triple-barrier, trend-scanning). + + The library's existing labelers are free functions and remain so. New + class-based labelers may subclass this to expose a uniform ``label`` entry + point and be registered in the ``labelers`` registry. See + ``EXTENDING.md`` for a worked example. + """ + + @abstractmethod + def label( + self, + prices: pd.Series, + events: pd.DataFrame | None = None, + **kwargs: Any, + ) -> pd.DataFrame: + """ + Produce labels for the given price series. + + Parameters + ---------- + prices : pd.Series + Price (or log-price) series indexed by timestamp. + events : pd.DataFrame, optional + Event definitions (e.g. event start times, vertical barriers, + target sizes) where applicable. + **kwargs + Method-specific options. + + Returns + ------- + pd.DataFrame + Labels, conventionally indexed by event time. + """ + raise NotImplementedError + + +class BaseBetSizer(ABC): + """ + Optional contract for a bet-sizing method. + + Maps model outputs (e.g. predicted probabilities) to a position size in + ``[-1, 1]``. New class-based bet sizers may subclass this and register in + the ``bet_sizers`` registry. + """ + + @abstractmethod + def bet_size(self, probabilities: pd.Series, **kwargs: Any) -> pd.Series: + """ + Convert prediction probabilities into signed bet sizes. + + Parameters + ---------- + probabilities : pd.Series + Predicted probabilities (or scores) indexed by timestamp. + **kwargs + Method-specific options. + + Returns + ------- + pd.Series + Signed bet sizes, conventionally in ``[-1, 1]``. + """ + raise NotImplementedError + + +class BasePortfolioOptimizer(ABC): + """ + Optional contract for a portfolio-construction method (e.g. HRP, NCO). + + New class-based optimizers may subclass this and register in the + ``portfolio_optimizers`` registry. + """ + + @abstractmethod + def weights(self, returns: pd.DataFrame, **kwargs: Any) -> pd.Series: + """ + Compute portfolio weights. + + Parameters + ---------- + returns : pd.DataFrame + Asset returns (rows = observations, columns = assets). Some + optimizers may accept a covariance matrix instead; document the + expectation in the concrete subclass. + **kwargs + Method-specific options. + + Returns + ------- + pd.Series + Portfolio weights indexed by asset. + """ + raise NotImplementedError + + +# --------------------------------------------------------------------------- # +# Lazy re-exports of the canonical interfaces that already live in the library. +# Importing them here eagerly would pull in the heavy data/backtest/features +# sub-package __init__ chains, defeating the lazy-import design. PEP 562 +# __getattr__ defers that cost until the name is actually accessed. +# --------------------------------------------------------------------------- # +_LAZY_REEXPORTS = { + "AbstractBars": "RiskLabAI.data.structures.abstract_bars:AbstractBars", + "BarBuilder": "RiskLabAI.data.structures.abstract_bars:AbstractBars", + "CrossValidator": ( + "RiskLabAI.backtest.validation.cross_validator_interface:CrossValidator" + ), + "FeatureImportanceStrategy": ( + "RiskLabAI.features.feature_importance.feature_importance_strategy" + ":FeatureImportanceStrategy" + ), +} + + +def __getattr__(name: str) -> Any: + target = _LAZY_REEXPORTS.get(name) + if target is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + import importlib + + module_path, _, attribute = target.partition(":") + obj = getattr(importlib.import_module(module_path), attribute) + globals()[name] = obj # cache for next access + return obj + + +def __dir__() -> list: + return sorted(__all__) diff --git a/src/RiskLabAI/core/registry.py b/src/RiskLabAI/core/registry.py new file mode 100644 index 0000000..ab47928 --- /dev/null +++ b/src/RiskLabAI/core/registry.py @@ -0,0 +1,402 @@ +""" +A small, dependency-free component registry. + +Several sub-packages in RiskLabAI already implement the same pattern by hand: +a string key is mapped to a class, and a factory instantiates the right class +from that key (``controller/bars_initializer.py``, +``backtest/validation/cross_validator_factory.py``, +``features/feature_importance/feature_importance_factory.py``). Each does it +slightly differently (different casing rules, different error behaviour, a +hand-written ``dict`` in three places). + +``Registry`` factors that pattern into one reusable, well-tested object so that: + +* every model family discovers and instantiates its components the same way; +* **new** models (e.g. an extension proposed in a RiskLabAI methods paper) can be + added with a single ``@registry.register("my_model")`` decorator, without + editing a central ``dict`` — the open/closed principle in practice; +* lookups fail loudly with a helpful message listing the valid keys, instead of + silently returning ``None`` or an empty result. + +The registry is intentionally tiny and imports nothing heavy, so importing it +(and the :mod:`RiskLabAI.core` package) stays cheap. Components may be registered +*lazily* by a ``"module.path:attribute"`` string, so registering the built-in +catalogue does not import pandas/numba/torch until a component is actually +created. +""" + +from __future__ import annotations + +import importlib +import inspect +import logging +from collections.abc import Iterator +from typing import ( + Any, + Callable, +) + +logger = logging.getLogger(__name__) + +__all__ = ["Registry"] + +# A factory is anything callable that returns a component instance: usually a +# class, but a plain function works too. +Factory = Callable[..., Any] + + +class _Entry: + """Internal record for one registered component (eager or lazy).""" + + __slots__ = ("key", "_obj", "_lazy_target", "metadata") + + def __init__( + self, + key: str, + obj: Factory | None = None, + lazy_target: str | None = None, + metadata: dict[str, Any] | None = None, + ) -> None: + self.key = key + self._obj = obj + self._lazy_target = lazy_target + self.metadata: dict[str, Any] = dict(metadata or {}) + + @property + def is_lazy(self) -> bool: + """True if the component has not been imported/resolved yet.""" + return self._obj is None + + def resolve(self) -> Factory: + """Return the factory, importing it on first use if registered lazily.""" + if self._obj is None: + if not self._lazy_target: + raise RuntimeError( + f"Registry entry {self.key!r} has neither an object nor a " + f"lazy import target." + ) + module_path, sep, attribute = self._lazy_target.partition(":") + if not sep: + raise ValueError( + f"Lazy target {self._lazy_target!r} for {self.key!r} must be " + f"of the form 'package.module:attribute'." + ) + module = importlib.import_module(module_path) + self._obj = getattr(module, attribute) + return self._obj + + +class Registry: + """ + A name -> component factory registry with case-insensitive lookup. + + Parameters + ---------- + name : str + Human-readable name of the registry (used in error messages), + e.g. ``"bars"`` or ``"cross_validators"``. + base : type, optional + An optional expected base class / interface for registered components. + It is **not** enforced (so existing free functions and duck-typed + components keep working), but it is recorded for documentation and can + be inspected via :attr:`base`. + + Examples + -------- + >>> from RiskLabAI.core.registry import Registry + >>> animals = Registry("animals") + >>> @animals.register("dog", aliases=("doggo",)) + ... class Dog: + ... def speak(self) -> str: + ... return "woof" + >>> animals.available() + ['dog'] + >>> animals.create("DOG").speak() # case-insensitive + 'woof' + >>> animals.create("doggo").speak() # alias + 'woof' + """ + + def __init__(self, name: str, *, base: type | None = None) -> None: + self.name = name + self.base = base + # canonical key -> entry + self._entries: dict[str, _Entry] = {} + # lower-cased key OR alias -> canonical key + self._index: dict[str, str] = {} + + # ------------------------------------------------------------------ # + # Registration + # ------------------------------------------------------------------ # + def register( + self, + key: Any = None, + obj: Factory | None = None, + *, + aliases: tuple[str, ...] = (), + metadata: dict[str, Any] | None = None, + override: bool = False, + ) -> Any: + """ + Register a component. Usable as a decorator or as a direct call. + + Decorator forms:: + + @reg.register("my_model") + class MyModel: ... + + @reg.register # key inferred from class name + class MyModel: ... + + Direct form:: + + reg.register("my_model", MyModel) + + Parameters + ---------- + key : str or callable, optional + The lookup key. When used as a bare decorator (``@reg.register``) + this is the class/function being decorated and the key is taken from + its ``__name__``. + obj : callable, optional + The factory (class or function). If given, this is a direct + registration and ``obj`` is returned unchanged. + aliases : tuple of str, optional + Additional names that resolve to the same component. + metadata : dict, optional + Arbitrary metadata stored alongside the entry (e.g. a description, + an AFML page reference, the family it belongs to). + override : bool, default False + Allow replacing an existing key. Otherwise a duplicate raises + :class:`KeyError`, which protects against accidental shadowing. + + Returns + ------- + The decorated object (decorator forms) or ``obj`` (direct form). + """ + # Bare decorator: @reg.register (key is the class/function itself) + if obj is None and callable(key) and not isinstance(key, str): + target = key + self._register( + target.__name__, + target, + aliases=aliases, + metadata=metadata, + override=override, + ) + return target + + # Direct call: reg.register("name", obj) + if obj is not None: + self._register( + key, + obj, + aliases=aliases, + metadata=metadata, + override=override, + ) + return obj + + # Decorator factory: @reg.register("name") + def decorator(target: Factory) -> Factory: + resolved_key = key if isinstance(key, str) else target.__name__ + self._register( + resolved_key, + target, + aliases=aliases, + metadata=metadata, + override=override, + ) + return target + + return decorator + + def register_lazy( + self, + key: str, + target: str, + *, + aliases: tuple[str, ...] = (), + metadata: dict[str, Any] | None = None, + override: bool = False, + ) -> None: + """ + Register a component by import path, deferring the import to first use. + + Parameters + ---------- + key : str + The lookup key. + target : str + Import path of the form ``"package.module:attribute"``. + aliases, metadata, override + See :meth:`register`. + """ + self._register( + key, + None, + lazy_target=target, + aliases=aliases, + metadata=metadata, + override=override, + ) + + def _register( + self, + key: str, + obj: Factory | None, + *, + lazy_target: str | None = None, + aliases: tuple[str, ...] = (), + metadata: dict[str, Any] | None = None, + override: bool = False, + ) -> None: + if not isinstance(key, str) or not key: + raise TypeError(f"Registry key must be a non-empty string, got {key!r}.") + lower = key.lower() + if lower in self._index and not override: + existing = self._index[lower] + raise KeyError( + f"{self.name!r} registry already has a component under " + f"{key!r} (canonical key {existing!r}). " + f"Pass override=True to replace it." + ) + # On override, drop any stale aliases pointing at the old canonical key. + if override and lower in self._index: + old_canonical = self._index[lower] + self._entries.pop(old_canonical, None) + self._index = {k: v for k, v in self._index.items() if v != old_canonical} + + self._entries[key] = _Entry( + key, + obj=obj, + lazy_target=lazy_target, + metadata=metadata, + ) + self._index[lower] = key + for alias in aliases: + if not isinstance(alias, str) or not alias: + raise TypeError(f"Alias must be a non-empty string, got {alias!r}.") + self._index[alias.lower()] = key + logger.debug("Registered %r in %r registry.", key, self.name) + + def unregister(self, key: str) -> None: + """Remove a component and all of its aliases. Raises if absent.""" + entry = self._lookup(key) + canonical = entry.key + self._entries.pop(canonical, None) + self._index = {k: v for k, v in self._index.items() if v != canonical} + + # ------------------------------------------------------------------ # + # Lookup & construction + # ------------------------------------------------------------------ # + def get(self, key: str) -> Factory: + """Return the registered factory (class/callable) for ``key``.""" + return self._lookup(key).resolve() + + def create( + self, + key: str, + *args: Any, + filter_unknown_kwargs: bool = False, + **kwargs: Any, + ) -> Any: + """ + Instantiate the component registered under ``key``. + + Parameters + ---------- + key : str + The lookup key (case-insensitive; aliases accepted). + *args, **kwargs + Forwarded to the component's constructor. + filter_unknown_kwargs : bool, default False + If True, silently drop keyword arguments the constructor does not + accept (unless it accepts ``**kwargs``). This mirrors the behaviour + of the existing cross-validator / feature-importance factories, + which let callers pass a shared bag of options. Off by default so + that typos surface as clear ``TypeError``s. + """ + factory = self.get(key) + if filter_unknown_kwargs: + kwargs = _filter_kwargs(factory, kwargs) + return factory(*args, **kwargs) + + def metadata(self, key: str) -> dict[str, Any]: + """Return the metadata dict registered alongside ``key``.""" + return dict(self._lookup(key).metadata) + + def is_lazy(self, key: str) -> bool: + """True if ``key`` is registered lazily and not yet imported.""" + return self._lookup(key).is_lazy + + def _lookup(self, key: str) -> _Entry: + try: + canonical = self._index[key.lower()] + except AttributeError: + raise TypeError( + f"Registry key must be a string, got {type(key).__name__}." + ) from None + except KeyError: + raise KeyError(self._not_found_message(key)) from None + return self._entries[canonical] + + def _not_found_message(self, key: str) -> str: + available = self.available() + listing = ", ".join(repr(k) for k in available) if available else "(none)" + return ( + f"{key!r} is not registered in the {self.name!r} registry. " + f"Available components: {listing}." + ) + + # ------------------------------------------------------------------ # + # Introspection / mapping protocol + # ------------------------------------------------------------------ # + def available(self) -> list[str]: + """Sorted list of canonical keys.""" + return sorted(self._entries.keys()) + + def keys(self) -> list[str]: + """Alias for :meth:`available` (mapping-like access).""" + return self.available() + + def aliases(self) -> dict[str, str]: + """Mapping of alias -> canonical key (excludes canonical keys themselves).""" + return { + alias: canonical + for alias, canonical in self._index.items() + if alias != canonical.lower() + } + + def __contains__(self, key: object) -> bool: + return isinstance(key, str) and key.lower() in self._index + + def __getitem__(self, key: str) -> Factory: + return self.get(key) + + def __iter__(self) -> Iterator[str]: + return iter(self.available()) + + def __len__(self) -> int: + return len(self._entries) + + def __repr__(self) -> str: # pragma: no cover - cosmetic + return f"" + + +def _filter_kwargs(factory: Factory, kwargs: dict[str, Any]) -> dict[str, Any]: + """ + Drop keyword arguments the factory's signature does not accept. + + If the signature contains ``**kwargs`` (VAR_KEYWORD), everything is kept. + """ + target = factory.__init__ if inspect.isclass(factory) else factory + try: + signature = inspect.signature(target) + except (TypeError, ValueError): # pragma: no cover - builtins without sig + return kwargs + params = signature.parameters.values() + if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params): + return kwargs + accepted = {p.name for p in params} + return {k: v for k, v in kwargs.items() if k in accepted} diff --git a/src/RiskLabAI/data/__init__.py b/src/RiskLabAI/data/__init__.py new file mode 100644 index 0000000..07cd3e3 --- /dev/null +++ b/src/RiskLabAI/data/__init__.py @@ -0,0 +1,34 @@ +""" +RiskLabAI Data Module + +This package contains all modules related to financial data processing, +including: +- Bar data structures (standard, time, imbalance, run) +- Labeling (triple-barrier, trend-scanning) +- Fractional differentiation +- Covariance matrix denoising +- Distance metrics +- Sample weighting +- Synthetic data generation +""" + +# Import sub-packages +from . import ( + denoise, + differentiation, + distance, + labeling, + structures, + synthetic_data, + weights, +) + +__all__ = [ + "denoise", + "differentiation", + "distance", + "labeling", + "structures", + "synthetic_data", + "weights", +] diff --git a/src/RiskLabAI/data/denoise/__init__.py b/src/RiskLabAI/data/denoise/__init__.py new file mode 100644 index 0000000..64ba8f1 --- /dev/null +++ b/src/RiskLabAI/data/denoise/__init__.py @@ -0,0 +1,39 @@ +""" +RiskLabAI Data Denoising Module + +Implements noise reduction techniques for covariance and correlation +matrices, based on Random Matrix Theory (RMT) and the +Marcenko-Pastur distribution. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapter 2. +""" + +from .denoising import ( + corr_to_cov, + cov_to_corr, + denoise_cov, + denoised_corr, + find_max_eval, + fit_kde, + marcenko_pastur_pdf, + optimal_portfolio, + pca, +) +from .nercome import nercome_denoised_covariance + +__all__ = [ + "marcenko_pastur_pdf", + "fit_kde", + "find_max_eval", + "pca", + "denoised_corr", + "denoised_corr2", + "cov_to_corr", + "corr_to_cov", + "denoise_cov", + "optimal_portfolio", + # NERCOME (Appraisal 24 admit) + "nercome_denoised_covariance", +] diff --git a/src/RiskLabAI/data/denoise/denoising.py b/src/RiskLabAI/data/denoise/denoising.py new file mode 100644 index 0000000..27d56d6 --- /dev/null +++ b/src/RiskLabAI/data/denoise/denoising.py @@ -0,0 +1,377 @@ +""" +Implements covariance matrix denoising using Random Matrix Theory (RMT). + +This module provides functions to: +1. Fit the Marcenko-Pastur distribution to the eigenvalues of a + correlation matrix. +2. Identify and remove eigenvalues associated with noise. +3. Reconstruct a "denoised" correlation and covariance matrix. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapter 2. +""" + +from typing import Optional + +import numpy as np +import pandas as pd +from scipy.optimize import minimize +from sklearn.neighbors import KernelDensity + +# --- FIX 5: Removed unused imports for LedoitWolf and block_diag --- + + +def marcenko_pastur_pdf(variance: float, q: float, num_points: int = 1000) -> pd.Series: + r""" + Compute the Marcenko-Pastur (MP) probability density function. + + This function defines the theoretical distribution of eigenvalues + for a random covariance matrix. + + .. math:: + f(\lambda) = \frac{q}{2\pi\sigma^2\lambda} \sqrt{(\lambda_{max} - \lambda) + (\lambda - \lambda_{min})} + + Parameters + ---------- + variance : float + Variance of the observations (\(\sigma^2\)). + q : float + Ratio T/N, where T is observations and N is features. + num_points : int, default=1000 + Number of points in the PDF. + + Returns + ------- + pd.Series + The Marcenko-Pastur PDF, indexed by eigenvalues (\(\lambda\)). + """ + lambda_min = variance * (1 - (1.0 / q) ** 0.5) ** 2 + lambda_max = variance * (1 + (1.0 / q) ** 0.5) ** 2 + + # --- FIX 1: Add epsilon to prevent division by zero if lambda_min=0 (when q=1) --- + e_min = max(lambda_min, 1e-10) + eigenvalues = np.linspace(e_min, lambda_max, num_points) + + pdf = (q / (2 * np.pi * variance * eigenvalues)) * ( + (lambda_max - eigenvalues) * (eigenvalues - lambda_min) + ) ** 0.5 + + # Set PDF to 0 where eigenvalues are outside the valid range (e.g., due to numerical precision) + pdf[np.isnan(pdf)] = 0 + + return pd.Series(pdf.flatten(), index=eigenvalues.flatten()) + + +def fit_kde( + observations: np.ndarray, + bandwidth: float = 0.01, + kernel: str = "gaussian", +) -> KernelDensity: + """ + Fit a Kernel Density Estimator (KDE) to observations. + + Parameters + ---------- + observations : np.ndarray + The observed data (e.g., eigenvalues). + bandwidth : float, default=0.01 + The bandwidth for the kernel. + kernel : str, default="gaussian" + The kernel to use. + + Returns + ------- + KernelDensity + A fitted `sklearn.neighbors.KernelDensity` object. + """ + observations = observations.reshape(-1, 1) + kde = KernelDensity(kernel=kernel, bandwidth=bandwidth).fit(observations) + return kde + + +def _mp_pdf_fit_error( + variance: float, q: float, eigenvalues: np.ndarray, bandwidth: float +) -> float: # <-- FIX 2: Added bandwidth + r""" + Error function for fitting the MP PDF to observed eigenvalues. + + Calculates the sum of squared errors between the theoretical + MP PDF and the empirical PDF (from KDE). + + Parameters + ---------- + variance : float + The \(\sigma^2\) parameter to test. + q : float + The T/N ratio. + eigenvalues : np.ndarray + The observed eigenvalues. + bandwidth : float + The KDE bandwidth. + + Returns + ------- + float + The sum of squared errors. + """ + # Ensure eigenvalues is 1D for PDF generation + if eigenvalues.ndim == 2: + eigenvalues = np.diag(eigenvalues) + + theoretical_pdf = marcenko_pastur_pdf(variance, q, num_points=eigenvalues.shape[0]) + + # Fit empirical PDF + # --- FIX 2: Pass bandwidth to fit_kde --- + kde = fit_kde(eigenvalues, bandwidth=bandwidth) + empirical_pdf = np.exp( + kde.score_samples(theoretical_pdf.index.values.reshape(-1, 1)) + ) + + # Calculate SSE + sse = np.sum((empirical_pdf - theoretical_pdf.values) ** 2) + return sse + + +def find_max_eval( + eigenvalues: np.ndarray, q: float, bandwidth: float +) -> tuple[float, float]: + r""" + Find the maximum theoretical eigenvalue (\(\lambda_{max}\)) + by fitting the Marcenko-Pastur distribution. + + Parameters + ---------- + eigenvalues : np.ndarray + The diagonal matrix (or 1D vector) of observed eigenvalues. + q : float + The T/N ratio. + bandwidth : float + The KDE bandwidth. + + Returns + ------- + Tuple[float, float] + - lambda_max: The maximum theoretical eigenvalue. + - variance: The fitted variance (\(\sigma^2\)). + """ + # --- FIX 3: Ensure we have a 1D array for fitting --- + if eigenvalues.ndim == 2: + eigenvalues_1d = np.diag(eigenvalues) + else: + eigenvalues_1d = eigenvalues + + # Minimize the SSE to find the best-fit variance + # --- FIX 2: Pass bandwidth to the objective function --- + def objective_func(*args): + return _mp_pdf_fit_error(args[0], q, eigenvalues_1d, bandwidth) + + optimizer_result = minimize( + objective_func, + x0=np.array([0.5]), # Initial variance guess + bounds=((1e-5, 1 - 1e-5),), + ) + + if optimizer_result.success: + variance = optimizer_result.x[0] + else: + variance = 1.0 # Fallback + + # Calculate lambda_max based on the fitted variance + lambda_max = variance * (1 + (1.0 / q) ** 0.5) ** 2 + return lambda_max, variance + + +def denoised_corr( + eigenvalues: np.ndarray, eigenvectors: np.ndarray, num_facts: int +) -> np.ndarray: + """ + Reconstruct the correlation matrix using only the eigenvalues + associated with signal (i.e., > lambda_max). + + Note: Assumes eigenvalues are sorted in descending order. + + Parameters + ---------- + eigenvalues : np.ndarray + The diagonal matrix of *all* eigenvalues (sorted descending). + eigenvectors : np.ndarray + The matrix of eigenvectors (corresponding to descending eigenvalues). + num_facts : int + The number of factors (signal eigenvalues) to keep. + + Returns + ------- + np.ndarray + The denoised correlation matrix. + """ + # 1. Get the eigenvalues and eigenvectors for signal + # --- FIX 3: This logic is now correct as eigenvalues are descending --- + eigenvalues_1d = np.diag(eigenvalues) + eigenvalues_signal = np.diag(eigenvalues_1d[:num_facts]) + + eigenvectors_signal = eigenvectors[:, :num_facts] + + # 2. Reconstruct the signal-only correlation matrix + corr1 = eigenvectors_signal @ eigenvalues_signal @ eigenvectors_signal.T + + # 3. Get the eigenvalues for noise and average them + if num_facts < eigenvalues.shape[0]: + # --- FIX 3: Correctly averages the smaller (noise) eigenvalues --- + avg_noise_eigenvalue = eigenvalues_1d[num_facts:].mean() + eigenvectors_noise = eigenvectors[:, num_facts:] + + # 4. Reconstruct the noise-only correlation matrix + corr2 = ( + eigenvectors_noise + @ (np.diag([avg_noise_eigenvalue] * (eigenvalues.shape[0] - num_facts))) + @ eigenvectors_noise.T + ) + + # 5. Add them back together + corr1 = corr1 + corr2 + + # 6. Rescale to be a valid correlation matrix + diag_inv_sqrt = 1.0 / np.sqrt(np.diag(corr1)) + corr1 = np.diag(diag_inv_sqrt) @ corr1 @ np.diag(diag_inv_sqrt) + np.fill_diagonal(corr1, 1.0) # Clean up numerical errors + return corr1 + + +# --- Utility Functions --- + + +def pca(matrix: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """ + Computes the principal component analysis of a Hermitian matrix. + Ensures eigenvalues are sorted descending. + + :param matrix: Hermitian matrix (e.g., correlation matrix) + :type matrix: np.ndarray + :return: (eigenvalues_vector, eigenvectors_matrix) + :rtype: Tuple[np.ndarray, np.ndarray] + """ + eigenvalues, eigenvectors = np.linalg.eigh(matrix) + indices = eigenvalues.argsort()[::-1] # Sort descending + eigenvalues = eigenvalues[indices] + eigenvectors = eigenvectors[:, indices] + return eigenvalues, eigenvectors + + +def cov_to_corr(cov: np.ndarray) -> np.ndarray: + """Convert covariance matrix to correlation matrix.""" + std = np.sqrt(np.diag(cov)) + # Handle division by zero if any std is 0 + std[std == 0] = 1.0 + corr = cov / np.outer(std, std) + corr[corr < -1] = -1.0 # Handle numerical errors + corr[corr > 1] = 1.0 + np.fill_diagonal(corr, 1.0) # Ensure diagonal is 1 + return corr + + +def corr_to_cov(corr: np.ndarray, std: np.ndarray) -> np.ndarray: + """Convert correlation matrix to covariance matrix.""" + return corr * np.outer(std, std) + + +def denoise_cov(cov0: np.ndarray, q: float, bandwidth: float = 0.01) -> np.ndarray: + """ + De-noises a covariance matrix. + + Parameters + ---------- + cov0 : np.ndarray + The original (noisy) covariance matrix. + q : float + The T/N ratio. + bandwidth : float, default=0.01 + The KDE bandwidth. + + Returns + ------- + np.ndarray + The de-noised covariance matrix. + """ + + corr0 = cov_to_corr(cov0) + + # --- FIX 3: Use pca helper to get DESCENDING eigenvalues/vectors --- + eigenvalues, eigenvectors = pca(corr0) + eigenvalues_diag = np.diag(eigenvalues) # 2D diag matrix (desc) + + # Find the noise cutoff + # --- FIX 2: Pass bandwidth down to find_max_eval --- + emax0, var0 = find_max_eval(eigenvalues_diag, q, bandwidth) + + # --- FIX 3: Correctly find num factors as count of evals > emax0 --- + n_facts0 = np.sum(eigenvalues > emax0) + + # Denoise the correlation matrix + corr1 = denoised_corr(eigenvalues_diag, eigenvectors, n_facts0) + + # Convert back to covariance + cov1 = corr_to_cov(corr1, np.diag(cov0) ** 0.5) + return cov1 + + +def optimal_portfolio(cov: np.ndarray, mu: Optional[np.ndarray] = None) -> np.ndarray: + """ + Compute the optimal (e.g., minimum variance) portfolio weights. + + (Note: This is duplicated in `optimization/nco.py`) + + Parameters + ---------- + cov : np.ndarray + Covariance matrix. + mu : np.ndarray, optional + Vector of expected returns. If None, computes GMV portfolio. + + Returns + ------- + np.ndarray + The optimal portfolio weights. + """ + inv_cov = np.linalg.inv(cov) + ones = np.ones(shape=(inv_cov.shape[0], 1)) + + if mu is None: + mu = ones + + w = inv_cov @ mu + w /= ones.T @ w + return w.flatten() + + +def optimal_portfolio_denoised( + cov: np.ndarray, + q: float, + mu: Optional[np.ndarray] = None, + bandwidth: float = 0.01, +) -> np.ndarray: + """ + Compute the optimal portfolio weights from a denoised covariance matrix. + + Parameters + ---------- + cov : np.ndarray + The *original* (noisy) covariance matrix. + q : float + The T/N ratio. + mu : np.ndarray, optional + Vector of expected returns. + bandwidth : float, default=0.01 + The KDE bandwidth. + + Returns + ------- + np.ndarray + The optimal, denoised portfolio weights. + """ + # --- FIX 4: Pass bandwidth to denoise_cov --- + cov_denoised = denoise_cov(cov, q, bandwidth) + + # Compute optimal portfolio on the denoised matrix + return optimal_portfolio(cov_denoised, mu) diff --git a/src/RiskLabAI/data/denoise/nercome.py b/src/RiskLabAI/data/denoise/nercome.py new file mode 100644 index 0000000..9899e68 --- /dev/null +++ b/src/RiskLabAI/data/denoise/nercome.py @@ -0,0 +1,133 @@ +r""" +NERCOME: nonparametric eigenvalue-regularized covariance estimation (Lam 2016). + +de Prado denoises the covariance by Marcenko-Pastur eigenvalue clipping (`denoising.denoise_cov`), which +assumes a clean noise bulk separated from the signal eigenvalues by a gap. When the eigenvalue spectrum +has no clean gap (a slowly-decaying bulk) or is non-stationary, that assumption breaks and clipping +degenerates toward the raw sample covariance. NERCOME (Lam 2016) instead regularizes the eigenvalues by +sample-splitting: it estimates the eigenvectors on one split of the data and the oracle eigenvalues by +projecting the held-out split's sample covariance onto those eigenvectors, averaging over many random +splits. This recovers the covariance more accurately and with better conditioning on no-gap / +non-stationary spectra, and converges to clipping where the gap assumption holds. + +Admitted in Appraisal 24 (CONTRIBUTIONS_LEDGER 2026-06-27) - portfolio's first admitted extension. Regime +tag, verbatim from the verdict: + + prefer NERCOME over MP clipping for covariance estimation when the eigenvalue spectrum has no clean + gap or is non-stationary (better accuracy and conditioning, and lower OOS volatility via NCO / + min-variance); it converges to clipping on clean-gap stationary spectra. It costs more turnover and + concentration than clipping, gives no risk-adjusted-return edge (none does - 1/N stands), and HRP + does not benefit (use it through NCO / min-variance). + +Held-out confirmed (appraisals/24_results, HELDOUT.md): on the sealed no-gap x non-stationary x T=30 +corner NERCOME has distinguishably lower covariance error (relative Frobenius 0.563 vs MP clipping 0.684) +and better conditioning (29.6 vs 46.3), with lower NCO realized risk (1.315 vs 1.553); HRP is insensitive. +Unlike `denoise_cov`, which cleans a covariance matrix, NERCOME is a data-driven (sample-splitting) +estimator, so it takes the return matrix. Evidence and caveats: appraisals/24_verdict.md. + +References +---------- +Lam, C. (2016) Nonparametric eigenvalue-regularized precision or covariance matrix estimator. + The Annals of Statistics, 44(3), 928-953. +Lopez de Prado, M. (2018) Advances in Financial Machine Learning, ch. 2 (the MP-clipping baseline). +""" + +from __future__ import annotations + +import numpy as np + +from .denoising import corr_to_cov, cov_to_corr + + +def _ensure_positive_definite(matrix: np.ndarray, eps: float = 1e-10) -> np.ndarray: + """Project a symmetric matrix to the nearest PD one by flooring its eigenvalues.""" + matrix = (matrix + matrix.T) / 2.0 + values, vectors = np.linalg.eigh(matrix) + floor = ( + max(eps, eps * float(values.max())) if values.size and values.max() > 0 else eps + ) + values = np.clip(values, floor, None) + out = (vectors * values) @ vectors.T + return (out + out.T) / 2.0 + + +def nercome_denoised_covariance( + returns: np.ndarray, + n_splits: int = 50, + split_fraction: float = 2.0 / 3.0, + random_state: int | None = None, +) -> np.ndarray: + r""" + NERCOME denoised covariance (Lam 2016), estimated by sample-splitting from a return matrix. + + For each of ``n_splits`` random row permutations the ``T`` observations are split into two halves; the + eigenvectors ``P`` come from the first half's sample covariance, and the regularized eigenvalues are + the oracle projection ``d_i = p_i' S2 p_i`` of the second half's sample covariance ``S2`` onto those + eigenvectors. The estimator ``P diag(d) P'`` is averaged over the splits (each term is PSD, so the + average is PSD). The returns are standardized to unit sample variance, NERCOME-cleaned in correlation + space, then scaled back to a covariance by the sample column standard deviations. + + Prefer NERCOME over MP eigenvalue clipping when the eigenvalue spectrum has no clean gap or is + non-stationary (better accuracy and conditioning, and lower out-of-sample volatility via NCO / + min-variance); it converges to clipping on clean-gap stationary spectra, costs more turnover and + concentration, gives no risk-adjusted-return edge, and HRP does not benefit (use it through NCO / + min-variance). See the module docstring for the full verbatim regime tag and appraisals/24_verdict.md. + Unlike :func:`denoising.denoise_cov`, this takes the return matrix, not a covariance. + + Parameters + ---------- + returns : np.ndarray + The ``T x p`` matrix of asset returns (rows = observations, columns = assets). + n_splits : int, default 50 + Number of random sample-splits averaged. + split_fraction : float, default 2/3 + Fraction ``m / T`` of rows used for the eigenvector (first) half (Lam's ~2T/3 default). + random_state : int, optional + Seed for the split permutations. + + Returns + ------- + np.ndarray + The ``p x p`` NERCOME denoised covariance (symmetric positive-definite). + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> returns = rng.standard_normal((120, 8)) + >>> cov = nercome_denoised_covariance(returns, n_splits=20, random_state=0) + >>> cov.shape == (8, 8) and np.allclose(cov, cov.T) + True + """ + returns = np.asarray(returns, dtype=float) + if returns.ndim != 2: + raise ValueError("returns must be a 2-D (T x p) array") + n, p = returns.shape + if n < 4: + raise ValueError("NERCOME needs at least 4 observations") + std = returns.std(axis=0, ddof=1) + std = np.where(std <= 0, 1.0, std) + z = returns / std + + m = max(p + 1, int(round(split_fraction * n))) + m = min(m, n - 2) + rng = np.random.default_rng(random_state) + accumulated = np.zeros((p, p)) + used = 0 + for _ in range(n_splits): + order = rng.permutation(n) + first, second = order[:m], order[m:] + if len(second) < 2: + continue + cov_first = np.cov(z[first], rowvar=False) + cov_second = np.cov(z[second], rowvar=False) + _, vectors = np.linalg.eigh(cov_first) + oracle = np.einsum( + "ij,jk,ik->i", vectors.T, cov_second, vectors.T + ) # p_i' S2 p_i + oracle = np.clip(oracle, 0.0, None) + accumulated += (vectors * oracle) @ vectors.T + used += 1 + estimate = accumulated / max(used, 1) + correlation = cov_to_corr(_ensure_positive_definite(estimate)) + return corr_to_cov(correlation, std) diff --git a/src/RiskLabAI/data/differentiation/__init__.py b/src/RiskLabAI/data/differentiation/__init__.py new file mode 100644 index 0000000..877c5a7 --- /dev/null +++ b/src/RiskLabAI/data/differentiation/__init__.py @@ -0,0 +1,7 @@ +from .adaptive_differentiation import ( + adaptive_differencing_order, + adaptive_fractional_difference, + rescaled_range_hurst, + wavelet_variance_hurst, +) +from .differentiation import * diff --git a/src/RiskLabAI/data/differentiation/adaptive_differentiation.py b/src/RiskLabAI/data/differentiation/adaptive_differentiation.py new file mode 100644 index 0000000..6eeaa2c --- /dev/null +++ b/src/RiskLabAI/data/differentiation/adaptive_differentiation.py @@ -0,0 +1,268 @@ +r""" +Adaptive Fractional Differencing (AFD), a bias-corrected differencing-order estimate. + +de Prado's fixed-width fractional differentiation (`differentiation.py`) finds the minimum order d that +makes a series stationary via an ADF search. On finite samples that min-d-via-ADF is biased and +under-differences: the under-powered ADF test passes while the series still carries spurious residual +long-memory weights, so the recovered order is too small. AFD instead estimates the differencing order +directly from the data's long-memory structure, with a finite-sample bias correction, and so recovers +the genuine order far more accurately when memory is strong and the sample is finite. + +Admitted in Appraisal 13 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tag, verbatim from the verdict: + + prefer AFD over fixed-width FFD when the differencing order itself must be right - strong long memory + and finite samples, where min-d-via-ADF under-differences; on weak memory the gap narrows. The + implemented AFD is a tractable clean-room approximation of the published wavelet-Hurst + ridge + + CV-truncation method (no public code). + +The real-data predictive-lift confirmation is a tracked obligation (``REAL_DATA_FOLLOWUPS.md``): the +order accuracy is established against ground truth, but the downstream predictive value of the more +accurate order is not yet decisive. Evidence and caveats: appraisals/13_verdict.md (and 13b if logged). + +This is a tractable clean-room approximation of the published AFD (IEEE Access 2025; no public code): +the differencing order is anchored at ``0.5 + d_hat`` where ``d_hat`` is a finite-sample-bias-corrected +Hurst estimate of the increments (a ridge-style equal-weight blend of a wavelet-variance and an R/S +Hurst estimate, which lowers the variance/bias of either alone), the fixed-width-FFD truncation is +chosen by cross-validation over weight thresholds, and the order is raised to the ADF stationarity +boundary if not yet stationary. ``pywt`` (PyWavelets) is an optional dependency: when present AFD uses +the wavelet + R/S Hurst blend, otherwise the R/S estimate alone (the blend is the validated default). + +References +---------- +Adaptive Fractional Differencing (2025) IEEE Access. (Clean-room approximation; no public source.) +Lopez de Prado, M. (2018) Advances in Financial Machine Learning, ch. 5 (the FFD baseline). +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +from statsmodels.tsa.stattools import adfuller + +from .differentiation import fractional_difference_fixed_single + +try: # optional wavelet dependency (enables the wavelet-variance Hurst component of the blend) + import pywt + + _HAS_PYWT = True +except ( + ImportError +): # pragma: no cover - exercised only in environments without PyWavelets + _HAS_PYWT = False + +DEFAULT_DELTA_GRID = [round(i * 0.05, 3) for i in range(0, 25)] # 0.00 .. 1.20 +DEFAULT_CV_THRESHOLDS = (1e-3, 1e-4, 1e-5) +DEFAULT_ADF_SIGNIFICANCE = 0.05 +DEFAULT_ADF_MAXLAG = 1 + + +def wavelet_variance_hurst( + series: np.ndarray, wavelet: str = "db2", skip_fine: int = 1 +) -> float: + r""" + Wavelet-variance Hurst estimate: :math:`\log_2 \mathrm{Var}(\text{detail}_j) \approx (2H-1)\,j`. + + Returns ``nan`` if ``pywt`` is not installed or the series is too short to form enough octaves. + """ + if not _HAS_PYWT: + return float("nan") + x = np.asarray(series, dtype=float) + x = x - x.mean() + n = x.size + dec_len = pywt.Wavelet(wavelet).dec_len + max_level = pywt.dwt_max_level(n, dec_len) + level = max(3, min(max_level, int(np.log2(n)) - 2)) + if level < 3: + return float("nan") + coefficients = pywt.wavedec(x, wavelet, level=level) + details = coefficients[1:] # coarsest .. finest + octaves = np.arange(level, 0, -1) + variances = np.array([np.mean(d**2) for d in details]) + mask = octaves >= (1 + skip_fine) + j = octaves[mask].astype(float) + v = np.where(variances[mask] <= 0, 1e-12, variances[mask]) + if j.size < 2: + return float("nan") + slope = np.polyfit(j, np.log2(v), 1)[0] + return (slope + 1.0) / 2.0 + + +def rescaled_range_hurst(series: np.ndarray) -> float: + """Classical rescaled-range (R/S) Hurst estimate (the log-log slope of R/S against window size).""" + x = np.asarray(series, dtype=float) + n = x.size + if n < 32: + return float("nan") + sizes = np.unique( + np.floor(np.logspace(np.log10(8), np.log10(n // 2), 8)).astype(int) + ) + rs_means, used = [], [] + for s in sizes: + k = n // s + if k < 1: + continue + values = [] + for i in range(k): + segment = x[i * s : (i + 1) * s] + deviations = np.cumsum(segment - segment.mean()) + spread = deviations.max() - deviations.min() + scale = segment.std() + if scale > 0: + values.append(spread / scale) + if values: + rs_means.append(np.mean(values)) + used.append(s) + if len(used) < 2: + return float("nan") + return float(np.polyfit(np.log(used), np.log(rs_means), 1)[0]) + + +def adaptive_differencing_order(increments: np.ndarray) -> float: + r""" + Bias-corrected memory order :math:`\hat{d}` of a series of increments. + + A ridge-style equal-weight blend of the wavelet-variance and R/S Hurst estimates, + :math:`\hat{d} = \bar{H} - 0.5`, clipped to ``[0, 0.99]``. Falls back to the R/S estimate alone if + ``pywt`` is unavailable. Clean-room approximation of the published wavelet-Hurst + ridge procedure. + """ + h_wavelet = wavelet_variance_hurst(increments) + h_rs = rescaled_range_hurst(increments) + estimates = [h for h in (h_wavelet, h_rs) if np.isfinite(h)] + if not estimates: + return float("nan") + h_blend = float(np.mean(estimates)) + return float(np.clip(h_blend - 0.5, 0.0, 0.99)) + + +def _adf_pvalue(values: np.ndarray, maxlag: int) -> float: + v = np.asarray(values, dtype=float) + v = v[~np.isnan(v)] + if v.size < 20 or np.allclose(v.std(), 0.0): + return 1.0 + try: + return float(adfuller(v, maxlag=maxlag, regression="c", autolag=None)[1]) + except Exception: # pragma: no cover + return 1.0 + + +def _memory_retained(differenced: pd.Series, level: pd.Series) -> float: + common = differenced.dropna() + if len(common) < 5: + return float("nan") + level_aligned = level.reindex(common.index) + ok = (~common.isna()) & (~level_aligned.isna()) + if ok.sum() < 5 or common[ok].std() == 0 or level_aligned[ok].std() == 0: + return float("nan") + return float( + abs(np.corrcoef(common[ok].to_numpy(), level_aligned[ok].to_numpy())[0, 1]) + ) + + +def adaptive_fractional_difference( + series: pd.Series, + delta_grid: list | None = None, + cv_thresholds: tuple = DEFAULT_CV_THRESHOLDS, + adf_significance: float = DEFAULT_ADF_SIGNIFICANCE, + adf_maxlag: int = DEFAULT_ADF_MAXLAG, +) -> dict: + r""" + Adaptive Fractional Differencing of a (price/level) series. + + Estimates the memory order from the series increments with a bias-corrected Hurst blend + (:func:`adaptive_differencing_order`), anchors the differencing order at that estimate plus one + (the integration order of a price), chooses the fixed-width-FFD weight-truncation threshold by + cross-validation (the threshold that retains the most memory subject to ADF stationarity), and + raises the order along ``delta_grid`` only as far as the ADF stationarity boundary if needed. + + Prefer AFD over fixed-width FFD when the differencing order itself must be right (strong long memory, + finite samples), where de Prado's min-d-via-ADF under-differences; on weak memory the gap narrows. + See the module docstring for the full regime tag, the clean-room-approximation note, the tracked + real-data follow-up, and appraisals/13_verdict.md. + + Parameters + ---------- + series : pd.Series + The price / level series to difference. + delta_grid : list, optional + Ascending grid of candidate orders. Defaults to 0.00 .. 1.20 in steps of 0.05. + cv_thresholds : tuple, default (1e-3, 1e-4, 1e-5) + The FFD weight-truncation thresholds searched by cross-validation. + adf_significance : float, default 0.05 + The ADF p-value below which the series is judged stationary. + adf_maxlag : int, default 1 + The ADF lag (matches the repo's fixed-width FFD search). + + Returns + ------- + dict + ``order`` (the differencing order applied), ``d_hat`` (the bias-corrected memory order of the + increments), ``adf_pvalue``, ``memory_retained`` (abs correlation of the differenced series with + the level), ``threshold`` (the chosen truncation), and ``series`` (the differenced pd.Series). + + Examples + -------- + >>> import numpy as np, pandas as pd + >>> rng = np.random.default_rng(0) + >>> price = pd.Series(np.cumsum(rng.standard_normal(500))) # an integrated series + >>> result = adaptive_fractional_difference(price) + >>> 0.0 <= result["order"] <= 2.0 and result["adf_pvalue"] <= 1.0 + True + """ + if delta_grid is None: + delta_grid = DEFAULT_DELTA_GRID + level = series.dropna() + increments = level.diff().dropna().to_numpy() + d_hat_increment = adaptive_differencing_order(increments) + if not np.isfinite(d_hat_increment): + return { + "order": float("nan"), + "d_hat": float("nan"), + "adf_pvalue": 1.0, + "memory_retained": float("nan"), + "threshold": float("nan"), + "series": pd.Series(dtype=float), + } + order_anchor = float(np.clip(0.5 + d_hat_increment, 0.0, max(delta_grid))) + + # CV truncation: the threshold with max retained memory among stationary diffs at the anchor order. + best_threshold, best = cv_thresholds[1], None + for threshold in cv_thresholds: + diff = fractional_difference_fixed_single( + series, order_anchor, threshold + ).dropna() + if len(diff) < 20: + continue + pvalue = _adf_pvalue(diff.to_numpy(), adf_maxlag) + memory = _memory_retained(diff, series) + candidate = ( + pvalue < adf_significance, + memory if np.isfinite(memory) else -1.0, + threshold, + pvalue, + diff, + ) + if best is None or (candidate[0], candidate[1]) > (best[0], best[1]): + best = candidate + if best is not None: + best_threshold = best[2] + + # Anchor at the order estimate; raise it upward only to reach the stationarity boundary. + order = order_anchor + diff = fractional_difference_fixed_single(series, order, best_threshold).dropna() + pvalue = _adf_pvalue(diff.to_numpy(), adf_maxlag) if len(diff) >= 20 else 1.0 + grid_up = [g for g in delta_grid if g > order_anchor] + while pvalue >= adf_significance and grid_up: + order = grid_up.pop(0) + diff = fractional_difference_fixed_single( + series, order, best_threshold + ).dropna() + pvalue = _adf_pvalue(diff.to_numpy(), adf_maxlag) if len(diff) >= 20 else 1.0 + + return { + "order": float(order), + "d_hat": float(d_hat_increment), + "adf_pvalue": float(pvalue), + "memory_retained": _memory_retained(diff, series), + "threshold": float(best_threshold), + "series": diff, + } diff --git a/src/RiskLabAI/data/differentiation/differentiation.py b/src/RiskLabAI/data/differentiation/differentiation.py new file mode 100644 index 0000000..8a7a049 --- /dev/null +++ b/src/RiskLabAI/data/differentiation/differentiation.py @@ -0,0 +1,399 @@ +""" +Implements fractional differentiation of time series, including both +expanding-window (standard) and fixed-width-window (FFD) methods. + +Includes functions to find the minimum differentiation factor 'd' that +results in a stationary series. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapter 5. +""" + +from typing import TYPE_CHECKING, Optional + +import numpy as np +import pandas as pd +from statsmodels.tsa.stattools import adfuller + +if TYPE_CHECKING: + import matplotlib.pyplot as plt # type hints only; optional at runtime + + +def calculate_weights_std(degree: float, size: int) -> np.ndarray: + """ + Compute weights for standard (expanding window) fractional differentiation. + + Reference: + Snippet 5.2, Page 82. + + Parameters + ---------- + degree : float + The degree of differentiation (d). + size : int + The number of weights to generate (e.g., the series length). + + Returns + ------- + np.ndarray + An array of weights, ordered for dot product (w_0 at the end). + """ + weights = [1.0] + for k in range(1, size): + weight = -weights[-1] / k * (degree - k + 1) + weights.append(weight) + + # Reverse for dot product: [w_k, ..., w_1, w_0] + return np.array(weights[::-1]).reshape(-1, 1) + + +def calculate_weights_ffd(degree: float, threshold: float = 1e-5) -> np.ndarray: + """ + Compute weights for Fixed-Width Window Fractional Differentiation (FFD). + + The weights are generated until they fall below the `threshold`. + + Reference: + Snippet 5.3, Page 83. + + Parameters + ---------- + degree : float + The degree of differentiation (d). + threshold : float, default=1e-5 + The minimum absolute weight to include. + + Returns + ------- + np.ndarray + An array of weights, ordered for dot product (w_0 at the end). + """ + weights = [1.0] + k = 1 + while True: + weight = -weights[-1] / k * (degree - k + 1) + if abs(weight) < threshold: + break + weights.append(weight) + k += 1 + + # Reverse for dot product: [w_k, ..., w_1, w_0] + return np.array(weights[::-1]).reshape(-1, 1) + + +def fractional_difference_std( + series: pd.DataFrame, degree: float, threshold: float = 0.01 +) -> pd.DataFrame: + """ + Compute the standard (expanding window) fractionally differentiated series. + + This method uses all available past data for each calculation. + + Reference: + Snippet 5.2, Page 82. + + Parameters + ---------- + series : pd.DataFrame + DataFrame of time series (one column per series). + degree : float + The degree of differentiation (d). + threshold : float, default=0.01 + The weight-loss threshold to determine the minimum number + of observations required (warm-up period). + + Returns + ------- + pd.DataFrame + DataFrame of fractionally differentiated series. + """ + # 1. Compute weights for the full series + weights = calculate_weights_std(degree, series.shape[0]) + + # 2. Determine warm-up period + weights_cumsum_abs = np.cumsum(np.abs(weights)) + weights_cumsum_abs /= weights_cumsum_abs[-1] + skip = np.searchsorted(weights_cumsum_abs, threshold) + + result_df = pd.DataFrame(index=series.index, columns=series.columns, dtype=float) + + # The expanding-window value result[t] = sum_j w_j * x[t - j] is exactly the + # first len(x) terms of a causal convolution of the series with the weights + # in natural order [w_0, w_1, ...]. Using np.convolve replaces the O(n^2) + # per-row growing-window dot product with a single O(n*k) pass per column, + # producing identical values (verified in test/test_performance.py). + weights_natural = weights[::-1].flatten() + + for name in series.columns: + # Use .ffill() - fillna(method=) is deprecated + series_ffill = series[[name]].ffill().dropna() + n_obs = series_ffill.shape[0] + if series_ffill.empty or n_obs < skip: + continue + + series_np = series_ffill.to_numpy().reshape(-1) + convolved = np.convolve(series_np, weights_natural[:n_obs])[:n_obs] + result_df.loc[series_ffill.index[skip:], name] = convolved[skip:] + + return result_df.dropna(how="all") + + +def fractional_difference_fixed( + series: pd.DataFrame, degree: float, threshold: float = 1e-5 +) -> pd.DataFrame: + """ + Compute the Fixed-Width Window (FFD) fractionally differentiated series. + + This method iterates over columns and applies a fast, convolved + FFD calculation to each one. + + Reference: + Snippet 5.3, Page 83. + + Parameters + ---------- + series : pd.DataFrame + DataFrame of time series (one column per series). + degree : float + The degree of differentiation (d). + threshold : float, default=1e-5 + Threshold to determine the fixed window width. + + Returns + ------- + pd.DataFrame + DataFrame of fractionally differentiated series. + """ + result_df = pd.DataFrame(index=series.index) + + for name in series.columns: + result_df[name] = fractional_difference_fixed_single( + series[name], degree, threshold + ) + + return result_df.dropna(how="all") + + +def fractional_difference_fixed_single( + series: pd.Series, degree: float, threshold: float = 1e-5 +) -> pd.Series: + """ + Compute the FFD series for a single `pd.Series` using np.convolve. + + Parameters + ---------- + series : pd.Series + Time series. + degree : float + The degree of differentiation (d). + threshold : float, default=1e-5 + Threshold to determine the fixed window width. + + Returns + ------- + pd.Series + The fractionally differentiated series. + """ + + # 1. Compute weights + # Reverse weights: calculate_weights_ffd returns [w_k, ..., w_0] + weights = calculate_weights_ffd(degree, threshold).flatten()[::-1] + width = len(weights) + + # 2. Prepare data (drop leading NaNs) + series_ffill = series.ffill().dropna() + + if series_ffill.empty or series_ffill.shape[0] < width: + # Not enough data to convolve + return pd.Series(index=series.index, dtype="float64") + + series_np = series_ffill.to_numpy() + + # 3. Apply convolution + # 'valid' mode computes the dot product only where the + # window fully overlaps the series. + convolved_vals = np.convolve(series_np, weights, mode="valid") + + # 4. Create result series on the valid index + # The result aligns with the *end* of the window + result_index = series_ffill.index[width - 1 :] + valid_results = pd.Series(convolved_vals, index=result_index) + + # 5. Reindex to original full index + # This correctly places NaNs at the start (warm-up) + # and in any gaps that were in the original series. + return valid_results.reindex(series.index) + + +def plot_weights( + degree_range: tuple[float, float], + number_degrees: int, + size: int, + ax: Optional["plt.Axes"] = None, +) -> "plt.Axes": + """ + Plot the weights of fractionally differentiated series for various degrees. + + Parameters + ---------- + degree_range : Tuple[float, float] + (min_degree, max_degree) to plot. + number_degrees : int + Number of 'd' values to plot in the range. + size : int + Length of the time series (number of weights). + ax : plt.Axes, optional + Matplotlib axes to plot on. If None, a new figure/axes is created. + + Returns + ------- + plt.Axes + The axes object with the plot. + """ + import matplotlib.pyplot as plt # optional dependency: RiskLabAI[plot] + + if ax is None: + fig, ax = plt.subplots(figsize=(10, 6)) + + weights_df = pd.DataFrame() + for degree in np.linspace(degree_range[0], degree_range[1], number_degrees): + degree = round(degree, 2) + weights = calculate_weights_std(degree, size) + weights_df[degree] = pd.Series(weights.flatten(), index=range(size - 1, -1, -1)) + + weights_df.plot(ax=ax) + ax.set_xlabel("Lag") + ax.set_ylabel("Weight") + ax.set_title("Fractional Differentiation Weights") + ax.legend(title="Degree (d)") + return ax + + +def find_optimal_ffd_simple( + input_series: pd.DataFrame, p_value_threshold: float = 0.05 +) -> pd.DataFrame: + """ + Find the minimum 'd' that passes the ADF test, for a range of d. + + This function tests `d` in 11 steps from 0 to 1. + + Reference: + Snippet 5.4, Page 85. + + Parameters + ---------- + input_series : pd.DataFrame + DataFrame containing a 'close' column. + p_value_threshold : float, default=0.05 + The ADF test p-value to pass. + + Returns + ------- + pd.DataFrame + DataFrame of ADF test results for each 'd'. + """ + results_list = [] + + # Resample to daily to ensure consistent lags + series_daily = np.log(input_series[["close"]]).resample("1D").last().dropna() + + for d in np.linspace(0, 1, 11): + differentiated = fractional_difference_fixed( + series_daily, d, threshold=0.01 + ).dropna() + + if differentiated.empty: + continue + + corr = np.corrcoef( + series_daily.loc[differentiated.index, "close"], differentiated["close"] + )[0, 1] + + try: + adf_result = adfuller( + differentiated["close"], maxlag=1, regression="c", autolag=None + ) + + results_list.append( + { + "d": d, + "adfStat": adf_result[0], + "pVal": adf_result[1], + "lags": adf_result[2], + "nObs": adf_result[3], + "95% conf": adf_result[4]["5%"], + "corr": corr, + } + ) + except Exception: + # Handle cases where ADF test fails (e.g., insufficient data) + continue + + if not results_list: + return pd.DataFrame() + + return pd.DataFrame(results_list).set_index("d") + + +def fractionally_differentiated_log_price( + input_series: pd.Series, + threshold: float = 1e-5, + step: float = 0.01, + p_value_threshold: float = 0.05, +) -> pd.Series: + """ + Find the minimum 'd' that makes a log-price series stationary. + + This function iteratively increases 'd' by 'step' until the ADF test + p-value falls below `p_value_threshold`. + + Parameters + ---------- + input_series : pd.Series + Time series of prices. + threshold : float, default=1e-5 + The weight threshold for FFD. + step : float, default=0.01 + The increment for testing 'd'. + p_value_threshold : float, default=0.05 + The significance level for the ADF test. + + Returns + ------- + pd.Series + The fractionally differentiated series with the minimum + stationary-passing 'd'. + """ + log_price = np.log(input_series) + degree = 0.0 + p_value = 1.0 + + differentiated_series = None + + while p_value > p_value_threshold: + degree += step + if degree > 2.0: # Safety break + raise ValueError("Failed to find stationary 'd' < 2.0") + + differentiated = fractional_difference_fixed_single( + log_price, degree, threshold=threshold + ).dropna() + + if differentiated.empty: + continue # Not enough data for this 'd' + + try: + adf_test = adfuller(differentiated, maxlag=1, regression="c", autolag=None) + p_value = adf_test[1] + except Exception: + p_value = 1.0 # Failed test, keep going + + if differentiated_series is None: + differentiated_series = differentiated # Store first valid series + + # Return the last computed series that passed + if differentiated_series is None: + raise ValueError("Could not generate any differentiated series.") + + return differentiated diff --git a/src/RiskLabAI/data/distance/__init__.py b/src/RiskLabAI/data/distance/__init__.py new file mode 100644 index 0000000..f2260bc --- /dev/null +++ b/src/RiskLabAI/data/distance/__init__.py @@ -0,0 +1,2 @@ +from .codependence import distance_correlation, ksg_mutual_information +from .distance_metric import * diff --git a/src/RiskLabAI/data/distance/codependence.py b/src/RiskLabAI/data/distance/codependence.py new file mode 100644 index 0000000..a9fdb48 --- /dev/null +++ b/src/RiskLabAI/data/distance/codependence.py @@ -0,0 +1,188 @@ +r""" +Nonparametric codependence kernels: the KSG mutual-information estimator and distance correlation. + +de Prado's codependence kernel (`distance_metric.calculate_mutual_information`) estimates mutual +information from a binned (histogram) joint distribution, whose bias and variance grow as the sample +shrinks relative to the bin count and which depends on an arbitrary bin choice. These two estimators +are binning-free alternatives for the codependence-kernel role of the variation-of-information metric +and ONC clustering: + +- the Kraskov-Stogbauer-Grassberger (KSG) k-nearest-neighbour mutual-information estimator, which uses + adaptive kNN distances rather than a fixed grid, so it is far less biased on short, noisy, or + nonlinear / heavy-tailed samples; and +- distance correlation, a tuning-free dependence index in [0, 1] that is zero only at independence and + detects nonlinear dependence with no estimation parameter. + +Admitted in Appraisal 11 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tags, verbatim from the verdict: + + KSG: prefer KSG over binned MI/VI on short, noisy, or nonlinear/heavy-tailed samples - it is far + less biased there and is essentially unbiased on linear dependence; on large samples with simple + near-linear dependence the binned estimator is at least as good, so KSG converges rather than + dominating everywhere. Use raw KSG; the surrogate de-bias adds nothing. + + Distance correlation: a tuning-free nonlinear dependence screen / codependence kernel - preferred + when a parameter-free nonlinear screen or maximally stable clustering is wanted (best real-data + cluster stability). It is a dependence index, not a metric on partitions like VI (keep VI/KSG for + the metric role). + +Evidence and caveats: appraisals/11_verdict.md. + +References +---------- +Kraskov, A., Stogbauer, H. and Grassberger, P. (2004) Estimating mutual information. + Physical Review E, 69(6), 066138. +Szekely, G. J., Rizzo, M. L. and Bakirov, N. K. (2007) Measuring and testing dependence by + correlation of distances. Annals of Statistics, 35(6), 2769-2794. +""" + +from __future__ import annotations + +import numpy as np +from scipy.spatial import cKDTree +from scipy.special import digamma + + +def _jitter(values: np.ndarray, rng: np.random.Generator) -> np.ndarray: + """Break exact ties with negligible data-scaled noise so kNN distances are well defined.""" + values = np.asarray(values, dtype=float).ravel() + scale = np.std(values) + if scale == 0.0: + scale = 1.0 + return values + rng.standard_normal(values.shape) * scale * 1e-10 + + +def ksg_mutual_information( + x: np.ndarray, + y: np.ndarray, + k: int = 4, + random_state: int | None = 0, +) -> float: + r""" + Kraskov-Stogbauer-Grassberger mutual information (algorithm 1), in nats. + + Uses the Chebyshev (max) norm in the joint space; for each point the distance to its k-th joint + neighbour sets the marginal search radius, and the marginal neighbour counts ``n_x``, ``n_y`` enter + the digamma estimator + + .. math:: + \hat{I}(X;Y) = \psi(k) + \psi(N) - \langle \psi(n_x+1) + \psi(n_y+1) \rangle . + + Being binning-free it is far less biased than histogram MI on short / nonlinear / heavy-tailed + samples, and essentially unbiased on linear dependence; on large near-linear samples binned MI is at + least as good, so prefer KSG in the former regime. It can return a slightly negative value for + (near-)independent data, which is a characterized property of the estimator, not an error. See the + module docstring for the full regime tag and appraisals/11_verdict.md. + + Parameters + ---------- + x, y : np.ndarray + The two 1-D samples (equal length). + k : int, default 4 + Number of nearest neighbours. + random_state : int, optional + Seed for the tie-breaking jitter (deterministic by default). + + Returns + ------- + float + The estimated mutual information in nats. + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> z = rng.standard_normal(2000) + >>> x = z + 0.1 * rng.standard_normal(2000) + >>> y = z + 0.1 * rng.standard_normal(2000) + >>> ksg_mutual_information(x, y) > 0.5 # strong dependence + True + """ + rng = np.random.default_rng(random_state) + x = _jitter(x, rng) + y = _jitter(y, rng) + n = x.size + if n < 2: + return 0.0 + k = min(k, n - 1) + z = np.column_stack([x, y]) + joint = cKDTree(z) + distances, _ = joint.query(z, k=k + 1, p=np.inf) # k+1 to exclude self + eps = distances[:, -1] + tree_x = cKDTree(x.reshape(-1, 1)) + tree_y = cKDTree(y.reshape(-1, 1)) + radius = eps * (1.0 - 1e-10) + n_x = ( + np.asarray( + tree_x.query_ball_point( + x.reshape(-1, 1), radius, p=np.inf, return_length=True + ) + ) + - 1 + ) + n_y = ( + np.asarray( + tree_y.query_ball_point( + y.reshape(-1, 1), radius, p=np.inf, return_length=True + ) + ) + - 1 + ) + n_x = np.maximum(n_x, 0) + n_y = np.maximum(n_y, 0) + mi = digamma(k) + digamma(n) - np.mean(digamma(n_x + 1) + digamma(n_y + 1)) + return float(mi) + + +def _double_center(distance_matrix: np.ndarray) -> np.ndarray: + row = distance_matrix.mean(axis=1, keepdims=True) + col = distance_matrix.mean(axis=0, keepdims=True) + return distance_matrix - row - col + distance_matrix.mean() + + +def distance_correlation(x: np.ndarray, y: np.ndarray) -> float: + r""" + Distance correlation (Szekely-Rizzo-Bakirov 2007), a dependence index in [0, 1]. + + Computed from the double-centered pairwise-distance matrices: with + :math:`A = \mathrm{centered}(|x_i - x_j|)` and :math:`B = \mathrm{centered}(|y_i - y_j|)`, + + .. math:: + \mathrm{dCor} = \sqrt{\frac{\overline{A \cdot B}}{\sqrt{\overline{A \cdot A}\,\overline{B \cdot B}}}}. + + It is zero only at population independence and detects nonlinear dependence with no estimation + parameter, so it is a tuning-free nonlinear dependence screen / codependence kernel. It is a + dependence index, not a metric on partitions like the variation of information (keep VI / KSG for the + metric role). See the module docstring for the full regime tag and appraisals/11_verdict.md. + + Parameters + ---------- + x, y : np.ndarray + The two 1-D samples (equal length). + + Returns + ------- + float + The distance correlation in [0, 1]. + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> x = rng.standard_normal(500) + >>> y = x ** 2 + 0.1 * rng.standard_normal(500) # nonlinear dependence + >>> distance_correlation(x, y) > 0.3 + True + """ + x = np.asarray(x, float).ravel() + y = np.asarray(y, float).ravel() + a = np.abs(x[:, None] - x[None, :]) + b = np.abs(y[:, None] - y[None, :]) + centered_a = _double_center(a) + centered_b = _double_center(b) + dcov2 = (centered_a * centered_b).mean() + dvar_x = (centered_a * centered_a).mean() + dvar_y = (centered_b * centered_b).mean() + denom = np.sqrt(dvar_x * dvar_y) + if denom <= 0: + return 0.0 + return float(np.sqrt(max(dcov2, 0.0) / denom)) diff --git a/src/RiskLabAI/data/distance/distance_metric.py b/src/RiskLabAI/data/distance/distance_metric.py new file mode 100644 index 0000000..0549235 --- /dev/null +++ b/src/RiskLabAI/data/distance/distance_metric.py @@ -0,0 +1,304 @@ +""" +Implements information-theoretic and other distance metrics for +financial data analysis. + +Includes: +- Variation of Information (VI) +- Mutual Information (MI) +- Optimal number of bins calculation +- Angular distance + +Reference: + De Prado, M. (2020) Advances in financial machine learning. + John Wiley & Sons, Chapter 3. +""" + +from typing import Optional + +import numpy as np +import scipy.stats as ss +from sklearn.metrics import mutual_info_score + + +def calculate_variation_of_information( + x: np.ndarray, y: np.ndarray, bins: int, norm: bool = False +) -> float: + """ + Calculate the Variation of Information (VI). + + VI(X, Y) = H(X) + H(Y) - 2 * I(X, Y) + + Reference: + Snippet 3.2, Page 44. + + Parameters + ---------- + x : np.ndarray + First data array. + y : np.ndarray + Second data array. + bins : int + Number of bins for the 2D histogram. + norm : bool, default=False + If True, normalize VI to [0, 1] by dividing by + the joint entropy H(X, Y). + + Returns + ------- + float + The Variation of Information. + """ + histogram_xy = np.histogram2d(x, y, bins)[0] + mutual_information = mutual_info_score(None, None, contingency=histogram_xy) + + marginal_x = ss.entropy(histogram_xy.sum(axis=1)) + marginal_y = ss.entropy(histogram_xy.sum(axis=0)) + + variation_xy = marginal_x + marginal_y - 2 * mutual_information + + if norm: + joint_xy = marginal_x + marginal_y - mutual_information + if joint_xy == 0: + return 0.0 # Avoid division by zero if entropies are 0 + variation_xy /= joint_xy + + return variation_xy + + +def calculate_number_of_bins( + num_observations: int, correlation: Optional[float] = None +) -> int: + """ + Calculate the optimal number of bins for discretization. + + Uses a formula optimized for either univariate (correlation=None) + or bivariate (correlation is provided) cases. + + Reference: + Snippet 3.3, Page 46. + + Parameters + ---------- + num_observations : int + Number of observations (N). + correlation : float, optional + Correlation between the two series. If None, + uses the univariate formula. + + Returns + ------- + int + The optimal number of bins. + """ + if correlation is None: + # Univariate formula + z = ( + 8 + + 324 * num_observations + + 12 * (36 * num_observations + 729 * num_observations**2) ** 0.5 + ) ** (1 / 3.0) + bins = round(z / 6.0 + 2.0 / (3 * z) + 1.0 / 3) + else: + if np.isclose(correlation, 1.0) or np.isclose(correlation, -1.0): + # Handle perfect correlation case by setting to almost 1 + correlation = np.sign(correlation) * (1.0 - 1e-10) + + if (1.0 - correlation**2) == 0: + # Handle numerical instability if correlation is still 1 + return calculate_number_of_bins(num_observations, correlation=None) + + # Bivariate formula + bins = round( + (2**-0.5) + * (1 + (1 + 24 * num_observations / (1.0 - correlation**2)) ** 0.5) ** 0.5 + ) + return int(bins) + + +def calculate_variation_of_information_extended( + x: np.ndarray, y: np.ndarray, norm: bool = False +) -> float: + """ + Calculate Variation of Information with an optimal number of bins. + + This function first calculates the optimal number of bins + using `calculate_number_of_bins` before computing VI. + + Reference: + Snippet 3.3, Page 46. + + Parameters + ---------- + x : np.ndarray + First data array. + y : np.ndarray + Second data array. + norm : bool, default=False + If True, normalize the VI. + + Returns + ------- + float + The Variation of Information. + """ + correlation = np.corrcoef(x, y)[0, 1] + num_bins = calculate_number_of_bins(x.shape[0], correlation=correlation) + + return calculate_variation_of_information(x, y, num_bins, norm) + + +def calculate_mutual_information( + x: np.ndarray, y: np.ndarray, norm: bool = False +) -> float: + """ + Calculate Mutual Information (MI) with an optimal number of bins. + + MI(X, Y) = H(X) + H(Y) - H(X, Y) + Normalized MI = MI(X, Y) / min(H(X), H(Y)) + + Reference: + Snippet 3.4, Page 48. + + Parameters + ---------- + x : np.ndarray + First data array. + y : np.ndarray + Second data array. + norm : bool, default=False + If True, normalize the MI to [0, 1]. + + Returns + ------- + float + The Mutual Information. + """ + correlation = np.corrcoef(x, y)[0, 1] + num_bins = calculate_number_of_bins(x.shape[0], correlation=correlation) + + histogram_xy = np.histogram2d(x, y, num_bins)[0] + mutual_information = mutual_info_score(None, None, contingency=histogram_xy) + + if norm: + marginal_x = ss.entropy(histogram_xy.sum(axis=1)) + marginal_y = ss.entropy(histogram_xy.sum(axis=0)) + + min_entropy = min(marginal_x, marginal_y) + if min_entropy == 0: + return 0.0 # Avoid division by zero + + mutual_information /= min_entropy + + return mutual_information + + +def calculate_distance(dependence: np.ndarray, metric: str = "angular") -> np.ndarray: + r""" + Calculate a distance matrix from a dependence matrix (e.g., correlation). + + Metrics: + - 'angular': \( D_A(i, j) = \sqrt{0.5 (1 - \rho_{ij})} \) + - 'absolute_angular': \( D_{AA}(i, j) = \sqrt{0.5 (1 - |\rho_{ij}|)} \) + + Parameters + ---------- + dependence : np.ndarray + The dependence (correlation) matrix. + metric : str, default="angular" + The metric to use: "angular" or "absolute_angular". + + Returns + ------- + np.ndarray + The resulting distance matrix. + """ + # Clip to handle potential floating point errors + dependence = np.clip(dependence, -1.0, 1.0) + + if metric == "angular": + distance = ((1 - dependence).round(6) / 2.0) ** 0.5 + elif metric == "absolute_angular": + distance = ((1 - np.abs(dependence)).round(6) / 2.0) ** 0.5 + else: + raise ValueError(f"Unknown metric: {metric}") + return distance + + +def calculate_kullback_leibler_divergence(p: np.ndarray, q: np.ndarray) -> float: + """ + Calculate Kullback-Leibler (KL) divergence. + + D_KL(P || Q) = -sum[ p_i * log(q_i / p_i) ] + = sum[ p_i * log(p_i / q_i) ] + + Parameters + ---------- + p : np.ndarray + The "true" probability distribution. + q : np.ndarray + The "approximating" probability distribution. + + Returns + ------- + float + The KL divergence. + """ + # Ensure probabilities sum to 1 + p = p / np.sum(p) + q = q / np.sum(q) + + # Filter for terms where p > 0 and q > 0 + # Where p_i = 0, the term is 0. + # Where q_i = 0 (and p_i > 0), the term is +inf. + mask = (p > 0) & (q > 0) + p_filtered = p[mask] + q_filtered = q[mask] + + if len(p_filtered) == 0: + return 0.0 # No overlapping support + + # Check if any p_i > 0 corresponds to q_i = 0 + if np.any(p[q == 0] > 0): + return np.inf + + divergence = -np.sum(p_filtered * np.log(q_filtered / p_filtered)) + return divergence + + +def calculate_cross_entropy(p: np.ndarray, q: np.ndarray) -> float: + """ + Calculate the cross-entropy between two distributions. + + H(P, Q) = -sum[ p_i * log(q_i) ] + + Parameters + ---------- + p : np.ndarray + The "true" probability distribution. + q : np.ndarray + The "approximating" probability distribution. + + Returns + ------- + float + The cross-entropy. + """ + p = p / np.sum(p) + q = q / np.sum(q) + + # Filter for terms where p > 0 and q > 0 + # Where p_i = 0, the term is 0. + # Where q_i = 0 (and p_i > 0), the term is +inf. + mask = (p > 0) & (q > 0) + p_filtered = p[mask] + q_filtered = q[mask] + + if len(p_filtered) == 0: + return 0.0 + + # Check if any p_i > 0 corresponds to q_i = 0 + if np.any(p[q == 0] > 0): + return np.inf + + entropy = -np.sum(p_filtered * np.log(q_filtered)) + return entropy diff --git a/src/RiskLabAI/data/labeling/__init__.py b/src/RiskLabAI/data/labeling/__init__.py new file mode 100644 index 0000000..fb5a3e8 --- /dev/null +++ b/src/RiskLabAI/data/labeling/__init__.py @@ -0,0 +1,49 @@ +""" +RiskLabAI Data Labeling Module + +Implements the core logic for financial time series labeling, including: +- The Triple-Barrier Method +- Meta-Labeling +- Trend-Scanning Labels +- CUSUM Filtering + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapters 3 & 4. +""" + +from .financial_labels import ( + calculate_t_value_linear_regression, + find_trend_using_trend_scanning, +) +from .labeling import ( + cusum_filter_events_dynamic_threshold, + daily_volatility_with_log_returns, + expand_call, + lin_parts, + meta_events, + meta_labeling, + process_jobs, + report_progress, + symmetric_cusum_filter, + triple_barrier, + vertical_barrier, +) + +__all__ = [ + # from labeling.py + "cusum_filter_events_dynamic_threshold", + "symmetric_cusum_filter", + "daily_volatility_with_log_returns", + "vertical_barrier", + "triple_barrier", + "meta_events", + "meta_labeling", + "lin_parts", + "process_jobs", + "expand_call", + "report_progress", + # from financial_labels.py + "calculate_t_value_linear_regression", + "find_trend_using_trend_scanning", +] diff --git a/src/RiskLabAI/data/labeling/financial_labels.py b/src/RiskLabAI/data/labeling/financial_labels.py new file mode 100644 index 0000000..295d349 --- /dev/null +++ b/src/RiskLabAI/data/labeling/financial_labels.py @@ -0,0 +1,137 @@ +""" +Implements the Trend-Scanning Labeling method. + +This method labels events based on the t-value of the linear regression +slope over various forward-looking windows, selecting the window +with the most significant trend. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapter 4. +""" + +import numpy as np +import pandas as pd +from scipy import stats + + +def calculate_t_value_linear_regression(prices: pd.Series) -> float: + """ + Calculate the t-value of the slope of a linear regression. + + Parameters + ---------- + prices : pd.Series + Time series of prices. + + Returns + ------- + float + The t-value of the regression slope. Returns np.nan if + regression fails (e.g., constant series or < 2 data points). + """ + # --- SUGGESTION: Explicitly handle insufficient data --- + if prices.shape[0] < 2: + return np.nan + + x = np.arange(prices.shape[0]) + try: + ols = stats.linregress(x, prices.values) + except ValueError: + return np.nan + + if ols.stderr == 0: + # Perfect trend: t -> +/- inf. Constant series: 0/0 -> undefined (NaN), + # consistent with the documented contract. + return np.sign(ols.slope) * np.inf if ols.slope != 0 else np.nan + + return ols.slope / ols.stderr + + +def find_trend_using_trend_scanning( + molecule: pd.Index, close: pd.Series, span: tuple[int, int] +) -> pd.DataFrame: + """ + Implement the trend-scanning method. + + For each event in `molecule`, this function scans forward over + various window lengths (defined by `span`). It computes the t-value + of the OLS slope for each window and identifies the window + that maximizes `|t_value|`. + + Reference: + Snippet 4.1, Page 69. + + Parameters + ---------- + molecule : pd.Index + The timestamps of the events to be labeled. + close : pd.Series + Time series of close prices. + span : Tuple[int, int] + A tuple of (min_span, max_span) defining the range of + window lengths to scan. `min_span` must be >= 2. + `max_span` is exclusive, as in `range()`. + + Returns + ------- + pd.DataFrame + DataFrame indexed by event, with columns: + - 'End Time': The timestamp of the *end* of the vertical barrier + (i.e., end of the max span). + - 't-Value': The t-value of the most significant trend found. + - 'Trend': The sign of the trend (-1, 0, or 1). + """ + outputs = pd.DataFrame(index=molecule, columns=["End Time", "t-Value", "Trend"]) + + # --- SUGGESTION: Add robustness check for span --- + # Ensure min_span < max_span and min_span is at least 2 for OLS. + if span[0] >= span[1] or span[0] < 2: + outputs["Trend"] = pd.to_numeric(outputs["Trend"], downcast="signed") + return outputs.dropna(subset=["Trend"]) + + spans = range(*span) + # Use span[1] - 1 directly. It's safer than max(spans) + # which fails on an empty range. + max_span_val = span[1] - 1 + + for index in molecule: + t_values = pd.Series(dtype="float64") + + try: + location = close.index.get_loc(index) + except KeyError: + continue # Event timestamp not in close index + + # Ensure we don't scan past the end of the series + if location + max_span_val >= close.shape[0]: + continue + + # Get timestamp for the vertical barrier (end of max span) + vertical_barrier_time = close.index[location + max_span_val] + + for span_val in spans: + # End of this specific window + tail_time = close.index[location + span_val - 1] + window_prices = close.loc[index:tail_time] + + t_values.loc[tail_time] = calculate_t_value_linear_regression(window_prices) + + if t_values.empty: + continue + + # Find the window end that maximized |t_value| + # Use idxmax on the absolute values, but get the original t-value + best_t_value_idx = t_values.replace([-np.inf, np.inf, np.nan], 0).abs().idxmax() + best_t_value = t_values[best_t_value_idx] + + outputs.loc[index] = [ + vertical_barrier_time, + best_t_value, + np.sign(best_t_value), + ] + + outputs["End Time"] = pd.to_datetime(outputs["End Time"]) + outputs["Trend"] = pd.to_numeric(outputs["Trend"], downcast="signed") + + return outputs.dropna(subset=["Trend"]) diff --git a/src/RiskLabAI/data/labeling/labeling.py b/src/RiskLabAI/data/labeling/labeling.py new file mode 100644 index 0000000..7a12491 --- /dev/null +++ b/src/RiskLabAI/data/labeling/labeling.py @@ -0,0 +1,453 @@ +""" +Implements the core financial labeling functions, including: +- CUSUM filters for event sampling. +- Volatility estimation. +- The Triple-Barrier Method (vertical and horizontal barriers). +- Meta-labeling. +- Re-exports of the parallel-execution helpers from ``RiskLabAI.hpc``. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapters 3 & 4. +""" + +from concurrent.futures import ProcessPoolExecutor +from typing import Optional + +import numpy as np +import pandas as pd + +# Parallel-processing helpers have a single source of truth in RiskLabAI.hpc. +# They were previously duplicated in this module; they are re-exported here +# under their historical names for backward compatibility. `lin_parts` maps to +# hpc's `linear_partitions` (identical for valid inputs, with an added guard). +from RiskLabAI.hpc import ( # noqa: F401 (re-exported for backward compat) + expand_call, + process_jobs, + report_progress, +) +from RiskLabAI.hpc import linear_partitions as lin_parts # noqa: F401 + + +def cusum_filter_events_dynamic_threshold( + prices: pd.Series, threshold: pd.Series +) -> pd.DatetimeIndex: + """ + Detect events using the Symmetric CUSUM filter with a dynamic threshold. + + This filter identifies timestamps where the cumulative sum of price + changes exceeds a dynamic, time-varying threshold. + + Reference: + Snippet 3.2, Page 48 (modified for dynamic threshold). + + Parameters + ---------- + prices : pd.Series + A pandas Series of prices. + threshold : pd.Series + A pandas Series containing the threshold values for each timestamp. + Must be aligned with the `prices` index. + + Returns + ------- + pd.DatetimeIndex + Timestamps of the detected events (when a barrier was touched). + """ + time_events = [] + shift_positive, shift_negative = 0.0, 0.0 + price_delta = prices.diff().dropna() + + # Align price changes with thresholds + price_delta, thresholds = price_delta.align(threshold, join="inner", copy=False) + + for (index, value), thresh_val in zip(price_delta.items(), thresholds.values): + shift_positive = max(0.0, shift_positive + value) + shift_negative = min(0.0, shift_negative + value) + + if shift_negative < -thresh_val: + shift_negative = 0.0 # Reset only this counter + time_events.append(index) + elif shift_positive > thresh_val: + shift_positive = 0.0 # Reset only this counter + time_events.append(index) + + return pd.DatetimeIndex(time_events) + + +def symmetric_cusum_filter(prices: pd.Series, threshold: float) -> pd.DatetimeIndex: + """ + Detect events using the Symmetric CUSUM filter with a fixed threshold. + + Reference: + Snippet 3.2, Page 48. + + Parameters + ---------- + prices : pd.Series + A pandas Series of prices. + threshold : float + The fixed threshold value. + + Returns + ------- + pd.DatetimeIndex + Timestamps of the detected events (when a barrier was touched). + """ + time_events = [] + shift_positive, shift_negative = 0.0, 0.0 + price_delta = prices.diff().dropna() + + for index, value in price_delta.items(): + shift_positive = max(0.0, shift_positive + value) + shift_negative = min(0.0, shift_negative + value) + + if shift_negative < -threshold: + shift_negative = 0.0 + time_events.append(index) + elif shift_positive > threshold: + shift_positive = 0.0 + time_events.append(index) + + return pd.DatetimeIndex(time_events) + + +def daily_volatility_with_log_returns(close: pd.Series, span: int = 100) -> pd.Series: + """ + Calculate daily volatility using log returns. + + This method computes the EWMA standard deviation of daily log returns. + + Reference: + Snippet 3.1, Page 44. + + Parameters + ---------- + close : pd.Series + Time series of close prices. + span : int, default=100 + The span parameter for the EWMA. + + Returns + ------- + pd.Series + Series of daily volatility estimates. + """ + # Find timestamps of 1 day prior + df = close.index.searchsorted(close.index - pd.Timedelta(days=1)) + df = df[df > 0] + df = pd.Series( + close.index[df - 1], + index=close.index[close.shape[0] - df.shape[0] :], + ) + + # Calculate log returns + returns = np.log(close.loc[df.index] / close.loc[df.values].values) + stds = returns.ewm(span=span).std().rename("std") + return stds + + +def vertical_barrier( + close: pd.Series, time_events: pd.DatetimeIndex, number_days: int +) -> pd.Series: + """ + Create a vertical barrier for the triple-barrier method. + + This function finds the timestamp that occurs `number_days` after + each event in `time_events`. + + Reference: + Snippet 3.4, Page 50. + + Parameters + ---------- + close : pd.Series + Time series of close prices (used for the index). + time_events : pd.DatetimeIndex + The timestamps of the events (e.g., from CUSUM filter). + number_days : int + The number of days to look forward for the vertical barrier. + + Returns + ------- + pd.Series + A Series where the index is the event timestamp and the + value is the vertical barrier timestamp. + """ + barrier_dates = time_events + pd.Timedelta(days=number_days) + + # Find the integer location of the barrier dates in the price index + timestamp_array = close.index.searchsorted(barrier_dates) + + # Filter out any indices that are out of bounds + valid_indices = timestamp_array < close.shape[0] + timestamp_array = timestamp_array[valid_indices] + + barrier_series = pd.Series( + close.index[timestamp_array], + index=time_events[valid_indices], + ) + return barrier_series + + +def triple_barrier( + close: pd.Series, + events: pd.DataFrame, + ptsl: list[float], + molecule: list[pd.Timestamp], +) -> pd.DataFrame: + """ + Apply the triple-barrier method for a subset of events. + + This is the core worker function for `meta_events`. It finds the + first touch time of the upper, lower, or vertical barrier. + + Reference: + Snippet 3.3, Page 50. + + Parameters + ---------- + close : pd.Series + Time series of close prices. + events : pd.DataFrame + DataFrame with event info. Must contain 'End Time' (vertical + barrier), 'Base Width' (volatility), and 'Side' (if applicable). + ptsl : List[float] + A list of two floats: [profit_taking_multiplier, stop_loss_multiplier]. + molecule : List[pd.Timestamp] + The subset of event timestamps this worker is responsible for. + + Returns + ------- + pd.DataFrame + A DataFrame with columns ['stop_loss', 'profit_taking'] + indexed by the event timestamp, indicating the first + touch time for each horizontal barrier. + """ + # Filter for events this worker owns + events_filtered = events.loc[molecule] + output = pd.DataFrame(index=events_filtered.index) + output["End Time"] = events_filtered["End Time"] # Use original end time + + # 1. Set horizontal barriers + if ptsl[0] > 0: + profit_taking = ptsl[0] * events_filtered["Base Width"] + else: + profit_taking = pd.Series(np.inf, index=events_filtered.index) + + if ptsl[1] > 0: + stop_loss = -ptsl[1] * events_filtered["Base Width"] + else: + stop_loss = pd.Series(-np.inf, index=events_filtered.index) + + # Get side if it exists, otherwise default to 1 (long) + side = events_filtered.get("Side", pd.Series(1.0, index=events_filtered.index)) + + # 2. Find first touch time. + # Positional numpy indexing replaces the per-event pandas label slicing and + # scalar `.loc` writes. For each event we slice the close-price array between + # the event start and its vertical barrier, compute side-adjusted log + # returns, and take the first index where each horizontal barrier is + # crossed. This yields the same first-touch timestamps as the previous + # implementation (verified in test/test_performance.py) but is ~10-70x + # faster on realistic sizes. + close_index = close.index + close_values = close.to_numpy(dtype=float) + close_index_values = close_index.values + + vertical_filled = events_filtered["End Time"].fillna(close_index[-1]) + start_positions = close_index.get_indexer(events_filtered.index) + end_positions = ( + close_index.searchsorted(vertical_filled.to_numpy(), side="right") - 1 + ) + + stop_loss_values = stop_loss.to_numpy() + profit_taking_values = profit_taking.to_numpy() + side_values = side.to_numpy(dtype=float) + + n_events = len(events_filtered) + stop_loss_touch = np.full(n_events, np.datetime64("NaT"), dtype="datetime64[ns]") + profit_taking_touch = np.full( + n_events, np.datetime64("NaT"), dtype="datetime64[ns]" + ) + + for i in range(n_events): + start, end = start_positions[i], end_positions[i] + segment = close_values[start : end + 1] + path_returns = np.log(segment / segment[0]) * side_values[i] + + below = path_returns < stop_loss_values[i] + if below.any(): + stop_loss_touch[i] = close_index_values[start + np.argmax(below)] + + above = path_returns > profit_taking_values[i] + if above.any(): + profit_taking_touch[i] = close_index_values[start + np.argmax(above)] + + # First barrier touched = earliest of {vertical, stop-loss, profit-taking}, + # ignoring NaT (the same semantics as the previous output.min(axis=1)). + candidates = pd.DataFrame( + np.vstack( + [ + events_filtered["End Time"].to_numpy().astype("datetime64[ns]"), + stop_loss_touch, + profit_taking_touch, + ] + ).T, + index=events_filtered.index, + ) + output["End Time"] = candidates.min(axis=1) + return output[["End Time"]] + + +def meta_events( + close: pd.Series, + time_events: pd.DatetimeIndex, + ptsl: list[float], + target: pd.Series, + return_min: float, + num_threads: int, + vertical_barrier_times: Optional[pd.Series] = None, + side: Optional[pd.Series] = None, +) -> pd.DataFrame: + """ + Generate events for meta-labeling using the triple-barrier method. + + This function sets up and executes the triple-barrier search in parallel. + + Reference: + Snippet 3.6, Page 51. + + Parameters + ---------- + close : pd.Series + Time series of close prices. + time_events : pd.DatetimeIndex + Timestamps of the events (e.g., from CUSUM filter). + ptsl : List[float] + [profit_taking_multiplier, stop_loss_multiplier]. + target : pd.Series + Series of volatility (or 'Base Width') for setting barrier width. + return_min : float + The minimum target volatility required to run a search. + num_threads : int + Number of parallel threads to use. + vertical_barrier_times : pd.Series, optional + A Series of vertical barrier timestamps. If None, barriers are not used. + side : pd.Series, optional + A Series of position sides (1 or -1). If None, assumes long-only. + + Returns + ------- + pd.DataFrame + The events DataFrame with columns: + - 'End Time': The timestamp of the *first* barrier touch. + - 'Base Width': The volatility used for the barriers. + - 'Side': (Optional) The side of the bet. + """ + # 1. Filter events by volatility + target = target.reindex(time_events) + target = target[target > return_min] + if target.empty: + return pd.DataFrame(columns=["End Time", "Base Width", "Side"]) + + # 2. Set up vertical barrier + if vertical_barrier_times is None: + vertical_barrier_times = pd.Series(pd.NaT, index=target.index) + else: + vertical_barrier_times = vertical_barrier_times.reindex(target.index) + + # 3. Set up sides + if side is None: + side_series = pd.Series(1.0, index=target.index) + ptsl_final = [ptsl[0], ptsl[0]] # Symmetric barriers + else: + side_series = side.reindex(target.index) + ptsl_final = ptsl[:2] # Asymmetric barriers + + # 4. Create base events DataFrame + events = pd.concat( + { + "End Time": vertical_barrier_times, + "Base Width": target, + "Side": side_series, + }, + axis=1, + ).dropna(subset=["Base Width"]) + + # 5. Run in parallel + molecule_subsets = np.array_split(events.index, num_threads) + + with ProcessPoolExecutor(max_workers=num_threads) as executor: + results = list( + executor.map( + triple_barrier, + [close] * num_threads, + [events] * num_threads, + [ptsl_final] * num_threads, + molecule_subsets, + ) + ) + + # Combine results and update End Time + first_touch_times = pd.concat(results, axis=0)["End Time"] + events["End Time"] = first_touch_times.reindex(events.index) + + if side is None: + events = events.drop("Side", axis=1) + + return events + + +def meta_labeling(events: pd.DataFrame, close: pd.Series) -> pd.DataFrame: + """ + Calculate returns and assign binary labels for meta-labeling. + + Reference: + Snippet 3.7, Page 52. + + Parameters + ---------- + events : pd.DataFrame + The events DataFrame from `meta_events`. + close : pd.Series + Time series of close prices. + + Returns + ------- + pd.DataFrame + DataFrame with columns: + - 'End Time': Time of first barrier touch. + - 'Return': The return of the trade. + - 'Label': The binary label (0 or 1 if 'Side' is present, -1 or 1 if not). + - 'Side': (Optional) The side of the bet. + """ + events_filtered = events.dropna(subset=["End Time"]) + + # Get all unique timestamps + all_dates = events_filtered.index.union( + events_filtered["End Time"].values + ).drop_duplicates() + + # Reindex prices to only the required dates + close_filtered = close.reindex(all_dates, method="bfill") + + out = pd.DataFrame(index=events_filtered.index) + out["End Time"] = events_filtered["End Time"] + + out["Return"] = np.log( + close_filtered.loc[events_filtered["End Time"].values].values + ) - np.log(close_filtered.loc[events_filtered.index].values) + + if "Side" in events_filtered: + out["Return"] *= events_filtered["Side"] + out["Side"] = events_filtered["Side"] + + # Assign labels + out["Label"] = np.sign(out["Return"]) + + if "Side" in events_filtered: + # Meta-labeling: 1 if profitable, 0 if not + out.loc[out["Return"] <= 0, "Label"] = 0.0 + + return out diff --git a/src/RiskLabAI/data/structures/__init__.py b/src/RiskLabAI/data/structures/__init__.py new file mode 100644 index 0000000..6ad32f4 --- /dev/null +++ b/src/RiskLabAI/data/structures/__init__.py @@ -0,0 +1,35 @@ +""" +RiskLabAI Data Structures Module +""" + +# Exports from standard_bars.py +# Exports from imbalance_bars.py +from .imbalance_bars import ExpectedImbalanceBars, FixedImbalanceBars + +# Exports from run_bars.py +from .run_bars import ExpectedRunBars, FixedRunBars +from .standard_bars import StandardBars + +# Exports from time_bars.py +from .time_bars import TimeBars + +# Hardened (anti-degeneracy) information-driven bars (Appraisal 19 robustness fix) +from .hardened_information_driven_bars import ( + HardenedExpectedImbalanceBars, + HardenedExpectedRunBars, +) + +# Note: Removed 'pca_weights' from here as it belongs in 'optimization', +# not 'data.structures'. + + +__all__ = [ + "StandardBars", + "TimeBars", + "ExpectedImbalanceBars", + "FixedImbalanceBars", + "ExpectedRunBars", + "FixedRunBars", + "HardenedExpectedImbalanceBars", + "HardenedExpectedRunBars", +] diff --git a/src/RiskLabAI/data/structures/abstract_bars.py b/src/RiskLabAI/data/structures/abstract_bars.py new file mode 100644 index 0000000..d71ea25 --- /dev/null +++ b/src/RiskLabAI/data/structures/abstract_bars.py @@ -0,0 +1,252 @@ +""" +A base class for the various bar types. Includes the logic shared between +classes, to minimise duplicated code. +""" + +from abc import ABC, abstractmethod +from collections.abc import Iterable +from typing import Any, Optional, Union + +import numpy as np + +from RiskLabAI.utils.constants import ( + CUMULATIVE_BUY_VOLUME, + CUMULATIVE_DOLLAR, + CUMULATIVE_TICKS, + CUMULATIVE_VOLUME, + N_TICKS_ON_BAR_FORMATION, + PREVIOUS_TICK_RULE, +) + +# Type hint for a single tick: (datetime, price, volume) +TickData = Union[list[Any], tuple[Any, ...], np.ndarray] + + +class AbstractBars(ABC): + """ + Abstract base class for all bar types. + + Contains the base properties and methods shared between all + bar structures, such as Standard, Time, and Information-Driven bars. + """ + + def __init__(self, bar_type: str): + """ + AbstractBars constructor. + + Parameters + ---------- + bar_type : str + The type of bar, used as a key for statistics. + e.g., 'dollar', 'volume', 'tick_imbalance', etc. + """ + # Main fields + self.bar_type = bar_type + self.tick_counter = 0 # Global tick counter + + # Cached fields for the current bar being built + self.previous_tick_price: Optional[float] = None + self.open_price: Optional[float] = None + self.close_price: Optional[float] = None + self.high_price: float = -np.inf + self.low_price: float = np.inf + + self.base_statistics: dict[str, Union[int, float]] = { + PREVIOUS_TICK_RULE: 0, + CUMULATIVE_TICKS: 0, + CUMULATIVE_DOLLAR: 0, + CUMULATIVE_VOLUME: 0, + CUMULATIVE_BUY_VOLUME: 0, + N_TICKS_ON_BAR_FORMATION: 0, + } + + @abstractmethod + def construct_bars_from_data(self, data: Iterable[TickData]) -> list[list[Any]]: + """ + Constructs bars from input tick data. + + This method must be implemented by all concrete subtypes. + + Parameters + ---------- + data : Iterable[TickData] + An iterable (list, tuple, generator) of tick data. + Each tick is expected to be a tuple/list of + (date_time, price, volume). + + Returns + ------- + List[List[Any]] + A list of the constructed bars, where each bar is a list + of aggregated values. + """ + pass + + def update_base_fields(self, price: float, tick_rule: int, volume: float): + """ + Update the base statistics for the current bar in progress. + + Parameters + ---------- + price : float + Price of the current tick. + tick_rule : int + Tick rule (1, -1, or 0) of the current tick. + volume : float + Volume of the current tick. + """ + dollar_value = price * volume + + if self.open_price is None: + self.open_price = price + + # Update high/low prices + self.high_price, self.low_price = self._high_and_low_price_update(price) + + # Update bar base statistics + self.base_statistics[CUMULATIVE_TICKS] += 1 + self.base_statistics[CUMULATIVE_DOLLAR] += dollar_value + self.base_statistics[CUMULATIVE_VOLUME] += volume + + if tick_rule == 1: + self.base_statistics[CUMULATIVE_BUY_VOLUME] += volume + + @abstractmethod + def _bar_construction_condition(self, threshold: float) -> bool: + """ + Check if the condition to sample a new bar has been met. + + Parameters + ---------- + threshold : float + The threshold value to check against. + + Returns + ------- + bool + True if the bar should be sampled, False otherwise. + """ + pass + + def _reset_cached_fields(self): + """ + Reset the cached fields when a new bar is sampled. + """ + self.open_price = None + self.high_price, self.low_price = -np.inf, np.inf + + self.base_statistics[CUMULATIVE_TICKS] = 0 + self.base_statistics[CUMULATIVE_DOLLAR] = 0 + self.base_statistics[CUMULATIVE_VOLUME] = 0 + self.base_statistics[CUMULATIVE_BUY_VOLUME] = 0 + + def _tick_rule(self, price: float) -> int: + """ + Compute the tick rule (sign of the price change). + + Reference: + Page 29, Advances in Financial Machine Learning. + + Parameters + ---------- + price : float + Price of the current tick. + + Returns + ------- + int + The signed tick (1, -1, or 0). + """ + if self.previous_tick_price is not None: + tick_difference = price - self.previous_tick_price + else: + tick_difference = 0.0 + + if tick_difference != 0: + signed_tick = int(np.sign(tick_difference)) + self.base_statistics[PREVIOUS_TICK_RULE] = signed_tick + else: + # Use the previous non-zero tick rule + signed_tick = int(self.base_statistics[PREVIOUS_TICK_RULE]) + + self.previous_tick_price = price + return signed_tick + + def _high_and_low_price_update(self, price: float) -> tuple[float, float]: + """ + Update the high and low prices for the current bar. + + Parameters + ---------- + price : float + Price of the current tick. + + Returns + ------- + Tuple[float, float] + (new_high_price, new_low_price) + """ + high_price = max(price, self.high_price) + low_price = min(price, self.low_price) + return high_price, low_price + + def _construct_next_bar( + self, + date_time: Any, + tick_index: int, + price: float, + high_price: float, + low_price: float, + threshold: float, + ) -> list[Any]: + """ + Format and return the newly constructed bar. + + Parameters + ---------- + date_time : Any + Timestamp for the bar. + tick_index : int + The global tick counter index. + price : float + The close price of the bar. + high_price : float + The high price of the bar. + low_price : float + The low price of the bar. + threshold : float + The threshold that triggered this bar's construction. + + Returns + ------- + List[Any] + A list containing all aggregated bar data. + """ + open_price = self.open_price if self.open_price is not None else price + close_price = price + low_price = min(low_price, open_price) + high_price = max(high_price, open_price) + + # Get cumulative statistics + cumulative_ticks = self.base_statistics[CUMULATIVE_TICKS] + cumulative_dollar = self.base_statistics[CUMULATIVE_DOLLAR] + cumulative_volume = self.base_statistics[CUMULATIVE_VOLUME] + cumulative_buy_volume = self.base_statistics[CUMULATIVE_BUY_VOLUME] + cumulative_sell_volume = cumulative_volume - cumulative_buy_volume + + next_bar = [ + date_time, + tick_index, + open_price, + high_price, + low_price, + close_price, + cumulative_volume, + cumulative_buy_volume, + cumulative_sell_volume, + cumulative_ticks, + cumulative_dollar, + threshold, + ] + + return next_bar diff --git a/src/RiskLabAI/data/structures/abstract_imbalance_bars.py b/src/RiskLabAI/data/structures/abstract_imbalance_bars.py new file mode 100644 index 0000000..7cf805a --- /dev/null +++ b/src/RiskLabAI/data/structures/abstract_imbalance_bars.py @@ -0,0 +1,178 @@ +""" +Abstract base class for Imbalance Bars (Fixed and Expected). +""" + +from abc import abstractmethod +from collections.abc import Iterable +from typing import Any, Optional + +import numpy as np + +from RiskLabAI.data.structures.abstract_bars import TickData +from RiskLabAI.data.structures.abstract_information_driven_bars import ( + AbstractInformationDrivenBars, +) +from RiskLabAI.utils.constants import ( + CUMULATIVE_THETA, + CUMULATIVE_TICKS, + EXPECTED_IMBALANCE, + EXPECTED_IMBALANCE_WINDOW, + EXPECTED_TICKS_NUMBER, + PREVIOUS_BARS_N_TICKS_LIST, + PREVIOUS_TICK_IMBALANCES_LIST, +) + + +class AbstractImbalanceBars(AbstractInformationDrivenBars): + """ + Abstract class for Imbalance Bars (Fixed and Expected). + + Implements the bar sampling logic based on cumulative imbalance (theta) + exceeding a dynamic threshold E[T] * |E[b_t]|. + + Reference: + Pages 29-30, Advances in Financial Machine Learning. + """ + + def __init__( + self, + bar_type: str, + window_size_for_expected_n_ticks_estimation: Optional[int], + window_size_for_expected_imbalance_estimation: int, + initial_estimate_of_expected_n_ticks_in_bar: int, + analyse_thresholds: bool, + ): + """ + Constructor. + (Parameters same as original) + """ + super().__init__( + bar_type, + window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation, + ) + + self.imbalance_bars_statistics = { + CUMULATIVE_THETA: 0.0, + EXPECTED_IMBALANCE: np.nan, + PREVIOUS_BARS_N_TICKS_LIST: [], + PREVIOUS_TICK_IMBALANCES_LIST: [], + } + + self.analyse_thresholds = [] if analyse_thresholds else None + + def construct_bars_from_data(self, data: Iterable[TickData]) -> list[list[Any]]: + """ + Constructs imbalance bars from input tick data. + (Parameters same as original) + """ + bars_list = [] + + # Keep track of last timestamp and threshold + date_time = None + threshold = np.inf + + for tick_data in data: + self.tick_counter += 1 + + date_time, price, volume = tick_data[0], tick_data[1], tick_data[2] + + # Update common fields + tick_rule = self._tick_rule(price) + self.update_base_fields(price, tick_rule, volume) + self.close_price = price + + # Calculate imbalance + imbalance = self._imbalance_at_tick(price, tick_rule, volume) + self.imbalance_bars_statistics[PREVIOUS_TICK_IMBALANCES_LIST].append( + imbalance + ) + self.imbalance_bars_statistics[CUMULATIVE_THETA] += imbalance + + # Warm-up E[b] if it's the first time + if np.isnan(self.imbalance_bars_statistics[EXPECTED_IMBALANCE]): + self.imbalance_bars_statistics[EXPECTED_IMBALANCE] = ( + self._ewma_expected_imbalance( + self.imbalance_bars_statistics[PREVIOUS_TICK_IMBALANCES_LIST], + self.information_driven_bars_statistics[ + EXPECTED_IMBALANCE_WINDOW + ], + warm_up=True, + ) + ) + + if self.analyse_thresholds is not None: + stats = { + **self.base_statistics, + **self.information_driven_bars_statistics, + **self.imbalance_bars_statistics, + "timestamp": date_time, + } + self.analyse_thresholds.append(stats) + + # Calculate threshold and check condition + expected_ticks = self.information_driven_bars_statistics[ + EXPECTED_TICKS_NUMBER + ] + expected_imbalance = self.imbalance_bars_statistics[EXPECTED_IMBALANCE] + + if np.isnan(expected_ticks) or np.isnan(expected_imbalance): + threshold = np.inf + else: + threshold = expected_ticks * np.abs(expected_imbalance) + + if self._bar_construction_condition(threshold): + next_bar = self._construct_next_bar( + date_time, + self.tick_counter, + self.close_price, + self.high_price, + self.low_price, + threshold, + ) + bars_list.append(next_bar) + + # Store T for E[T] update + self.imbalance_bars_statistics[PREVIOUS_BARS_N_TICKS_LIST].append( + self.base_statistics[CUMULATIVE_TICKS] + ) + + # Update E[T] + self.information_driven_bars_statistics[EXPECTED_TICKS_NUMBER] = ( + self._expected_number_of_ticks() + ) + + # Update E[b] + self.imbalance_bars_statistics[EXPECTED_IMBALANCE] = ( + self._ewma_expected_imbalance( + self.imbalance_bars_statistics[PREVIOUS_TICK_IMBALANCES_LIST], + self.information_driven_bars_statistics[ + EXPECTED_IMBALANCE_WINDOW + ], + warm_up=False, + ) + ) + + # Reset cached fields + self._reset_cached_fields() + + return bars_list + + def _bar_construction_condition(self, threshold: float) -> bool: + """Check if cumulative imbalance |theta| exceeds the threshold.""" + if np.isnan(threshold) or np.isinf(threshold): + return False + + cumulative_theta = self.imbalance_bars_statistics[CUMULATIVE_THETA] + return np.abs(cumulative_theta) >= threshold + + def _reset_cached_fields(self): + """Reset base fields and cumulative theta.""" + super()._reset_cached_fields() + self.imbalance_bars_statistics[CUMULATIVE_THETA] = 0.0 + + @abstractmethod + def _expected_number_of_ticks(self) -> float: + """Calculate E[T] when a new bar is sampled.""" + pass diff --git a/src/RiskLabAI/data/structures/abstract_information_driven_bars.py b/src/RiskLabAI/data/structures/abstract_information_driven_bars.py new file mode 100644 index 0000000..adbfd1c --- /dev/null +++ b/src/RiskLabAI/data/structures/abstract_information_driven_bars.py @@ -0,0 +1,129 @@ +""" +Abstract base class for Information-Driven Bars (Imbalance and Run bars). +""" + +from abc import abstractmethod +from typing import Optional + +import numpy as np + +from RiskLabAI.data.structures.abstract_bars import AbstractBars +from RiskLabAI.utils.constants import ( + EXPECTED_IMBALANCE_WINDOW, + EXPECTED_TICKS_NUMBER, +) +from RiskLabAI.utils.ewma import ewma + + +class AbstractInformationDrivenBars(AbstractBars): + """ + Abstract class for Information-Driven Bars (Imbalance and Run). + + Implements the shared logic for bars that sample based on + dynamic thresholds derived from tick imbalances. + + Reference: + Pages 29-32, Advances in Financial Machine Learning. + """ + + def __init__( + self, + bar_type: str, + window_size_for_expected_n_ticks_estimation: Optional[int], + initial_estimate_of_expected_n_ticks_in_bar: int, + window_size_for_expected_imbalance_estimation: int, + ): + """ + Constructor. + + Parameters + ---------- + bar_type : str + e.g., 'dollar_imbalance', 'tick_run', etc. + window_size_for_expected_n_ticks_estimation : int, optional + Window size for EWMA of E[T]. (None for Fixed bars). + initial_estimate_of_expected_n_ticks_in_bar : int + Initial guess for E[T]. + window_size_for_expected_imbalance_estimation : int + Window size for EWMA of E[b]. + """ + super().__init__(bar_type) + self.information_driven_bars_statistics = { + EXPECTED_TICKS_NUMBER: float(initial_estimate_of_expected_n_ticks_in_bar), + EXPECTED_IMBALANCE_WINDOW: window_size_for_expected_imbalance_estimation, + } + self.window_size_for_expected_n_ticks_estimation = ( + window_size_for_expected_n_ticks_estimation + ) + + def _ewma_expected_imbalance( + self, array: list, window: int, warm_up: bool = False + ) -> float: + """ + Calculate the EWMA of the expected imbalance. + + Parameters + ---------- + array : list + List of observed imbalances. + window : int + EWMA window. + warm_up : bool, default=False + If True, wait for `E[T]` ticks before calculating. + + Returns + ------- + float + The EWMA of the expected imbalance. + """ + if warm_up: + expected_ticks = self.information_driven_bars_statistics[ + EXPECTED_TICKS_NUMBER + ] + if np.isnan(expected_ticks) or len(array) < expected_ticks: + return np.nan + + ewma_window = int(min(len(array), window)) + if ewma_window == 0: + return np.nan + + return ewma(np.array(array[-ewma_window:], dtype=float), window=ewma_window)[-1] + + def _imbalance_at_tick( + self, price: float, signed_tick: int, volume: float + ) -> float: + """ + Calculate the imbalance (theta) for the current tick. + + Parameters + ---------- + price : float + Current tick price. + signed_tick : int + Current tick rule (1, -1, 0). + volume : float + Current tick volume. + + Returns + ------- + float + The calculated imbalance. + """ + if self.bar_type in ("tick_imbalance", "tick_run"): + imbalance = float(signed_tick) + elif self.bar_type in ("dollar_imbalance", "dollar_run"): + imbalance = float(signed_tick * volume * price) + elif self.bar_type in ("volume_imbalance", "volume_run"): + imbalance = float(signed_tick * volume) + else: + raise ValueError(f"Unknown bar_type for imbalance: {self.bar_type}") + + return imbalance + + @abstractmethod + def _expected_number_of_ticks(self) -> float: + """ + Abstract method to update the expected number of ticks (E[T]). + This is implemented differently for "fixed" vs. "expected" bars. + """ + pass diff --git a/src/RiskLabAI/data/structures/abstract_run_bars.py b/src/RiskLabAI/data/structures/abstract_run_bars.py new file mode 100644 index 0000000..5320bce --- /dev/null +++ b/src/RiskLabAI/data/structures/abstract_run_bars.py @@ -0,0 +1,261 @@ +""" +Abstract base class for Run Bars (Fixed and Expected). +""" + +from abc import abstractmethod +from collections.abc import Iterable +from typing import Any, Optional + +import numpy as np + +from RiskLabAI.data.structures.abstract_bars import TickData +from RiskLabAI.data.structures.abstract_information_driven_bars import ( + AbstractInformationDrivenBars, +) +from RiskLabAI.utils.constants import ( + BUY_TICKS_NUMBER, + CUMULATIVE_BUY_THETA, + CUMULATIVE_SELL_THETA, + CUMULATIVE_TICKS, + EXPECTED_BUY_IMBALANCE, + EXPECTED_BUY_TICKS_PROPORTION, + EXPECTED_IMBALANCE_WINDOW, + EXPECTED_SELL_IMBALANCE, + EXPECTED_TICKS_NUMBER, + PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST, + PREVIOUS_BARS_N_TICKS_LIST, + PREVIOUS_TICK_IMBALANCES_BUY_LIST, + PREVIOUS_TICK_IMBALANCES_SELL_LIST, +) +from RiskLabAI.utils.ewma import ewma + + +class AbstractRunBars(AbstractInformationDrivenBars): + """ + Abstract class for Run Bars (Fixed and Expected). + (Docstrings same as original) + """ + + def __init__( + self, + bar_type: str, + window_size_for_expected_n_ticks_estimation: Optional[int], + window_size_for_expected_imbalance_estimation: int, + initial_estimate_of_expected_n_ticks_in_bar: int, + analyse_thresholds: bool, + ): + """ + Constructor. + (Parameters same as original) + """ + super().__init__( + bar_type, + window_size_for_expected_n_ticks_estimation, + initial_estimate_of_expected_n_ticks_in_bar, + window_size_for_expected_imbalance_estimation, + ) + + self.run_bars_statistics = { + CUMULATIVE_BUY_THETA: 0.0, + CUMULATIVE_SELL_THETA: 0.0, + EXPECTED_BUY_IMBALANCE: np.nan, + EXPECTED_SELL_IMBALANCE: np.nan, + EXPECTED_BUY_TICKS_PROPORTION: np.nan, + BUY_TICKS_NUMBER: 0, + PREVIOUS_BARS_N_TICKS_LIST: [], + PREVIOUS_TICK_IMBALANCES_BUY_LIST: [], + PREVIOUS_TICK_IMBALANCES_SELL_LIST: [], + PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST: [], + } + + self.analyse_thresholds = [] if analyse_thresholds else None + + def construct_bars_from_data(self, data: Iterable[TickData]) -> list[list[Any]]: + """ + Constructs run bars from input tick data. + (Parameters same as original) + """ + bars_list = [] + + # Keep track of last timestamp and threshold + date_time = None + threshold = np.inf + + for tick_data in data: + self.tick_counter += 1 + + date_time, price, volume = tick_data[0], tick_data[1], tick_data[2] + + # Update common fields + tick_rule = self._tick_rule(price) + self.update_base_fields(price, tick_rule, volume) + self.close_price = price + + # Calculate imbalance + imbalance = self._imbalance_at_tick(price, tick_rule, volume) + + if imbalance > 0: + self.run_bars_statistics[PREVIOUS_TICK_IMBALANCES_BUY_LIST].append( + imbalance + ) + self.run_bars_statistics[CUMULATIVE_BUY_THETA] += imbalance + self.run_bars_statistics[BUY_TICKS_NUMBER] += 1 + elif imbalance < 0: + self.run_bars_statistics[PREVIOUS_TICK_IMBALANCES_SELL_LIST].append( + -imbalance + ) + self.run_bars_statistics[CUMULATIVE_SELL_THETA] += -imbalance + + # Warm-up E[theta_buy], E[theta_sell], and P[buy] + warm_up_stats = [ + self.run_bars_statistics[EXPECTED_BUY_IMBALANCE], + self.run_bars_statistics[EXPECTED_SELL_IMBALANCE], + self.run_bars_statistics[EXPECTED_BUY_TICKS_PROPORTION], + ] + + if np.isnan(warm_up_stats).any(): + self.run_bars_statistics[EXPECTED_BUY_IMBALANCE] = ( + self._ewma_expected_imbalance( + self.run_bars_statistics[PREVIOUS_TICK_IMBALANCES_BUY_LIST], + self.information_driven_bars_statistics[ + EXPECTED_IMBALANCE_WINDOW + ], + warm_up=True, + ) + ) + self.run_bars_statistics[EXPECTED_SELL_IMBALANCE] = ( + self._ewma_expected_imbalance( + self.run_bars_statistics[PREVIOUS_TICK_IMBALANCES_SELL_LIST], + self.information_driven_bars_statistics[ + EXPECTED_IMBALANCE_WINDOW + ], + warm_up=True, + ) + ) + + # Update P[buy] + if self.base_statistics[CUMULATIVE_TICKS] > 0: + buy_ticks_num = self.run_bars_statistics[BUY_TICKS_NUMBER] + cum_ticks = self.base_statistics[CUMULATIVE_TICKS] + self.run_bars_statistics[EXPECTED_BUY_TICKS_PROPORTION] = ( + buy_ticks_num / cum_ticks + ) + + if self.analyse_thresholds is not None: + stats = { + **self.base_statistics, + **self.information_driven_bars_statistics, + **self.run_bars_statistics, + "timestamp": date_time, + } + self.analyse_thresholds.append(stats) + + # Calculate threshold and check condition + threshold = self._calculate_run_threshold() + + if self._bar_construction_condition(threshold): + next_bar = self._construct_next_bar( + date_time, + self.tick_counter, + self.close_price, + self.high_price, + self.low_price, + threshold, + ) + bars_list.append(next_bar) + + # Store T and P[buy] for E[T] and E[P[buy]] updates + cum_ticks = self.base_statistics[CUMULATIVE_TICKS] + buy_ticks_num = self.run_bars_statistics[BUY_TICKS_NUMBER] + + self.run_bars_statistics[PREVIOUS_BARS_N_TICKS_LIST].append(cum_ticks) + + # Avoid division by zero if bar has 0 ticks (should be rare) + buy_proportion = (buy_ticks_num / cum_ticks) if cum_ticks > 0 else 0 + self.run_bars_statistics[ + PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST + ].append(buy_proportion) + + # Update E[T] + self.information_driven_bars_statistics[EXPECTED_TICKS_NUMBER] = ( + self._expected_number_of_ticks() + ) + + # Update E[P[buy]] + window = ( + self.window_size_for_expected_n_ticks_estimation + or self.information_driven_bars_statistics[ + EXPECTED_IMBALANCE_WINDOW + ] + ) + + prob_buy_list = self.run_bars_statistics[ + PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST + ] + + self.run_bars_statistics[EXPECTED_BUY_TICKS_PROPORTION] = ewma( + np.array(prob_buy_list[-window:], dtype=float), + window, + )[-1] + + # Update E[theta_buy] and E[theta_sell] + self.run_bars_statistics[EXPECTED_BUY_IMBALANCE] = ( + self._ewma_expected_imbalance( + self.run_bars_statistics[PREVIOUS_TICK_IMBALANCES_BUY_LIST], + self.information_driven_bars_statistics[ + EXPECTED_IMBALANCE_WINDOW + ], + ) + ) + self.run_bars_statistics[EXPECTED_SELL_IMBALANCE] = ( + self._ewma_expected_imbalance( + self.run_bars_statistics[PREVIOUS_TICK_IMBALANCES_SELL_LIST], + self.information_driven_bars_statistics[ + EXPECTED_IMBALANCE_WINDOW + ], + ) + ) + + # Reset cached fields + self._reset_cached_fields() + + return bars_list + + def _calculate_run_threshold(self) -> float: + """Helper function to calculate the dynamic run threshold.""" + e_t = self.information_driven_bars_statistics[EXPECTED_TICKS_NUMBER] + e_p_buy = self.run_bars_statistics[EXPECTED_BUY_TICKS_PROPORTION] + e_theta_buy = self.run_bars_statistics[EXPECTED_BUY_IMBALANCE] + e_theta_sell = self.run_bars_statistics[EXPECTED_SELL_IMBALANCE] + + if np.isnan([e_t, e_p_buy, e_theta_buy, e_theta_sell]).any(): + return np.inf + + # Threshold = E[T] * max(P[buy] * E[theta_buy], (1-P[buy]) * E[theta_sell]) + buy_threshold = e_p_buy * e_theta_buy + sell_threshold = (1 - e_p_buy) * e_theta_sell + + return e_t * max(buy_threshold, sell_threshold) + + def _bar_construction_condition(self, threshold: float) -> bool: + """Check if cumulative buy or sell run exceeds the threshold.""" + if np.isinf(threshold) or np.isnan(threshold): + return False + + max_theta = max( + self.run_bars_statistics[CUMULATIVE_BUY_THETA], + self.run_bars_statistics[CUMULATIVE_SELL_THETA], + ) + return max_theta >= threshold + + def _reset_cached_fields(self): + """Reset base fields and cumulative run counters.""" + super()._reset_cached_fields() + self.run_bars_statistics[CUMULATIVE_BUY_THETA] = 0.0 + self.run_bars_statistics[CUMULATIVE_SELL_THETA] = 0.0 + self.run_bars_statistics[BUY_TICKS_NUMBER] = 0 + + @abstractmethod + def _expected_number_of_ticks(self) -> float: + """Calculate E[T] when a new bar is sampled.""" + pass diff --git a/src/RiskLabAI/data/structures/hardened_information_driven_bars.py b/src/RiskLabAI/data/structures/hardened_information_driven_bars.py new file mode 100644 index 0000000..0862eeb --- /dev/null +++ b/src/RiskLabAI/data/structures/hardened_information_driven_bars.py @@ -0,0 +1,147 @@ +r""" +Hardened (anti-degeneracy) imbalance and run bars. + +de Prado's information-driven bars (AFML ch.2) sample a new bar when the cumulative signed imbalance +(or run) exceeds the product of the expected number of ticks E[T] and the expected absolute imbalance. +With an EWMA estimate of E[T] and the imbalance, the naive construction degenerates under some +parameterizations: the bar size can collapse toward a single tick (when the running imbalance estimate +makes the threshold tiny) or diverge toward extremely long bars (when E[T] runs away), so the bar series +is no longer usable. The hardened variant adds two guards that bound the bar size without touching the +well-behaved regime. + +Admitted in Appraisal 19 (CONTRIBUTIONS_LEDGER 2026-06-27) as a correctness / robustness fix. Scope +note, verbatim from the verdict: + + replace the naive imbalance/run-bar construction with the guarded version - it removes the + documented degeneracy (1-tick collapse / divergence) while leaving well-behaved parameterizations + unchanged. This is a robustness admission, not a performance claim. + +The guards are: (1) never close a bar before ``min_ticks`` ticks (prevents the 1-tick collapse), and +(2) force-close a bar at ``max_ticks`` ticks (prevents divergence); E[T] is additionally clamped to +``[min_ticks, max_ticks]`` via the base class's ``expected_ticks_number_bounds``. The naive +``ExpectedImbalanceBars`` / ``ExpectedRunBars`` are left unchanged and remain available; the hardened +classes only subclass them. Held-out confirmed on the sealed tick segment (appraisals/19_results, +HELDOUT.md): naive imbalance collapses (86.9% of bars <= 2 ticks, min 1) while hardened holds (0% +<= 2-tick, min 20); naive run diverges (11 bars) while hardened stays bounded; a well-behaved run +configuration is unchanged (6.2% bar-count difference). Evidence and caveats: appraisals/19_verdict.md. + +References +---------- +Lopez de Prado, M. (2018) Advances in Financial Machine Learning, ch. 2 (information-driven bars). +""" + +from __future__ import annotations + +from RiskLabAI.utils.constants import CUMULATIVE_TICKS + +from .imbalance_bars import ExpectedImbalanceBars +from .run_bars import ExpectedRunBars + + +class HardenedExpectedImbalanceBars(ExpectedImbalanceBars): + r""" + Expected-imbalance bars with anti-degeneracy guards (Appraisal 19 robustness fix). + + Subclasses :class:`ExpectedImbalanceBars` and overrides only the bar-construction condition: a bar + cannot close before ``min_ticks`` ticks (removes the 1-tick collapse) and is force-closed at + ``max_ticks`` ticks (removes the divergence); E[T] is clamped to ``[min_ticks, max_ticks]``. Between + the guards the original de Prado condition is used unchanged, so a well-parameterized construction is + left essentially as-is. See the module docstring for the verbatim scope note and + appraisals/19_verdict.md. + + Parameters + ---------- + bar_type : str + e.g. ``'dollar_imbalance'`` / ``'tick_imbalance'`` (passed through to the base class). + min_ticks : int + Minimum number of ticks before a bar may close (the collapse guard). + max_ticks : int + Maximum number of ticks before a bar is force-closed (the divergence guard). + **kwargs + Forwarded to :class:`ExpectedImbalanceBars` (``window_size_for_expected_n_ticks_estimation``, + ``initial_estimate_of_expected_n_ticks_in_bar``, ``window_size_for_expected_imbalance_estimation``, + ``analyse_thresholds``). ``expected_ticks_number_bounds`` is set to ``(min_ticks, max_ticks)``. + + Examples + -------- + >>> bars = HardenedExpectedImbalanceBars( + ... bar_type="dollar_imbalance", + ... initial_estimate_of_expected_n_ticks_in_bar=200, + ... min_ticks=20, + ... max_ticks=5000, + ... ) + >>> data = [("2020-01-01 10:00:00", 100.0, 10.0)] * 60 + >>> out = bars.construct_bars_from_data(data) # no 1-tick collapse + >>> isinstance(out, list) + True + """ + + def __init__(self, *args, min_ticks: int, max_ticks: int, **kwargs): + if min_ticks <= 0 or max_ticks < min_ticks: + raise ValueError("require 0 < min_ticks <= max_ticks") + kwargs["expected_ticks_number_bounds"] = (min_ticks, max_ticks) + super().__init__(*args, **kwargs) + self.min_ticks = min_ticks + self.max_ticks = max_ticks + + def _bar_construction_condition(self, threshold: float) -> bool: + n_ticks = self.base_statistics[CUMULATIVE_TICKS] + if n_ticks < self.min_ticks: + return False + if n_ticks >= self.max_ticks: + return True + return super()._bar_construction_condition(threshold) + + +class HardenedExpectedRunBars(ExpectedRunBars): + r""" + Expected-run bars with anti-degeneracy guards (Appraisal 19 robustness fix). + + Subclasses :class:`ExpectedRunBars` and overrides only the bar-construction condition with the same + two guards as :class:`HardenedExpectedImbalanceBars`: no close before ``min_ticks`` ticks, force-close + at ``max_ticks`` ticks, E[T] clamped to ``[min_ticks, max_ticks]``. See the module docstring for the + verbatim scope note and appraisals/19_verdict.md. + + Parameters + ---------- + bar_type : str + e.g. ``'dollar_run'`` / ``'tick_run'``. + min_ticks : int + Minimum ticks before a bar may close (collapse guard). + max_ticks : int + Maximum ticks before a bar is force-closed (divergence guard). + **kwargs + Forwarded to :class:`ExpectedRunBars`; ``expected_ticks_number_bounds`` is set to + ``(min_ticks, max_ticks)``. + + Examples + -------- + >>> bars = HardenedExpectedRunBars( + ... bar_type="dollar_run", + ... window_size_for_expected_n_ticks_estimation=200, + ... initial_estimate_of_expected_n_ticks_in_bar=200, + ... window_size_for_expected_imbalance_estimation=10000, + ... min_ticks=20, + ... max_ticks=5000, + ... ) + >>> data = [("2020-01-01 10:00:00", 100.0, 10.0)] * 60 + >>> out = bars.construct_bars_from_data(data) + >>> isinstance(out, list) + True + """ + + def __init__(self, *args, min_ticks: int, max_ticks: int, **kwargs): + if min_ticks <= 0 or max_ticks < min_ticks: + raise ValueError("require 0 < min_ticks <= max_ticks") + kwargs["expected_ticks_number_bounds"] = (min_ticks, max_ticks) + super().__init__(*args, **kwargs) + self.min_ticks = min_ticks + self.max_ticks = max_ticks + + def _bar_construction_condition(self, threshold: float) -> bool: + n_ticks = self.base_statistics[CUMULATIVE_TICKS] + if n_ticks < self.min_ticks: + return False + if n_ticks >= self.max_ticks: + return True + return super()._bar_construction_condition(threshold) diff --git a/src/RiskLabAI/data/structures/imbalance_bars.py b/src/RiskLabAI/data/structures/imbalance_bars.py new file mode 100644 index 0000000..749dee0 --- /dev/null +++ b/src/RiskLabAI/data/structures/imbalance_bars.py @@ -0,0 +1,134 @@ +""" +Concrete implementations of Imbalance Bars: +- ExpectedImbalanceBars +- FixedImbalanceBars +""" + +from typing import Optional + +import numpy as np + +from RiskLabAI.data.structures.abstract_imbalance_bars import AbstractImbalanceBars +from RiskLabAI.utils.constants import ( + EXPECTED_TICKS_NUMBER, + PREVIOUS_BARS_N_TICKS_LIST, +) +from RiskLabAI.utils.ewma import ewma + + +class ExpectedImbalanceBars(AbstractImbalanceBars): + """ + Concrete class for Imbalance Bars with a dynamic, EWMA-based + Expected Number of Ticks (E[T]). + """ + + def __init__( + self, + bar_type: str, + window_size_for_expected_n_ticks_estimation: int = 10000, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + ): + """ + Constructor. + + Parameters + ---------- + bar_type : str + e.g., 'dollar_imbalance', 'tick_imbalance'. + window_size_for_expected_n_ticks_estimation : int + Window size for EWMA of E[T]. + initial_estimate_of_expected_n_ticks_in_bar : int + Initial guess for E[T]. + window_size_for_expected_imbalance_estimation : int + Window size for EWMA of E[b]. + expected_ticks_number_bounds : Tuple[float, float], optional + (Lower, Upper) bounds to clamp E[T]. + analyse_thresholds : bool, default=False + If True, store threshold data for analysis. + """ + super().__init__( + bar_type, + window_size_for_expected_n_ticks_estimation, + window_size_for_expected_imbalance_estimation, + initial_estimate_of_expected_n_ticks_in_bar, + analyse_thresholds, + ) + + if expected_ticks_number_bounds is None: + self.expected_ticks_number_lower_bound = 0.0 + self.expected_ticks_number_upper_bound = np.inf + else: + self.expected_ticks_number_lower_bound = expected_ticks_number_bounds[0] + self.expected_ticks_number_upper_bound = expected_ticks_number_bounds[1] + + def _expected_number_of_ticks(self) -> float: + """ + Calculate E[T] using an EWMA of previous bar tick counts. + """ + prev_ticks_list = self.imbalance_bars_statistics[PREVIOUS_BARS_N_TICKS_LIST] + + window = self.window_size_for_expected_n_ticks_estimation + if window is None or window <= 0: + # Fallback to simple mean if window is invalid + return np.mean(prev_ticks_list) + + if not prev_ticks_list: + # No bars yet, return initial estimate + return self.information_driven_bars_statistics[EXPECTED_TICKS_NUMBER] + + ewma_ticks = ewma( + np.array(prev_ticks_list[-window:], dtype=float), window=window + )[-1] + + return min( + max(ewma_ticks, self.expected_ticks_number_lower_bound), + self.expected_ticks_number_upper_bound, + ) + + +class FixedImbalanceBars(AbstractImbalanceBars): + """ + Concrete class for Imbalance Bars with a fixed (constant) + Expected Number of Ticks (E[T]). + """ + + def __init__( + self, + bar_type: str, + initial_estimate_of_expected_n_ticks_in_bar: int = 20000, + window_size_for_expected_imbalance_estimation: int = 10000, + analyse_thresholds: bool = False, + window_size_for_expected_n_ticks_estimation: Optional[int] = None, + ): + """ + Constructor. + + Parameters + ---------- + bar_type : str + e.g., 'dollar_imbalance', 'tick_imbalance'. + initial_estimate_of_expected_n_ticks_in_bar : int + The *fixed* value for E[T]. + window_size_for_expected_imbalance_estimation : int + Window size for EWMA of E[b]. + analyse_thresholds : bool, default=False + If True, store threshold data for analysis. + window_size_for_expected_n_ticks_estimation : int, optional + Ignored. Kept for consistent interface with factory. + """ + super().__init__( + bar_type, + window_size_for_expected_n_ticks_estimation=None, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + analyse_thresholds=analyse_thresholds, + ) + + def _expected_number_of_ticks(self) -> float: + """ + Return the fixed E[T] value. + """ + return self.information_driven_bars_statistics[EXPECTED_TICKS_NUMBER] diff --git a/src/RiskLabAI/data/structures/run_bars.py b/src/RiskLabAI/data/structures/run_bars.py new file mode 100644 index 0000000..ac1fb5c --- /dev/null +++ b/src/RiskLabAI/data/structures/run_bars.py @@ -0,0 +1,132 @@ +""" +Concrete implementations of Run Bars: +- ExpectedRunBars +- FixedRunBars +""" + +from typing import Optional + +import numpy as np + +from RiskLabAI.data.structures.abstract_run_bars import AbstractRunBars +from RiskLabAI.utils.constants import ( + EXPECTED_TICKS_NUMBER, + PREVIOUS_BARS_N_TICKS_LIST, +) +from RiskLabAI.utils.ewma import ewma + + +class ExpectedRunBars(AbstractRunBars): + """ + Concrete class for Run Bars with a dynamic, EWMA-based + Expected Number of Ticks (E[T]). + """ + + def __init__( + self, + bar_type: str, + window_size_for_expected_n_ticks_estimation: int, + initial_estimate_of_expected_n_ticks_in_bar: int, + window_size_for_expected_imbalance_estimation: int, + expected_ticks_number_bounds: Optional[tuple[float, float]] = None, + analyse_thresholds: bool = False, + ): + """ + Constructor. + + Parameters + ---------- + bar_type : str + e.g., 'dollar_run', 'tick_run'. + window_size_for_expected_n_ticks_estimation : int + Window size for EWMA of E[T]. + initial_estimate_of_expected_n_ticks_in_bar : int + Initial guess for E[T]. + window_size_for_expected_imbalance_estimation : int + Window size for EWMA of E[theta]. + expected_ticks_number_bounds : Tuple[float, float], optional + (Lower, Upper) bounds to clamp E[T]. + analyse_thresholds : bool, default=False + If True, store threshold data for analysis. + """ + super().__init__( + bar_type, + window_size_for_expected_n_ticks_estimation, + window_size_for_expected_imbalance_estimation, + initial_estimate_of_expected_n_ticks_in_bar, + analyse_thresholds, + ) + + if expected_ticks_number_bounds is None: + self.expected_ticks_number_lower_bound = 0.0 + self.expected_ticks_number_upper_bound = np.inf + else: + self.expected_ticks_number_lower_bound = expected_ticks_number_bounds[0] + self.expected_ticks_number_upper_bound = expected_ticks_number_bounds[1] + + def _expected_number_of_ticks(self) -> float: + """ + Calculate E[T] using an EWMA of previous bar tick counts. + """ + prev_ticks_list = self.run_bars_statistics[PREVIOUS_BARS_N_TICKS_LIST] + + window = self.window_size_for_expected_n_ticks_estimation + if window is None or window <= 0: + return np.mean(prev_ticks_list) + + if not prev_ticks_list: + return self.information_driven_bars_statistics[EXPECTED_TICKS_NUMBER] + + ewma_ticks = ewma( + np.array(prev_ticks_list[-window:], dtype=float), window=window + )[-1] + + return min( + max(ewma_ticks, self.expected_ticks_number_lower_bound), + self.expected_ticks_number_upper_bound, + ) + + +class FixedRunBars(AbstractRunBars): + """ + Concrete class for Run Bars with a fixed (constant) + Expected Number of Ticks (E[T]). + """ + + def __init__( + self, + bar_type: str, + initial_estimate_of_expected_n_ticks_in_bar: int, + window_size_for_expected_imbalance_estimation: int, + analyse_thresholds: bool = False, + window_size_for_expected_n_ticks_estimation: Optional[int] = None, # Not used + ): + """ + Constructor. + + Parameters + ---------- + bar_type : str + e.g., 'dollar_run', 'tick_run'. + initial_estimate_of_expected_n_ticks_in_bar : int + The *fixed* value for E[T]. + window_size_for_expected_imbalance_estimation : int + Window size for EWMA of E[theta]. + analyse_thresholds : bool, default=False + If True, store threshold data for analysis. + window_size_for_expected_n_ticks_estimation : int, optional + Ignored. Kept for consistent interface with factory. + """ + super().__init__( + bar_type, + window_size_for_expected_n_ticks_estimation=None, + window_size_for_expected_imbalance_estimation=window_size_for_expected_imbalance_estimation, + initial_estimate_of_expected_n_ticks_in_bar=initial_estimate_of_expected_n_ticks_in_bar, + analyse_thresholds=analyse_thresholds, + ) + + def _expected_number_of_ticks(self) -> float: + """ + Return the fixed E[T] value. + """ + return self.information_driven_bars_statistics[EXPECTED_TICKS_NUMBER] diff --git a/src/RiskLabAI/data/structures/standard_bars.py b/src/RiskLabAI/data/structures/standard_bars.py new file mode 100644 index 0000000..110a3b9 --- /dev/null +++ b/src/RiskLabAI/data/structures/standard_bars.py @@ -0,0 +1,87 @@ +""" +Implements Standard Bars (Tick, Volume, Dollar). +""" + +from collections.abc import Iterable +from typing import Any + +from RiskLabAI.data.structures.abstract_bars import AbstractBars, TickData + + +class StandardBars(AbstractBars): + """ + Concrete class for Standard Bars (Tick, Volume, Dollar). + + Generates a new bar whenever a cumulative threshold of ticks, + volume, or dollars is reached. + """ + + def __init__(self, bar_type: str, threshold: float = 50000): + """ + StandardBars constructor. + + Parameters + ---------- + bar_type : str + Type of bar. Must be one of: + `CUMULATIVE_TICKS`, `CUMULATIVE_VOLUME`, `CUMULATIVE_DOLLAR`. + threshold : float, default=50000 + The threshold value to trigger bar construction. + """ + super().__init__(bar_type) + self.threshold = threshold + + def construct_bars_from_data(self, data: Iterable[TickData]) -> list[list[Any]]: + """ + Constructs standard bars from input tick data. + + Parameters + ---------- + data : Iterable[TickData] + An iterable (list, tuple, generator) of tick data. + Each tick is (date_time, price, volume). + + Returns + ------- + List[List[Any]] + A list of the constructed standard bars. + """ + bars_list = [] + + # Keep track of last timestamp for final bar + date_time = None + + for tick_data in data: + self.tick_counter += 1 + + # Unpack data + date_time, price, volume = tick_data[0], tick_data[1], tick_data[2] + + # Update common fields + tick_rule = self._tick_rule(price) + self.update_base_fields(price, tick_rule, volume) + self.close_price = price # Update close price continuously + + # Check if bar construction condition is met + if self._bar_construction_condition(self.threshold): + next_bar = self._construct_next_bar( + date_time, + self.tick_counter, + self.close_price, + self.high_price, + self.low_price, + self.threshold, + ) + bars_list.append(next_bar) + + # Reset cached fields for the next bar + self._reset_cached_fields() + + return bars_list + + def _bar_construction_condition(self, threshold: float) -> bool: + """ + Check if the cumulative value of the `bar_type` has + exceeded the threshold. + """ + return self.base_statistics[self.bar_type] >= threshold diff --git a/src/RiskLabAI/data/structures/time_bars.py b/src/RiskLabAI/data/structures/time_bars.py new file mode 100644 index 0000000..835e7a3 --- /dev/null +++ b/src/RiskLabAI/data/structures/time_bars.py @@ -0,0 +1,135 @@ +""" +Implements Time Bars. +""" + +from collections.abc import Iterable +from typing import Any + +import numpy as np +import pandas as pd + +from RiskLabAI.data.structures.abstract_bars import AbstractBars, TickData + + +class TimeBars(AbstractBars): + """ + Concrete class for Time Bars. + + Generates a new bar whenever a fixed time duration has passed. + """ + + def __init__(self, resolution_type: str, resolution_units: int): + """ + TimeBars constructor. + + Parameters + ---------- + resolution_type : str + Type of bar resolution: 'S', 'MIN', 'H', 'D', 'W'. + resolution_units : int + Number of units for the resolution (e.g., 5 for 5-minutes). + """ + super().__init__(bar_type="time") + + self.resolution_to_n_seconds = { + "S": 1, + "MIN": 60, + "H": 3600, + "D": 86400, + "W": 604800, + } + + if resolution_type.upper() not in self.resolution_to_n_seconds: + raise ValueError( + f"Invalid resolution_type. Use one of {list(self.resolution_to_n_seconds.keys())}" + ) + + self.resolution_type = resolution_type.upper() + self.resolution_units = resolution_units + self.threshold_in_seconds = ( + self.resolution_units * self.resolution_to_n_seconds[self.resolution_type] + ) + + self.current_bar_timestamp = np.nan + self.current_bar_end_timestamp = np.nan + + def construct_bars_from_data(self, data: Iterable[TickData]) -> list[list[Any]]: + """ + Constructs time bars from input tick data. + + Parameters + ---------- + data : Iterable[TickData] + An iterable (list, tuple, generator) of tick data. + Each tick is (date_time, price, volume). + `date_time` must be a pandas Timestamp. + + Returns + ------- + List[List[Any]] + A list of the constructed time bars. + """ + bars_list = [] + for tick_data in data: + self.tick_counter += 1 + + # Unpack data + date_time, price, volume = tick_data[0], tick_data[1], tick_data[2] + + # Get tick timestamp in seconds + try: + tick_timestamp_sec = date_time.timestamp() + except AttributeError as err: + raise TypeError( + "TimeBars require `date_time` to be a pandas Timestamp " + "or datetime object with a .timestamp() method." + ) from err + + # Determine the "floor" timestamp for this bar + bar_start_timestamp_sec = ( + int(tick_timestamp_sec // self.threshold_in_seconds) + * self.threshold_in_seconds + ) + + # Initialize first bar + if np.isnan(self.current_bar_timestamp): + self.current_bar_timestamp = bar_start_timestamp_sec + self.current_bar_end_timestamp = ( + bar_start_timestamp_sec + self.threshold_in_seconds + ) + + # Check if this tick belongs to a new bar + if self._bar_construction_condition(tick_timestamp_sec): + # Construct the *previous* bar + bar_end_time = pd.to_datetime(self.current_bar_end_timestamp, unit="s") + + next_bar = self._construct_next_bar( + bar_end_time, + self.tick_counter - 1, # Index of the *previous* tick + self.close_price, # Close price from *previous* tick + self.high_price, + self.low_price, + self.current_bar_end_timestamp, + ) + bars_list.append(next_bar) + + # Reset for the new bar + self._reset_cached_fields() + self.current_bar_timestamp = bar_start_timestamp_sec + self.current_bar_end_timestamp = ( + bar_start_timestamp_sec + self.threshold_in_seconds + ) + + # Update fields with current tick data + tick_rule = self._tick_rule(price) + self.update_base_fields(price, tick_rule, volume) + self.close_price = price + + return bars_list + + def _bar_construction_condition(self, tick_timestamp_sec: float) -> bool: + """ + Check if the current tick's timestamp has crossed + the end-time of the current bar. + """ + return tick_timestamp_sec >= self.current_bar_end_timestamp diff --git a/src/RiskLabAI/data/synthetic_data/__init__.py b/src/RiskLabAI/data/synthetic_data/__init__.py new file mode 100644 index 0000000..b06e36e --- /dev/null +++ b/src/RiskLabAI/data/synthetic_data/__init__.py @@ -0,0 +1,35 @@ +""" +RiskLabAI Synthetic Data Module + +Provides tools for generating synthetic financial data, from +simple block-diagonal covariance matrices to complex, +regime-switching Heston-Merton price paths. +""" + +from .drift_burst_hypothesis import drift_volatility_burst +from .simulation import ( + form_block_matrix, + form_true_matrix, + random_cov, + simulates_cov_mu, +) +from .synthetic_controlled_environment import ( + align_params_length, + compute_log_returns, + generate_prices_from_regimes, + heston_merton_log_returns, + parallel_generate_prices, +) + +__all__ = [ + "drift_volatility_burst", + "compute_log_returns", + "heston_merton_log_returns", + "align_params_length", + "generate_prices_from_regimes", + "parallel_generate_prices", + "random_cov", + "form_block_matrix", + "form_true_matrix", + "simulates_cov_mu", +] diff --git a/src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py b/src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py new file mode 100644 index 0000000..7dd5156 --- /dev/null +++ b/src/RiskLabAI/data/synthetic_data/drift_burst_hypothesis.py @@ -0,0 +1,94 @@ +""" +Implements the Drift-Burst Hypothesis (DBH) model for synthetic data. + +This model generates drift and volatility parameters for a bubble scenario, +featuring an "explosion" at the midpoint. +""" + +import numpy as np + + +def drift_volatility_burst( + bubble_length: int, + a_before: float, + a_after: float, + b_before: float, + b_after: float, + alpha: float, + beta: float, + explosion_filter_width: float = 0.1, +) -> tuple[np.ndarray, np.ndarray]: + r""" + Compute the drift and volatility for a DBH burst scenario. + + The model defines drift and volatility as a function of time `t` + (represented by `steps` from 0 to 1), with an explosion at `t=0.5`. + + .. math:: + drift(t) = \frac{a(t)}{|t - 0.5|^\alpha} + vol(t) = \frac{b(t)}{|t - 0.5|^\beta} + + To prevent division by zero, an `explosion_filter_width` is used + to clamp the denominator near `t=0.5`. + + Parameters + ---------- + bubble_length : int + The number of timesteps in the bubble. + a_before : float + Drift coefficient 'a' for `t < 0.5`. + a_after : float + Drift coefficient 'a' for `t > 0.5`. + b_before : float + Volatility coefficient 'b' for `t < 0.5`. + b_after : float + Volatility coefficient 'b' for `t > 0.5`. + alpha : float + Exponent for the drift denominator. + beta : float + Exponent for the volatility denominator. + explosion_filter_width : float, default=0.1 + Width of the "safe" area around the midpoint (0.5) + to prevent division by zero. + + Returns + ------- + Tuple[np.ndarray, np.ndarray] + - drifts: The array of drift values. + - volatilities: The array of volatility values. + """ + steps = np.linspace(0, 1, bubble_length) + + # Clamp values near the midpoint to avoid division by zero + before_mask = (steps >= (0.5 - explosion_filter_width)) & (steps < 0.5) + after_mask = (steps > 0.5) & (steps <= (0.5 + explosion_filter_width)) + + steps_filtered = np.copy(steps) + steps_filtered[before_mask] = 0.5 - explosion_filter_width + steps_filtered[after_mask] = 0.5 + explosion_filter_width + + # Assign 'a' and 'b' parameters based on position + a_values = np.where(steps <= 0.5, a_before, a_after) + b_values = np.where(steps <= 0.5, b_before, b_after) + + # Calculate denominator, with NaN at the exact midpoint + denominators = np.abs(steps_filtered - 0.5) + denominators[steps == 0.5] = np.nan + + drifts = a_values / (denominators**alpha) + volatilities = b_values / (denominators**beta) + + # Handle the NaN at the midpoint by filling with 0 drift + # and the preceding volatility value. + nan_mask = np.isnan(denominators) + if np.any(nan_mask): + drifts[nan_mask] = 0.0 + # Find first NaN index and fill with previous value + nan_index = np.where(nan_mask)[0][0] + if nan_index > 0: + volatilities[nan_mask] = volatilities[nan_index - 1] + else: + # Handle case where midpoint is the first element + volatilities[nan_mask] = b_before # Fallback + + return drifts, volatilities diff --git a/src/RiskLabAI/data/synthetic_data/simulation.py b/src/RiskLabAI/data/synthetic_data/simulation.py new file mode 100644 index 0000000..973c1c2 --- /dev/null +++ b/src/RiskLabAI/data/synthetic_data/simulation.py @@ -0,0 +1,103 @@ +""" +Functions for generating synthetic, structured data for financial simulations. +Includes functions for creating random covariance matrices and +simulating multivariate normal observations. +""" + +import numpy as np +import pandas as pd +from scipy.linalg import block_diag +from sklearn.covariance import LedoitWolf + +# Import the utility from the denoising module +from RiskLabAI.data.denoise.denoising import corr_to_cov + + +def random_cov(num_columns: int, num_factors: int) -> np.ndarray: + """ + Generate a random covariance matrix. + + :param num_columns: Number of columns in the covariance matrix + :type num_columns: int + :param num_factors: Number of factors for random covariance matrix + :type num_factors: int + :return: Random covariance matrix + :rtype: np.ndarray + """ + w = np.random.normal(size=(num_columns, num_factors)) + cov = np.dot(w, w.T) + cov += np.diag(np.random.uniform(size=num_columns)) + return cov + + +def form_block_matrix( + n_blocks: int, block_size: int, block_correlation: float +) -> np.ndarray: + """ + Forms a block diagonal correlation matrix. + + :param n_blocks: Number of blocks + :type n_blocks: int + :param block_size: Size of each block + :type block_size: int + :param block_correlation: Correlation within each block + :type block_correlation: float + :return: Block diagonal correlation matrix + :rtype: np.ndarray + """ + block = np.ones((block_size, block_size)) * block_correlation + block[range(block_size), range(block_size)] = 1 + corr = block_diag(*([block] * n_blocks)) + return corr + + +def form_true_matrix( + n_blocks: int, block_size: int, block_correlation: float +) -> tuple[np.ndarray, np.ndarray]: + """ + Forms a shuffled block diagonal correlation matrix and the + corresponding covariance matrix. + + :param n_blocks: Number of blocks + :type n_blocks: int + :param block_size: Size of each block + :type block_size: int + :param block_correlation: Correlation within each block + :type block_correlation: float + :return: (mu0, cov0) Mean vector and Covariance matrix + :rtype: Tuple[np.ndarray, np.ndarray] + """ + corr0 = form_block_matrix(n_blocks, block_size, block_correlation) + corr0 = pd.DataFrame(corr0) + cols = corr0.columns.tolist() + np.random.shuffle(cols) + corr0 = corr0[cols].loc[cols].copy(deep=True).values + + std0 = np.random.uniform(0.05, 0.2, corr0.shape[0]) + cov0 = corr_to_cov(corr0, std0) + mu0 = np.random.normal(std0, std0, cov0.shape[0]).reshape(-1, 1) + return mu0, cov0 + + +def simulates_cov_mu( + mu0: np.ndarray, cov0: np.ndarray, n_obs: int, shrink: bool = False +) -> tuple[np.ndarray, np.ndarray]: + """ + Simulates multivariate normal observations and computes the + sample mean and covariance. + + :param mu0: True mean + :type mu0: np.ndarray + :param cov0: True covariance matrix + :type cov0: np.ndarray + :param n_obs: Number of observations + :type n_obs: int + :param shrink: Whether to use Ledoit-Wolf shrinkage, default False + :type shrink: bool + :return: (mu1, cov1) Sample mean and covariance matrix + :rtype: Tuple[np.ndarray, np.ndarray] + """ + x = np.random.multivariate_normal(mu0.flatten(), cov0, size=n_obs) + mu1 = x.mean(axis=0).reshape(-1, 1) + cov1 = LedoitWolf().fit(x).covariance_ if shrink else np.cov(x, rowvar=0) + return mu1, cov1 diff --git a/src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py b/src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py new file mode 100644 index 0000000..1372afe --- /dev/null +++ b/src/RiskLabAI/data/synthetic_data/synthetic_controlled_environment.py @@ -0,0 +1,431 @@ +""" +Generates synthetic price data using a Heston-Merton model +with Markov-switching regimes. +""" + +from typing import Optional, Union + +import numpy as np +import pandas as pd +from joblib import Parallel, delayed + +try: + from numba import jit +except ImportError: + def jit(*decorator_args, **decorator_kwargs): + """Return the original function when optional acceleration is absent.""" + if ( + len(decorator_args) == 1 + and callable(decorator_args[0]) + and not decorator_kwargs + ): + return decorator_args[0] + + def decorator(function): + return function + + return decorator + +# Type hint for regime parameters +RegimeParams = dict[str, Union[float, list[float]]] + + +@jit(nopython=True) +def compute_log_returns( + n_steps: int, + mu_vector: np.ndarray, + kappa_vector: np.ndarray, + theta_vector: np.ndarray, + xi_vector: np.ndarray, + dw_stock: np.ndarray, + dw_vol: np.ndarray, + jump_comp: np.ndarray, + poisson_var: np.ndarray, + dt: float, + sqrt_dt: float, + lambda_vector: np.ndarray, + m_vector: np.ndarray, + v_vector: np.ndarray, + regime_change: np.ndarray, +) -> np.ndarray: + r""" + Compute log returns using the Heston-Merton model (Numba-optimized). + + Heston Model (Volatility): + .. math:: + v_{i+1} = v_i + \kappa_i (\theta_i - \max(v_i, 0)) dt + + \xi_i \sqrt{\max(v_i, 0)} dwV_i \sqrt{dt} + + Merton Jump-Diffusion (Price): + .. math:: + \log(S_{i+1}/S_i) = \left(\mu_i - 0.5 v_i - \lambda_i (m_i + + \frac{v^2_i}{2})\right) dt + + \sqrt{v_i} dwS_i \sqrt{dt} + dJ_i + + Parameters + ---------- + n_steps : int + Number of steps (N). + mu_vector : np.ndarray + Drift vector. + kappa_vector : np.ndarray + Mean-reversion speed vector. + theta_vector : np.ndarray + Long-term mean vector for volatility. + xi_vector : np.ndarray + Volatility of volatility vector. + dw_stock : np.ndarray + Wiener process for stock. + dw_vol : np.ndarray + Wiener process for volatility. + jump_comp : np.ndarray + Jump component (Y). + poisson_var : np.ndarray + Poisson random variable vector (n). + dt : float + Time step. + sqrt_dt : float + Square root of the time step. + lambda_vector : np.ndarray + Jump intensity vector. + m_vector : np.ndarray + Mean of jump size vector. + v_vector : np.ndarray + Variance of jump size vector. + regime_change : np.ndarray + Boolean array, True if regime changed at this step. + + Returns + ------- + np.ndarray + Array of log returns. + """ + v = np.empty(n_steps + 1) + jump_events = np.multiply(poisson_var, jump_comp) + v[0] = theta_vector[0] # Initial volatility + log_returns = np.empty(n_steps) + + for i in range(n_steps): + if regime_change[i]: + v[i] = theta_vector[i] # Reset vol to new regime's long-term mean + + # Ensure v[i] is non-negative for sqrt + v_i_safe = max(v[i], 0.0) + + # Volatility process (Heston) + v[i + 1] = ( + v[i] + + kappa_vector[i] * (theta_vector[i] - v_i_safe) * dt + + xi_vector[i] * np.sqrt(v_i_safe) * dw_vol[i] * sqrt_dt + ) + + # Log return (Merton Jump-Diffusion) + log_returns[i] = ( + ( + mu_vector[i] + - 0.5 * v_i_safe + - lambda_vector[i] * (m_vector[i] + (v_vector[i] ** 2) / 2.0) + ) + * dt + + np.sqrt(v_i_safe) * dw_stock[i] * sqrt_dt + + jump_events[i] + ) + + return log_returns + + +def heston_merton_log_returns( + total_time: float, + n_steps: int, + mu_vector: np.ndarray, + kappa_vector: np.ndarray, + theta_vector: np.ndarray, + xi_vector: np.ndarray, + rho_vector: np.ndarray, + lambda_vector: np.ndarray, + m_vector: np.ndarray, + v_vector: np.ndarray, + regime_change: np.ndarray, + random_state: Optional[int] = None, +) -> np.ndarray: + """ + Generate Heston-Merton log returns. + + This function sets up the correlated random numbers and calls + the Numba-jitted `compute_log_returns`. + + Parameters + ---------- + total_time : float + Total time (T). + n_steps : int + Number of steps (N). + *..._vector : np.ndarray + Arrays of model parameters (see `compute_log_returns`). + regime_change : np.ndarray + Boolean array, True if regime changed at this step. + random_state : int, optional + Seed for the random number generator. + + Returns + ------- + np.ndarray + Array of log returns. + """ + # Ensure vectors are of length n_steps + params = [ + mu_vector, + kappa_vector, + theta_vector, + xi_vector, + rho_vector, + lambda_vector, + m_vector, + v_vector, + ] + if not all(len(p) == n_steps for p in params): + raise ValueError("All parameter vectors must have length `n_steps`") + + rng = np.random.default_rng(random_state) + dt = total_time / n_steps + sqrt_dt = np.sqrt(dt) + + # Generate correlated random numbers + z = np.zeros((n_steps, 3)) + n = np.zeros(n_steps) + for i in range(n_steps): + cov_matrix = np.array( + [ + [1.0, rho_vector[i], 0.0], + [rho_vector[i], 1.0, 0.0], + [0.0, 0.0, v_vector[i] ** 2], + ] + ) + z[i] = rng.multivariate_normal([0.0, 0.0, m_vector[i]], cov_matrix) + n[i] = rng.poisson(lambda_vector[i] * dt) + + dw_stock = z[:, 0] + dw_vol = z[:, 1] + jump_comp = z[:, 2] + + return compute_log_returns( + n_steps, + mu_vector, + kappa_vector, + theta_vector, + xi_vector, + dw_stock, + dw_vol, + jump_comp, + n, + dt, + sqrt_dt, + lambda_vector, + m_vector, + v_vector, + regime_change, + ) + + +def align_params_length( + regime_params: RegimeParams, +) -> tuple[dict[str, list[float]], int]: + """ + Align the parameter lists within a regime to be the same length. + + If a parameter is a float, it's broadcast to a list. + If a parameter is a list shorter than the max, it's extended + by repeating its last value. + + Parameters + ---------- + regime_params : RegimeParams + Dictionary of regime parameters. + + Returns + ------- + Tuple[Dict[str, List[float]], int] + - The aligned regime parameter dictionary. + - The maximum length (number of steps) for this regime. + """ + max_len = max(len(v) if isinstance(v, list) else 1 for v in regime_params.values()) + + aligned_params: dict[str, list[float]] = {} + for key, value in regime_params.items(): + if isinstance(value, list): + if len(value) < max_len: + # Extend list by repeating last value + aligned_params[key] = value + [value[-1]] * (max_len - len(value)) + else: + aligned_params[key] = value[:max_len] # Truncate if too long + else: + # Broadcast float to list + aligned_params[key] = [value] * max_len + + return aligned_params, max_len + + +def generate_prices_from_regimes( + regimes: dict[str, RegimeParams], + transition_matrix: np.ndarray, + total_time: float, + n_steps: int, + random_state: Optional[int] = None, +) -> tuple[pd.Series, np.ndarray]: + """ + Generate a price series from a Markov-switching regime model. + + Parameters + ---------- + regimes : Dict[str, RegimeParams] + Dictionary mapping regime names to their parameter sets. + transition_matrix : np.ndarray + The Markov Chain transition matrix. + total_time : float + Total time for the simulation (e.g., 1.0 for 1 year). + n_steps : int + Number of discrete steps in the simulation. + random_state : int, optional + Seed for random number generation. + + Returns + ------- + Tuple[pd.Series, np.ndarray] + - The generated price series, indexed by business days. + - The array of simulated regime names for each step. + """ + rng = np.random.default_rng(random_state) + + # 1. Simulate the Markov Chain + import quantecon.markov as qe # optional dependency: RiskLabAI[synth] + + regime_names = list(regimes.keys()) + markov_chain = qe.MarkovChain(transition_matrix, state_values=regime_names) + simulated_regimes = markov_chain.simulate(ts_length=n_steps, random_state=rng) + + # 2. Unpack parameters based on simulated regimes + param_lists: dict[str, list[float]] = { + "mu": [], + "kappa": [], + "theta": [], + "xi": [], + "rho": [], + "lam": [], + "m": [], + "v": [], + } + + regime_path_expanded = [] + + current_step = 0 + while current_step < n_steps: + regime_name = simulated_regimes[current_step] + params, regime_len = align_params_length(regimes[regime_name].copy()) + + steps_to_take = min(regime_len, n_steps - current_step) + + for key in param_lists: + param_lists[key].extend(params[key][:steps_to_take]) + + regime_path_expanded.extend([regime_name] * steps_to_take) + current_step += steps_to_take + + # 3. Finalize parameter arrays and regime path + simulated_regimes_final = np.array(regime_path_expanded) + param_arrays: dict[str, np.ndarray] = { + key: np.array(val) for key, val in param_lists.items() + } + + # 4. Identify regime change points + regime_change = simulated_regimes_final[1:] != simulated_regimes_final[:-1] + regime_change = np.pad( + regime_change, (1, 0), mode="constant", constant_values=False + ) + + # 5. Generate log returns + log_returns = heston_merton_log_returns( + total_time, + n_steps, + param_arrays["mu"], + param_arrays["kappa"], + param_arrays["theta"], + param_arrays["xi"], + param_arrays["rho"], + param_arrays["lam"], + param_arrays["m"], + param_arrays["v"], + regime_change, + random_state=rng.integers(0, 1e6), + ) + + # 6. Create price series with a Business Day index + start_day = "2000-01-01" + business_days = pd.date_range(start=start_day, periods=n_steps, freq="B") + + price_series = pd.Series(log_returns, index=business_days).ffill() + prices = 100 * np.exp(price_series.cumsum()) + prices.name = "Price" + + return prices, simulated_regimes_final + + +def parallel_generate_prices( + number_of_paths: int, + regimes: dict[str, RegimeParams], + transition_matrix: np.ndarray, + total_time: float, + n_steps: int, + random_state: Optional[int] = None, + n_jobs: int = 1, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Parallel generation of price paths. + + Parameters + ---------- + number_of_paths : int + The number of paths to generate. + regimes : Dict[str, RegimeParams] + Dictionary of regime parameters. + transition_matrix : np.ndarray + Markov Chain transition matrix. + total_time : float + Total time for the simulation. + number_of_steps : int + Number of steps in the simulation. + random_state : int, optional + Seed for *main* random number generator (which generates seeds). + n_jobs : int, default=1 + Number of parallel jobs to run. + + Returns + ------- + Tuple[pd.DataFrame, pd.DataFrame] + - prices_df: DataFrame of (n_steps x number_of_paths) prices. + - regimes_df: DataFrame of (n_steps x number_of_paths) regimes. + """ + rng = np.random.default_rng(random_state) + # Generate unique seeds for each parallel job + random_states = rng.integers(0, 10 * number_of_paths, size=number_of_paths) + + results = Parallel(n_jobs=n_jobs)( + delayed(generate_prices_from_regimes)( + regimes, + transition_matrix, + total_time, + n_steps, + seed, + ) + for seed in random_states + ) + + prices, simulated_regimes = zip(*results) + + prices_df = pd.concat(prices, axis=1) + prices_df.columns = range(number_of_paths) + + simulated_regimes_df = pd.DataFrame(simulated_regimes).T + simulated_regimes_df.columns = range(number_of_paths) + simulated_regimes_df.index = prices_df.index + + return prices_df, simulated_regimes_df diff --git a/src/RiskLabAI/data/weights/__init__.py b/src/RiskLabAI/data/weights/__init__.py new file mode 100644 index 0000000..5788242 --- /dev/null +++ b/src/RiskLabAI/data/weights/__init__.py @@ -0,0 +1,24 @@ +""" +RiskLabAI Sample Weights Module + +Implements sample weighting techniques for financial machine learning, +focusing on uniqueness (concurrency) and time decay. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapter 4. +""" + +from .sample_weights import ( + calculate_average_uniqueness, + calculate_time_decay, + expand_label_for_meta_labeling, + sample_weight_absolute_return_meta_labeling, +) + +__all__ = [ + "expand_label_for_meta_labeling", + "calculate_average_uniqueness", + "sample_weight_absolute_return_meta_labeling", + "calculate_time_decay", +] diff --git a/src/RiskLabAI/data/weights/sample_weights.py b/src/RiskLabAI/data/weights/sample_weights.py new file mode 100644 index 0000000..d85988d --- /dev/null +++ b/src/RiskLabAI/data/weights/sample_weights.py @@ -0,0 +1,234 @@ +""" +Implements sample weighting techniques for financial machine learning, +focusing on uniqueness (concurrency) and time decay. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapter 4. +""" + +import numpy as np +import pandas as pd + + +def expand_label_for_meta_labeling( + close_index: pd.Index, + timestamp: pd.Series, + molecule: pd.Index, +) -> pd.Series: + """ + Compute the number of concurrent events for each timestamp. + + This function expands the event start/end times (`timestamp`) + to a full series indicating how many events are active + at each point in time. + + Reference: + Based on Snippet 4.1, Page 60 (label concurrency). + + Parameters + ---------- + close_index : pd.Index + The master index of all timestamps (e.g., from price bars). + timestamp : pd.Series + Series where index is event start time, value is event end time. + molecule : pd.Index + The subset of event start times to process. + + Returns + ------- + pd.Series + A Series indexed by `close_index` where each value is the + count of active events at that timestamp. + """ + # Filter events that are relevant to this molecule + ts = timestamp.fillna(close_index[-1]) + ts = ts[ts.index.isin(molecule)] + ts = ts[ts > molecule[0]] + + if ts.empty: + # Return an empty series; align in the caller will handle it + return pd.Series(dtype=float) + + # Find min/max index locations + iloc_min = close_index.searchsorted(ts.index[0]) + iloc_max = close_index.searchsorted(ts.max()) + + # Create a count series over the relevant time span + count = pd.Series(0, index=close_index[iloc_min : iloc_max + 1]) + + for t_in, t_out in ts.items(): + # Ensure t_out is within the count index + # (it might be larger than iloc_max if fillna was used) + t_out = min(t_out, count.index[-1]) + count.loc[t_in:t_out] += 1 + + return count.loc[molecule[0] : ts.max()] + + +def calculate_average_uniqueness( + index_matrix: pd.DataFrame, +) -> pd.Series: + """ + Calculate the average uniqueness of each event. + + Uniqueness is calculated as 1/c_t, averaged over the + duration of the event. + + Reference: + Snippet 4.2, Page 62. + + Parameters + ---------- + index_matrix : pd.DataFrame + An indicator matrix (T x N) where T is timestamps + and N is the number of events. + + Returns + ------- + pd.Series + A Series of average uniqueness for each event (column). + """ + # c_t: Concurrency at each timestamp + concurrency = index_matrix.sum(axis=1) + + # 1/c_t: Uniqueness at each timestamp + # This is a (T x N) DataFrame, 0 where event is not active + uniqueness = index_matrix.div(concurrency, axis=0).fillna(0) + + # Sum of 1/c_t for each event + total_uniqueness = uniqueness.sum(axis=0) + + # Number of active periods for each event + event_duration = (index_matrix > 0).sum(axis=0) + + # Average uniqueness: sum(1/c_t) / sum(I) + average_uniqueness = total_uniqueness / event_duration + + # Handle events that never occurred (duration 0) + average_uniqueness = average_uniqueness.fillna(0) + + return average_uniqueness + + +def sample_weight_absolute_return_meta_labeling( + timestamp: pd.Series, price: pd.Series, molecule: pd.Index +) -> pd.Series: + """ + Calculate sample weights based on absolute log-return attribution. + + The weight of a sample is proportional to the sum of the absolute + log-returns that occurred during the event, divided by the + concurrency of those returns. + + w_i = sum_{t=t_i,0}^{t_i,1} [ |r_t| / c_t ] + + Weights are then normalized to sum to the number of samples. + + Reference: + Based on Snippet 4.10, Page 70. + + Parameters + ---------- + timestamp : pd.Series + Series where index is event start time, value is event end time. + price : pd.Series + Series of prices. + molecule : pd.Index + The subset of event start times to process. + + Returns + ------- + pd.Series + A Series of sample weights, normalized to sum to N. + """ + # 1. Compute concurrency + concurrency_events = expand_label_for_meta_labeling( + price.index, timestamp, molecule + ) + + # 2. Compute absolute log returns + log_return = np.log(price).diff().abs() + + # Align returns and concurrency + # Use 'left' join to keep the log_return (price) index + log_return, concurrency_events = log_return.align( + concurrency_events, join="left", fill_value=0 + ) + + weight = pd.Series(index=molecule, dtype=float) + + # 3. Calculate weighted returns + for t_in, t_out in timestamp.loc[weight.index].items(): + if t_out not in log_return.index: + # Find the closest preceding index + t_out = log_return.index[log_return.index.searchsorted(t_out) - 1] + + # r_t / c_t + # Filter concurrency > 0 to avoid division by zero + relevant_concurrency = concurrency_events.loc[t_in:t_out] + relevant_log_return = log_return.loc[t_in:t_out] + + active_periods = relevant_concurrency > 0 + if active_periods.any(): + weighted_return = ( + relevant_log_return[active_periods] + / relevant_concurrency[active_periods] + ) + weight.loc[t_in] = weighted_return.sum() + else: + weight.loc[t_in] = 0.0 + + weight = weight.abs() + + # 4. Normalize + if weight.sum() == 0: + # Avoid division by zero if all weights are 0 + return pd.Series(1.0, index=molecule) + + weight *= len(weight) / weight.sum() + return weight + + +def calculate_time_decay(weight: pd.Series, clf_last_weight: float = 1.0) -> pd.Series: + """ + Apply a time-decay factor to sample weights. + + This function applies a linearly decaying weight, where the + most recent observation has weight 1.0 and the oldest + has weight `clf_last_weight`. + + Reference: + Snippet 4.11, Page 71. + + Parameters + ---------- + weight : pd.Series + The original sample weights (e.g., from uniqueness). + clf_last_weight : float, default=1.0 + The weight to assign to the oldest observation. + If 1.0, all weights are 1 (no decay). + If 0.0, weights decay linearly to 0. + + Returns + ------- + pd.Series + The new weights with time decay applied. + """ + clf_weight = weight.sort_index().cumsum() + + if clf_last_weight < 0 or clf_last_weight > 1: + raise ValueError("clf_last_weight must be between 0 and 1") + + # Calculate slope + if clf_last_weight == 1.0 or clf_weight.empty or clf_weight.iloc[-1] == 0: + slope = 0.0 + const = 1.0 + else: + slope = (1.0 - clf_last_weight) / clf_weight.iloc[-1] + const = 1.0 - slope * clf_weight.iloc[-1] + + clf_weight = const + slope * clf_weight + clf_weight[clf_weight < 0] = 0.0 # Should not happen if clf_last_weight >= 0 + + return clf_weight diff --git a/src/RiskLabAI/ensemble/__init__.py b/src/RiskLabAI/ensemble/__init__.py new file mode 100644 index 0000000..8ff0b5c --- /dev/null +++ b/src/RiskLabAI/ensemble/__init__.py @@ -0,0 +1,24 @@ +""" +RiskLabAI Ensemble Module + +Implements ensemble-related methods, such as the theoretical +accuracy of bagging classifiers. +""" + +# From the theoretical function file (bagging_classifier_accuracy.py) +from .bagging_classifier_accuracy import bagging_classifier_accuracy + +# From the empirical class file (empirical_bagging_accuracy.py) +from .empirical_bagging_accuracy import ( + BaggingClassifierAccuracy, + calculate_bootstrap_accuracy, + plot_bootstrap_accuracy_distribution, +) + +# Expose all names for import +__all__ = [ + "bagging_classifier_accuracy", # The function + "BaggingClassifierAccuracy", # The class + "calculate_bootstrap_accuracy", + "plot_bootstrap_accuracy_distribution", +] diff --git a/src/RiskLabAI/ensemble/bagging_classifier_accuracy.py b/src/RiskLabAI/ensemble/bagging_classifier_accuracy.py new file mode 100644 index 0000000..141d580 --- /dev/null +++ b/src/RiskLabAI/ensemble/bagging_classifier_accuracy.py @@ -0,0 +1,45 @@ +# RiskLabAI/ensemble/bagging_classifier_accuracy.py +""" +Implements the theoretical accuracy of a bagging classifier, +as described in "Advances in Financial Machine Learning" by de Prado (2018), +Chapter 6, Section 6.2, p. 86. +""" + +# import numpy as np <-- Removed unused import +from scipy.stats import binom + + +def bagging_classifier_accuracy(N: int, p: float) -> float: + """ + Calculates the theoretical accuracy of a bagging classifier + assuming independent classifiers. + + The bagging classifier is correct if the majority of + classifiers are correct. + + Parameters + ---------- + N : int + Number of classifiers (must be odd). + p : float + Accuracy of a single classifier (0 <= p <= 1). + + Returns + ------- + float + The accuracy of the ensemble. + """ + if N % 2 == 0: + raise ValueError(f"Number of estimators N must be odd. Got {N}.") + + # The majority threshold. + # e.g., if N=101, k=50. We need 51 or more correct votes. + k = (N - 1) // 2 + + # Probability of k or fewer successes (P(X <= k)) + prob_majority_wrong = binom.cdf(k, N, p) + + # Probability of more than k successes (P(X > k)) + prob_majority_correct = 1.0 - prob_majority_wrong + + return prob_majority_correct diff --git a/src/RiskLabAI/ensemble/empirical_bagging_accuracy.py b/src/RiskLabAI/ensemble/empirical_bagging_accuracy.py new file mode 100644 index 0000000..e59cb5d --- /dev/null +++ b/src/RiskLabAI/ensemble/empirical_bagging_accuracy.py @@ -0,0 +1,377 @@ +""" +Implements the BaggingClassifierAccuracy class to evaluate the +performance of a bagging classifier using various weighting schemes, +as described in "Advances in Financial Machine Learning" by de Prado (2018), +Chapter 6, Section 6.5, pp. 91-92. +""" + +from typing import TYPE_CHECKING, Optional + +import numpy as np +import pandas as pd +from scipy.stats import norm + +if TYPE_CHECKING: + import matplotlib.pyplot as plt # type hints only; optional at runtime +from sklearn.ensemble import BaggingClassifier +from sklearn.exceptions import NotFittedError +from sklearn.metrics import accuracy_score +from sklearn.tree import DecisionTreeClassifier + + +class BaggingClassifierAccuracy: + """ + Evaluates a bagging classifier's accuracy using different + weighting schemes based on decision tree c_i scores. + + Methods: + - fit: Fits the bagging classifier. + - calculate_c_i: Calculates the c_i score for each tree. + - calculate_weights: Computes weights (uniform, c_i, 1-c_i^2). + - predict: Predicts class labels using specified weights. + - evaluate_all_schemes: Gets accuracy for all weighting schemes. + """ + + def __init__( + self, + n_estimators: int = 1000, + max_samples: int = 100, + max_features: float = 1.0, + random_state: Optional[int] = None, + ): + """ + Initializes the BaggingClassifier. + + Note: This class uses a specific base_estimator: + DecisionTreeClassifier(criterion='entropy', max_features=1, + class_weight='balanced') as per the book's context. + + Parameters + ---------- + n_estimators : int, default=1000 + Number of trees in the ensemble. + max_samples : int, default=100 + Number of samples to draw for training each tree. + max_features : float, default=1.0 + Number of features to draw for training each tree. + random_state : int, optional + Seed for reproducibility. + """ + self.n_estimators = n_estimators + self.max_samples = max_samples + self.max_features = max_features + self.random_state = random_state + + self.base_estimator = DecisionTreeClassifier( + criterion="entropy", + max_features=1, # Trees vote on one feature + class_weight="balanced", + ) + + self.clf = BaggingClassifier( + estimator=self.base_estimator, + n_estimators=self.n_estimators, + max_samples=self.max_samples, + max_features=self.max_features, + random_state=self.random_state, + ) + + self.estimators_ = None + self.weights_ = None + self.c_i_scores_ = None + # <-- ADDED: Store class labels + self.class_0_ = None + self.class_1_ = None + + def fit(self, X: pd.DataFrame, y: pd.Series) -> "BaggingClassifierAccuracy": + """ + Fits the bagging classifier on the training data. + + Parameters + ---------- + X : pd.DataFrame + Training features. + y : pd.Series + Training labels. + + Returns + ------- + self + Fitted object. + """ + self.clf.fit(X, y) + self.estimators_ = self.clf.estimators_ + + # <-- ADDED: Check for binary classification and store classes + if len(self.clf.classes_) != 2: + raise ValueError("This class only supports binary classification.") + + self.class_0_ = self.clf.classes_[0] + self.class_1_ = self.clf.classes_[1] + + return self + + def calculate_c_i(self, X: pd.DataFrame, y: pd.Series) -> np.ndarray: + """ + Calculates the c_i score (accuracy) for each decision tree + on the full training set. + + Parameters + ---------- + X : pd.DataFrame + Training features. + y : pd.Series + Training labels. + + Returns + ------- + np.ndarray + Array of c_i scores for each estimator. + """ + if self.estimators_ is None: + raise NotFittedError("Classifier must be fitted first. Call .fit()") + + c_i_scores = [] + for tree in self.estimators_: + y_pred = tree.predict(X) + acc = accuracy_score(y, y_pred) + c_i_scores.append(acc) + + self.c_i_scores_ = np.array(c_i_scores) + return self.c_i_scores_ + + def calculate_weights(self, X: pd.DataFrame, y: pd.Series) -> dict[str, np.ndarray]: + """ + Calculates weights for each estimator based on three schemes: + 1. Uniform (w_i = 1/N) + 2. c_i (w_i = c_i / sum(c_i)) + 3. 1 - c_i^2 (w_i = (1 - c_i^2) / sum(1 - c_i^2)) + + Parameters + ---------- + X : pd.DataFrame + Training features. + y : pd.Series + Training labels. + + Returns + ------- + Dict[str, np.ndarray] + Dictionary of weight arrays for each scheme. + """ + if self.c_i_scores_ is None: + # calculate_c_i also checks if model is fitted + self.calculate_c_i(X, y) + + c_i = self.c_i_scores_ + n = len(c_i) + + # 1. Uniform weights + w_uniform = np.ones(n) / n + + # 2. c_i weights + sum_c_i = np.sum(c_i) + w_c_i = c_i / sum_c_i if sum_c_i != 0 else w_uniform + + # 3. 1 - c_i^2 weights (proportional to variance) + c_i_squared = c_i**2 + w_variance = 1.0 - c_i_squared + sum_w_var = np.sum(w_variance) + w_variance = w_variance / sum_w_var if sum_w_var != 0 else w_uniform + + self.weights_ = {"uniform": w_uniform, "c_i": w_c_i, "variance": w_variance} + return self.weights_ + + def predict(self, X: pd.DataFrame, weight_scheme: str = "uniform") -> np.ndarray: + """ + Predicts class labels for X using the specified weighting scheme. + + Parameters + ---------- + X : pd.DataFrame + Features to predict on. + weight_scheme : str, default='uniform' + The weighting scheme to use ('uniform', 'c_i', 'variance'). + + Returns + ------- + np.ndarray + Predicted class labels. + """ + # <-- UPDATED: Check fit status first + if self.estimators_ is None: + raise NotFittedError("Classifier must be fitted first. Call .fit()") + + if self.weights_ is None: + # <-- UPDATED: More specific error + raise NotFittedError( + "Weights must be calculated first. Call .calculate_weights()" + ) + + if weight_scheme not in self.weights_: + raise ValueError( + f"Unknown weight_scheme: {weight_scheme}. " + f"Must be one of {list(self.weights_.keys())}" + ) + + weights = self.weights_[weight_scheme] + + # Get predictions from each tree + # (N_samples, N_estimators) + tree_preds = np.array([tree.predict(X) for tree in self.estimators_]).T + + # <-- UPDATED: Convert labels {class_0, class_1} to {-1, 1} + # Map class_1 to 1, and class_0 to -1 + tree_preds_signed = np.where(tree_preds == self.class_1_, 1, -1) + + # Calculate weighted average vote + # (N_samples, N_estimators) * (N_estimators,) -> (N_samples,) + weighted_votes = np.dot(tree_preds_signed, weights) + + # <-- UPDATED: Convert vote back to {class_0, class_1} + # Positive vote -> class_1, Negative or Zero vote -> class_0 + y_pred = np.where(weighted_votes > 0, self.class_1_, self.class_0_) + + return y_pred + + def evaluate_all_schemes( + self, + X_test: pd.DataFrame, + y_test: pd.Series, + X_train: pd.DataFrame, + y_train: pd.Series, + ) -> dict[str, float]: + """ + Fits, calculates weights, and evaluates accuracy for all + three weighting schemes. + + Parameters + ---------- + X_test : pd.DataFrame + Test features. + y_test : pd.Series + Test labels. + X_train : pd.DataFrame + Training features (for calculating weights). + y_train : pd.Series + Training labels (for calculating weights). + + Returns + ------- + Dict[str, float] + Dictionary of accuracy scores for each scheme. + """ + # Fit classifier + self.fit(X_train, y_train) + + # Calculate weights (uses X_train, y_train implicitly) + self.calculate_weights(X_train, y_train) + + accuracies = {} + for scheme in self.weights_.keys(): + y_pred = self.predict(X_test, weight_scheme=scheme) + acc = accuracy_score(y_test, y_pred) + accuracies[scheme] = acc + + return accuracies + + +# --- Standalone Functions for Bootstrap Analysis --- + + +def calculate_bootstrap_accuracy( + clf: BaggingClassifier, X: pd.DataFrame, y: pd.Series, n_bootstraps: int = 1000 +) -> tuple[np.ndarray, float, float]: + """ + Calculates the accuracy of a bagging classifier over multiple + bootstrapped samples of the test set. + + This helps estimate the standard deviation of the accuracy (std(a_n)). + + Parameters + ---------- + clf : BaggingClassifier + A fitted BaggingClassifier instance. + X : pd.DataFrame + Test features. + y : pd.Series + Test labels. + n_bootstraps : int, default=1000 + Number of bootstrap iterations. + + Returns + ------- + Tuple[np.ndarray, float, float] + - a_n_values: Array of accuracy scores from each bootstrap. + - a_n_mean: The mean accuracy. + - a_n_std: The standard deviation of the accuracies. + """ + a_n_values = [] + n_samples = len(y) + + # Use indices from the original X/y DataFrames/Series + indices = X.index + + # --- CHANGE: Fixed typo n_bootstraMps -> n_bootstraps --- + for _ in range(n_bootstraps): + # --- END CHANGE --- + # Sample test set with replacement + boot_indices = np.random.choice(indices, n_samples, replace=True) + X_boot = X.loc[boot_indices] + y_boot = y.loc[boot_indices] + + # Predict on the bootstrapped sample + y_pred = clf.predict(X_boot) + acc = accuracy_score(y_boot, y_pred) + a_n_values.append(acc) + + a_n_values = np.array(a_n_values) + a_n_mean = np.mean(a_n_values) + a_n_std = np.std(a_n_values, ddof=1) + + return a_n_values, a_n_mean, a_n_std + + +def plot_bootstrap_accuracy_distribution( + a_n_values: np.ndarray, + a_n_mean: float, + a_n_std: float, + ax: Optional["plt.Axes"] = None, +) -> "plt.Axes": + """ + Plots the distribution of bootstrapped accuracy scores. + + Parameters + ---------- + a_n_values : np.ndarray + Array of accuracy scores from each bootstrap. + a_n_mean : float + The mean accuracy. + a_n_std : float + The standard deviation of the accuracies. + ax : plt.Axes, optional + Matplotlib Axes object to plot on. If None, creates a new one. + + Returns + ------- + plt.Axes + The Axes object with the plot. + """ + import matplotlib.pyplot as plt # optional dependency: RiskLabAI[plot] + import seaborn as sns # optional dependency: RiskLabAI[plot] + + if ax is None: + fig, ax = plt.subplots(figsize=(10, 6)) + + sns.histplot( + a_n_values, kde=True, ax=ax, stat="density", label="Empirical Distribution" + ) + + # Overlay a normal distribution + x_min, x_max = ax.get_xlim() + x = np.linspace(x_min, x_max, 100) + p = norm.pdf(x, a_n_mean, a_n_std) + ax.plot(x, p, "k", linewidth=2, label=f"Normal(μ={a_n_mean:.3f}, σ={a_n_std:.3f})") + + ax.legend() + return ax diff --git a/src/RiskLabAI/features/__init__.py b/src/RiskLabAI/features/__init__.py new file mode 100644 index 0000000..511d313 --- /dev/null +++ b/src/RiskLabAI/features/__init__.py @@ -0,0 +1,6 @@ +from . import ( + entropy_features, + feature_importance, + microstructural_features, + structural_breaks, +) diff --git a/src/RiskLabAI/features/entropy_features/__init__.py b/src/RiskLabAI/features/entropy_features/__init__.py new file mode 100644 index 0000000..96576ca --- /dev/null +++ b/src/RiskLabAI/features/entropy_features/__init__.py @@ -0,0 +1,31 @@ +""" +RiskLabAI Entropy Features Module + +Provides various entropy estimators for financial time series, such +as Shannon, Lempel-Ziv, and Kontoyiannis, plus the bias-corrected family +(Miller-Madow, Grassberger, NSB) for undersampled symbol counts. +""" + +from .bias_corrected import ( + grassberger_entropy, + miller_madow_entropy, + nsb_entropy, +) +from .kontoyiannis import kontoyiannis_entropy, longest_match_length +from .lempel_ziv import lempel_ziv_entropy +from .plug_in import plug_in_entropy_estimator +from .pmf import ngram_counts, probability_mass_function +from .shannon import shannon_entropy + +__all__ = [ + "shannon_entropy", + "lempel_ziv_entropy", + "probability_mass_function", + "ngram_counts", + "plug_in_entropy_estimator", + "kontoyiannis_entropy", + "longest_match_length", + "miller_madow_entropy", + "grassberger_entropy", + "nsb_entropy", +] diff --git a/src/RiskLabAI/features/entropy_features/bias_corrected.py b/src/RiskLabAI/features/entropy_features/bias_corrected.py new file mode 100644 index 0000000..507c8ed --- /dev/null +++ b/src/RiskLabAI/features/entropy_features/bias_corrected.py @@ -0,0 +1,293 @@ +""" +Bias-corrected Shannon entropy estimators (Miller-Madow, Grassberger, NSB). + +de Prado's plug-in / maximum-likelihood entropy (`plug_in_entropy_estimator`, AFML ch.18) is +systematically biased downward when the number of distinct symbols (the alphabet, here the number of +possible n-grams) is large relative to the sample, because unobserved symbols contribute zero. The +bias is approximately -(K - 1) / (2N), growing with the alphabet K and shrinking with the sample N. +These estimators add back the missing mass: Miller-Madow analytically to first order, Grassberger to +higher order, and NSB by integrating over a near-uniform-entropy Bayesian prior. + +All three operate on the same n-gram counts as the plug-in (via +:func:`~RiskLabAI.features.entropy_features.pmf.ngram_counts`) and return entropy in bits, normalized +by the word length, so they are drop-in comparable with `plug_in_entropy_estimator`. The plug-in's +behaviour is unchanged. + +Admitted in Appraisal 06 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tag (preferred-when / avoid-when), +verbatim from the ledger: + + Prefer a bias-corrected estimator over the plug-in whenever the symbol counts are undersampled (a + large effective alphabet relative to the sample: long words, fine encodings, or short windows). + NSB is most accurate in deep undersampling; Grassberger is close at lower cost (the practical + default when undersampled); Miller-Madow is the cheap first-order fix. All converge to the plug-in + when N is much larger than K with no over-correction, so a correction is a safe default and the + plug-in remains fine when well-sampled. The gain is negligible for coarse encodings (e.g. binary), + and a gap caused by non-stationarity (e.g. sigma encoding) is not an entropy-bias problem the + correction can fix. + +Evidence and caveats: appraisals/06_verdict.md. + +References +---------- +Miller, G. (1955) Note on the bias of information estimates. In Information Theory in Psychology. +Grassberger, P. (2008) Entropy estimates from insufficient samplings. arXiv:physics/0307138. +Nemenman, I., Shafee, F. and Bialek, W. (2002) Entropy and inference, revisited. NIPS 14. +Wolpert, D. H. and Wolf, D. R. (1995) Estimating functions of probability distributions from a + finite set of samples. Physical Review E, 52(6), 6841. +""" + +from functools import lru_cache +from typing import Optional + +import numpy as np +from scipy import integrate, optimize +from scipy.special import gammaln, psi + +from .pmf import ngram_counts + +_LN2 = np.log(2.0) + + +def _counts_array(message: str, approximate_word_length: int) -> np.ndarray: + """Positive n-gram counts of a message as a float array (empty if the message is too short).""" + counts = ngram_counts(message, approximate_word_length) + if not counts: + return np.zeros(0) + return np.asarray(list(counts.values()), dtype=np.float64) + + +def _plugin_bits(counts: np.ndarray) -> float: + """Plug-in / maximum-likelihood entropy (bits) of a count vector: H = -sum p log2 p.""" + n = counts.sum() + if n <= 0: + return 0.0 + p = counts / n + return float(-np.sum(p * np.log2(p))) + + +def miller_madow_entropy(message: str, approximate_word_length: int = 1) -> float: + """ + Miller-Madow bias-corrected Shannon entropy, normalized by word length (bits). + + The plug-in entropy plus the first-order analytic correction (K_hat - 1) / (2N), where K_hat is + the number of observed n-grams and N the number of n-grams counted. The cheapest correction; it + captures most of the gain at moderate undersampling. + + Prefer a bias-corrected estimator over the plug-in whenever the symbol counts are undersampled (a + large effective alphabet relative to the sample: long words, fine encodings, or short windows). + Miller-Madow is the cheap first-order fix; it converges to the plug-in when N is much larger than + K, with no over-correction. The gain is negligible for coarse encodings (e.g. binary). See the + module docstring for the full regime tag and appraisals/06_verdict.md. + + Parameters + ---------- + message : str + Input string (e.g. a discretized time series "110100..."). + approximate_word_length : int, default 1 + The n-gram length; the result is normalized by it, matching `plug_in_entropy_estimator`. + + Returns + ------- + float + The Miller-Madow entropy in bits per symbol. + + Examples + -------- + >>> from RiskLabAI.features.entropy_features import miller_madow_entropy + >>> miller_madow_entropy("110100110101", approximate_word_length=1) >= 0 + True + """ + counts = _counts_array(message, approximate_word_length) + if counts.size == 0: + return 0.0 + n = counts.sum() + k_hat = counts.size + corrected = _plugin_bits(counts) + (k_hat - 1.0) / (2.0 * n * _LN2) + return corrected / approximate_word_length + + +def _grassberger_g(n: np.ndarray) -> np.ndarray: + """G(n) = psi(n) + 0.5 (-1)^n [psi((n+1)/2) - psi(n/2)] (Grassberger 2008), in nats.""" + sign = np.where(n.astype(np.int64) % 2 == 0, 1.0, -1.0) + return psi(n) + 0.5 * sign * (psi((n + 1.0) / 2.0) - psi(n / 2.0)) + + +def grassberger_entropy(message: str, approximate_word_length: int = 1) -> float: + """ + Grassberger (2008) bias-corrected Shannon entropy, normalized by word length (bits). + + Uses the digamma finite-sample correction H = ln N - (1/N) sum n_i G(n_i), converted to bits. A + higher-order correction with no tuning, intermediate in cost between Miller-Madow and NSB. + + Prefer a bias-corrected estimator over the plug-in whenever the symbol counts are undersampled (a + large effective alphabet relative to the sample: long words, fine encodings, or short windows). + Grassberger is close to NSB at lower cost (the practical default when undersampled); it converges + to the plug-in when N is much larger than K, with no over-correction. The gain is negligible for + coarse encodings (e.g. binary). See the module docstring for the full regime tag and + appraisals/06_verdict.md. + + Parameters + ---------- + message : str + Input string. + approximate_word_length : int, default 1 + The n-gram length; the result is normalized by it. + + Returns + ------- + float + The Grassberger entropy in bits per symbol. + + Examples + -------- + >>> from RiskLabAI.features.entropy_features import grassberger_entropy + >>> grassberger_entropy("110100110101", approximate_word_length=1) >= 0 + True + """ + counts = _counts_array(message, approximate_word_length) + if counts.size == 0: + return 0.0 + n = counts.sum() + h_nats = np.log(n) - (1.0 / n) * np.sum(counts * _grassberger_g(counts)) + return float(h_nats / _LN2) / approximate_word_length + + +# --- NSB (Nemenman-Shafee-Bialek 2002) ------------------------------------------------------- +# +# Posterior mean entropy at a fixed Dirichlet concentration beta (Wolpert-Wolf 1995), integrated over +# beta against the NSB prior P(beta) proportional to d/dbeta E[H | beta], which makes the a priori mean +# entropy xi(beta) = psi0(K beta + 1) - psi0(beta + 1) near-uniform on (0, ln K). K is the true +# alphabet size; integration is in the xi variable (uniform prior) for stability, and the beta(xi) +# inversion depends only on (K, n_points) so it is cached. + + +def _xi_of_beta(beta: float, k: int) -> float: + """A priori mean entropy (nats) implied by concentration beta over K symbols.""" + return psi(k * beta + 1.0) - psi(beta + 1.0) + + +@lru_cache(maxsize=256) +def _nsb_beta_grid( + k: int, n_points: int +) -> tuple[tuple[float, ...], tuple[float, ...]]: + """ + Quadrature nodes for the NSB xi-integral over (0, ln K): cached ``(xi_nodes, beta_nodes)``. + + xi(beta) is monotone increasing from 0 (beta -> 0) to ln K (beta -> inf); each node is inverted + once with Brent's method. Depends only on (K, n_points), so it is cached across calls (the result + is returned as tuples to keep it hashable / immutable). + """ + xi_max = np.log(k) + xi_nodes = np.linspace( + xi_max / (n_points + 1), xi_max * n_points / (n_points + 1), n_points + ) + beta_nodes = np.empty(n_points) + for i, xi in enumerate(xi_nodes): + + def _f(beta: float, target: float = xi) -> float: + return _xi_of_beta(beta, k) - target + + beta_nodes[i] = optimize.brentq(_f, 1e-8, 1e8, xtol=1e-12) + return tuple(xi_nodes.tolist()), tuple(beta_nodes.tolist()) + + +def _log_evidence(beta: np.ndarray, k: int, n: float, counts: np.ndarray) -> np.ndarray: + """Unnormalised log marginal likelihood at each concentration in ``beta`` (nats, vectorised).""" + b = beta[:, None] + obs_term = np.sum(gammaln(counts[None, :] + b) - gammaln(b), axis=1) + return gammaln(k * beta) - gammaln(n + k * beta) + obs_term + + +def _mean_entropy_given_beta( + beta: np.ndarray, k: int, n: float, counts: np.ndarray, k_obs: int +) -> np.ndarray: + """ + Posterior mean entropy (nats) at each concentration in ``beta`` (Wolpert-Wolf 1995). + + As beta -> 0 this tends to psi(N + 1) - (1/N) sum n_i psi(n_i + 1), the maximum-likelihood-like + Bayesian limit; as beta -> inf it tends to the entropy of the uniform distribution over K. + """ + denom = n + k * beta + b = beta[:, None] + observed = np.sum((counts[None, :] + b) * psi(counts[None, :] + b + 1.0), axis=1) + unobserved = (k - k_obs) * beta * psi(beta + 1.0) + return psi(denom + 1.0) - (observed + unobserved) / denom + + +def _nsb_from_counts(counts: np.ndarray, k: int, n_points: int) -> float: + """NSB posterior mean entropy (bits) for a count vector and true alphabet size K.""" + n = counts.sum() + if n <= 0 or k <= 1: + return 0.0 + k_obs = counts.size + + xi_tuple, beta_tuple = _nsb_beta_grid(k, n_points) + xi_nodes = np.asarray(xi_tuple) + beta = np.asarray(beta_tuple) + + log_ev = _log_evidence(beta, k, n, counts) + mean_h = _mean_entropy_given_beta(beta, k, n, counts, k_obs) + + # Uniform prior in xi; stabilise the evidence by its maximum before exponentiating. + log_ev -= np.max(log_ev) + weight = np.exp(log_ev) + numerator = integrate.trapezoid(weight * mean_h, xi_nodes) + normaliser = integrate.trapezoid(weight, xi_nodes) + if normaliser <= 0 or not np.isfinite(normaliser): + return float("nan") + return float((numerator / normaliser) / _LN2) + + +def nsb_entropy( + message: str, + approximate_word_length: int = 1, + alphabet_size: Optional[int] = None, + n_points: int = 80, +) -> float: + """ + NSB (Nemenman-Shafee-Bialek 2002) Bayesian Shannon entropy, normalized by word length (bits). + + The Wolpert-Wolf posterior mean entropy integrated over the Dirichlet concentration against a + prior chosen so the implied prior on entropy is near-uniform. NSB needs the true alphabet size; it + is the most accurate estimator in deep undersampling, and the most expensive. + + Prefer a bias-corrected estimator over the plug-in whenever the symbol counts are undersampled (a + large effective alphabet relative to the sample: long words, fine encodings, or short windows). + NSB is most accurate in deep undersampling; it converges to the plug-in when N is much larger than + K, with no over-correction. The gain is negligible for coarse encodings (e.g. binary), and a gap + caused by non-stationarity (e.g. sigma encoding) is not an entropy-bias problem the correction can + fix. See the module docstring for the full regime tag and appraisals/06_verdict.md. + + Parameters + ---------- + message : str + Input string. + approximate_word_length : int, default 1 + The n-gram length; the result is normalized by it. + alphabet_size : int, optional + The number of distinct base symbols the encoding can produce. The effective alphabet of words + is ``alphabet_size ** approximate_word_length``. Defaults to the number of distinct symbols + observed in the message (a lower bound; pass the encoding's true alphabet when known). + n_points : int, default 80 + Number of quadrature nodes for the Bayesian integral. + + Returns + ------- + float + The NSB entropy in bits per symbol. + + Examples + -------- + >>> from RiskLabAI.features.entropy_features import nsb_entropy + >>> nsb_entropy("110100110101", approximate_word_length=1, alphabet_size=2) >= 0 + True + """ + counts = _counts_array(message, approximate_word_length) + if counts.size == 0: + return 0.0 + base = alphabet_size if alphabet_size is not None else len(set(message)) + base = max(int(base), 1) + k_eff = base**approximate_word_length + # The effective alphabet cannot be smaller than the number of distinct words observed. + k_eff = max(k_eff, counts.size) + return _nsb_from_counts(counts, k_eff, n_points) / approximate_word_length diff --git a/src/RiskLabAI/features/entropy_features/kontoyiannis.py b/src/RiskLabAI/features/entropy_features/kontoyiannis.py new file mode 100644 index 0000000..aa976fe --- /dev/null +++ b/src/RiskLabAI/features/entropy_features/kontoyiannis.py @@ -0,0 +1,109 @@ +""" +Implements the Kontoyiannis Entropy estimator (LZ-based). +""" + +from math import log2 +from typing import Optional + + +def longest_match_length(message: str, i: int, n: int) -> tuple[int, str]: + """ + Find the length of the longest match for the substring starting at `i`. + + This function searches for the longest match of `message[i:i+l]` + within the preceding window `message[max(0, i-n):i]`. + + Parameters + ---------- + message : str + Input string. + i : int + The starting index of the substring to match. + n : int + The length of the look-back window. + + Returns + ------- + Tuple[int, str] + - The length of the longest match (L_i) + 1. + - The matched substring. + """ + longest_match = "" + # Iterate through possible match lengths + for length in range(1, n + 1): + pattern = message[i : i + length] + + # Stop if pattern goes beyond message length + if i + length > len(message): + break + + found = False + # Look back in the window [max(0, i-n), i-1] + for j in range(max(0, i - n), i): + candidate = message[j : j + length] + if pattern == candidate: + longest_match = pattern + found = True + break + + # If a pattern of this length was not found, the + # longest match was one shorter. + if not found: + break + + return len(longest_match) + 1, longest_match + + +def kontoyiannis_entropy(message: str, window: Optional[int] = None) -> float: + r""" + Calculate Kontoyiannis Entropy (an LZ78-based estimator). + + .. math:: + H_k(n) = \frac{1}{\sum_{i} 1} \sum_{i} \frac{\log_2(n_i)}{L_i(n)} + + Reference: + Kontoyiannis, I. (1998). "Pointwise redundancy in Lempel-Ziv + parsing." + + Parameters + ---------- + message : str + Input string. + window : int, optional + If None, uses an expanding window (full lookback, n_i = i). + If set, uses a rolling window (n_i = window). + + Returns + ------- + float + The calculated Kontoyiannis Entropy (H_k). + """ + message_length = len(message) + sum_h = 0.0 + num_points = 0 + + if window is None: + # Expanding window: n = i + points = range(2, message_length) + else: + # Rolling window: n = window + window = min(window, message_length - 1) + points = range(window, message_length) + + if not points: + return 0.0 + + for i in points: + n = i if window is None else window + if n == 0: + continue # Avoid log2(0) + + l_i, _ = longest_match_length(message, i, n) + + sum_h += log2(n) / l_i + num_points += 1 + + if num_points == 0: + return 0.0 + + return sum_h / num_points diff --git a/src/RiskLabAI/features/entropy_features/lempel_ziv.py b/src/RiskLabAI/features/entropy_features/lempel_ziv.py new file mode 100644 index 0000000..38b5887 --- /dev/null +++ b/src/RiskLabAI/features/entropy_features/lempel_ziv.py @@ -0,0 +1,41 @@ +""" +Implements the Lempel-Ziv (LZ) Entropy estimator. +""" + + +def lempel_ziv_entropy(message: str) -> float: + """ + Calculate the Lempel-Ziv (LZ) complexity as an entropy estimator. + + This function calculates the number of unique substrings encountered + during a one-pass traversal of the message, normalized by the + message length. + + Parameters + ---------- + message : str + Input string. + + Returns + ------- + float + The calculated Lempel-Ziv Entropy. + """ + if not message: + return 0.0 + + library = set() + message_length = len(message) + i = 0 + + while i < message_length: + j = i + # Find the longest substring starting at `i` that is *not* in the library + while j < message_length and message[i : j + 1] in library: + j += 1 + + # Add the new, unseen substring to the library + library.add(message[i : j + 1]) + i = j + 1 + + return len(library) / message_length diff --git a/src/RiskLabAI/features/entropy_features/plug_in.py b/src/RiskLabAI/features/entropy_features/plug_in.py new file mode 100644 index 0000000..ae5052e --- /dev/null +++ b/src/RiskLabAI/features/entropy_features/plug_in.py @@ -0,0 +1,39 @@ +""" +Implements the Plug-in Entropy estimator. +""" + +from math import log2 + +from .pmf import probability_mass_function + + +def plug_in_entropy_estimator(message: str, approximate_word_length: int = 1) -> float: + """ + Calculate the Plug-in Entropy Estimator (based on n-gram PMF). + + This is the Shannon entropy applied to the PMF of n-grams + (words of `approximate_word_length`). + + Parameters + ---------- + message : str + Input string. + approximate_word_length : int, default=1 + The n-gram length. + + Returns + ------- + float + The calculated Plug-in Entropy, normalized by word length. + """ + if not message: + return 0.0 + + pmf = probability_mass_function(message, approximate_word_length) + if not pmf: + return 0.0 + + plug_in_entropy = -sum(p * log2(p) for p in pmf.values() if p > 0) + + # Normalize by word length + return plug_in_entropy / approximate_word_length diff --git a/src/RiskLabAI/features/entropy_features/pmf.py b/src/RiskLabAI/features/entropy_features/pmf.py new file mode 100644 index 0000000..75294dc --- /dev/null +++ b/src/RiskLabAI/features/entropy_features/pmf.py @@ -0,0 +1,64 @@ +""" +Implements the n-gram count and Probability Mass Function (PMF) calculation. +""" + +from collections import Counter + + +def ngram_counts(message: str, approximate_word_length: int) -> dict[str, int]: + """ + Count the overlapping n-grams (words) in a message. + + This is the shared counting path used by both the plug-in estimator (via + :func:`probability_mass_function`) and the bias-corrected estimators, which need + the raw counts and the sample size rather than the normalized probabilities. + + Parameters + ---------- + message : str + Input string. + approximate_word_length : int + The length of the "words" or n-grams to analyze (e.g., 1, 2, 3). + + Returns + ------- + Dict[str, int] + A dictionary mapping each observed n-gram to its integer count. + """ + if not message or len(message) < approximate_word_length: + return {} + + return dict( + Counter( + message[i : i + approximate_word_length] + for i in range(len(message) - approximate_word_length + 1) + ) + ) + + +def probability_mass_function( + message: str, approximate_word_length: int +) -> dict[str, float]: + """ + Calculate the Probability Mass Function (PMF) of n-grams. + + Parameters + ---------- + message : str + Input string. + approximate_word_length : int + The length of the "words" or n-grams to analyze (e.g., 1, 2, 3). + + Returns + ------- + Dict[str, float] + A dictionary mapping each n-gram to its probability. + """ + counts = ngram_counts(message, approximate_word_length) + + # The total number of n-grams (windows). + num_windows = float(sum(counts.values())) + if num_windows == 0: + return {} + + return {key: count / num_windows for key, count in counts.items()} diff --git a/src/RiskLabAI/features/entropy_features/shannon.py b/src/RiskLabAI/features/entropy_features/shannon.py new file mode 100644 index 0000000..937fb63 --- /dev/null +++ b/src/RiskLabAI/features/entropy_features/shannon.py @@ -0,0 +1,34 @@ +""" +Implements the Shannon Entropy estimator. +""" + +from collections import Counter +from math import log2 + + +def shannon_entropy(message: str) -> float: + """ + Calculate the Shannon Entropy of a message. + + Parameters + ---------- + message : str + Input string (e.g., a discretized time series "1-1101..."). + + Returns + ------- + float + The calculated Shannon Entropy (in bits). + """ + if not message: + return 0.0 + + character_counts = Counter(message) + message_length = len(message) + + entropy = -sum( + (count / message_length) * log2(count / message_length) + for count in character_counts.values() + ) + + return entropy diff --git a/src/RiskLabAI/features/feature_importance/__init__.py b/src/RiskLabAI/features/feature_importance/__init__.py new file mode 100644 index 0000000..118b1cc --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/__init__.py @@ -0,0 +1,66 @@ +""" +RiskLabAI Feature Importance Module + +Provides a framework for calculating feature importance using +various methods (MDI, MDA, SFI) and clustering. +""" + +# Import from other RiskLabAI modules +from RiskLabAI.cluster.clustering import ( + cluster_k_means_top, + random_block_correlation, +) +from RiskLabAI.data.synthetic_data.simulation import ( + form_true_matrix, + simulates_cov_mu, +) + +from .clustered_feature_importance_mda import ClusteredFeatureImportanceMDA +from .clustered_feature_importance_mdi import ClusteredFeatureImportanceMDI +from .debiased_importance import ( + conditional_predictive_impact, + mdi_plus_importance, +) +from .feature_importance_controller import FeatureImportanceController +from .feature_importance_factory import FeatureImportanceFactory +from .feature_importance_mda import FeatureImportanceMDA + +# Strategy Implementations +from .feature_importance_mdi import FeatureImportanceMDI +from .feature_importance_sfi import FeatureImportanceSFI + +# Core Strategy Pattern +from .feature_importance_strategy import FeatureImportanceStrategy + +# Utility Functions for this module +from .generate_synthetic_data import get_test_dataset +from .orthogonal_features import _compute_eigenvectors, orthogonal_features +from .weighted_tau import calculate_weighted_tau + +# Placeholder for future modules (from original file) +# from . import swefi + +__all__ = [ + # Core Pattern + "FeatureImportanceStrategy", + "FeatureImportanceFactory", + "FeatureImportanceController", + # Strategy Implementations + "FeatureImportanceMDI", + "ClusteredFeatureImportanceMDI", + "FeatureImportanceMDA", + "ClusteredFeatureImportanceMDA", + "FeatureImportanceSFI", + # Debiased / conditional importance (Appraisal 10) + "mdi_plus_importance", + "conditional_predictive_impact", + # Imported utilities from *other* modules + "cluster_k_means_top", + "random_block_correlation", + "form_true_matrix", + "simulates_cov_mu", + # Utilities *from this* module + "get_test_dataset", + "orthogonal_features", + "calculate_weighted_tau", +] diff --git a/src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py b/src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py new file mode 100644 index 0000000..a88bb0c --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py @@ -0,0 +1,157 @@ +""" +Computes Clustered Mean Decrease Accuracy (MDA) feature importance. +""" + +import logging +from typing import Any + +import numpy as np +import pandas as pd +from sklearn.metrics import log_loss +from sklearn.model_selection import KFold + +from .feature_importance_strategy import FeatureImportanceStrategy + +logger = logging.getLogger(__name__) + + +class ClusteredFeatureImportanceMDA(FeatureImportanceStrategy): + """ + Computes clustered feature importance using MDA. + + This method shuffles entire *clusters* of features at a time + and measures the decrease in model performance. + """ + + def __init__( + self, + classifier: object, + clusters: dict[str, list[str]], + n_splits: int = 10, + random_state: int = 42, + ): + """ + Initialize the strategy. + + Parameters + ---------- + classifier : object + An *untrained* scikit-learn classifier. + clusters : Dict[str, List[str]] + Dictionary mapping cluster names to lists of feature names. + n_splits : int, default=10 + Number of splits for cross-validation. + random_state : int, default=42 + Seed for KFold and shuffling for reproducibility. + """ + self.classifier = classifier + self.clusters = clusters + self.n_splits = n_splits + self.random_state = random_state + + def compute(self, x: pd.DataFrame, y: pd.Series, **kwargs: Any) -> pd.DataFrame: + """ + Compute Clustered MDA feature importance. + + Parameters + ---------- + x : pd.DataFrame + The feature data. + y : pd.Series + The target data. + **kwargs : Any + - 'train_sample_weights': Optional sample weights for training. + - 'score_sample_weights': Optional sample weights for scoring. + + Returns + ------- + pd.DataFrame + DataFrame with "Mean" and "StandardDeviation" of importance + for each *cluster*. + """ + train_weights = kwargs.get("train_sample_weights") + score_weights = kwargs.get("score_sample_weights") + + if train_weights is None: + train_weights = np.ones(x.shape[0]) + if score_weights is None: + score_weights = np.ones(x.shape[0]) + + cv_generator = KFold( + n_splits=self.n_splits, shuffle=True, random_state=self.random_state + ) + baseline_scores = pd.Series(dtype=float) + shuffled_scores = pd.DataFrame(columns=self.clusters.keys(), dtype=float) + + for i, (train_idx, test_idx) in enumerate(cv_generator.split(X=x)): + logger.debug("Fold %d start ...", i) + + x_train, y_train, w_train = ( + x.iloc[train_idx, :], + y.iloc[train_idx], + train_weights[train_idx], + ) + x_test, y_test, w_test = ( + x.iloc[test_idx, :], + y.iloc[test_idx], + score_weights[test_idx], + ) + + classifier_fit = self.classifier.fit( + X=x_train, y=y_train, sample_weight=w_train + ) + prediction_probability = classifier_fit.predict_proba(x_test) + + baseline_scores.loc[i] = -log_loss( + y_test, + prediction_probability, + labels=self.classifier.classes_, + sample_weight=w_test, + ) + + # Get scores for each shuffled *cluster* + rng = np.random.default_rng(self.random_state + i) + for cluster_name in shuffled_scores.columns: + x_test_shuffled = x_test.copy(deep=True) + + # --- CORRECTED SHUFFLING LOGIC --- + # Get all feature names for this cluster + cluster_cols = self.clusters[cluster_name] + + if not cluster_cols: # Skip if cluster is empty + shuffled_scores.loc[i, cluster_name] = baseline_scores.loc[i] + continue + + # Get the underlying numpy array for these columns + cluster_data = x_test_shuffled[cluster_cols].values.copy() + + # Shuffle the rows of this array in-place. + # This applies the *same* permutation to all features + # in the cluster, preserving intra-cluster correlation. + rng.shuffle(cluster_data) + + # Assign the shuffled data back + x_test_shuffled[cluster_cols] = cluster_data + # --- END CORRECTION --- + + prob = classifier_fit.predict_proba(x_test_shuffled) + shuffled_scores.loc[i, cluster_name] = -log_loss( + y_test, prob, labels=self.classifier.classes_, sample_weight=w_test + ) + + # Calculate importance as the simple drop in score + importances = shuffled_scores.rsub(baseline_scores, axis=0) + + # Central Limit Theorem for standard deviation + importances_summary = pd.concat( + { + "Mean": importances.mean(), + "StandardDeviation": ( + importances.std() * (importances.shape[0] ** -0.5) + ), + }, + axis=1, + ) + + importances_summary.index = [f"C_{i}" for i in importances_summary.index] + return importances_summary diff --git a/src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py b/src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py new file mode 100644 index 0000000..dd7092d --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py @@ -0,0 +1,108 @@ +""" +Computes Clustered Mean Decrease Impurity (MDI) feature importance. +""" + +from typing import Any + +import numpy as np +import pandas as pd +from sklearn.ensemble import BaseEnsemble + +from .feature_importance_strategy import FeatureImportanceStrategy + + +class ClusteredFeatureImportanceMDI(FeatureImportanceStrategy): + """ + Computes Clustered MDI feature importance. + + Aggregates MDI importance for pre-defined clusters of features. + """ + + def __init__( + self, + classifier: BaseEnsemble, + clusters: dict[str, list[str]], + ): + """ + Initialize the strategy. + + Parameters + ---------- + classifier : BaseEnsemble + An *untrained* scikit-learn ensemble model. + clusters : Dict[str, List[str]] + Dictionary mapping cluster names to lists of feature names. + """ + if not isinstance(classifier, BaseEnsemble): + raise TypeError("Classifier must be an ensemble (e.g., RandomForest).") + + self.classifier = classifier + self.clusters = clusters + + def _group_mean_std( + self, dataframe: pd.DataFrame, clusters: dict[str, list[str]] + ) -> pd.DataFrame: + """ + Calculate the mean and standard deviation for cluster importances. + """ + output = pd.DataFrame(columns=["Mean", "StandardDeviation"]) + + for cluster_name, feature_names in clusters.items(): + # Sum importance for all features in the cluster + cluster_data = dataframe[feature_names].sum(axis=1) + + cluster_mean = cluster_data.mean() + cluster_std = cluster_data.std() + + output.loc[f"C_{cluster_name}", "Mean"] = cluster_mean + output.loc[f"C_{cluster_name}", "StandardDeviation"] = cluster_std * ( + cluster_data.shape[0] ** -0.5 + ) + return output + + def compute(self, x: pd.DataFrame, y: pd.Series, **kwargs: Any) -> pd.DataFrame: + """ + Compute Clustered MDI feature importance. + + Parameters + ---------- + x : pd.DataFrame + The feature data. + y : pd.Series + The target data. + **kwargs : Any + Keyword arguments for the classifier's `fit` method + (e.g., `sample_weight`). + + Returns + ------- + pd.DataFrame + DataFrame with "Mean" and "StandardDeviation" of importance + for each *cluster*. + """ + train_sample_weights = kwargs.get("sample_weight") + + # Fit the classifier + self.classifier.fit(x, y, sample_weight=train_sample_weights) + + # Get importance from each tree + importances_dict = { + i: tree.feature_importances_ + for i, tree in enumerate(self.classifier.estimators_) + } + importances_df = pd.DataFrame.from_dict(importances_dict, orient="index") + + if hasattr(self.classifier, "feature_names_in_"): + importances_df.columns = self.classifier.feature_names_in_ + else: + importances_df.columns = x.columns + + # Replace 0 with NaN + importances_df.replace(0, np.nan, inplace=True) + + # Group by cluster + aggregated_importances = self._group_mean_std(importances_df, self.clusters) + + # Normalize + aggregated_importances /= aggregated_importances["Mean"].sum() + return aggregated_importances diff --git a/src/RiskLabAI/features/feature_importance/debiased_importance.py b/src/RiskLabAI/features/feature_importance/debiased_importance.py new file mode 100644 index 0000000..a601ebc --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/debiased_importance.py @@ -0,0 +1,277 @@ +r""" +Debiased and conditional feature importance: MDI+ and the Conditional Predictive Impact (CPI). + +de Prado's MDI counts impurity reduction, which inflates with split opportunities (high cardinality) +and with in-sample reuse, so it over-credits noisy / continuous features and can rank noise above +signal; MDA has no significance test. Two published fixes extend the baselines in +`features.feature_importance`: + +- MDI+ (Agarwal et al. 2023) recasts each tree's MDI as a regularized GLM on the tree's decision-stump + representation and scores a feature by its out-of-sample partial contribution, removing the + in-sample / cardinality inflation; and +- the Conditional Predictive Impact (CPI, Watson and Wright 2021) measures a feature's importance as + the model-loss increase when the true feature is replaced by a conditionally-resampled knockoff, with + a paired significance test, the calibrated test MDA lacks. + +Admitted in Appraisal 10 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tags, verbatim from the verdict: + + MDI+: prefer MDI+ over MDI when features are noisy, high-cardinality, or mixed-type - it rejects + noise/cardinality inflation (noise-reject 0.94 vs 0.50) and recovers true relevance better; + converges to MDI when features are orthogonal and high-SNR. + + CPI: prefer CPI when you need a statistically valid test of whether a feature matters (MDA has + none); it holds nominal size with good power. + +The implemented MDI+ is the simplified faithful variant admitted (per-tree decision-stump ridge GLM +with an out-of-bag partial-variance score), NOT the full leave-one-out / similarity-weighted paper +algorithm, which is a logged refinement. CPI's test holds nominal size in the analysis regimes; on the +hardest held-out corner (high correlation x low SNR) it is mildly elevated (~0.067 vs 0.05), a small +finite-sample effect. Evidence and caveats: appraisals/10_verdict.md. + +References +---------- +Agarwal, A. et al. (2023) MDI+: A flexible random forest based feature importance framework. + arXiv:2307.01932. +Watson, D. S. and Wright, M. N. (2021) Testing conditional independence in supervised learning + algorithms. Machine Learning, 110, 2107-2129. (The Conditional Predictive Impact.) +""" + +from __future__ import annotations + +from numbers import Integral + +import numpy as np +import pandas as pd +from scipy.stats import ttest_1samp +from sklearn.ensemble import RandomForestClassifier +from sklearn.linear_model import Ridge +from sklearn.model_selection import KFold +from sklearn.utils import check_random_state + +_DEFAULT_N_ESTIMATORS = 100 +_DEFAULT_MAX_DEPTH = 6 +_DEFAULT_TREES_DECOMP = 60 + + +def _bootstrap_sample_indices( + random_state: int, n_samples: int, max_samples: int | float | None +) -> np.ndarray: + """Reproduce documented bootstrap sampling without private library APIs.""" + if max_samples is None: + n_bootstrap = n_samples + elif isinstance(max_samples, Integral): + n_bootstrap = int(max_samples) + else: + n_bootstrap = max(round(n_samples * float(max_samples)), 1) + return check_random_state(random_state).randint( + 0, n_samples, n_bootstrap + ) + + +def _default_forest( + random_state: int, n_estimators: int = _DEFAULT_N_ESTIMATORS +) -> RandomForestClassifier: + return RandomForestClassifier( + n_estimators=n_estimators, + max_depth=_DEFAULT_MAX_DEPTH, + random_state=random_state, + n_jobs=1, + bootstrap=True, + ) + + +def mdi_plus_importance( + x: pd.DataFrame, + y: pd.Series, + random_state: int = 0, + n_trees: int = _DEFAULT_TREES_DECOMP, + classifier: RandomForestClassifier | None = None, +) -> pd.Series: + r""" + MDI+ feature importance (Agarwal et al. 2023, simplified faithful variant). + + For each tree it builds a centered decision-stump basis (one indicator ``1{x_f <= threshold}`` per + internal node, grouped by the splitting feature), fits a ridge GLM of the label on that basis using + the tree's in-bag rows, and scores each feature by the variance of its block's partial prediction + evaluated on the tree's out-of-bag rows; the per-tree scores are averaged. The tree representation + plus regularized GLM plus out-of-sample partial contribution remove the in-sample / cardinality + inflation of plain MDI. + + Prefer MDI+ over MDI when features are noisy, high-cardinality, or mixed-type; it converges to MDI + when features are orthogonal and high-SNR. This is the simplified faithful variant (not the full + leave-one-out paper algorithm). See the module docstring for the full regime tag and + appraisals/10_verdict.md. + + Parameters + ---------- + x : pd.DataFrame + The design matrix. + y : pd.Series + The binary target. + random_state : int, default 0 + Forest seed. + n_trees : int, default 60 + Number of trees used by the per-tree decomposition. + classifier : RandomForestClassifier, optional + A bootstrap RandomForest to use instead of the default (depth-6, 100-tree) forest. + + Returns + ------- + pd.Series + MDI+ importance per feature (indexed by column; higher is more important). + + Examples + -------- + >>> import numpy as np, pandas as pd + >>> rng = np.random.default_rng(0) + >>> x = pd.DataFrame(rng.standard_normal((400, 4)), columns=list("abcd")) + >>> y = pd.Series((x["a"] + 0.3 * rng.standard_normal(400) > 0).astype(int)) + >>> imp = mdi_plus_importance(x, y) + >>> imp.idxmax() == "a" + True + """ + forest = classifier if classifier is not None else _default_forest(random_state) + forest.fit(x, y) + xv = x.to_numpy(float) + yv = y.to_numpy(float) + n, n_features = xv.shape + accumulated = np.zeros(n_features) + used = 0 + for estimator in forest.estimators_[:n_trees]: + tree = estimator.tree_ + internal = np.where(tree.feature >= 0)[0] + if internal.size == 0: + continue + basis = (xv[:, tree.feature[internal]] <= tree.threshold[internal]).astype( + float + ) + basis = basis - basis.mean(axis=0, keepdims=True) + groups = tree.feature[internal] + in_bag = np.unique( + _bootstrap_sample_indices(estimator.random_state, n, forest.max_samples) + ) + out_of_bag = np.setdiff1d(np.arange(n), in_bag, assume_unique=False) + if out_of_bag.size < 5: + continue + ridge = Ridge(alpha=1.0, fit_intercept=True) + ridge.fit(basis[in_bag], yv[in_bag]) + coefficients = ridge.coef_ + for feature in np.unique(groups): + mask = groups == feature + partial = basis[out_of_bag][:, mask] @ coefficients[mask] + accumulated[feature] += partial.var() + used += 1 + accumulated = accumulated / max(used, 1) + return pd.Series(accumulated, index=x.columns) + + +def _per_sample_log_loss( + y_true: np.ndarray, proba: np.ndarray, classes: np.ndarray +) -> np.ndarray: + eps = 1e-15 + proba = np.clip(proba, eps, 1 - eps) + class_to_column = {c: k for k, c in enumerate(classes)} + columns = np.array([class_to_column[v] for v in y_true]) + chosen = proba[np.arange(len(y_true)), columns] + return -np.log(chosen) + + +def conditional_predictive_impact( + x: pd.DataFrame, + y: pd.Series, + random_state: int = 0, + n_splits: int = 4, + n_estimators: int = 80, + classifier: RandomForestClassifier | None = None, +) -> pd.DataFrame: + r""" + Conditional Predictive Impact (CPI; Watson and Wright 2021), with a calibrated significance test. + + For each cross-validation fold a classifier is fit on the training rows; on the test rows each + feature is replaced by a conditional Gaussian knockoff (the feature regressed on the others by ridge + plus resampled residual noise), and the per-sample increase in log loss is recorded. Pooled across + folds, the mean increase is the CPI statistic and a one-sided paired t-test gives a p-value, so the + method tests whether a feature matters - the calibrated test MDA lacks. + + Prefer CPI when you need a statistically valid test of whether a feature matters; it holds nominal + size with good power (mildly elevated on the hardest high-correlation x low-SNR corner). See the + module docstring for the full regime tag and appraisals/10_verdict.md. + + Parameters + ---------- + x : pd.DataFrame + The design matrix. + y : pd.Series + The target. + random_state : int, default 0 + Seed for the folds, forest, and knockoff noise. + n_splits : int, default 4 + Cross-validation folds. + n_estimators : int, default 80 + Trees in the per-fold forest (used when ``classifier`` is None). + classifier : RandomForestClassifier, optional + A classifier template cloned per fold (defaults to a depth-6 forest). + + Returns + ------- + pd.DataFrame + Indexed by feature with columns ``importance`` (mean log-loss increase; higher = more important) + and ``p_value`` (one-sided paired t-test of importance > 0). + + Examples + -------- + >>> import numpy as np, pandas as pd + >>> rng = np.random.default_rng(1) + >>> x = pd.DataFrame(rng.standard_normal((300, 3)), columns=list("abc")) + >>> y = pd.Series((x["a"] + 0.3 * rng.standard_normal(300) > 0).astype(int)) + >>> cpi = conditional_predictive_impact(x, y) + >>> bool(cpi.loc["a", "p_value"] < cpi.loc["c", "p_value"]) + True + """ + from sklearn.base import clone + + columns = list(x.columns) + folds = KFold(n_splits=n_splits, shuffle=True, random_state=random_state) + xv = x.to_numpy(float) + yv = y.to_numpy() + differences: dict[str, list] = {c: [] for c in columns} + template = ( + classifier + if classifier is not None + else _default_forest(random_state, n_estimators) + ) + for i, (train, test) in enumerate(folds.split(x)): + model = clone(template).fit(x.iloc[train], y.iloc[train]) + classes = model.classes_ + loss_true = _per_sample_log_loss( + yv[test], model.predict_proba(x.iloc[test]), classes + ) + rng = np.random.default_rng(random_state + 100 + i) + for j, c in enumerate(columns): + others = [k for k in range(len(columns)) if k != j] + ridge = Ridge(alpha=1.0).fit(xv[train][:, others], xv[train][:, j]) + residual = xv[train][:, j] - ridge.predict(xv[train][:, others]) + sigma = residual.std() + 1e-9 + knockoff = ridge.predict(xv[test][:, others]) + sigma * rng.standard_normal( + len(test) + ) + x_knockoff = xv[test].copy() + x_knockoff[:, j] = knockoff + loss_knockoff = _per_sample_log_loss( + yv[test], + model.predict_proba(pd.DataFrame(x_knockoff, columns=columns)), + classes, + ) + differences[c].append(loss_knockoff - loss_true) + importance, p_value = {}, {} + for c in columns: + d = np.concatenate(differences[c]) + importance[c] = float(d.mean()) + if d.std() < 1e-12: + p_value[c] = 1.0 + else: + t_stat, p_two = ttest_1samp(d, 0.0) + p_value[c] = float(p_two / 2 if t_stat > 0 else 1.0 - p_two / 2) + return pd.DataFrame( + {"importance": pd.Series(importance), "p_value": pd.Series(p_value)} + ).reindex(columns) diff --git a/src/RiskLabAI/features/feature_importance/feature_importance_controller.py b/src/RiskLabAI/features/feature_importance/feature_importance_controller.py new file mode 100644 index 0000000..8c9a568 --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/feature_importance_controller.py @@ -0,0 +1,42 @@ +""" +Deprecated controller for managing feature-importance strategies. + +Use the core component registry (:data:`RiskLabAI.core.FEATURE_IMPORTANCE`) +directly instead. This controller is retained for backward compatibility and +delegates to that registry; it is scheduled for removal in 4.0.0. +""" + +import warnings +from typing import Any + +import pandas as pd + +from .feature_importance_strategy import FeatureImportanceStrategy + + +class FeatureImportanceController: + """ + Deprecated. Use ``RiskLabAI.core.FEATURE_IMPORTANCE.create(...)`` instead. + + Scheduled for removal in 4.0.0. + """ + + def __init__(self, strategy_type: str, **kwargs: Any): + warnings.warn( + "FeatureImportanceController is deprecated and will be removed in " + "4.0.0; use RiskLabAI.core.FEATURE_IMPORTANCE.create(strategy_type, " + "...) instead.", + DeprecationWarning, + stacklevel=2, + ) + from RiskLabAI.core import FEATURE_IMPORTANCE + + self.strategy_instance: FeatureImportanceStrategy = FEATURE_IMPORTANCE.create( + strategy_type, filter_unknown_kwargs=True, **kwargs + ) + + def calculate_importance( + self, x: pd.DataFrame, y: pd.Series, **kwargs: Any + ) -> pd.DataFrame: + """Calculate feature importance using the configured strategy.""" + return self.strategy_instance.compute(x, y, **kwargs) diff --git a/src/RiskLabAI/features/feature_importance/feature_importance_factory.py b/src/RiskLabAI/features/feature_importance/feature_importance_factory.py new file mode 100644 index 0000000..6065abb --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/feature_importance_factory.py @@ -0,0 +1,50 @@ +""" +Deprecated factory for creating feature-importance strategy objects. + +The core component registry (:data:`RiskLabAI.core.FEATURE_IMPORTANCE`) is now +the single source of truth. This factory is retained for backward compatibility +and delegates to that registry; it is scheduled for removal in 4.0.0. +""" + +import warnings +from typing import Any + +from .feature_importance_strategy import FeatureImportanceStrategy + + +class FeatureImportanceFactory: + """ + Deprecated. Use ``RiskLabAI.core.FEATURE_IMPORTANCE.create(...)`` instead. + + Scheduled for removal in 4.0.0. + """ + + @staticmethod + def create_feature_importance( + strategy_type: str, **kwargs: Any + ) -> FeatureImportanceStrategy: + """ + Deprecated. Create a feature-importance strategy via the core registry. + + Raises + ------ + ValueError + If ``strategy_type`` is not a known strategy. + """ + warnings.warn( + "FeatureImportanceFactory is deprecated and will be removed in " + "4.0.0; use RiskLabAI.core.FEATURE_IMPORTANCE.create(strategy_type, " + "...) instead.", + DeprecationWarning, + stacklevel=2, + ) + from RiskLabAI.core import FEATURE_IMPORTANCE + + if strategy_type.lower() not in FEATURE_IMPORTANCE: + valid = list(FEATURE_IMPORTANCE.available()) + raise ValueError( + f"Invalid strategy_type: {strategy_type}. Valid types are: {valid}" + ) + return FEATURE_IMPORTANCE.create( + strategy_type, filter_unknown_kwargs=True, **kwargs + ) diff --git a/src/RiskLabAI/features/feature_importance/feature_importance_mda.py b/src/RiskLabAI/features/feature_importance/feature_importance_mda.py new file mode 100644 index 0000000..b989249 --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/feature_importance_mda.py @@ -0,0 +1,135 @@ +""" +Computes Mean Decrease Accuracy (MDA) feature importance. +""" + +import logging +from typing import Any + +import numpy as np +import pandas as pd +from sklearn.metrics import log_loss +from sklearn.model_selection import KFold + +from .feature_importance_strategy import FeatureImportanceStrategy + +logger = logging.getLogger(__name__) + + +class FeatureImportanceMDA(FeatureImportanceStrategy): + """ + Computes feature importance using Mean Decrease Accuracy (MDA). + + This method shuffles each feature one by one and measures how + much the model's performance (e.g., log loss) decreases. + """ + + def __init__(self, classifier: object, n_splits: int = 10, random_state: int = 42): + """ + Initialize the strategy. + + Parameters + ---------- + classifier : object + An *untrained* scikit-learn classifier. + n_splits : int, default=10 + Number of splits for cross-validation. + """ + self.classifier = classifier + self.n_splits = n_splits + self.random_state = random_state + + def compute(self, x: pd.DataFrame, y: pd.Series, **kwargs: Any) -> pd.DataFrame: + """ + Compute MDA feature importance. + + Parameters + ---------- + x : pd.DataFrame + The feature data. + y : pd.Series + The target data. + **kwargs : Any + - 'train_sample_weights': Optional sample weights for training. + - 'score_sample_weights': Optional sample weights for scoring. + + Returns + ------- + pd.DataFrame + DataFrame with "Mean" and "StandardDeviation" of importance. + """ + train_weights = kwargs.get("train_sample_weights") + score_weights = kwargs.get("score_sample_weights") + + if train_weights is None: + train_weights = np.ones(x.shape[0]) + if score_weights is None: + score_weights = np.ones(x.shape[0]) + + cv_generator = KFold( + n_splits=self.n_splits, shuffle=True, random_state=self.random_state + ) + baseline_scores = pd.Series(dtype=float) + shuffled_scores = pd.DataFrame(columns=x.columns, dtype=float) + + for i, (train_idx, test_idx) in enumerate(cv_generator.split(x)): + logger.debug("Fold %d start ...", i) + + x_train, y_train, w_train = ( + x.iloc[train_idx, :], + y.iloc[train_idx], + train_weights[train_idx], + ) + x_test, y_test, w_test = ( + x.iloc[test_idx, :], + y.iloc[test_idx], + score_weights[test_idx], + ) + + # Fit classifier and get baseline score + fitted_classifier = self.classifier.fit( + X=x_train, y=y_train, sample_weight=w_train + ) + pred_proba = fitted_classifier.predict_proba(x_test) + + baseline_scores.loc[i] = -log_loss( + y_test, + pred_proba, + labels=self.classifier.classes_, + sample_weight=w_test, + ) + + # Get scores for each shuffled feature + rng = np.random.default_rng(self.random_state + i) + for feature in x.columns: + x_test_shuffled = x_test.copy(deep=True) + # Shuffle a copy and assign back: with pandas copy-on-write + # (default since pandas 3.0), column .values is a read-only + # view and in-place shuffling raises ValueError. + shuffled_values = x_test_shuffled[feature].to_numpy(copy=True) + rng.shuffle(shuffled_values) # seeded shuffle + x_test_shuffled[feature] = shuffled_values + + shuffled_proba = fitted_classifier.predict_proba(x_test_shuffled) + + shuffled_scores.loc[i, feature] = -log_loss( + y_test, + shuffled_proba, + labels=self.classifier.classes_, + sample_weight=w_test, + ) + + # Calculate importance as the simple drop in score + importances = shuffled_scores.rsub(baseline_scores, axis=0) + + # Calculate mean and std dev + importances_summary = pd.concat( + { + "Mean": importances.mean(), + "StandardDeviation": ( + importances.std() * (importances.shape[0] ** -0.5) + ), + }, + axis=1, + ) + + return importances_summary diff --git a/src/RiskLabAI/features/feature_importance/feature_importance_mdi.py b/src/RiskLabAI/features/feature_importance/feature_importance_mdi.py new file mode 100644 index 0000000..c1c1be9 --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/feature_importance_mdi.py @@ -0,0 +1,88 @@ +""" +Computes Mean Decrease Impurity (MDI) feature importance. +""" + +from typing import Any + +import numpy as np +import pandas as pd +from sklearn.ensemble import BaseEnsemble + +from .feature_importance_strategy import FeatureImportanceStrategy + + +class FeatureImportanceMDI(FeatureImportanceStrategy): + """ + Computes feature importance using Mean Decrease Impurity (MDI). + + This method is specific to tree-based ensembles (like RandomForest) + and measures importance as the average impurity decrease. + """ + + def __init__(self, classifier: BaseEnsemble): + """ + Initialize the strategy. + + Parameters + ---------- + classifier : BaseEnsemble + An *untrained* scikit-learn ensemble model (e.g., RandomForestClassifier). + """ + if not isinstance(classifier, BaseEnsemble): + raise TypeError("Classifier must be an ensemble (e.g., RandomForest).") + + self.classifier = classifier + + def compute(self, x: pd.DataFrame, y: pd.Series, **kwargs: Any) -> pd.DataFrame: + """ + Compute MDI feature importance. + + Parameters + ---------- + x : pd.DataFrame + The feature data. + y : pd.Series + The target data. + **kwargs : Any + Keyword arguments for the classifier's `fit` method + (e.g., `sample_weight`). + + Returns + ------- + pd.DataFrame + DataFrame with "Mean" and "StandardDeviation" of importance. + """ + train_sample_weights = kwargs.get("sample_weight") + + # Fit the classifier + self.classifier.fit(x, y, sample_weight=train_sample_weights) + + # Get importance from each tree + importances_dict = { + i: tree.feature_importances_ + for i, tree in enumerate(self.classifier.estimators_) + } + importances_df = pd.DataFrame.from_dict(importances_dict, orient="index") + + # Ensure correct feature names + if hasattr(self.classifier, "feature_names_in_"): + importances_df.columns = self.classifier.feature_names_in_ + else: + importances_df.columns = x.columns + + # Replace 0 with NaN (as per user's original code) + importances_df.replace(0, np.nan, inplace=True) + + importances = pd.concat( + { + "Mean": importances_df.mean(), + "StandardDeviation": ( + importances_df.std() * (importances_df.shape[0] ** -0.5) + ), + }, + axis=1, + ) + + # Normalize + importances /= importances["Mean"].sum() + return importances diff --git a/src/RiskLabAI/features/feature_importance/feature_importance_sfi.py b/src/RiskLabAI/features/feature_importance/feature_importance_sfi.py new file mode 100644 index 0000000..e311d96 --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/feature_importance_sfi.py @@ -0,0 +1,119 @@ +""" +Computes Single Feature Importance (SFI). +""" + +from typing import Any + +import numpy as np +import pandas as pd +from sklearn.metrics import accuracy_score, log_loss +from sklearn.model_selection import KFold + +from .feature_importance_strategy import FeatureImportanceStrategy + + +class FeatureImportanceSFI(FeatureImportanceStrategy): + """ + Computes Single Feature Importance (SFI). + + This method calculates the importance of each feature by training + and evaluating a model using *only that single feature*. + """ + + def __init__( + self, + classifier: object, + n_splits: int = 10, + scoring: str = "log_loss", + ): + """ + Initialize the strategy. + + Parameters + ---------- + classifier : object + An *untrained* scikit-learn classifier. + n_splits : int, default=10 + Number of splits for cross-validation. + scoring : str, default="log_loss" + Scoring method ("log_loss" or "accuracy"). + """ + self.classifier = classifier + self.n_splits = n_splits + self.scoring = scoring + + def compute(self, x: pd.DataFrame, y: pd.Series, **kwargs: Any) -> pd.DataFrame: + """ + Compute SFI feature importance. + + Parameters + ---------- + x : pd.DataFrame + The feature data. + y : pd.Series + The target data. + **kwargs : Any + - 'train_sample_weights': Optional sample weights for training. + - 'score_sample_weights': Optional sample weights for scoring. + + Returns + ------- + pd.DataFrame + DataFrame with "FeatureName", "Mean", and "StandardDeviation" + of the SFI scores. + """ + train_sample_weights = kwargs.get("train_sample_weights") + score_sample_weights = kwargs.get("score_sample_weights") + + if train_sample_weights is None: + train_sample_weights = np.ones(x.shape[0]) + if score_sample_weights is None: + score_sample_weights = np.ones(x.shape[0]) + + cv_generator = KFold(n_splits=self.n_splits) + feature_names = x.columns + importances = [] + + for feature_name in feature_names: + scores = [] + feature_data = x[[feature_name]] + + for train_idx, test_idx in cv_generator.split(feature_data): + x_train, y_train, w_train = ( + feature_data.iloc[train_idx], + y.iloc[train_idx], + train_sample_weights[train_idx], + ) + x_test, y_test, w_test = ( + feature_data.iloc[test_idx], + y.iloc[test_idx], + score_sample_weights[test_idx], + ) + + self.classifier.fit(x_train, y_train, sample_weight=w_train) + + if self.scoring == "log_loss": + pred_proba = self.classifier.predict_proba(x_test) + score = -log_loss( + y_test, + pred_proba, + sample_weight=w_test, + labels=self.classifier.classes_, + ) + elif self.scoring == "accuracy": + pred = self.classifier.predict(x_test) + score = accuracy_score(y_test, pred, sample_weight=w_test) + else: + raise ValueError(f"'{self.scoring}' method not defined.") + + scores.append(score) + + importances.append( + { + "FeatureName": feature_name, + "Mean": np.mean(scores), + "StandardDeviation": np.std(scores, ddof=1) * (len(scores) ** -0.5), + } + ) + + return pd.DataFrame(importances).set_index("FeatureName") diff --git a/src/RiskLabAI/features/feature_importance/feature_importance_strategy.py b/src/RiskLabAI/features/feature_importance/feature_importance_strategy.py new file mode 100644 index 0000000..bd59e3c --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/feature_importance_strategy.py @@ -0,0 +1,37 @@ +""" +Abstract Base Class for feature importance strategies. +""" + +from abc import ABC, abstractmethod +from typing import Any + +import pandas as pd + + +class FeatureImportanceStrategy(ABC): + """ + Abstract Base Class for computing feature importance. + + Derived classes must implement the `compute` method. + """ + + @abstractmethod + def compute(self, x: pd.DataFrame, y: pd.Series, **kwargs: Any) -> pd.DataFrame: + """ + Abstract method to compute feature importance. + + Parameters + ---------- + x : pd.DataFrame + The feature data. + y : pd.Series + The target data. + **kwargs : Any + Additional keyword arguments (e.g., sample_weights). + + Returns + ------- + pd.DataFrame + A DataFrame containing feature importances. + """ + pass diff --git a/src/RiskLabAI/features/feature_importance/generate_synthetic_data.py b/src/RiskLabAI/features/feature_importance/generate_synthetic_data.py new file mode 100644 index 0000000..67e625e --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/generate_synthetic_data.py @@ -0,0 +1,80 @@ +""" +Generates a synthetic dataset for feature importance testing. + +Based on the method from De Prado (2018). +""" + +import numpy as np +import pandas as pd +from sklearn.datasets import make_classification + + +def get_test_dataset( + n_features: int = 100, + n_informative: int = 25, + n_redundant: int = 25, + n_samples: int = 10000, + random_state: int = 0, + sigma_std: float = 0.0, +) -> tuple[pd.DataFrame, pd.Series]: + """ + Generate a synthetic dataset with informative, redundant, and noise features. + + Parameters + ---------- + n_features : int, default=100 + Total number of features to generate. + n_informative : int, default=25 + Number of informative features. + n_redundant : int, default=25 + Number of redundant features. These are created by adding + Gaussian noise to informative features. + n_samples : int, default=10000 + Number of samples (rows) to generate. + random_state : int, default=0 + Random state for reproducibility. + sigma_std : float, default=0.0 + Standard deviation of the noise added to create redundant features. + + Returns + ------- + Tuple[pd.DataFrame, pd.Series] + - X (pd.DataFrame): The generated feature matrix. + - y (pd.Series): The generated labels. + """ + rng = np.random.default_rng(random_state) + + # 1. Generate informative and noise features + n_noise = n_features - n_informative - n_redundant + x_array, y_array = make_classification( + n_samples=n_samples, + n_features=n_informative + n_noise, + n_informative=n_informative, + n_redundant=0, + shuffle=False, + random_state=random_state, + ) + + # 2. Create DataFrame + columns = [f"I_{i}" for i in range(n_informative)] + columns += [f"N_{i}" for i in range(n_noise)] + x_df = pd.DataFrame(x_array, columns=columns) + y_series = pd.Series(y_array, name="Target") + + # 3. Create redundant features + # Randomly pick informative features to copy + redundant_indices = rng.choice(range(n_informative), size=n_redundant, replace=True) + + for i, orig_idx in enumerate(redundant_indices): + orig_feature_name = f"I_{orig_idx}" + new_feature_name = f"R_{i}" + + # Add noise to the original informative feature + noise = ( + rng.normal(loc=0.0, scale=sigma_std, size=n_samples) + * x_df[orig_feature_name].std() + ) + + x_df[new_feature_name] = x_df[orig_feature_name] + noise + + return x_df[sorted(x_df.columns)], y_series diff --git a/src/RiskLabAI/features/feature_importance/orthogonal_features.py b/src/RiskLabAI/features/feature_importance/orthogonal_features.py new file mode 100644 index 0000000..8136c8b --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/orthogonal_features.py @@ -0,0 +1,99 @@ +""" +Implements a method for feature orthogonalization using PCA. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. +""" + +import numpy as np +import pandas as pd + + +def _compute_eigenvectors( + dot_product: np.ndarray, explained_variance_threshold: float +) -> pd.DataFrame: + """ + Compute eigenvalues and eigenvectors and filter by explained variance. + + Parameters + ---------- + dot_product : np.ndarray + The dot product matrix (e.g., X.T @ X). + explained_variance_threshold : float + The cumulative variance threshold to filter eigenvectors. + + Returns + ------- + pd.DataFrame + DataFrame with sorted eigenvalues, eigenvectors, and cumulative variance. + """ + eigenvalues, eigenvectors = np.linalg.eigh(dot_product) + + # Sort in descending order + indices_sorted = eigenvalues.argsort()[::-1] + eigenvalues = eigenvalues[indices_sorted] + eigenvectors = eigenvectors[:, indices_sorted] + + eigen_dataframe = pd.DataFrame( + { + "EigenValue": eigenvalues, + "EigenVector": [ev for ev in eigenvectors.T], + }, + index=[f"PC_{i+1}" for i in range(len(eigenvalues))], + ) + + # Calculate cumulative variance + cumulative_variance = eigenvalues.cumsum() / eigenvalues.sum() + eigen_dataframe["CumulativeVariance"] = cumulative_variance + + # Find the index where cumulative variance crosses the threshold + index = cumulative_variance.searchsorted(explained_variance_threshold) + + # Keep components up to and including the one that crosses the threshold + eigen_dataframe = eigen_dataframe.iloc[: index + 1, :] + + return eigen_dataframe + + +def orthogonal_features( + features: pd.DataFrame, variance_threshold: float = 0.95 +) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Compute orthogonal features using PCA. + + Parameters + ---------- + features : pd.DataFrame + The feature DataFrame. + variance_threshold : float, default=0.95 + Cumulative explained variance threshold to keep. + + Returns + ------- + Tuple[pd.DataFrame, pd.DataFrame] + - orthogonal_features_df: The transformed (orthogonal) features. + - eigen_dataframe: DataFrame with eigenvalues and vectors. + """ + # 1. Normalize features (z-score) + normalized_features = (features - features.mean(axis=0)) / features.std(axis=0) + normalized_features = normalized_features.dropna(axis=1) # Drop constant cols + + # 2. Compute dot product (proportional to covariance) + dot_product = normalized_features.T @ normalized_features + + # 3. Get principal components + eigen_dataframe = _compute_eigenvectors(dot_product, variance_threshold) + + # 4. Get transformation matrix (stacking eigenvectors) + transformation_matrix = np.vstack(eigen_dataframe["EigenVector"].values).T + + # 5. Transform features + orthogonal_features_arr = normalized_features.values @ transformation_matrix + + orthogonal_features_df = pd.DataFrame( + orthogonal_features_arr, + index=features.index, + columns=eigen_dataframe.index, + ) + + return orthogonal_features_df, eigen_dataframe diff --git a/src/RiskLabAI/features/feature_importance/weighted_tau.py b/src/RiskLabAI/features/feature_importance/weighted_tau.py new file mode 100644 index 0000000..b5f271e --- /dev/null +++ b/src/RiskLabAI/features/feature_importance/weighted_tau.py @@ -0,0 +1,36 @@ +""" +Calculates the weighted Kendall's tau. +""" + +import numpy as np +import scipy.stats as stats + + +def calculate_weighted_tau( + feature_importances: np.ndarray, principal_component_ranks: np.ndarray +) -> float: + r""" + Calculate the weighted Kendall's tau (\(\tau\)). + + This function calculates a weighted correlation between two rankings. + Here, it's used to compare feature importances to their ranks, + weighted by the inverse of the rank (giving more weight to + disagreements at the top). + + Parameters + ---------- + feature_importances : np.ndarray + Vector of feature importances. + principal_component_ranks : np.ndarray + Vector of ranks (e.g., 1, 2, 3, ...). + + Returns + ------- + float + The weighted Kendall's tau correlation coefficient. + """ + # Weights are the inverse of the rank + weights = 1.0 / principal_component_ranks + + tau, _ = stats.weightedtau(feature_importances, weights) + return tau diff --git a/src/RiskLabAI/features/microstructural_features/__init__.py b/src/RiskLabAI/features/microstructural_features/__init__.py new file mode 100644 index 0000000..ed25a06 --- /dev/null +++ b/src/RiskLabAI/features/microstructural_features/__init__.py @@ -0,0 +1,29 @@ +""" +RiskLabAI Microstructural Features Module + +Implements estimators for market microstructure features, such as the +Corwin-Schultz spread, the Bekker-Parkinson volatility, and the EDGE +effective bid-ask spread estimator. +""" + +from .bekker_parkinson_volatility_estimator import ( + bekker_parkinson_volatility_estimates, + sigma_estimates, +) +from .corwin_schultz import ( + alpha_estimates, + beta_estimates, + corwin_schultz_estimator, + gamma_estimates, +) +from .edge import edge_estimator + +__all__ = [ + "beta_estimates", + "gamma_estimates", + "alpha_estimates", + "corwin_schultz_estimator", + "sigma_estimates", + "bekker_parkinson_volatility_estimates", + "edge_estimator", +] diff --git a/src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py b/src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py new file mode 100644 index 0000000..ad513f0 --- /dev/null +++ b/src/RiskLabAI/features/microstructural_features/bekker_parkinson_volatility_estimator.py @@ -0,0 +1,77 @@ +""" +Implements the Bekker-Parkinson volatility estimator, which adjusts +the Parkinson volatility by the Corwin-Schultz spread. + +Reference: + De Prado, M. (2018) Advances in Financial Machine Learning, + page 286, snippet 19.2. +""" + +from math import pi + +import numpy as np +import pandas as pd + +from .corwin_schultz import _DENOMINATOR, beta_estimates, gamma_estimates + + +def sigma_estimates(beta: pd.Series, gamma: pd.Series) -> pd.Series: + r""" + Compute Bekker-Parkinson volatility \(\sigma\) estimates. + + .. math:: + k_2 = \sqrt{8 / \pi} + d = 3 - 2\sqrt{2} + \sigma = \frac{(\sqrt{2} - 1) \sqrt{\beta}}{d} + + \sqrt{\frac{\gamma}{k_2^2 d}} + + Parameters + ---------- + beta : pd.Series + \(\beta\) estimates vector from Corwin-Schultz. + gamma : pd.Series + \(\gamma\) estimates vector from Corwin-Schultz. + + Returns + ------- + pd.Series + Bekker-Parkinson volatility \(\sigma\) estimates. + """ + k2 = (8 / pi) ** 0.5 + + term1 = (2**0.5 - 1) * (beta**0.5) / _DENOMINATOR + term2 = (gamma / (k2**2 * _DENOMINATOR)) ** 0.5 + + # Floor at zero + sigma = np.maximum(term1 + term2, 0) + + return sigma + + +def bekker_parkinson_volatility_estimates( + high_prices: pd.Series, low_prices: pd.Series, window_span: int = 20 +) -> pd.Series: + r""" + Compute Bekker-Parkinson volatility estimates from high and low prices. + + This function first calculates the Corwin-Schultz \(\beta\) and \(\gamma\) + parameters and then uses them to compute the volatility estimates. + + Parameters + ---------- + high_prices : pd.Series + Time series of high prices. + low_prices : pd.Series + Time series of low prices. + window_span : int, default=20 + Rolling window span for \(\beta\) estimation. + + Returns + ------- + pd.Series + Bekker-Parkinson volatility estimates. + """ + beta = beta_estimates(high_prices, low_prices, window_span) + gamma = gamma_estimates(high_prices, low_prices) + + return sigma_estimates(beta, gamma) diff --git a/src/RiskLabAI/features/microstructural_features/corwin_schultz.py b/src/RiskLabAI/features/microstructural_features/corwin_schultz.py new file mode 100644 index 0000000..b773650 --- /dev/null +++ b/src/RiskLabAI/features/microstructural_features/corwin_schultz.py @@ -0,0 +1,135 @@ +""" +Implements the Corwin and Schultz (2012) bid-ask spread estimator. + +Reference: + Corwin, S. A., & Schultz, P. (2012). A simple way to estimate + bid-ask spreads from daily high and low prices. + The Journal of Finance, 67(2), 719-760. +""" + +import numpy as np +import pandas as pd + +# Constant 'd' from the paper (denominator) +_DENOMINATOR = 3 - 2 * (2**0.5) + + +def beta_estimates( + high_prices: pd.Series, low_prices: pd.Series, window_span: int +) -> pd.Series: + r""" + Estimate \(\beta\) (sum of squared high-low log-ratios). + + .. math:: + \beta_t = \sum_{j=t-1}^{t} [\ln(H_j / L_j)]^2 + + This is then averaged over the `window_span`. + + Parameters + ---------- + high_prices : pd.Series + Time series of high prices. + low_prices : pd.Series + Time series of low prices. + window_span : int + Rolling window span for averaging. + + Returns + ------- + pd.Series + The estimated \(\beta\) vector. + """ + log_ratios_sq = np.log(high_prices / low_prices) ** 2 + + # Sum of current and previous day's squared log-ratio + beta = log_ratios_sq.rolling(window=2).sum() + + # Average over the window span + beta = beta.rolling(window=window_span).mean() + return beta + + +def gamma_estimates(high_prices: pd.Series, low_prices: pd.Series) -> pd.Series: + r""" + Estimate \(\gamma\) (squared log-ratio of two-day high/low). + + .. math:: + \gamma_t = [\ln(\max(H_t, H_{t-1}) / \min(L_t, L_{t-1}))]^2 + + Parameters + ---------- + high_prices : pd.Series + Time series of high prices. + low_prices : pd.Series + Time series of low prices. + + Returns + ------- + pd.Series + The estimated \(\gamma\) vector. + """ + high_prices_max = high_prices.rolling(window=2).max() + low_prices_min = low_prices.rolling(window=2).min() + gamma = np.log(high_prices_max / low_prices_min) ** 2 + return gamma + + +def alpha_estimates(beta: pd.Series, gamma: pd.Series) -> pd.Series: + r""" + Estimate \(\alpha\) from \(\beta\) and \(\gamma\). + + .. math:: + d = 3 - 2\sqrt{2} + \alpha = \frac{(\sqrt{2} - 1)\sqrt{\beta}}{d} + - \sqrt{\frac{\gamma}{d}} + + Parameters + ---------- + beta : pd.Series + \(\beta\) estimates vector. + gamma : pd.Series + \(\gamma\) estimates vector. + + Returns + ------- + pd.Series + The estimated \(\alpha\) vector, floored at 0. + """ + term1 = ((2**0.5) - 1) * (beta**0.5) / _DENOMINATOR + term2 = (gamma / _DENOMINATOR) ** 0.5 + + # Floor at zero + alpha = np.maximum(term1 - term2, 0) + return alpha + + +def corwin_schultz_estimator( + high_prices: pd.Series, low_prices: pd.Series, window_span: int = 20 +) -> pd.Series: + r""" + Estimate the bid-ask spread using the Corwin and Schultz (2012) method. + + .. math:: + S = \frac{2(e^\alpha - 1)}{1 + e^\alpha} + + Parameters + ---------- + high_prices : pd.Series + Time series of high prices. + low_prices : pd.Series + Time series of low prices. + window_span : int, default=20 + Rolling window span for \(\beta\) estimation. + + Returns + ------- + pd.Series + The estimated spread vector. + """ + beta = beta_estimates(high_prices, low_prices, window_span) + gamma = gamma_estimates(high_prices, low_prices) + alpha = alpha_estimates(beta, gamma) + + # Calculate spread + spread = 2 * (np.exp(alpha) - 1) / (1 + np.exp(alpha)) + return spread diff --git a/src/RiskLabAI/features/microstructural_features/edge.py b/src/RiskLabAI/features/microstructural_features/edge.py new file mode 100644 index 0000000..8366f0a --- /dev/null +++ b/src/RiskLabAI/features/microstructural_features/edge.py @@ -0,0 +1,181 @@ +""" +Implements the EDGE effective bid-ask spread estimator +(Ardia, Guidotti & Kroencke 2024). + +Admitted in Appraisal 03 (CONTRIBUTIONS_LEDGER row 1, 2026-06-26). Regime tag, +verbatim from the verdict: + + Prefer EDGE over Roll and Abdi-Ranaldo for low-frequency spread estimation in + all regimes; over Corwin-Schultz at small spreads (the edge narrows at very high + illiquidity and very large spreads). + +EDGE pools the open, high, low and close prices and corrects for discrete, +infrequently traded data, giving lower bias and variance than the close-to-close +Roll (1984) estimator and the two-day high-low Corwin-Schultz (2012) estimator, and +never returning an invalid (negative) point estimate. The result is a proportional +spread (0.01 is a 1% spread). Real-data confirmation is a logged follow-up pending an +adequate public intraday / quote dataset (DECISION_LOG 2026-06-26). Evidence and +caveats: appraisals/03_verdict.md (03_results/RESULTS.md + HELDOUT.md). + +Reference: + Ardia, D., Guidotti, E., & Kroencke, T. A. (2024). Efficient estimation of + bid-ask spreads from open, high, low, and close prices. Journal of Financial + Economics, 161, 103916. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import numpy as np + +_ArrayLike = Sequence[float] | np.ndarray + + +def _lag_nan(values: np.ndarray) -> np.ndarray: + """Shift by one period; the first element (no predecessor) becomes NaN.""" + lagged = np.empty_like(values) + lagged[0] = np.nan + lagged[1:] = values[:-1] + return lagged + + +def edge_estimator( + open: _ArrayLike, + high: _ArrayLike, + low: _ArrayLike, + close: _ArrayLike, + sign: bool = False, +) -> float: + r""" + Estimate the effective bid-ask spread using the EDGE estimator + (Ardia, Guidotti & Kroencke 2024). + + The estimator forms two unbiased squared-spread moment conditions from the log + open/high/low/close prices and a tick-rule trade indicator, then combines them + with the GMM-optimal, variance-weighted average: + + .. math:: + \hat{S}^2 = \frac{v_2 \, e_1 + v_1 \, e_2}{v_1 + v_2}, \qquad + \hat{S} = \sqrt{|\hat{S}^2|} + + where :math:`e_k` and :math:`v_k` are the mean and variance of the two estimators + (equal weight if the total variance is not positive). + + Parameters + ---------- + open, high, low, close : array-like + Four equal-length sequences of bar open, high, low and close prices + (arrays of bar prices, not a DataFrame). + sign : bool, default=False + If True, return a signed estimate (negative when the estimated squared + spread is negative); otherwise the spread magnitude is returned. + + Returns + ------- + float + The estimated proportional spread (0.01 is a 1% spread), or ``NaN`` when the + estimate is undefined (fewer than three observations, fewer than two traded + periods, or a degenerate / zero-variance input). + + Raises + ------ + ValueError + If the open, high, low and close sequences are not all the same length. + + Examples + -------- + >>> o = [100.0, 101.2, 100.5, 102.1, 101.8, 103.0] + >>> h = [101.0, 101.9, 101.3, 102.8, 102.5, 103.6] + >>> l = [99.4, 100.6, 99.9, 101.3, 101.0, 102.2] + >>> c = [101.1, 100.7, 101.0, 102.3, 101.6, 103.2] + >>> round(edge_estimator(o, h, l, c), 6) # doctest: +SKIP + 0.011176 + """ + open_prices = np.asarray(open, dtype=float) + high_prices = np.asarray(high, dtype=float) + low_prices = np.asarray(low, dtype=float) + close_prices = np.asarray(close, dtype=float) + + n = open_prices.shape[0] + if not ( + high_prices.shape[0] == n + and low_prices.shape[0] == n + and close_prices.shape[0] == n + ): + raise ValueError("open, high, low, and close must have the same length.") + if n < 3: + return float("nan") + + o = np.log(open_prices) + h = np.log(high_prices) + ll = np.log(low_prices) + c = np.log(close_prices) + m = (h + ll) / 2.0 + + h1 = _lag_nan(h) + l1 = _lag_nan(ll) + c1 = _lag_nan(c) + m1 = _lag_nan(m) + + # Log-returns; r1's first element is masked to align with the lagged quantities. + r1 = m - o + r1[0] = np.nan + r2 = o - m1 + r3 = m - c1 + r4 = c1 - m1 + r5 = o - c1 + + # Trade indicator: the bar traded if its range is non-zero or the low differs + # from the previous close. NaN where any required input is missing. + valid_tau = ~(np.isnan(h) | np.isnan(ll) | np.isnan(c1)) + tau = np.full(n, np.nan) + tau[valid_tau] = ( + (h[valid_tau] != ll[valid_tau]) | (ll[valid_tau] != c1[valid_tau]) + ).astype(float) + + def _indicator(a: np.ndarray, b: np.ndarray) -> np.ndarray: + """(tau and a != b) as 0/1, NaN where tau or any required input is missing.""" + out = np.full(n, np.nan) + valid = ~(np.isnan(tau) | np.isnan(a) | np.isnan(b)) + out[valid] = ((tau[valid] == 1.0) & (a[valid] != b[valid])).astype(float) + return out + + po1 = _indicator(o, h) + po2 = _indicator(o, ll) + pc1 = _indicator(c1, h1) + pc2 = _indicator(c1, l1) + + pt = np.nanmean(tau) + po = np.nanmean(po1) + np.nanmean(po2) + pc = np.nanmean(pc1) + np.nanmean(pc2) + + if np.nansum(tau) < 2 or po == 0.0 or pc == 0.0: + return float("nan") + + # De-meaned log-returns, weighted by the trade indicator and trade probability. + d1 = r1 - (np.nanmean(r1) / pt) * tau + d3 = r3 - (np.nanmean(r3) / pt) * tau + d5 = r5 - (np.nanmean(r5) / pt) * tau + + # Two unbiased squared-spread estimators, from the open and the previous close. + x1 = (-4.0 / po) * d1 * r2 + (-4.0 / pc) * d3 * r4 + x2 = (-4.0 / po) * d1 * r5 + (-4.0 / pc) * d5 * r4 + + e1 = np.nanmean(x1) + e2 = np.nanmean(x2) + v1 = np.nanmean(x1 * x1) - e1 * e1 + v2 = np.nanmean(x2 * x2) - e2 * e2 + + # Variance-weighted (GMM-optimal) average of the two estimators; equal weight if + # the total variance is not positive. + total_variance = v1 + v2 + if total_variance > 0.0: + squared_spread = (v2 * e1 + v1 * e2) / total_variance + else: + squared_spread = (e1 + e2) / 2.0 + + spread = float(np.sqrt(np.abs(squared_spread))) + if sign and squared_spread < 0.0: + spread = -spread + return spread diff --git a/src/RiskLabAI/features/structural_breaks/__init__.py b/src/RiskLabAI/features/structural_breaks/__init__.py new file mode 100644 index 0000000..3e64182 --- /dev/null +++ b/src/RiskLabAI/features/structural_breaks/__init__.py @@ -0,0 +1,41 @@ +""" +RiskLabAI Structural Breaks Features Module + +Implements methods for detecting structural breaks in time series, +including the Augmented Dickey-Fuller (ADF) and (G)SADF tests +as described by de Prado, plus the Phillips-Shi-Yu (2015) GSADF / BSADF +multiple-bubble detector and date-stamping. +""" + +from .pelt_change_points import pelt_change_points +from .structural_breaks import ( + compute_beta, + get_bsadf_sequence, + get_bsadf_statistic, + get_bubble_episodes, + get_expanding_window_adf, + get_gsadf_statistic, + get_sadf_sequence, + lag_dataframe, + prepare_data, + psy_minimum_window, + simulate_psy_critical_values, +) +from .volatility_robust_sadf import volatility_robust_sadf + +__all__ = [ + "lag_dataframe", + "prepare_data", + "compute_beta", + "get_expanding_window_adf", + "get_bsadf_statistic", + "psy_minimum_window", + "get_sadf_sequence", + "get_bsadf_sequence", + "get_gsadf_statistic", + "get_bubble_episodes", + "simulate_psy_critical_values", + # Appraisal 26 admits (alongside SADF/GSADF/CUSUM) + "volatility_robust_sadf", + "pelt_change_points", +] diff --git a/src/RiskLabAI/features/structural_breaks/pelt_change_points.py b/src/RiskLabAI/features/structural_breaks/pelt_change_points.py new file mode 100644 index 0000000..dbb9a2b --- /dev/null +++ b/src/RiskLabAI/features/structural_breaks/pelt_change_points.py @@ -0,0 +1,98 @@ +r""" +PELT exact multiple change-point detection (Killick, Fearnhead and Eckley 2012). + +de Prado's CUSUM (Chu-Stinchcombe-White) structural-break test targets a single mean shift and is, by +construction, blind to a pure variance change. PELT (Pruned Exact Linear Time) finds the exact set of +multiple change-points that minimizes a segment cost plus a per-change penalty, so with a Gaussian +(mean-and-variance) cost it detects and dates multiple and variance change-points that CUSUM misses, +without over-segmenting when penalized appropriately. + +Admitted in Appraisal 26 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tag, verbatim from the verdict: + + prefer it over CUSUM for detecting and dating multiple and/or variance change-points (which CUSUM + misses), without over-segmenting; CUSUM remains adequate for a single mean shift. + +Held-out confirmed (appraisals/26_results, HELDOUT.md): on the sealed mixed mean+variance corner (3 true +change-points) PELT recovers 2.65 of 3 on average with a 0.000 over-segmentation rate (it never exceeds +the true count), where CUSUM recovers far fewer of these mean+variance change-points. The CUSUM baseline +is unchanged. Evidence: appraisals/26_verdict.md. + +PELT is provided through the ``ruptures`` package (BSD-2-Clause), an optional dependency imported lazily; +the GPL-3 ``exuber`` reference was not used. Install it with ``pip install ruptures``. + +References +---------- +Killick, R., Fearnhead, P. and Eckley, I. A. (2012) Optimal detection of changepoints with a linear + computational cost. Journal of the American Statistical Association, 107(500), 1590-1598. +""" + +from __future__ import annotations + +import numpy as np + + +def pelt_change_points( + series, + model: str = "normal", + penalty_multiplier: float = 3.0, + min_size: int = 10, + jump: int = 5, +) -> list: + r""" + Detect and date change-points with PELT (exact multiple change-point detection). + + Fits a PELT model with a BIC-style penalty ``penalty_multiplier * log(T)`` and returns the interior + change indices (the segment boundaries, excluding the series endpoints). With ``model="normal"`` the + Gaussian cost captures both mean and variance changes, so PELT recovers multiple and variance + change-points that CUSUM misses, without over-segmenting under an adequate penalty. + + Prefer PELT over CUSUM for detecting and dating multiple and/or variance change-points; CUSUM remains + adequate for a single mean shift. See the module docstring for the verbatim regime tag and + appraisals/26_verdict.md. + + Parameters + ---------- + series : array-like + The 1-D series to segment. + model : str, default "normal" + ``ruptures`` cost model (``"normal"`` for mean+variance, ``"l2"`` for mean only, ``"rbf"`` ...). + penalty_multiplier : float, default 3.0 + Penalty ``= penalty_multiplier * log(T)`` (higher penalizes more change-points). + min_size : int, default 10 + Minimum segment length. + jump : int, default 5 + Grid stride for candidate change-points (``ruptures`` speed knob). + + Returns + ------- + list of int + The interior change-point indices, ascending (empty if none). + + Raises + ------ + ImportError + If ``ruptures`` is not installed (it is an optional dependency). + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> x = np.concatenate([rng.standard_normal(120), rng.standard_normal(120) + 3.0]) + >>> cps = pelt_change_points(x) # doctest: +SKIP + >>> any(100 < c < 140 for c in cps) # doctest: +SKIP + True + """ + try: + import ruptures as rpt + except ImportError as exc: # pragma: no cover - optional dependency + raise ImportError( + "pelt_change_points requires ruptures (pip install ruptures)" + ) from exc + + x = np.asarray(series, dtype=float).reshape(-1, 1) + n = x.shape[0] + if n < 2 * int(min_size): + return [] + algorithm = rpt.Pelt(model=model, min_size=int(min_size), jump=int(jump)).fit(x) + breakpoints = algorithm.predict(pen=float(penalty_multiplier) * np.log(n)) + return [int(b) for b in breakpoints if 0 < b < n] diff --git a/src/RiskLabAI/features/structural_breaks/structural_breaks.py b/src/RiskLabAI/features/structural_breaks/structural_breaks.py new file mode 100644 index 0000000..4ce4a5e --- /dev/null +++ b/src/RiskLabAI/features/structural_breaks/structural_breaks.py @@ -0,0 +1,708 @@ +""" +Implements the core logic for the (G)SADF (Generalized) Supreme +Augmented Dickey-Fuller test for structural breaks. + +References: + De Prado, M. (2018) Advances in financial machine learning. + John Wiley & Sons, Chapter 17. (SADF, Phillips-Wu-Yu 2011.) + Phillips, P. C. B., Shi, S. and Yu, J. (2015) Testing for multiple bubbles: + historical episodes of exuberance and collapse in the S&P 500. + International Economic Review, 56(4), 1043-1078. (GSADF / BSADF.) +""" + +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd + + +def lag_dataframe( + market_data: pd.DataFrame, lags: Union[int, list[int]] +) -> pd.DataFrame: + """ + Apply lags to a DataFrame. + + Reference: + Snippet 17.3, Page 253. + + Parameters + ---------- + market_data : pd.DataFrame + DataFrame of price or log price. + lags : Union[int, List[int]] + An integer number of lags (e.g., 3 creates lags 0, 1, 2, 3) + or a specific list of lags to create. + + Returns + ------- + pd.DataFrame + A DataFrame with lagged columns, e.g., 'price_0', 'price_1', ... + """ + lagged_parts = [] + + if isinstance(lags, int): + lags_list = range(lags + 1) + else: + lags_list = [int(lag) for lag in lags] + + for lag in lags_list: + lagged_data = market_data.shift(lag) + lagged_data.columns = [f"{col}_{lag}" for col in market_data.columns] + lagged_parts.append(lagged_data) + + return pd.concat(lagged_parts, axis=1) + + +def prepare_data( + log_price_series: pd.Series, constant: str, lags: int +) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Prepare the y and X matrices for ADF regression. + + Reference: + Snippet 17.3, Page 253. + + Parameters + ---------- + log_price_series : pd.Series # <-- CHANGED: Accept Series + Series of log prices. + constant : str + Type of regression constant ('nc', 'c', 'ct', 'ctt'). + lags : int + Number of lags to include. + + Returns + ------- + Tuple[pd.DataFrame, pd.DataFrame] + - y_df: The dependent variable (delta log price). + - x_df: The independent variables (lagged level, lagged deltas, constants). + """ + # <-- ADDED: Convert univariate series to frame for internal processing + log_price = log_price_series.to_frame() + + price_diff = log_price.diff().dropna() + y_df = price_diff + + # 1. Lagged level + x_df = log_price.shift(1).copy() + x_df.columns = ["level_l1"] + + # 2. Lagged deltas + if lags > 0: + lagged_deltas = price_diff.shift(1) + lagged_deltas.columns = ["delta_l1"] + + if lags > 1: + for i in range(2, lags + 1): + lagged_deltas[f"delta_l{i}"] = price_diff.shift(i) + + x_df = x_df.join(lagged_deltas, how="outer") + + # 3. Add constants + if constant == "c": + x_df["constant"] = 1 + elif constant == "ct": + x_df["constant"] = 1 + x_df["trend"] = np.arange(1, len(x_df) + 1) + elif constant == "ctt": + x_df["constant"] = 1 + x_df["trend"] = np.arange(1, len(x_df) + 1) + x_df["trend_sq"] = x_df["trend"] ** 2 + + # Align y and X by dropping NaNs created by lagging + combined = y_df.join(x_df, how="inner").dropna() + + y_df = combined.iloc[:, [0]] + x_df = combined.iloc[:, 1:] + + return y_df, x_df + + +def compute_beta( + y_window: np.ndarray, x_window: np.ndarray +) -> tuple[np.ndarray, np.ndarray]: + """ + Compute OLS beta coefficients and their variance. + + Reference: + Snippet 17.2, Page 251. + + Parameters + ---------- + y_window : np.ndarray + Window of the dependent variable. + x_window : np.ndarray + Window of the independent variables. + + Returns + ------- + Tuple[np.ndarray, np.ndarray] + - beta_mean: The OLS coefficients. + - beta_variance: The variance-covariance matrix of the coefficients. + """ + try: + xt_x_inv = np.linalg.inv(x_window.T @ x_window) + xt_y = x_window.T @ y_window + + beta_mean = xt_x_inv @ xt_y + + error = y_window - (x_window @ beta_mean) + variance_e = (error.T @ error) / (x_window.shape[0] - x_window.shape[1]) + beta_variance = variance_e * xt_x_inv + + return beta_mean, beta_variance + + except np.linalg.LinAlgError: + # Handle singular matrix + return np.full((x_window.shape[1], 1), np.nan), np.full( + (x_window.shape[1], x_window.shape[1]), np.nan + ) + + +def get_expanding_window_adf( + log_price: pd.Series, + min_sample_length: int, + constant: str, + lags: int, +) -> pd.Series: + """ + Compute the ADF t-statistic over an expanding window. + + This is useful for plotting the evolution of the test statistic over time. + + Reference: + Based on Snippet 17.2, Page 251. + + Parameters + ---------- + log_price : pd.Series + Series of log prices. + min_sample_length : int + The minimum number of samples to start the expanding window. + constant : str + Type of regression constant ('nc', 'c', 'ct', 'ctt'). + lags : int + Number of lags to include in the regression. + + Returns + ------- + pd.Series + A Series of ADF t-statistics, indexed by timestamp. + """ + # <-- CHANGED: Pass Series directly to prepare_data + y_df, x_df = prepare_data(log_price, constant=constant, lags=lags) + + adf_stats = [] + timestamps = [] + + for i in range(min_sample_length, y_df.shape[0] + 1): + y_window = y_df.iloc[:i].values + x_window = x_df.iloc[:i].values + + beta_mean, beta_variance = compute_beta(y_window, x_window) + + if np.isnan(beta_variance[0, 0]): + t_stat = np.nan + else: + beta_std_level = beta_variance[0, 0] ** 0.5 + if beta_std_level == 0: + t_stat = -np.inf if beta_mean[0, 0] < 0 else np.inf + else: + t_stat = beta_mean[0, 0] / beta_std_level + + adf_stats.append(t_stat) + timestamps.append(y_df.index[i - 1]) + + return pd.Series(adf_stats, index=timestamps) + + +def get_bsadf_statistic( + log_price: pd.Series, # <-- CHANGED: Accept Series + min_sample_length: int, + constant: str, + lags: int, +) -> dict[str, Any]: + """ + Compute the Backward Supremum ADF (BSADF) statistic. + + This test runs an expanding ADF test starting from every possible + point in the series and finds the supremum (highest) t-statistic. + This is used to detect the *origination* of a bubble. + + Reference: + Snippet 17.4, Page 253. (Renamed from `adf` for clarity). + + Parameters + ---------- + log_price : pd.Series # <-- CHANGED + Series of log prices. + min_sample_length : int + Minimum sample length for each ADF test. + constant : str + Type of regression constant ('nc', 'c', 'ct', 'ctt'). + lags : int + Number of lags to include. + + Returns + ------- + Dict[str, Any] + A dictionary containing: + - 'Time': The timestamp of the end of the series. + - 'bsadf': The BSADF statistic (the supremum ADF). + """ + # 1. Prepare the full X, y matrices + # <-- CHANGED: Pass Series directly + y, x = prepare_data(log_price, constant=constant, lags=lags) + + # 2. Define all possible start points + # <-- BUG FIX: Removed '+ lags' from the range + start_points = range(0, y.shape[0] - min_sample_length + 1) + bsadf = -np.inf # Supremum ADF + + y_np, x_np = y.values, x.values + + # 3. Loop over all expanding windows + for start in start_points: + y_window, x_window = y_np[start:], x_np[start:] + + # 4. Compute ADF regression for this window + beta_mean, beta_variance = compute_beta(y_window, x_window) + + if np.isnan(beta_variance[0, 0]): + continue + + # 5. Get t-statistic for the first coefficient (the level) + beta_mean_level = beta_mean[0, 0] + beta_std_level = beta_variance[0, 0] ** 0.5 + + if beta_std_level == 0: + t_stat = -np.inf if beta_mean_level < 0 else np.inf + else: + t_stat = beta_mean_level / beta_std_level + + if t_stat > bsadf: + bsadf = t_stat + + return {"Time": log_price.index[-1], "bsadf": bsadf} + + +# ---------------------------------------------------------------------------------- +# GSADF / BSADF (Phillips-Shi-Yu 2015): the generalized sup-ADF and its backward +# sequence, with date-stamping and a finite-sample critical-value simulator. +# +# de Prado (AFML ch.17) ships the single-window SADF: the forward-expanding ADF is +# anchored at the sample origin, so several explosive episodes dilute and mask one +# another and SADF collapses them to about one. GSADF varies *both* window endpoints +# and the backward BSADF sequence dates each origination and collapse, recovering and +# counting multiple bubbles where SADF sees one. The functions below build directly +# on the validated ``get_bsadf_statistic`` / ``get_expanding_window_adf`` above (and +# agree with them to ~1e-9 on shared windows; see the structural-breaks tests). +# +# Admitted in Appraisal 05 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tag, verbatim: +# Prefer GSADF/BSADF over single-window SADF when a series may contain more than +# one explosive episode (it recovers and counts each). For a single suspected +# bubble, SADF is at least as good. Use seasonally-adjusted data and the simulated +# finite-sample critical values; GSADF over-flags strongly trending or seasonal +# series and is mildly oversized in large samples, so treat broad flags cautiously. +# Evidence and caveats: appraisals/05_verdict.md. +# ---------------------------------------------------------------------------------- + + +def psy_minimum_window(sample_length: int) -> int: + """ + Phillips-Shi-Yu minimum window length, in observations. + + PSY (2015) set the minimum window fraction to r0 = 0.01 + 1.8 / sqrt(T) and take + the minimum window as round(r0 * T), the standard rule used to simulate the + finite-sample critical values. + + Parameters + ---------- + sample_length : int + The series length T (number of price observations). + + Returns + ------- + int + The minimum window length in observations (at least 3). + """ + r0 = 0.01 + 1.8 / (sample_length**0.5) + return max(int(round(r0 * sample_length)), 3) + + +def _psy_sadf_bsadf_sequences( + y: np.ndarray, min_sample_length: int +) -> tuple[np.ndarray, np.ndarray]: + """ + Fast prefix-sum kernel for the PSY-standard ADF specification (intercept, no lag + augmentation: Delta y_t = alpha + beta y_{t-1} + e_t). + + Returns ``(sadf_seq, bsadf_seq)``, each of length ``len(y)`` and indexed by the + window endpoint (price index), with ``NaN`` before the first valid endpoint: + + - ``sadf_seq[r2]`` = ADF(0, r2) (forward-expanding, PWY) + - ``bsadf_seq[r2]`` = sup_{r1 in [0, r2 - nmin]} ADF(r1, r2) (backward sup, PSY) + + The ADF t-statistic of the lagged-level coefficient for any window is computed in + O(1) from prefix sums of the regression observations, so the whole sequence is + O(T^2). Clean-room from Phillips-Shi-Yu (2015); it agrees with the loop-based + ``get_bsadf_statistic`` / ``get_expanding_window_adf`` to ~1e-9 on shared windows. + """ + y = np.ascontiguousarray(np.asarray(y, dtype=np.float64)) + length = y.shape[0] + sadf = np.full(length, np.nan) + bsadf = np.full(length, np.nan) + if length < min_sample_length + 1: + return sadf, bsadf + + # Regression observation i (0-based) uses level x_i = y[i] and delta d_i = y[i+1]-y[i]. + x = y[:-1] + d = y[1:] - y[:-1] + + # Prefix sums (length m+1, leading 0) so any window's sufficient statistics are O(1). + cx = np.concatenate(([0.0], np.cumsum(x))) + cxx = np.concatenate(([0.0], np.cumsum(x * x))) + cd = np.concatenate(([0.0], np.cumsum(d))) + cxd = np.concatenate(([0.0], np.cumsum(x * d))) + cdd = np.concatenate(([0.0], np.cumsum(d * d))) + + for r2 in range(min_sample_length, length): + # Window [r1, r2] uses regression observations [r1, r2); n = r2 - r1. + r1 = np.arange(0, r2 - min_sample_length + 1) + n = (r2 - r1).astype(np.float64) + sx = cx[r2] - cx[r1] + sxx = cxx[r2] - cxx[r1] + sd = cd[r2] - cd[r1] + sxd = cxd[r2] - cxd[r1] + sdd = cdd[r2] - cdd[r1] + den = n * sxx - sx * sx + with np.errstate(divide="ignore", invalid="ignore"): + beta = (n * sxd - sx * sd) / den + alpha = (sd - beta * sx) / n + ssr = np.maximum(sdd - alpha * sd - beta * sxd, 0.0) + var_beta = (ssr / (n - 2)) * n / den + t_stat = beta / np.sqrt(var_beta) + valid = (den > 0.0) & (n >= 3) & (var_beta > 0.0) + t_stat = np.where(valid, t_stat, -np.inf) + best = t_stat.max() + bsadf[r2] = best if np.isfinite(best) else np.nan + # r1 == 0 is the forward-expanding (origin-anchored) ADF = SADF date-stamp. + sadf[r2] = t_stat[0] if valid[0] else np.nan + + return sadf, bsadf + + +def get_sadf_sequence( + log_price: pd.Series, + min_sample_length: int, + constant: str = "c", + lags: int = 0, +) -> pd.Series: + """ + Forward-expanding ADF (SADF) sequence, indexed by window endpoint. + + This is the date-stamping sequence for de Prado's single-window SADF: each point + is the ADF t-statistic of an expanding window anchored at the sample origin. It is + a thin wrapper over :func:`get_expanding_window_adf` (and the fast prefix-sum + kernel for the standard intercept-only, no-lag specification) so the SADF and GSADF + date-stamps share an aligned index. + + Parameters + ---------- + log_price : pd.Series + Series of log prices. ``NaN`` values (e.g. FRED holiday gaps) are dropped. + min_sample_length : int + Minimum window length in observations (see :func:`psy_minimum_window`). + constant : str, default "c" + Regression constant ('nc', 'c', 'ct', 'ctt'). + lags : int, default 0 + Number of lagged-difference terms (PSY simulate with 0). + + Returns + ------- + pd.Series + The SADF t-statistic at each endpoint, indexed by timestamp. + """ + log_price = log_price.dropna() + if constant == "c" and lags == 0: + sadf, _ = _psy_sadf_bsadf_sequences(log_price.to_numpy(), min_sample_length) + endpoints = log_price.index[min_sample_length:] + return pd.Series(sadf[min_sample_length:], index=endpoints) + return get_expanding_window_adf(log_price, min_sample_length, constant, lags) + + +def get_bsadf_sequence( + log_price: pd.Series, + min_sample_length: int, + constant: str = "c", + lags: int = 0, +) -> pd.Series: + """ + Backward sup-ADF (BSADF) sequence, indexed by window endpoint (Phillips-Shi-Yu). + + For each endpoint the statistic is the supremum of the ADF t-statistic over all + window start points, so a later episode is tested on its own sub-sample rather than + diluted by the earlier sample. Comparing this sequence against a critical-value + sequence (see :func:`simulate_psy_critical_values`) date-stamps each bubble in real + time; see :func:`get_bubble_episodes`. + + For the standard intercept-only, no-lag specification this uses a fast O(T^2) + prefix-sum kernel; for other constants or ``lags > 0`` it reuses the validated + :func:`get_bsadf_statistic` on expanding prefixes. Both agree to ~1e-9. + + Parameters + ---------- + log_price : pd.Series + Series of log prices. ``NaN`` values are dropped. + min_sample_length : int + Minimum window length in observations (see :func:`psy_minimum_window`). + constant : str, default "c" + Regression constant ('nc', 'c', 'ct', 'ctt'). + lags : int, default 0 + Number of lagged-difference terms. + + Returns + ------- + pd.Series + The BSADF t-statistic at each endpoint, indexed by timestamp. + """ + log_price = log_price.dropna() + if constant == "c" and lags == 0: + _, bsadf = _psy_sadf_bsadf_sequences(log_price.to_numpy(), min_sample_length) + endpoints = log_price.index[min_sample_length:] + return pd.Series(bsadf[min_sample_length:], index=endpoints) + + values, timestamps = [], [] + first_end = min_sample_length + lags + 1 + for end in range(first_end, len(log_price) + 1): + result = get_bsadf_statistic( + log_price.iloc[:end], min_sample_length, constant, lags + ) + values.append(result["bsadf"]) + timestamps.append(result["Time"]) + return pd.Series(values, index=timestamps) + + +def get_gsadf_statistic( + log_price: pd.Series, + min_sample_length: int, + constant: str = "c", + lags: int = 0, +) -> float: + """ + Generalized sup-ADF (GSADF) statistic (Phillips-Shi-Yu 2015). + + The GSADF is the supremum of the backward sup-ADF (BSADF) sequence over all + flexible windows, i.e. ``max`` of :func:`get_bsadf_sequence`. A series is flagged + as containing at least one explosive episode when the GSADF exceeds its + finite-sample critical value (:func:`simulate_psy_critical_values`). + + Prefer GSADF/BSADF over single-window SADF when a series may contain more than one + explosive episode (it recovers and counts each); for a single suspected bubble, + SADF is at least as good. Use seasonally-adjusted data and the simulated + finite-sample critical values; GSADF over-flags strongly trending or seasonal + series and is mildly oversized in large samples, so treat broad flags cautiously. + (Admitted in Appraisal 05; see appraisals/05_verdict.md.) + + Parameters + ---------- + log_price : pd.Series + Series of log prices. ``NaN`` values are dropped. + min_sample_length : int + Minimum window length in observations (see :func:`psy_minimum_window`). + constant : str, default "c" + Regression constant ('nc', 'c', 'ct', 'ctt'). + lags : int, default 0 + Number of lagged-difference terms. + + Returns + ------- + float + The GSADF statistic, or ``NaN`` if the series is too short. + + Examples + -------- + >>> import numpy as np, pandas as pd + >>> rng = np.random.default_rng(0) + >>> y = np.cumsum(rng.standard_normal(200)) # random walk + >>> for t in range(120, 150): # embed an explosive episode + ... y[t] = 1.06 * y[t - 1] + rng.standard_normal() + >>> price = pd.Series(y) + >>> nmin = psy_minimum_window(len(price)) + >>> gsadf = get_gsadf_statistic(price, nmin) + >>> cv = simulate_psy_critical_values(len(price), nmin, n_simulations=200) + >>> bool(gsadf > cv["gsadf_global_cv"]) # bubble detected + True + """ + bsadf_seq = get_bsadf_sequence(log_price, min_sample_length, constant, lags) + values = bsadf_seq.to_numpy() + if values.size == 0 or not np.isfinite(values).any(): + return float("nan") + return float(np.nanmax(values)) + + +def get_bubble_episodes( + statistic_sequence: pd.Series, + critical_value_sequence: Union[pd.Series, np.ndarray, float], + min_duration: int = 1, +) -> list[tuple[Any, Any]]: + """ + Date-stamp explosive episodes from a sup-ADF sequence (Phillips-Shi-Yu 2015). + + An episode is a maximal run of endpoints where the statistic exceeds its + critical value and that lasts at least ``min_duration`` periods. The origination + is the first such period; the collapse is the first period the statistic falls + back below the critical value (or the last endpoint if it is still above at the + end of the sample). The same rule applied to the BSADF sequence date-stamps GSADF + episodes and applied to the SADF sequence date-stamps the single-window detector, + so the two are directly comparable. + + Parameters + ---------- + statistic_sequence : pd.Series + A BSADF or SADF sequence (see :func:`get_bsadf_sequence`, + :func:`get_sadf_sequence`). + critical_value_sequence : pd.Series, np.ndarray or float + The per-endpoint critical-value sequence (aligned with + ``statistic_sequence``) or a single scalar critical value. + min_duration : int, default 1 + Minimum run length, in periods, for a run to count as an episode. PSY use a + small minimum such as ``round(log(T))``. + + Returns + ------- + list of (origination, collapse) + Index labels (timestamps) bounding each detected episode, in order. + """ + stat = statistic_sequence.to_numpy(dtype=float) + index = statistic_sequence.index + n = stat.shape[0] + if isinstance(critical_value_sequence, pd.Series): + cv = critical_value_sequence.reindex(index).to_numpy(dtype=float) + elif np.isscalar(critical_value_sequence): + cv = np.full(n, float(critical_value_sequence)) + else: + cv = np.asarray(critical_value_sequence, dtype=float) + + above = np.isfinite(stat) & np.isfinite(cv) & (stat > cv) + episodes: list[tuple[Any, Any]] = [] + t = 0 + while t < n: + if above[t]: + start = t + while t < n and above[t]: + t += 1 + end = t # first endpoint back below the critical value (exclusive) + if end - start >= min_duration: + collapse = index[end] if end < n else index[-1] + episodes.append((index[start], collapse)) + else: + t += 1 + return episodes + + +_CRITICAL_VALUE_CACHE: dict[tuple, dict[str, Any]] = {} + + +def simulate_psy_critical_values( + sample_length: int, + min_sample_length: Optional[int] = None, + constant: str = "c", + lags: int = 0, + n_simulations: int = 2000, + level: float = 0.95, + seed: int = 0, +) -> dict[str, Any]: + """ + Finite-sample critical values for SADF and GSADF, by simulating the random-walk + null (the standard Phillips-Shi-Yu approach), with caching. + + The ADF-family statistics are pivotal under the random-walk null, so the values + depend only on the sample length, the minimum window and the regression + specification, not on the data's level or scale. For each of ``n_simulations`` + independent random-walk paths the SADF and BSADF sequences are computed; the + ``level`` quantile of the global statistic across paths gives the global critical + value (used for detection), and the ``level`` quantile per endpoint gives the + critical-value sequence (used for date-stamping; see :func:`get_bubble_episodes`). + + Parameters + ---------- + sample_length : int + The series length T to simulate. + min_sample_length : int, optional + Minimum window length in observations. Defaults to + :func:`psy_minimum_window`. + constant : str, default "c" + Regression constant ('nc', 'c', 'ct', 'ctt'). + lags : int, default 0 + Number of lagged-difference terms. + n_simulations : int, default 2000 + Number of random-walk paths. + level : float, default 0.95 + Quantile of the null distribution (a 95% test targets a 5% size). + seed : int, default 0 + Base seed; path ``s`` uses ``seed + s`` for reproducibility. + + Returns + ------- + dict + Keys ``sample_length``, ``min_sample_length``, ``level``, + ``sadf_global_cv``, ``gsadf_global_cv`` (floats), and + ``sadf_sequence_cv``, ``bsadf_sequence_cv`` (per-endpoint critical-value + arrays aligned with :func:`get_sadf_sequence` / :func:`get_bsadf_sequence`). + """ + if min_sample_length is None: + min_sample_length = psy_minimum_window(sample_length) + + cache_key = ( + sample_length, + min_sample_length, + constant, + lags, + n_simulations, + level, + seed, + ) + if cache_key in _CRITICAL_VALUE_CACHE: + return _CRITICAL_VALUE_CACHE[cache_key] + + fast = constant == "c" and lags == 0 + sadf_paths = np.full((n_simulations, sample_length), np.nan) + bsadf_paths = np.full((n_simulations, sample_length), np.nan) + for s in range(n_simulations): + rng = np.random.default_rng(seed + s) + y = np.cumsum(rng.standard_normal(sample_length)) # random-walk null + if fast: + sadf, bsadf = _psy_sadf_bsadf_sequences(y, min_sample_length) + else: + price = pd.Series(y) + sadf = ( + get_sadf_sequence(price, min_sample_length, constant, lags) + .reindex(range(sample_length)) + .to_numpy() + ) + bsadf = ( + get_bsadf_sequence(price, min_sample_length, constant, lags) + .reindex(range(sample_length)) + .to_numpy() + ) + sadf_paths[s] = sadf + bsadf_paths[s] = bsadf + + global_sadf = np.nanmax(sadf_paths, axis=1) + global_gsadf = np.nanmax(bsadf_paths, axis=1) + # Quantiles only over the valid endpoint columns (the leading columns are all NaN). + sadf_seq_cv = np.nanquantile(sadf_paths[:, min_sample_length:], level, axis=0) + bsadf_seq_cv = np.nanquantile(bsadf_paths[:, min_sample_length:], level, axis=0) + + result = { + "sample_length": sample_length, + "min_sample_length": min_sample_length, + "level": level, + "sadf_global_cv": float(np.nanquantile(global_sadf, level)), + "gsadf_global_cv": float(np.nanquantile(global_gsadf, level)), + "sadf_sequence_cv": sadf_seq_cv, + "bsadf_sequence_cv": bsadf_seq_cv, + } + _CRITICAL_VALUE_CACHE[cache_key] = result + return result diff --git a/src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py b/src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py new file mode 100644 index 0000000..c8096a7 --- /dev/null +++ b/src/RiskLabAI/features/structural_breaks/volatility_robust_sadf.py @@ -0,0 +1,126 @@ +r""" +Volatility-robust SADF / GSADF via wild-bootstrap critical values (Harvey-Leybourne-Sollis-Taylor 2016). + +The admitted SADF / GSADF bubble detector (`structural_breaks.get_sadf_sequence` / +`get_gsadf_statistic`) uses a sup-ADF statistic whose null distribution assumes homoskedastic errors. +Under non-stationary volatility the test over-rejects, flagging spurious bubbles (the mild oversizing +noted in Appraisal 05). The volatility-robust variant computes the SAME sup-ADF statistics but calibrates +the critical values by a wild bootstrap (Rademacher sign-flip of the first-difference residuals), which +preserves the series' own volatility pattern while destroying any explosive autocorrelation, restoring +correct size under non-stationary volatility. + +Admitted in Appraisal 26 (CONTRIBUTIONS_LEDGER 2026-06-27). Regime tag, verbatim from the verdict: + + prefer it over plain SADF/GSADF when the series' volatility may be non-stationary: it holds nominal + size where plain SADF over-rejects ~9x, at a modest power cost, and converges to plain SADF under + constant volatility. Pairs with the admitted GSADF/BSADF. + +Held-out confirmed (appraisals/26_results, HELDOUT.md): on the sealed variance-break-6x bubble null the +volatility-robust SADF holds nominal size (~0.067) where plain SADF over-rejects (~0.667, the homoskedastic +critical value flags spurious bubbles), and it matches plain SADF under constant volatility at a modest +power cost. The plain SADF / GSADF baseline is unchanged. Evidence: appraisals/26_verdict.md. + +References +---------- +Harvey, D. I., Leybourne, S. J., Sollis, R. and Taylor, A. M. R. (2016) Tests for explosive financial + bubbles in the presence of non-stationary volatility. Journal of Empirical Finance, 38, 548-574. +Phillips, P. C. B., Shi, S. and Yu, J. (2015) Testing for multiple bubbles. International Economic Review. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from .structural_breaks import _psy_sadf_bsadf_sequences, psy_minimum_window + + +def _sup_adf_statistics(log_price: np.ndarray, min_length: int) -> tuple: + sadf_seq, bsadf_seq = _psy_sadf_bsadf_sequences( + np.asarray(log_price, float), min_length + ) + sadf = float(np.nanmax(sadf_seq)) if np.isfinite(sadf_seq).any() else np.nan + gsadf = float(np.nanmax(bsadf_seq)) if np.isfinite(bsadf_seq).any() else np.nan + return sadf, gsadf + + +def volatility_robust_sadf( + log_price: np.ndarray | pd.Series, + min_length: int | None = None, + n_bootstrap: int = 99, + random_state: int | None = None, +) -> dict: + r""" + Volatility-robust SADF / GSADF wild-bootstrap test for an explosive (bubble) episode. + + Computes the observed SADF and GSADF sup-ADF statistics, then calibrates their p-values by a wild + bootstrap: each bootstrap path multiplies the demeaned first-difference residuals by an independent + Rademacher (+/- 1) sign and re-cumulates, preserving the series' (possibly non-stationary) volatility + while removing any explosive autocorrelation; the sup-ADF statistics are recomputed on each path to + form the null distribution. The p-value is the share of bootstrap statistics at least as large as the + observed one. + + Prefer this over plain SADF / GSADF when the series' volatility may be non-stationary: it holds nominal + size where the plain test over-rejects, at a modest power cost, and converges to the plain test under + constant volatility. It pairs with the admitted GSADF / BSADF (the same sup-ADF statistics). See the + module docstring for the verbatim regime tag and appraisals/26_verdict.md. + + Parameters + ---------- + log_price : np.ndarray or pd.Series + The log-price level series. + min_length : int, optional + Minimum (PSY) window for the sup-ADF recursion; defaults to ``psy_minimum_window(len)``. + n_bootstrap : int, default 99 + Wild-bootstrap replications. + random_state : int, optional + Seed for the Rademacher signs. + + Returns + ------- + dict + ``sadf``, ``gsadf`` (observed statistics), ``sadf_pvalue``, ``gsadf_pvalue`` (wild-bootstrap), + and ``reject_sadf`` / ``reject_gsadf`` (at the 5% level). + + Examples + -------- + >>> import numpy as np + >>> rng = np.random.default_rng(0) + >>> y = np.cumsum(rng.standard_normal(200)) # a random walk (no bubble) + >>> res = volatility_robust_sadf(y, n_bootstrap=49, random_state=0) + >>> set(res) == {"sadf", "gsadf", "sadf_pvalue", "gsadf_pvalue", "reject_sadf", "reject_gsadf"} + True + """ + y = np.asarray( + log_price.values if isinstance(log_price, pd.Series) else log_price, dtype=float + ) + if y.ndim != 1 or y.size < 8: + raise ValueError("log_price must be a 1-D series of length >= 8") + nmin = int(min_length) if min_length is not None else psy_minimum_window(y.size) + sadf_obs, gsadf_obs = _sup_adf_statistics(y, nmin) + + residual = np.diff(y) + residual = residual - residual.mean() + y0 = y[0] + rng = np.random.default_rng(random_state) + ge_sadf = ge_gsadf = 0 + for _ in range(n_bootstrap): + signs = rng.integers(0, 2, size=residual.shape[0]) * 2 - 1 + path = np.empty_like(y) + path[0] = y0 + path[1:] = y0 + np.cumsum(signs * residual) + sadf_b, gsadf_b = _sup_adf_statistics(path, nmin) + if np.isfinite(sadf_b) and np.isfinite(sadf_obs) and sadf_b >= sadf_obs: + ge_sadf += 1 + if np.isfinite(gsadf_b) and np.isfinite(gsadf_obs) and gsadf_b >= gsadf_obs: + ge_gsadf += 1 + p_sadf = (1 + ge_sadf) / (n_bootstrap + 1) + p_gsadf = (1 + ge_gsadf) / (n_bootstrap + 1) + return { + "sadf": sadf_obs, + "gsadf": gsadf_obs, + "sadf_pvalue": float(p_sadf), + "gsadf_pvalue": float(p_gsadf), + "reject_sadf": bool(p_sadf < 0.05), + "reject_gsadf": bool(p_gsadf < 0.05), + } diff --git a/src/RiskLabAI/hpc/__init__.py b/src/RiskLabAI/hpc/__init__.py new file mode 100644 index 0000000..b62b19e --- /dev/null +++ b/src/RiskLabAI/hpc/__init__.py @@ -0,0 +1,27 @@ +""" +RiskLabAI High-Performance Computing (HPC) Module + +Provides utilities for multiprocessing and parallel execution. +""" + +from .hpc import ( + expand_call, + linear_partitions, + mp_pandas_obj, + nested_partitions, + parallel_run, + process_jobs, + process_jobs_sequential, + report_progress, +) + +__all__ = [ + "report_progress", + "expand_call", + "process_jobs", + "process_jobs_sequential", + "linear_partitions", + "nested_partitions", + "mp_pandas_obj", + "parallel_run", +] diff --git a/src/RiskLabAI/hpc/hpc.py b/src/RiskLabAI/hpc/hpc.py new file mode 100644 index 0000000..03b3026 --- /dev/null +++ b/src/RiskLabAI/hpc/hpc.py @@ -0,0 +1,379 @@ +""" +High-Performance Computing (HPC) utilities for parallel processing. + +Provides a set of functions to parallelize operations, especially +on pandas objects, using Python's multiprocessing and Joblib. +""" + +import datetime as dt +import logging +import multiprocessing as mp +import time +from collections.abc import Iterable +from typing import Any, Callable, Optional, Union + +import joblib # <-- Added missing import +import numpy as np +import pandas as pd + +logger = logging.getLogger(__name__) + + +def parallel_run( + func: Callable[..., Any], + iterable: Iterable[Any], + num_cpus: int = -1, + lin_partition: bool = False, + **kwargs, +) -> list[Any]: + """ + Executes a function in parallel over an iterable using Joblib. + + Parameters + ---------- + func : Callable + The function to be parallelized. + iterable : Iterable + The iterable to loop over. + num_cpus : int, default=-1 + The number of CPUs to use. -1 means all available CPUs. + lin_partition : bool, default=False + How to partition the work: + - If False (default): Item-by-item. `func` receives one item + from `iterable` at a time. + - If True: Chunked. `iterable` is split into `num_cpus` chunks + (by index). `func` must be designed to receive a list of + indices (e.g., [0, 1, 2]) and process them. + **kwargs : + Additional keyword arguments to be passed to `func`. + + Returns + ------- + List[Any] + A list of the results from all parallel executions. + """ + if num_cpus == -1: + num_cpus = mp.cpu_count() + + if lin_partition: + # --- Chunked Partitioning --- + # func must accept a list of indices + # NOTE: This implies `iterable` must support `len()`. + try: + iterable_len = len(iterable) + except TypeError: + iterable = list(iterable) + iterable_len = len(iterable) + + num_atoms = num_cpus + iterable_partition = np.array_split(range(iterable_len), num_atoms) + jobs = (iterable_partition[i] for i in range(num_atoms)) + + results = joblib.Parallel(n_jobs=num_cpus)( + joblib.delayed(func)(job, **kwargs) for job in jobs + ) + + # Flatten the list of lists + return [item for sublist in results for item in sublist] + + else: + # --- Item-by-Item Partitioning (Standard) --- + # func accepts a single item + # <-- FIXED: Pass iterable directly. Joblib handles generators + # and lists efficiently. The original implementation was buggy. + results = joblib.Parallel(n_jobs=num_cpus)( + joblib.delayed(func)(job, **kwargs) for job in iterable + ) + return results + + +def report_progress( + job_number: int, total_jobs: int, start_time: float, task: str +) -> None: + """ + Report the progress of a computing task to the console. + + Parameters + ---------- + job_number : int + The current job number (e.g., `i` in a loop). + total_jobs : int + The total number of jobs. + start_time : float + The time the process started (e.g., `time.time()`). + task : str + A description of the task being performed. + """ + progress = job_number / total_jobs + elapsed_time_min = (time.time() - start_time) / 60.0 + + # Avoid division by zero if progress is 0 + remaining_time_min = 0.0 + if progress > 0: + remaining_time_min = elapsed_time_min * (1 / progress - 1) + + timestamp = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + message = ( + f"{timestamp} {progress*100:.2f}% {task} done after " + f"{elapsed_time_min:.2f} minutes. Remaining {remaining_time_min:.2f} minutes." + ) + + # Progress is emitted via logging (configure the 'RiskLabAI' logger to see + # it; the library is silent by default). + logger.info(message) + + +def expand_call(kargs: dict[str, Any]) -> Any: + """ + Wrapper function to expand keyword arguments for a callback. + + This is used by `process_jobs` to unpack a dictionary of arguments + and call the target function. + + Parameters + ---------- + kargs : dict + A dictionary of arguments, which *must* include a 'func' key + mapping to the function to be called. + + Returns + ------- + Any + The output of the callback function. + """ + func = kargs.pop("func") + return func(**kargs) + + +def process_jobs( + jobs: list[dict[str, Any]], task: Optional[str] = None, num_threads: int = -1 +) -> list[Any]: + """ + Process a list of jobs in parallel using multiprocessing. + + Parameters + ---------- + jobs : List[Dict[str, Any]] + A list of job dictionaries. Each dict must contain a 'func' key + and all arguments required by that function. + task : str, optional + A name for the task, used for progress reporting. If None, + the function name from the first job is used. + num_threads : int, default=-1 + The number of parallel processes to spawn. + -1 means use all available CPUs. + 1 means run sequentially (for debugging). + + Returns + ------- + List[Any] + A list containing the results from all jobs. + """ + if task is None: + task = jobs[0]["func"].__name__ + + # <-- Handle sequential case for debugging + if num_threads == 1: + logger.debug("Running %d jobs sequentially for debugging.", len(jobs)) + return process_jobs_sequential(jobs) + + # <-- Handle -1 for all CPUs + if num_threads == -1: + num_threads = mp.cpu_count() + + with mp.Pool(processes=num_threads) as pool: + outputs = [] + start_time = time.time() + + # Use imap_unordered for efficient processing + imap_results = pool.imap_unordered(expand_call, jobs) + + # Process results as they complete + for i, result in enumerate(imap_results, 1): + outputs.append(result) + report_progress(i, len(jobs), start_time, task) + + return outputs + + +def process_jobs_sequential(jobs: list[dict[str, Any]]) -> list[Any]: + """ + Run jobs sequentially (single-thread). Useful for debugging. + + Parameters + ---------- + jobs : List[Dict[str, Any]] + A list of job dictionaries. + + Returns + ------- + List[Any] + A list containing the results from all jobs. + """ + output = [] + start_time = time.time() + task = "Sequential Processing" + if jobs: + task = jobs[0].get("func", lambda: None).__name__ + + for i, job in enumerate(jobs, 1): + output_ = expand_call(job) + output.append(output_) + report_progress(i, len(jobs), start_time, task) + + return output + + +def linear_partitions(num_atoms: int, num_threads: int) -> np.ndarray: + """ + Generate linear partitions (indices) for parallel computation. + + Splits `num_atoms` into `num_threads` (or fewer) roughly equal parts. + + Parameters + ---------- + num_atoms : int + The total number of items to split. + num_threads : int + The desired number of partitions (threads). + + Returns + ------- + np.ndarray + An array of partition boundary indices. + """ + n_parts = min(num_threads, num_atoms) + if n_parts == 0: + return np.array([0]) + + partitions = np.linspace(0, num_atoms, n_parts + 1) + partitions = np.ceil(partitions).astype(int) + return partitions + + +def nested_partitions( + num_atoms: int, num_threads: int, upper_triangle: bool = False +) -> np.ndarray: + r""" + Generate nested partitions, useful for tasks with nested loops. + + The formula is designed to create partitions of varying sizes, + which can be more efficient for tasks where work is not + distributed linearly (e.g., matrix operations). + + .. math:: + p_i = \frac{-1 + \sqrt{1 + 4(p_{i-1}^2 + p_{i-1} + N(N+1)/K)}}{2} + + Parameters + ---------- + num_atoms : int + Number of atoms (N). + num_threads : int + Number of threads (K). + upper_triangle : bool, default=False + If True, sort partitions to be heaviest first. + + Returns + ------- + np.ndarray + An array of partition boundary indices. + """ + partitions = [0] + n_threads_ = min(num_threads, num_atoms) + + if n_threads_ == 0: + return np.array([0]) + + for _ in range(n_threads_): + last_part = partitions[-1] + part_size = 1 + 4 * ( + last_part**2 + last_part + num_atoms * (num_atoms + 1.0) / n_threads_ + ) + part_val = (-1 + part_size**0.5) / 2.0 + partitions.append(part_val) + + partitions = np.round(partitions).astype(int) + + if upper_triangle: # The first rows are the heaviest + partitions = np.cumsum(np.diff(partitions)[::-1]) + partitions = np.append(np.array([0]), partitions) + + return partitions + + +def mp_pandas_obj( + func: Callable[..., pd.Series], + pandas_object: tuple[str, pd.Index], + num_threads: int = -1, + mp_batches: int = 1, + linear_partition: bool = True, + **kwargs: Any, +) -> Union[pd.DataFrame, pd.Series, list[Any]]: + """ + Parallelize a function call on a pandas object (DataFrame/Series). + + This function splits the `pandas_object` index into partitions and + calls `func` on each partition in parallel. + + Parameters + ---------- + func : Callable + The function to be parallelized. It must accept a pandas + object as its first argument. + pandas_object : Tuple[str, pd.Index] + A tuple where: + - [0] (str): The name of the argument in `func` that + receives the partition (e.g., 'molecule'). + - [1] (pd.Index): The index to be partitioned. + num_threads : int, default=-1 + Number of parallel processes. + -1 means use all available CPUs. + 1 means run sequentially (for debugging). + mp_batches : int, default=1 + Number of batches to split the jobs into. + linear_partition : bool, default=True + If True, use `linear_partitions`. + If False, use `nested_partitions`. + **kwargs : Any + Other keyword arguments to be passed to `func`. + + Returns + ------- + Union[pd.DataFrame, pd.Series, List[Any]] + The concatenated results from all parallel calls. + Returns a List if the output is not a pandas object. + """ + # <-- Resolve num_threads here to correctly handle mp_batches + if num_threads == -1: + num_threads = mp.cpu_count() + + total_parts = num_threads * mp_batches + + if linear_partition: + parts = linear_partitions(len(pandas_object[1]), total_parts) + else: + parts = nested_partitions(len(pandas_object[1]), total_parts) + + jobs = [] + for i in range(1, len(parts)): + job = { + pandas_object[0]: pandas_object[1][parts[i - 1] : parts[i]], + "func": func, + } + job.update(kwargs) + jobs.append(job) + + # <-- Pass num_threads to process_jobs, which now handles 1 correctly + out = process_jobs(jobs, num_threads=num_threads) + + if not out: + return pd.DataFrame() # Return empty DataFrame if no results + + if isinstance(out[0], pd.DataFrame): + result_df = pd.concat(out) + elif isinstance(out[0], pd.Series): + result_df = pd.concat(out) + else: + return out # Return list of other objects + + return result_df.sort_index() diff --git a/src/RiskLabAI/optimization/__init__.py b/src/RiskLabAI/optimization/__init__.py new file mode 100644 index 0000000..eab6496 --- /dev/null +++ b/src/RiskLabAI/optimization/__init__.py @@ -0,0 +1,48 @@ +""" +RiskLabAI Portfolio Optimization Module + +Implements advanced portfolio optimization techniques, including: +- Hierarchical Risk Parity (HRP) +- Nested Clustered Optimisation (NCO) +- PCA-Based Hedging +- Custom Hyper-Parameter Tuning +""" + +from .hedging import ( + pca_weights, +) +from .hrp import ( + cluster_variance, + hrp, + quasi_diagonal, + recursive_bisection, +) +from .hyper_parameter_tuning import ( + MyPipeline, # deprecated alias (scheduled for removal in 4.0.0) + SampleWeightedPipeline, + clf_hyper_fit, +) +from .nco import ( + # cluster_kmeans_base is imported into nco.py, not defined there. + # It should be imported from RiskLabAI.cluster.clustering directly + # in any file that needs it, not from here. + get_optimal_portfolio_weights, + get_optimal_portfolio_weights_nco, +) + +__all__ = [ + # hrp.py + "cluster_variance", + "quasi_diagonal", + "recursive_bisection", + "hrp", + # nco.py + "get_optimal_portfolio_weights", + "get_optimal_portfolio_weights_nco", + # hedging.py + "pca_weights", + # hyper_parameter_tuning.py + "SampleWeightedPipeline", + "MyPipeline", # deprecated alias (scheduled for removal in 4.0.0) + "clf_hyper_fit", +] diff --git a/src/RiskLabAI/optimization/hedging.py b/src/RiskLabAI/optimization/hedging.py new file mode 100644 index 0000000..b4a4986 --- /dev/null +++ b/src/RiskLabAI/optimization/hedging.py @@ -0,0 +1,69 @@ +""" +Implements PCA-based portfolio hedging techniques. +""" + +from typing import Optional + +import numpy as np + + +def pca_weights( + cov: np.ndarray, + risk_distribution: Optional[np.ndarray] = None, + risk_target: float = 1.0, +) -> np.ndarray: + r""" + Calculate hedging weights using PCA. + + This function uses PCA to determine portfolio weights that + match a target risk profile. + + Reference: + De Prado, M. (2018) Advances in financial machine learning. + Methodology 36. + + .. math:: + w = V \cdot \sqrt{\frac{\rho T}{\lambda}} + + Where: + - \(w\) are the weights. + - \(V\) is the matrix of eigenvectors. + - \(\rho\) is the risk distribution. + - \(T\) is the risk target. + - \(\lambda\) are the eigenvalues. + + Parameters + ---------- + cov : np.ndarray + Covariance matrix. + risk_distribution : np.ndarray, optional + Target risk distribution across principal components. + If None, all risk is allocated to the component with the + smallest eigenvalue (minimum variance portfolio). + risk_target : float, default=1.0 + The total risk target (T). + + Returns + ------- + np.ndarray + The calculated portfolio weights. + """ + # Calculate eigenvalues and eigenvectors + eigen_values, eigen_vectors = np.linalg.eigh(cov) + + # Sort in descending order + indices = eigen_values.argsort()[::-1] + eigen_values, eigen_vectors = eigen_values[indices], eigen_vectors[:, indices] + + # If no risk distribution, target minimum variance + if risk_distribution is None: + risk_distribution = np.zeros(cov.shape[0]) + risk_distribution[-1] = 1.0 # All risk on last component + + # Compute loads (allocation in the orthogonal basis) + loads = risk_target * (risk_distribution / eigen_values) ** 0.5 + + # Calculate weights in the original basis + weights = np.dot(eigen_vectors, np.reshape(loads, (-1, 1))) + + return weights.flatten() diff --git a/src/RiskLabAI/optimization/hrp.py b/src/RiskLabAI/optimization/hrp.py new file mode 100644 index 0000000..6e1b461 --- /dev/null +++ b/src/RiskLabAI/optimization/hrp.py @@ -0,0 +1,215 @@ +""" +Implements the Hierarchical Risk Parity (HRP) portfolio +optimization algorithm. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. +""" + +import numpy as np +import pandas as pd +import scipy.cluster.hierarchy as sch +import scipy.spatial.distance as scd + + +def inverse_variance_weights(covariance_matrix: pd.DataFrame) -> np.ndarray: + """ + Compute the inverse-variance portfolio weights. + + Parameters + ---------- + covariance_matrix : pd.DataFrame + Covariance matrix of asset returns. + + Returns + ------- + np.ndarray + Array of portfolio weights. + """ + inv_var_weights = 1.0 / np.diag(covariance_matrix.values) + inv_var_weights /= inv_var_weights.sum() + return inv_var_weights + + +def cluster_variance( + covariance_matrix: pd.DataFrame, clustered_items: list[str] +) -> float: + """ + Compute the variance of a cluster using inverse-variance weighting. + + Parameters + ---------- + covariance_matrix : pd.DataFrame + Full covariance matrix of asset returns. + clustered_items : List[str] + List of asset names (index/columns) in the cluster. + + Returns + ------- + float + Variance of the cluster. + """ + cov_slice = covariance_matrix.loc[clustered_items, clustered_items] + weights = inverse_variance_weights(cov_slice).reshape(-1, 1) + + # V_cluster = w' * C * w + cluster_var = np.dot(np.dot(weights.T, cov_slice), weights)[0, 0] + return cluster_var + + +def quasi_diagonal(linkage_matrix: np.ndarray) -> list[int]: + """ + Return a sorted list of original item indices for a quasi-diagonal matrix. + + Parameters + ---------- + linkage_matrix : np.ndarray + The linkage matrix from `scipy.cluster.hierarchy.linkage`. + + Returns + ------- + List[int] + Sorted list of original item indices. + """ + link = linkage_matrix.astype(int) + num_items = link[-1, 3] # Total number of original items + + # Get the top-level clusters + items_to_process = [link[-1, 0], link[-1, 1]] + sorted_items = [] + + while len(items_to_process) > 0: + item = items_to_process.pop(0) # Process items recursively (depth-first) + + if item >= num_items: + # This is a cluster, add its children to the processing list + cluster_id = item - num_items + # Add children in their linkage order + items_to_process.insert(0, link[cluster_id, 1]) # Right child + items_to_process.insert(0, link[cluster_id, 0]) # Left child + else: + # This is an original item + sorted_items.append(item) + + return sorted_items + + +def recursive_bisection( + covariance_matrix: pd.DataFrame, sorted_items: list[str] +) -> pd.Series: + """ + Compute the Hierarchical Risk Parity (HRP) weights + using recursive bisection. + + Parameters + ---------- + covariance_matrix : pd.DataFrame + Covariance matrix of asset returns. + sorted_items : List[str] + Sorted list of asset names from `quasi_diagonal`. + + Returns + ------- + pd.Series + DataFrame of asset weights. + """ + weights = pd.Series(1.0, index=sorted_items) + clustered_items = [sorted_items] + + while len(clustered_items) > 0: + # Bisection + clustered_items = [ + i[j:k] + for i in clustered_items + for j, k in ((0, len(i) // 2), (len(i) // 2, len(i))) + if len(i) > 1 + ] + + # Process pairs of clusters + for i in range(0, len(clustered_items), 2): + cluster_0 = clustered_items[i] + cluster_1 = clustered_items[i + 1] + + # 1. Calculate variance for each cluster + variance_0 = cluster_variance(covariance_matrix, cluster_0) + variance_1 = cluster_variance(covariance_matrix, cluster_1) + + # 2. Calculate allocation factor (alpha) + if variance_0 + variance_1 == 0: + alpha = 0.5 # Default to equal weight if both variances are zero + else: + alpha = 1 - variance_0 / (variance_0 + variance_1) + + # 3. Apply weights + weights[cluster_0] *= alpha + weights[cluster_1] *= 1 - alpha + + return weights + + +def distance_corr(corr_matrix: np.ndarray) -> np.ndarray: + """ + Compute the distance matrix based on correlation. + d = sqrt(0.5 * (1 - p)) + + Parameters + ---------- + corr_matrix : np.ndarray + Correlation matrix. + + Returns + ------- + np.ndarray + Distance matrix. + """ + distance_matrix = ((1 - corr_matrix) / 2.0) ** 0.5 + return distance_matrix + + +def hrp(cov: pd.DataFrame, corr: pd.DataFrame) -> pd.Series: + """ + Main HRP algorithm. + + Constructs a hierarchical portfolio by: + 1. Calculating correlation-based distance. + 2. Hierarchical clustering (single linkage). + 3. Quasi-diagonalization of the linkage matrix. + 4. Recursive bisection to determine weights. + + Parameters + ---------- + cov : pd.DataFrame + Covariance matrix. + corr : pd.DataFrame + Correlation matrix. + + Returns + ------- + pd.Series + The HRP portfolio weights, sorted by original index. + """ + corr_df, cov_df = corr, cov + + # 1. Calculate distance + distance = distance_corr(corr_df.values) + + # Enforce exact symmetry: a correlation matrix produced by ``cov_to_corr`` or + # by denoising can be asymmetric at the floating-point level, which makes + # ``squareform`` reject it ("Distance matrix must be symmetric"). Averaging + # with the transpose removes that asymmetry without changing the clustering. + distance = (distance + distance.T) / 2.0 + np.fill_diagonal(distance, 0.0) + + dist_condensed = scd.squareform(distance, force="tovector") + + # 2. Cluster + link = sch.linkage(dist_condensed, "single") + + # 3. Quasi-diagonalize + sorted_items_idx = quasi_diagonal(link) + sorted_items_names = corr_df.index[sorted_items_idx].tolist() + + # 4. Recursive bisection + hrp_portfolio = recursive_bisection(cov_df, sorted_items_names) + + return hrp_portfolio.sort_index() diff --git a/src/RiskLabAI/optimization/hyper_parameter_tuning.py b/src/RiskLabAI/optimization/hyper_parameter_tuning.py new file mode 100644 index 0000000..ee87c64 --- /dev/null +++ b/src/RiskLabAI/optimization/hyper_parameter_tuning.py @@ -0,0 +1,197 @@ +""" +Hyperparameter tuning module that integrates with scikit-learn +and the custom PurgedKFold cross-validators. +""" + +import warnings +from typing import Any, Optional, Union + +import numpy as np +import pandas as pd +from sklearn.ensemble import BaggingClassifier +from sklearn.model_selection import GridSearchCV, RandomizedSearchCV +from sklearn.pipeline import Pipeline + +# Cross-validators are created from the core registry (2.0.0 single source). +from RiskLabAI.core import CROSS_VALIDATORS + + +class SampleWeightedPipeline(Pipeline): + """ + Custom pipeline class to correctly pass `sample_weight` to the + final estimator's `fit` method. + """ + + def fit( + self, + X: pd.DataFrame, + y: pd.Series, + sample_weight: Optional[np.ndarray] = None, + **fit_params, + ) -> "SampleWeightedPipeline": + """ + Fit the pipeline, passing `sample_weight` to the final step. + + Parameters + ---------- + X : pd.DataFrame + Feature data. + y : pd.Series + Labels. + sample_weight : np.ndarray, optional + Sample weights. + **fit_params : Any + Additional fit parameters. + + Returns + ------- + SampleWeightedPipeline + The fitted pipeline. + """ + if sample_weight is not None: + # Add sample_weight to fit_params for the *last step* + step_name = self.steps[-1][0] + fit_params[f"{step_name}__sample_weight"] = sample_weight + + return super().fit(X, y, **fit_params) + + +def clf_hyper_fit( + feature_data: pd.DataFrame, + label: pd.Series, + times: pd.Series, + pipe_clf: Pipeline, + param_grid: dict[str, Any], + validator_type: str = "purgedkfold", + validator_params: Optional[dict[str, Any]] = None, + bagging: Optional[list[Union[int, float]]] = None, + rnd_search_iter: int = 0, + n_jobs: int = -1, + **fit_params, +) -> Union[GridSearchCV, RandomizedSearchCV, Pipeline]: + """ + Perform hyperparameter tuning using a specified cross-validator. + + Parameters + ---------- + feature_data : pd.DataFrame + Feature data (X). + label : pd.Series + Labels (y). + times : pd.Series + Series of event start and end times for purging. + pipe_clf : Pipeline + The scikit-learn pipeline (or `SampleWeightedPipeline`) to tune. + param_grid : dict + Parameter grid for the search. + validator_type : str, default='purgedkfold' + The key for the cross-validator to use + (e.g., 'purgedkfold', 'combinatorialpurged'). + validator_params : dict, optional + Parameters for the cross-validator (e.g., `n_splits`, `embargo`). + bagging : List, optional + If provided, wraps the best estimator in a `BaggingClassifier`. + Format: [n_estimators, max_samples, max_features]. + Default [0, -1, 1.0] from original code seems incorrect, + so changed to None. + rnd_search_iter : int, default=0 + If 0, use `GridSearchCV`. + If > 0, use `RandomizedSearchCV` with this many iterations. + n_jobs : int, default=-1 + Number of parallel jobs for the search. + **fit_params : Any + Additional fit parameters (e.g., `sample_weight`). + + Returns + ------- + Union[GridSearchCV, RandomizedSearchCV, Pipeline] + The fitted grid search object, or a fitted Bagging pipeline. + """ + if bagging is None: + bagging = [0, 0.0, 1.0] # Default to no bagging + + if set(label.unique()) == {0, 1}: + scoring = "f1" # F1-score for meta-labeling + else: + scoring = "neg_log_loss" + + if validator_params is None: + validator_params = { + "times": times, + "n_splits": 5, + "embargo": 0.01, + } + else: + # Ensure 'times' is passed if not already present + if "times" not in validator_params: + validator_params["times"] = times + + # 1. Set up the custom cross-validator + inner_cv = CROSS_VALIDATORS.create( + validator_type, filter_unknown_kwargs=True, **validator_params + ) + + # 2. Set up the hyperparameter search + if rnd_search_iter == 0: + gs = GridSearchCV( + estimator=pipe_clf, + param_grid=param_grid, + scoring=scoring, + cv=inner_cv, + n_jobs=n_jobs, + ) + else: + gs = RandomizedSearchCV( + estimator=pipe_clf, + param_distributions=param_grid, + scoring=scoring, + cv=inner_cv, + n_jobs=n_jobs, + n_iter=rnd_search_iter, + ) + + # 3. Fit the search + gs = gs.fit(feature_data, label, **fit_params) + + # 4. (Optional) Fit bagging classifier on the best model + if bagging[0] > 0: + best_estimator = gs.best_estimator_ + + # Create a new pipeline with the best estimator's steps + bag_pipe = SampleWeightedPipeline(best_estimator.steps) + + bag_clf = BaggingClassifier( + estimator=bag_pipe, + n_estimators=int(bagging[0]), + max_samples=float(bagging[1]) if bagging[1] > 0 else 1.0, + max_features=float(bagging[2]), + n_jobs=n_jobs, + ) + + # Fit the bagging classifier + bag_clf = bag_clf.fit(feature_data, label, **fit_params) + + # Return as a pipeline + return Pipeline([("bag", bag_clf)]) + + # 5. Return the best estimator found + return gs.best_estimator_ + + +class MyPipeline(SampleWeightedPipeline): + """ + Deprecated alias for :class:`SampleWeightedPipeline` (scheduled for removal in 4.0.0). + + Implemented as an explicit subclass rather than via ``deprecated_class`` so + that the scikit-learn ``__init__`` signature is preserved (estimators may + not use ``*args``/``**kwargs`` in their constructor). + """ + + def __init__(self, steps, *, memory=None, verbose=False): + warnings.warn( + "MyPipeline is deprecated and will be removed in 4.0.0; " + "use SampleWeightedPipeline instead.", + DeprecationWarning, + stacklevel=2, + ) + super().__init__(steps, memory=memory, verbose=verbose) diff --git a/src/RiskLabAI/optimization/nco.py b/src/RiskLabAI/optimization/nco.py new file mode 100644 index 0000000..267807d --- /dev/null +++ b/src/RiskLabAI/optimization/nco.py @@ -0,0 +1,130 @@ +""" +Implements the Nested Clustered Optimization (NCO) algorithm. + +Reference: + De Prado, M. (2018) Advances in financial machine learning. +""" + +from typing import Optional + +import numpy as np +import pandas as pd + +# Import canonical implementations instead of duplicating +from RiskLabAI.cluster.clustering import cluster_k_means_base, covariance_to_correlation + + +def get_optimal_portfolio_weights( + covariance: np.ndarray, mu: Optional[np.ndarray] = None +) -> np.ndarray: + """ + Compute the optimal portfolio weights (Markowitz). + + If `mu` is not provided, computes the Global Minimum Variance (GMV) portfolio. + If `mu` is provided, computes the Mean-Variance Optimization (MVO) portfolio. + + Parameters + ---------- + covariance : np.ndarray + Covariance matrix. + mu : np.ndarray, optional + Vector of expected returns. + + Returns + ------- + np.ndarray + Optimal portfolio weights. + """ + inverse_covariance = np.linalg.inv(covariance) + ones = np.ones(shape=(inverse_covariance.shape[0], 1)) + + if mu is None: + mu = ones # For GMV portfolio + + weights = np.dot(inverse_covariance, mu) + weights /= np.dot(ones.T, weights) # Normalize weights to sum to 1 + return weights + + +def get_optimal_portfolio_weights_nco( + covariance: np.ndarray, + mu: Optional[np.ndarray] = None, + number_clusters: Optional[int] = None, +) -> np.ndarray: + """ + Compute optimal portfolio weights using the NCO algorithm. + + NCO works in three steps: + 1. Cluster assets based on their correlation matrix. + 2. Compute optimal (e.g., GMV) weights *within* each cluster. + 3. Compute optimal weights *between* the clusters (treating each + cluster as a single asset). + 4. Combine the inter-cluster and intra-cluster weights. + + Parameters + ---------- + covariance : np.ndarray + Covariance matrix. + mu : np.ndarray, optional + Vector of expected returns. If None, uses GMV. + number_clusters : int, optional + Maximum number of clusters. If None, defaults to N/2. + + Returns + ------- + np.ndarray + The NCO-optimized portfolio weights. + """ + covariance = pd.DataFrame(covariance) + correlation = covariance_to_correlation(covariance.to_numpy()) + correlation = pd.DataFrame( + correlation, index=covariance.index, columns=covariance.columns + ) + + if mu is not None: + mu = pd.Series(mu.flatten(), index=covariance.index) + + if number_clusters is None: + number_clusters = int(correlation.shape[0] / 2) + + # 1. Cluster assets + _, clusters, _ = cluster_k_means_base( + correlation, max_clusters=number_clusters, iterations=10 + ) + + # 2. Compute intra-cluster weights + weights_intra_cluster = pd.DataFrame( + 0.0, index=covariance.index, columns=clusters.keys() + ) + for i, cluster_assets in clusters.items(): + cov_intra = covariance.loc[cluster_assets, cluster_assets].values + + mu_intra = None + if mu is not None: + mu_intra = mu.loc[cluster_assets].values.reshape(-1, 1) + + weights_intra_cluster.loc[cluster_assets, i] = get_optimal_portfolio_weights( + cov_intra, mu_intra + ).flatten() + + # 3. Compute inter-cluster weights + # Reduce covariance matrix using intra-cluster weights + covariance_inter_cluster = weights_intra_cluster.T.dot( + covariance.dot(weights_intra_cluster) + ) + + mu_inter_cluster = None + if mu is not None: + mu_inter_cluster = weights_intra_cluster.T.dot(mu.values) + mu_inter_cluster = mu_inter_cluster.values.reshape(-1, 1) + + weights_inter_cluster = pd.Series( + get_optimal_portfolio_weights( + covariance_inter_cluster.values, mu_inter_cluster + ).flatten(), + index=covariance_inter_cluster.index, + ) + + # 4. Combine weights + weights_nco = weights_intra_cluster.mul(weights_inter_cluster, axis=1).sum(axis=1) + return weights_nco.values.reshape(-1, 1) diff --git a/src/RiskLabAI/pde/__init__.py b/src/RiskLabAI/pde/__init__.py new file mode 100644 index 0000000..2f5feb6 --- /dev/null +++ b/src/RiskLabAI/pde/__init__.py @@ -0,0 +1,75 @@ +""" +RiskLabAI PDE Solver Module + +Implements a Deep BSDE (Backward Stochastic Differential Equation) +solver for various financial PDEs. + +This sub-package requires PyTorch, which is an optional dependency of +RiskLabAI. The base install stays torch-free: ``import RiskLabAI`` never pulls +in this sub-package (it is loaded lazily). Importing ``RiskLabAI.pde`` without +torch installed raises a clear, actionable error instead of a bare +``ModuleNotFoundError``. +""" + +try: + import torch as _torch # noqa: F401 (presence check only) +except ImportError as _exc: # pragma: no cover - exercised only without torch + raise ImportError( + "RiskLabAI.pde requires PyTorch, which is an optional dependency. " + "Install it with: pip install 'RiskLabAI[pde]' (or: pip install torch)." + ) from _exc + +from .equation import ( + HJBLQ, + BlackScholesBarenblatt, + Equation, + PricingDefaultRisk, + PricingDiffRate, +) +from .model import ( + ISAB, + MAB, + PMA, + SAB, + DeepBSDE, + DeepTimeSetTransformer, + FBSNNNetwork, + Net1, + TimeDependentNetwork, + TimeDependentNetworkMonteCarlo, + TimeNet, + TimeNetForSet, +) +from .solver import ( + FBSDESolver, + FBSNNolver, # deprecated alias (scheduled for removal in 4.0.0) + FBSNNSolver, + initialize_weights, +) + +__all__ = [ + # Equations + "Equation", + "PricingDefaultRisk", + "HJBLQ", + "BlackScholesBarenblatt", + "PricingDiffRate", + # Models + "TimeNet", + "Net1", + "MAB", + "SAB", + "ISAB", + "PMA", + "TimeNetForSet", + "DeepTimeSetTransformer", + "FBSNNNetwork", + "DeepBSDE", + "TimeDependentNetwork", + "TimeDependentNetworkMonteCarlo", + # Solvers + "initialize_weights", + "FBSDESolver", + "FBSNNSolver", + "FBSNNolver", # deprecated alias (scheduled for removal in 4.0.0) +] diff --git a/src/RiskLabAI/pde/equation.py b/src/RiskLabAI/pde/equation.py new file mode 100644 index 0000000..5426077 --- /dev/null +++ b/src/RiskLabAI/pde/equation.py @@ -0,0 +1,316 @@ +""" +Base class and concrete implementations for Partial Differential Equations +(PDEs) to be solved by the Deep BSDE solver. + +Based on: https://github.com/frankhan91/DeepBSDE +""" + +from typing import Optional, Union + +import numpy as np +import torch +import torch.nn as nn +from torch import Tensor + + +class Equation: + """ + Base class for defining PDE-related functions. + """ + + def __init__(self, eqn_config: dict): + """ + Initialize the PDE configuration. + + Parameters + ---------- + eqn_config : dict + Dictionary containing PDE parameters: + - 'dim': Dimensionality of the problem (int) + - 'total_time': Total time horizon (float) + - 'num_time_interval': Number of time steps (int) + """ + self.dim: int = eqn_config["dim"] + self.total_time: float = eqn_config["total_time"] + self.num_time_interval: int = eqn_config["num_time_interval"] + self.delta_t: float = self.total_time / self.num_time_interval + self.sqrt_delta_t: float = np.sqrt(self.delta_t) + self.y_init: Optional[float] = None + + def sample(self, num_sample: int) -> tuple[np.ndarray, np.ndarray]: + """ + Sample the forward SDE (e.g., the underlying asset path). + + Parameters + ---------- + num_sample : int + Number of sample paths to generate. + + Returns + ------- + Tuple[np.ndarray, np.ndarray] + - dw_sample: Wiener process increments [num_sample, dim, num_time_interval] + - x_sample: Asset paths [num_sample, dim, num_time_interval+1] + """ + raise NotImplementedError + + def r_u(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + """ + The driver function 'f' of the BSDE, or the generator of the PDE. + Represents the non-linear interest rate and dividend term. + + Parameters + ---------- + t : float + Current time. + x : Tensor + Space coordinates [batch_size, dim]. + y : Tensor + Function value (price) [batch_size, 1]. + z : Tensor + Gradient (delta) [batch_size, dim]. + + Returns + ------- + Tensor + Value of the driver `f` [batch_size, 1]. + """ + raise NotImplementedError + + def h_z(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + """ + The Hamiltonian term H(z) in the PDE, or the volatility-related + term in the BSDE driver. + + Parameters + ---------- + t : float + Current time. + x : Tensor + Space coordinates [batch_size, dim]. + y : Tensor + Function value (price) [batch_size, 1]. + z : Tensor + Gradient (delta) [batch_size, dim]. + + Returns + ------- + Tensor + Value of the Hamiltonian `H(z)` [batch_size, 1]. + """ + raise NotImplementedError + + def terminal(self, t: float, x: Tensor) -> Tensor: + """ + Terminal condition (payoff) of the PDE. + + Parameters + ---------- + t : float + Current time (should be total_time). + x : Tensor + Space coordinates [batch_size, dim]. + + Returns + ------- + Tensor + Terminal payoff value [batch_size, 1]. + """ + raise NotImplementedError + + def sigma_matrix(self, x: Tensor) -> Tensor: + """Helper to get the volatility matrix sigma(x).""" + raise NotImplementedError + + def terminal_for_sample(self, x: Tensor) -> Tensor: + """Terminal condition for a multi-sample path.""" + raise NotImplementedError + + +class PricingDefaultRisk(Equation): + """ + PDE for pricing with default risk. + """ + + def __init__(self, eqn_config: dict): + super().__init__(eqn_config) + self.x_init = np.ones(self.dim) * 100.0 + self.sigma = 0.2 + self.rate = 0.02 # R + self.delta = 2.0 / 3 + self.gammah = 0.2 + self.gammal = 0.02 + self.mu_bar = 0.02 + self.K = 100 + self.vh = 50.0 + self.vl = 70.0 + self.slope = (self.gammah - self.gammal) / (self.vh - self.vl) + + def sample(self, num_sample: int) -> tuple[np.ndarray, np.ndarray]: + dw_sample = ( + np.random.normal(size=[num_sample, self.dim, self.num_time_interval]) + * self.sqrt_delta_t + ) + x_sample = np.zeros([num_sample, self.dim, self.num_time_interval + 1]) + x_sample[:, :, 0] = np.ones([num_sample, self.dim]) * self.x_init + for i in range(self.num_time_interval): + x_sample[:, :, i + 1] = (1 + self.mu_bar * self.delta_t) * x_sample[ + :, :, i + ] + (self.sigma_matrix(x_sample[:, :, i]) * dw_sample[:, :, i]) + return dw_sample, x_sample + + def r_u(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + piecewise_linear = ( + nn.ReLU()(nn.ReLU()(y - self.vh) * self.slope + self.gammah - self.gammal) + + self.gammal + ) + return (1 - self.delta) * piecewise_linear + self.rate + + def h_z(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + return torch.zeros((x.size()[0], 1), device=x.device) + + def sigma_matrix(self, x: Union[np.ndarray, Tensor]) -> Union[np.ndarray, Tensor]: + return self.sigma * x + + def terminal(self, t: float, x: Tensor) -> Tensor: + # torch.min(x, 1) returns (values, indices) + return nn.ReLU()(torch.min(x, 1, keepdim=True)[0]) + + def terminal_for_sample(self, x: Tensor) -> Tensor: + return nn.ReLU()(torch.min(x, 2, keepdim=True)[0]) + + +class HJBLQ(Equation): + """ + Hamilton-Jacobi-Bellman (HJB) equation with Linear-Quadratic (LQ) cost. + """ + + def __init__(self, eqn_config: dict): + super().__init__(eqn_config) + self.x_init = np.zeros(self.dim) + self.sigma = np.sqrt(2.0) + self.lambd = 1.0 + + def sample(self, num_sample: int) -> tuple[np.ndarray, np.ndarray]: + dw_sample = ( + np.random.normal(size=[num_sample, self.dim, self.num_time_interval]) + * self.sqrt_delta_t + ) + x_sample = np.zeros([num_sample, self.dim, self.num_time_interval + 1]) + x_sample[:, :, 0] = np.ones([num_sample, self.dim]) * self.x_init + for i in range(self.num_time_interval): + x_sample[:, :, i + 1] = x_sample[:, :, i] + self.sigma * dw_sample[:, :, i] + return dw_sample, x_sample + + def r_u(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + return torch.zeros((x.size()[0], 1), device=x.device) + + def h_z(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + return torch.sum(torch.square(z), dim=1, keepdim=True) / (self.sigma**2) + + def terminal(self, t: float, x: Tensor) -> Tensor: + return torch.log(0.5 * (1 + torch.norm(x, p=2, dim=1, keepdim=True) ** 2)) + + def sigma_matrix(self, x: Union[np.ndarray, Tensor]) -> float: + return self.sigma + + def terminal_for_sample(self, x: Tensor) -> Tensor: + # Used by Monte-Carlo solver, needs to match HJB terminal + return torch.log(0.5 * (1 + torch.norm(x, p=2, dim=2, keepdim=True) ** 2)) + + +class BlackScholesBarenblatt(Equation): + """ + Black-Scholes-Barenblatt equation. + """ + + def __init__(self, eqn_config: dict): + super().__init__(eqn_config) + self.x_init = np.ones(self.dim) * np.array( + [1.0 / (1.0 + i % 2) for i in range(self.dim)] + ) + self.sigma = 0.4 + self.rate = 0.05 # interest rate R + self.mu_bar = 0.0 + + def sample(self, num_sample: int) -> tuple[np.ndarray, np.ndarray]: + dw_sample = ( + np.random.normal(size=(num_sample, self.dim, self.num_time_interval)) + * self.sqrt_delta_t + ) + x_sample = np.zeros((num_sample, self.dim, self.num_time_interval + 1)) + x_sample[:, :, 0] = np.ones((num_sample, self.dim)) * self.x_init + for i in range(self.num_time_interval): + x_sample[:, :, i + 1] = (1 + self.mu_bar * self.delta_t) * x_sample[ + :, :, i + ] + (self.sigma_matrix(x_sample[:, :, i]) * dw_sample[:, :, i]) + return dw_sample, x_sample + + def r_u(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + return torch.ones((x.size()[0], 1), device=x.device) * self.rate + + def h_z(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + return -1 * torch.sum(z, dim=1, keepdim=True) * self.rate / self.sigma + + def terminal(self, t: float, x: Tensor) -> Tensor: + return torch.sum(x**2, dim=1, keepdim=True) + + def sigma_matrix(self, x: Union[np.ndarray, Tensor]) -> Union[np.ndarray, Tensor]: + return self.sigma * x + + def terminal_for_sample(self, x: Tensor) -> Tensor: + return torch.sum(x**2, dim=2, keepdim=True) + + +class PricingDiffRate(Equation): + """ + Nonlinear Black-Scholes with different interest rates for + borrowing and lending. + """ + + def __init__(self, eqn_config: dict): + super().__init__(eqn_config) + self.x_init = np.ones(self.dim) * 100 + self.sigma = 0.2 + self.mu_bar = 0.06 + self.rl = 0.04 + self.rb = 0.06 + self.alpha = 1.0 / self.dim + + def sample(self, num_sample: int) -> tuple[np.ndarray, np.ndarray]: + dw_sample = ( + np.random.normal(size=[num_sample, self.dim, self.num_time_interval]) + * self.sqrt_delta_t + ) + x_sample = np.zeros([num_sample, self.dim, self.num_time_interval + 1]) + x_sample[:, :, 0] = np.ones([num_sample, self.dim]) * self.x_init + factor = np.exp((self.mu_bar - (self.sigma**2) / 2) * self.delta_t) + for i in range(self.num_time_interval): + x_sample[:, :, i + 1] = ( + factor * np.exp(self.sigma * dw_sample[:, :, i]) + ) * x_sample[:, :, i] + return dw_sample, x_sample + + def r_u(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + temp = torch.sum(z, dim=1, keepdim=True) / self.sigma - y + return torch.where(temp > 0, self.rb, self.rl) + + def h_z(self, t: float, x: Tensor, y: Tensor, z: Tensor) -> Tensor: + temp = torch.sum(z, dim=1, keepdim=True) / self.sigma - y + sum_z = torch.sum(z, dim=1, keepdim=True) + return torch.where( + temp > 0, + (self.mu_bar - self.rb) * sum_z / self.sigma, + (self.mu_bar - self.rl) * sum_z / self.sigma, + ) + + def terminal(self, t: float, x: Tensor) -> Tensor: + temp = torch.max(x, 1, keepdim=True)[0] + return nn.ReLU()(temp - 120) - 2 * nn.ReLU()(temp - 150) + + def sigma_matrix(self, x: Union[np.ndarray, Tensor]) -> Union[np.ndarray, Tensor]: + return self.sigma * x + + def terminal_for_sample(self, x: Tensor) -> Tensor: + temp = torch.max(x, 2, keepdim=True)[0] + return torch.maximum(temp - 120, torch.tensor(0.0, device=x.device)) diff --git a/src/RiskLabAI/pde/model.py b/src/RiskLabAI/pde/model.py new file mode 100644 index 0000000..beef487 --- /dev/null +++ b/src/RiskLabAI/pde/model.py @@ -0,0 +1,372 @@ +""" +Defines the neural network architectures used by the PDE solvers. + +Includes: +- Standard feed-forward networks (Net1, TimeNet) +- Set-Transformer components (MAB, SAB, ISAB, PMA) +- Deep BSDE models (DeepBSDE, FBSNNNetwork, TimeDependentNetwork) + +Set-Transformer code based on: https://github.com/juho-lee/set_transformer +""" + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor +from torch.nn import BatchNorm1d, Linear, Module, ReLU, Tanh + +# --- Set Transformer Components --- + + +class MAB(Module): + """Multi-Head Attention Block (MAB).""" + + def __init__( + self, dim_q: int, dim_k: int, dim_v: int, num_heads: int, ln: bool = False + ): + super().__init__() + self.dim_v = dim_v + self.num_heads = num_heads + self.fc_q = Linear(dim_q, dim_v) + self.fc_k = Linear(dim_k, dim_v) + self.fc_v = Linear(dim_k, dim_v) + self.ln0 = nn.LayerNorm(dim_v) if ln else None + self.ln1 = nn.LayerNorm(dim_v) if ln else None + self.fc_o = Linear(dim_v, dim_v) + + def forward(self, q: Tensor, k: Tensor) -> Tensor: + q_fc = self.fc_q(q) + k_fc, v_fc = self.fc_k(k), self.fc_v(k) + + dim_split = self.dim_v // self.num_heads + q_ = torch.cat(q_fc.split(dim_split, 2), 0) + k_ = torch.cat(k_fc.split(dim_split, 2), 0) + v_ = torch.cat(v_fc.split(dim_split, 2), 0) + + attention = torch.softmax(q_.bmm(k_.transpose(1, 2)) / math.sqrt(self.dim_v), 2) + out = torch.cat((q_ + attention.bmm(v_)).split(q.size(0), 0), 2) + + out = self.ln0(out) if self.ln0 is not None else out + out = out + F.relu(self.fc_o(out)) + out = self.ln1(out) if self.ln1 is not None else out + return out + + +class SAB(Module): + """Self-Attention Block (SAB).""" + + def __init__(self, dim_in: int, dim_out: int, num_heads: int, ln: bool = False): + super().__init__() + self.mab = MAB(dim_in, dim_in, dim_out, num_heads, ln=ln) + + def forward(self, x: Tensor) -> Tensor: + return self.mab(x, x) + + +class ISAB(Module): + """Induced Self-Attention Block (ISAB).""" + + def __init__( + self, dim_in: int, dim_out: int, num_heads: int, num_inds: int, ln: bool = False + ): + super().__init__() + self.i = nn.Parameter(torch.Tensor(1, num_inds, dim_out)) + nn.init.xavier_uniform_(self.i) + self.mab0 = MAB(dim_out, dim_in, dim_out, num_heads, ln=ln) + self.mab1 = MAB(dim_in, dim_out, dim_out, num_heads, ln=ln) + + def forward(self, x: Tensor) -> Tensor: + h = self.mab0(self.i.repeat(x.size(0), 1, 1), x) + return self.mab1(x, h) + + +class PMA(Module): + """Pooling Multi-Head Attention (PMA).""" + + def __init__(self, dim: int, num_heads: int, num_seeds: int, ln: bool = False): + super().__init__() + self.s = nn.Parameter(torch.Tensor(1, num_seeds, dim)) + nn.init.xavier_uniform_(self.s) + self.mab = MAB(dim, dim, dim, num_heads, ln=ln) + + def forward(self, x: Tensor) -> Tensor: + return self.mab(self.s.repeat(x.size(0), 1, 1), x) + + +class TimeNetForSet(Module): + """ + Time-dependent feature transformation for SetTransformer. + + Applies separate linear layers to time (t) and features (x), + then combines them as exp(t) * x. + """ + + def __init__(self, in_features: int = 1, out_features: int = 64): + super().__init__() + self.feature_layer = Linear(in_features, out_features) + + self.time_layer1 = Linear(1, 10) + self.time_layer2 = Linear(10, 10) + self.time_layer3 = Linear(10, 10) + self.time_layer4 = Linear(10, out_features) + + self.relu_stack = nn.Sequential(ReLU(), ReLU(), ReLU()) + + def forward(self, t: Tensor, x: Tensor) -> Tensor: + t_feat = self.relu_stack[0](self.time_layer1(t)) + t_feat = self.relu_stack[1](self.time_layer2(t_feat)) + t_feat = self.relu_stack[2](self.time_layer3(t_feat)) + t_feat = self.time_layer4(t_feat) + + x_feat = self.feature_layer(x) + + return torch.exp(t_feat) * x_feat + + +class DeepTimeSetTransformer(Module): + """ + Full Deep Time Set Transformer model. + """ + + def __init__(self, input_dim: int): + super().__init__() + + # Feature extractor layers + self.layer1 = Linear(input_dim, 32) + self.layer2 = Linear(32, 32) + self.layer3 = Linear(32, 32) + self.layer4 = Linear(32, 32) + self.layer5 = Linear(32, 32) + self.activation = ReLU() + + # Pooling layer + self.regressor = PMA(dim=32, num_heads=4, num_seeds=1) + + # Time-dependent output layers + self.time_layer2 = TimeNetForSet(in_features=32, out_features=32) + self.time_layer3 = TimeNetForSet(in_features=32, out_features=32) + self.time_layer4 = TimeNetForSet(in_features=32, out_features=32) + self.time_layer5 = TimeNetForSet(in_features=32, out_features=1) + + def forward(self, t: Tensor, x: Tensor) -> Tensor: + x = self.activation(self.layer1(x)) + x = x - torch.mean(x, 2, keepdim=True) + + x = self.activation(self.layer2(x)) + x = x - torch.mean(x, 2, keepdim=True) + + x = self.activation(self.layer3(x)) + x = x - torch.mean(x, 2, keepdim=True) + + x = self.activation(self.layer4(x)) + x = x - torch.mean(x, 2, keepdim=True) + + x = self.activation(self.layer5(x)) + + output = self.regressor(x) + output = torch.squeeze(output) + + output = self.activation(self.time_layer2(t, output)) + output = self.activation(self.time_layer3(t, output)) + output = self.activation(self.time_layer4(t, output)) + output = self.time_layer5(t, output) + + return output + + +# --- Standard Networks --- + + +class TimeNet(Module): + """Simple feed-forward network for time features.""" + + def __init__(self, output_dim: int): + super().__init__() + self.layers = nn.ModuleList( + [ + Linear(4, 100), + Linear(100, 150), + Linear(150, 200), + Linear(200, 300), + Linear(300, 200), + Linear(200, output_dim), + ] + ) + self.tanh_stack = nn.ModuleList([Tanh() for _ in range(6)]) + + def forward(self, x: Tensor) -> Tensor: + for i in range(len(self.layers)): + x = self.tanh_stack[i](self.layers[i](x)) + return x + + +class Net1(Module): + """A simple Linear + BatchNorm layer.""" + + def __init__(self, input_dim: int, output_dim: int): + super().__init__() + self.layer = Linear(input_dim, output_dim) + self.bn = BatchNorm1d(output_dim) # Note: bn is not used in forward + + def forward(self, x: Tensor) -> Tensor: + return self.layer(x) + + +# --- BSDE Solver Networks --- + + +class FBSNNNetwork(Module): + """Feed-forward network for the FBSNN solver.""" + + def __init__(self, layer_sizes: list[int]): + super().__init__() + self.n_layer = len(layer_sizes) - 1 + self.layers = nn.ModuleList([]) + + for i in range(self.n_layer): + self.layers.append(Linear(layer_sizes[i], layer_sizes[i + 1])) + + def forward(self, x: Tensor) -> Tensor: + for i in range(self.n_layer): + x = self.layers[i](x) + if i < self.n_layer - 1: + x = torch.sin(x) # Use sin activation + return x + + +class DeepBSDE(Module): + """Network for the Deep BSDE method (one net per time step).""" + + def __init__(self, layer_sizes: list[int]): + super().__init__() + self.n_layer = len(layer_sizes) - 1 + self.layers = nn.ModuleList([]) + self.batch_layer = nn.ModuleList([]) + + for i in range(self.n_layer): + self.layers.append(Linear(layer_sizes[i], layer_sizes[i + 1], bias=False)) + self.batch_layer.append( + BatchNorm1d(layer_sizes[i], eps=1e-06, momentum=0.01) + ) + + self.batch_layer.append(BatchNorm1d(layer_sizes[-1], eps=1e-06, momentum=0.01)) + self.activation = ReLU() + + def forward(self, x: Tensor) -> Tensor: + # Note: Original code doesn't use batch_layer[0] + for i in range(self.n_layer - 1): + x = self.layers[i](x) + # x = self.batch_layer[i+1](x) # Original code commented this out + x = self.activation(x) + + x = self.layers[-1](x) + # x = self.batch_layer[-1](x) # Original code commented this out + return x + + +class TimeDependentNetwork(Module): + """Time-dependent network for BSDE solver.""" + + def __init__(self, indim: int, layersize: list[int], outdim: int): + super().__init__() + self.n_layer = len(layersize) + self.layers = nn.ModuleList([]) + self.time_layer = nn.ModuleList([]) + self.batch_layer = nn.ModuleList([]) + + # Input layer + self.layers.append(Net1(indim, layersize[0])) + self.time_layer.append(TimeNet(indim)) + self.batch_layer.append(BatchNorm1d(indim, eps=1e-06, momentum=0.01)) + self.batch_layer.append(BatchNorm1d(layersize[0], eps=1e-06, momentum=0.01)) + + # Hidden layers + for i in range(len(layersize) - 1): + self.layers.append(Net1(layersize[i], layersize[i + 1])) + self.time_layer.append(TimeNet(layersize[i])) + self.batch_layer.append( + BatchNorm1d(layersize[i + 1], eps=1e-06, momentum=0.01) + ) + + # Output layer + self.time_layer.append(TimeNet(outdim)) + self.linear = Linear(layersize[-1], outdim) + self.batch_layer.append(BatchNorm1d(outdim, eps=1e-06, momentum=0.01)) + + def forward(self, t: Tensor, x: Tensor) -> Tensor: + for i in range(self.n_layer): + # (t, t**2, t**3, exp(t)) + time_features = torch.cat((t, t**2, t**3, torch.exp(t)), 1) + time_weight = self.time_layer[i](time_features) + + x = self.batch_layer[i](x) + x = x * (1 + time_weight) + x = self.layers[i](x) + x = torch.sin(x) + + x = self.linear(x) + # x = self.batch_layer[-1](x) # Original code commented this out + return x + + +class TimeDependentNetworkMonteCarlo(Module): + """Time-dependent network for BSDE solver with Monte Carlo gradient.""" + + def __init__(self, indim: int, layersize: list[int], outdim: int, sigma: float): + super().__init__() + self.n_layer = len(layersize) + self.layers = nn.ModuleList([]) + self.time_layer = nn.ModuleList([]) + self.batch_layer = nn.ModuleList([]) + self.sigma = sigma + + # Input layer + self.layers.append(Net1(indim, layersize[0])) + self.time_layer.append(TimeNet(indim)) + self.batch_layer.append(BatchNorm1d(indim)) + self.batch_layer.append(BatchNorm1d(layersize[0])) + + # Hidden layers + for i in range(len(layersize) - 1): + self.layers.append(Net1(layersize[i], layersize[i + 1])) + self.time_layer.append(TimeNet(layersize[i])) + self.batch_layer.append(BatchNorm1d(layersize[i + 1])) + + # Output layer + self.time_layer.append(TimeNet(outdim)) + self.linear = Linear(layersize[-1], outdim) + self.activation = ReLU() + + def forward(self, t: Tensor, x: Tensor, y_mc: Tensor) -> Tensor: + """ + Forward pass. + + Parameters + ---------- + t : Tensor + Time tensor. + x : Tensor + Input features (asset price). + y_mc : Tensor + Monte Carlo estimate of the gradient (Z). + """ + x_prim = x # Store original input + + for i in range(self.n_layer): + time_features = torch.cat((t, t**2, t**3, torch.exp(t)), 1) + time_weight = self.time_layer[i](time_features) + + x = x * (1 + time_weight) + x = self.layers[i](x) + x = self.activation(x) + + # Output layer + time_features = torch.cat((t, t**2, t**3, torch.exp(t)), 1) + time_weight = self.time_layer[self.n_layer](time_features) + + # Combine network output with MC estimate + # Z = sigma * S * Network(t,S) + (1 - time_weight) * Z_MC + x = self.sigma * x_prim * self.linear(x) + (1 - time_weight) * y_mc + return x diff --git a/src/RiskLabAI/pde/solver.py b/src/RiskLabAI/pde/solver.py new file mode 100644 index 0000000..28003d6 --- /dev/null +++ b/src/RiskLabAI/pde/solver.py @@ -0,0 +1,412 @@ +""" +Implements the main FBSDE (Forward-Backward Stochastic Differential +Equation) solver classes. +""" + +import logging + +import torch +import torch.autograd as autograd +import torch.nn as nn + +from RiskLabAI._deprecation import deprecated_class +from RiskLabAI.pde.equation import Equation +from RiskLabAI.pde.model import * + +logger = logging.getLogger(__name__) + + +def initialize_weights(m: nn.Module) -> None: + """ + Initializes the weights of a Linear layer. + + Parameters + ---------- + m : nn.Module + The module to initialize. + """ + if isinstance(m, nn.Linear): + torch.nn.init.normal_(m.weight, 0, 0.1) + m.bias.data.fill_(0.00) + + +class FBSDESolver: + """ + Solver for FBSDEs using various deep learning methods. + """ + + def __init__( + self, + pde: Equation, + layer_sizes: list[int], + learning_rate: float, + solving_method: str, + device: torch.device, + ): + """ + Initialize the FBSDE Solver. + + Parameters + ---------- + pde : Equation + The PDE to solve (from equation.py). + layer_sizes : List[int] + List of hidden layer sizes for the network. + learning_rate : float + Learning rate for the optimizer. + solving_method : str + Method to use: 'Monte-Carlo', 'Deep-Time-SetTransformer', + 'DTNN', or 'DeepBSDE'. + device : torch.device + The device ('cpu' or 'cuda') to run on. + """ + self.pde = pde + self.layer_size = layer_sizes + self.learning_rate = learning_rate + self.method = solving_method + self.device = device + + # 1. Initialize the correct network model + if solving_method == "Monte-Carlo": + self.solver = TimeDependentNetworkMonteCarlo( + pde.dim, self.layer_size, pde.dim, pde.sigma + ).to(device) + elif solving_method == "Deep-Time-SetTransformer": + self.solver = DeepTimeSetTransformer(1).to(device) # Assumes dim=1 + elif solving_method == "DTNN": + self.solver = TimeDependentNetwork(pde.dim, self.layer_size, pde.dim).to( + device + ) + elif solving_method == "DeepBSDE": + # Create a separate network for each time step + self.solver = nn.ModuleList( + [DeepBSDE(layer_sizes).to(device) for _ in range(pde.num_time_interval)] + ) + else: + raise ValueError(f"Unknown solving_method: {solving_method}") + + # 2. Initialize the optimizer + if solving_method != "DeepBSDE": + self.optimizer = torch.optim.Adam( + self.solver.parameters(), lr=self.learning_rate, betas=(0.9, 0.99) + ) + else: + self.optimizer = torch.optim.Adam( + self.solver.parameters(), lr=self.learning_rate, betas=(0.9, 0.99) + ) + + def compute_loss( + self, y_path: Tensor, dw_path: Tensor, t: Tensor, init_y: Tensor, init_z: Tensor + ) -> tuple[Tensor, ...]: + """ + Compute the loss for a single batch. + """ + batch_size = y_path.size()[0] + y_terminal = init_y.expand(batch_size, 1) + + coef = torch.ones((batch_size, 1), device=self.device) + dw_coef = torch.zeros((batch_size, 1), device=self.device) + + for z in range(self.pde.num_time_interval): + S0 = y_path[:, :, z] + t0 = t * self.pde.delta_t * z + + if self.method == "DeepBSDE": + if z > 0: + out_z = self.solver[z](S0) + else: + # Use initial gradient Z_0 + out_z = init_z.expand(batch_size, self.pde.dim) + else: + # Other models are time-dependent + out_z = self.solver(t0, S0) # This assumes other models return Z + + samp_dw = dw_path[:, :, z] + + # Get driver components + interest_rate = self.pde.r_u(t0, S0, y_terminal, out_z) + hz = self.pde.h_z(t0, S0, y_terminal, out_z) + + # Update path + y_terminal = ( + y_terminal * (1 + interest_rate * self.pde.delta_t) + + hz * self.pde.delta_t + + torch.sum(out_z * samp_dw, dim=1, keepdim=True) + ) + + # Update coefficients for loss calculation + if z > 0: + dw_coef = ( + dw_coef * (1 + interest_rate * self.pde.delta_t) + + torch.sum(out_z * samp_dw, dim=1, keepdim=True) + + hz * self.pde.delta_t + ) + else: + dw_coef = ( + torch.sum(out_z * samp_dw, dim=1, keepdim=True) + + hz * self.pde.delta_t + ) + + coef = coef * (1 + interest_rate * self.pde.delta_t) + + # 4. Calculate terminal payoff and loss + payoff = self.pde.terminal(t * self.pde.total_time, y_path[:, :, -1]) + + loss = torch.mean(torch.square(payoff - y_terminal)) + + return loss, coef, dw_coef, payoff + + def solve( + self, num_iterations: int, batch_size: int, init_y: float + ) -> tuple[list[float], list[float]]: + """ + Solves the PDE. + + Parameters + ---------- + num_iterations : int + Number of training iterations. + batch_size : int + Batch size for training. + init_y : float + The initial guess for Y_0 (the price at t=0). + + Returns + ------- + Tuple[List[float], List[float]] + - losses: List of losses at each iteration. + - inits: List of Y_0 values at each iteration. + """ + losses = [] + inits = [] + + # Y_0 and Z_0 are trainable parameters for DeepBSDE + y0 = torch.tensor([init_y], device=self.device).requires_grad_(True) + z0 = torch.zeros(1, self.pde.dim, device=self.device).requires_grad_(True) + + if self.method == "DeepBSDE": + init_opt = torch.optim.Adam([y0], lr=0.1, betas=(0.9, 0.99)) + init_grad_opt = torch.optim.Adam([z0], lr=0.001, betas=(0.9, 0.99)) + + # Validation data + dw_val, y_val = self.pde.sample(128) + y_val = torch.tensor(y_val, dtype=torch.float32, device=self.device) + dw_val = torch.tensor(dw_val, dtype=torch.float32, device=self.device) + t_val = torch.ones((128, 1), device=self.device) + + # Set model to train mode + if self.method != "DeepBSDE": + self.solver.train() + else: + for model in self.solver: + model.train() + + for j in range(num_iterations): + logger.info("Iteration %d/%d", j + 1, num_iterations) + + dw_train, y_train = self.pde.sample(batch_size) + y_train = torch.tensor(y_train, dtype=torch.float32, device=self.device) + dw_train = torch.tensor(dw_train, dtype=torch.float32, device=self.device) + t_train = torch.ones((batch_size, 1), device=self.device) + + self.optimizer.zero_grad() + if self.method == "DeepBSDE": + init_opt.zero_grad() + init_grad_opt.zero_grad() + + loss, _, _, _ = self.compute_loss(y_train, dw_train, t_train, y0, z0) + + loss.backward() + + self.optimizer.step() + if self.method == "DeepBSDE": + init_opt.step() + init_grad_opt.step() + + # Validation and logging + with torch.no_grad(): + val_loss, coef, dw_coef, payoff = self.compute_loss( + y_val, dw_val, t_val, y0, z0 + ) + + if self.method != "DeepBSDE": + # For non-DeepBSDE, Y_0 is not a param but computed + # from the loss function's components. + y0_new = torch.mean((payoff - dw_coef) / coef) + inits.append(y0_new.item()) + logger.info("Loss: %.4f, Y_0: %.4f", val_loss.item(), y0_new.item()) + else: + inits.append(y0.item()) + logger.info("Loss: %.4f, Y_0: %.4f", val_loss.item(), y0.item()) + + losses.append(val_loss.item()) + + return losses, inits + + +class FBSNNSolver: + """ + Solver for FBSNN (Forward-Backward Stochastic Neural Network). + + This method solves the PDE by learning the solution Y_t directly + at each time step and minimizing the difference between the + predicted Y_{t+1} and the one-step-ahead approximation. + """ + + def __init__( + self, + pde: Equation, + layer_sizes: list[int], + learning_rate: float, + device: torch.device, + ): + """ + Initialize the FBSNN Solver. + + Parameters + ---------- + pde : Equation + The PDE to solve. + layer_sizes : List[int] + Layer sizes for the FBSNNNetwork. + learning_rate : float + Learning rate for the optimizer. + device : torch.device + The device ('cpu' or 'cuda') to run on. + """ + self.pde = pde + self.layer_size = layer_sizes + self.learning_rate = learning_rate + self.device = device + + self.solver = FBSNNNetwork(self.layer_size).to(self.device) + self.optimizer = torch.optim.Adam( + self.solver.parameters(), lr=self.learning_rate, betas=(0.99, 0.999) + ) + + def compute_loss( + self, y_path: Tensor, dw_path: Tensor, t: Tensor, init_y: Tensor + ) -> Tensor: + """ + Compute the loss for the FBSNN method. + """ + batch_size = y_path.size()[0] + init_y.expand(batch_size, 1) # Not used, but kept for signature + loss = torch.tensor(0.0, device=self.device) + + for z in range(self.pde.num_time_interval): + S0 = y_path[:, :, z].requires_grad_(True) + S1 = y_path[:, :, z + 1] + + t0 = t * self.pde.delta_t * z + t1 = t * self.pde.delta_t * (z + 1) + + # 1. Get network output Y_t and Y_{t+1} + y_0 = self.solver(torch.cat((t0, S0), dim=1)) + y_1 = self.solver(torch.cat((t1, S1), dim=1)) + + # 2. Calculate the gradient Z_t = dY_t/dS_t * sigma(S_t) + grad_y0 = autograd.grad( + outputs=torch.sum(y_0), inputs=S0, create_graph=True + )[0] + Z = grad_y0 * self.pde.sigma_matrix(S0) + + samp_dw = dw_path[:, :, z] + + # 3. Get driver components + interest_rate = self.pde.r_u(t0, S0, y_0, Z) + hz = self.pde.h_z(t0, S0, y_0, Z) + + # 4. Calculate the one-step approximation Y_hat_{t+1} + y_1_hat = ( + y_0 * (1 + interest_rate * self.pde.delta_t) + + hz * self.pde.delta_t + + torch.sum(Z * samp_dw, dim=1, keepdim=True) + ) + + # 5. Add to total loss + loss += torch.mean(torch.square(y_1_hat - y_1)) + + # 6. Add terminal condition loss + S_T = y_path[:, :, -1] + t_T = t * self.pde.total_time + + payoff = self.pde.terminal(t_T, S_T) + y_T = self.solver(torch.cat((t_T, S_T), dim=1)) + + loss += torch.mean(torch.square(payoff - y_T)) + + return loss + + def solve( + self, + num_iterations: int, + batch_size: int, + init_y: float, # Note: init_y is not used by FBSNN, but kept for API + ) -> tuple[list[float], list[float]]: + """ + Solves the PDE using the FBSNN method. + + Parameters + ---------- + num_iterations : int + Number of training iterations. + batch_size : int + Batch size for training. + init_y : float + Initial value (not used by this solver). + + Returns + ------- + Tuple[List[float], List[float]] + - losses: List of losses at each iteration. + - inits: List of Y_0 values at each iteration. + """ + losses = [] + inits = [] + + self.solver.train() + + # Validation data + dw_val, y_val = self.pde.sample(128) + y_val = torch.tensor(y_val, dtype=torch.float32, device=self.device) + dw_val = torch.tensor(dw_val, dtype=torch.float32, device=self.device) + t_val = torch.ones((128, 1), device=self.device) + + for j in range(num_iterations): + logger.info("Iteration %d/%d", j + 1, num_iterations) + + dw_train, y_train = self.pde.sample(batch_size) + y_train = torch.tensor(y_train, dtype=torch.float32, device=self.device) + dw_train = torch.tensor(dw_train, dtype=torch.float32, device=self.device) + t_train = torch.ones((batch_size, 1), device=self.device) + + self.optimizer.zero_grad() + + # init_y is not used, pass a dummy tensor + dummy_init = torch.tensor(0.0, device=self.device) + + loss = self.compute_loss(y_train, dw_train, t_train, dummy_init) + loss.backward() + self.optimizer.step() + + # Validation and logging + with torch.no_grad(): + val_loss = self.compute_loss(y_val, dw_val, t_val, dummy_init) + losses.append(val_loss.item()) + + # Get Y_0 from the trained network + S0_val = y_val[:, :, 0] + t0_val = t_val * self.pde.delta_t * 0 + y0_val = self.solver(torch.cat((t0_val, S0_val), dim=1)) + + y0_mean = torch.mean(y0_val).item() + inits.append(y0_mean) + + logger.info("Loss: %.4f, Y_0: %.4f", val_loss.item(), y0_mean) + + return losses, inits + + +# Deprecated alias for the misspelled original name; scheduled for removal in 4.0.0. +FBSNNolver = deprecated_class(FBSNNSolver, "FBSNNolver", removed_in="4.0.0") diff --git a/src/RiskLabAI/utils/__init__.py b/src/RiskLabAI/utils/__init__.py new file mode 100644 index 0000000..2ca8a25 --- /dev/null +++ b/src/RiskLabAI/utils/__init__.py @@ -0,0 +1,108 @@ +""" +RiskLabAI Utilities Package + +This module provides common helper functions used across the library, +including: +- Constants +- EWMA calculation +- Progress bar +- Strategy side determination +- Plotly figure layout helpers +- Publication-quality Matplotlib plotting +""" + +from RiskLabAI._deprecation import warn_deprecated_name + +from .constants import * +from .ewma import ewma +from .momentum_mean_reverting_strategy_sides import determine_strategy_side +from .progress import progress_bar + +# Plotting helpers are imported lazily (PEP 562) so that the base install +# does not require matplotlib/seaborn/plotly (RiskLabAI[plot] extra). +_LAZY_PLOTTING = { + "update_figure_layout": ("update_figure_layout", "update_figure_layout"), + "setup_publication_style": ("publication_plots", "setup_publication_style"), + "apply_plot_style": ("publication_plots", "apply_plot_style"), + "finalize_plot": ("publication_plots", "finalize_plot"), +} + + +# Deprecated non-ASCII constant identifiers -> their ASCII replacements. +# Accessing these via RiskLabAI.utils warns and returns the new value; the +# identifiers are scheduled for removal in 4.0.0 (string values are unchanged regardless). +_DEPRECATED_CONSTANTS = { + "CUMULATIVE_θ": "CUMULATIVE_THETA", + "CUMULATIVE_BUY_θ": "CUMULATIVE_BUY_THETA", + "CUMULATIVE_SELL_θ": "CUMULATIVE_SELL_THETA", +} + + +def __getattr__(name): + if name in _LAZY_PLOTTING: + from importlib import import_module + + module_name, attr = _LAZY_PLOTTING[name] + value = getattr(import_module(f".{module_name}", __name__), attr) + globals()[name] = value + return value + if name in _DEPRECATED_CONSTANTS: + new = _DEPRECATED_CONSTANTS[name] + warn_deprecated_name(name, new, removed_in="4.0.0") + return globals()[new] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +# --- Alias for backward compatibility --- +# The historical `compute_exponential_weighted_moving_average` name now maps to +# the canonical, numba-jitted `ewma` (the former `smoothing_average.py` +# duplicate has been removed). +compute_exponential_weighted_moving_average = ewma + +__all__ = [ + # constants + "DATE_TIME", + "TIMESTAMP", + "TICK_NUMBER", + "OPEN_PRICE", + "HIGH_PRICE", + "LOW_PRICE", + "CLOSE_PRICE", + "CUMULATIVE_TICKS", + "CUMULATIVE_DOLLAR", + "THRESHOLD", + "CUMULATIVE_VOLUME", + "CUMULATIVE_BUY_VOLUME", + "CUMULATIVE_SELL_VOLUME", + "CUMULATIVE_THETA", + "CUMULATIVE_BUY_THETA", + "CUMULATIVE_SELL_THETA", + "EXPECTED_IMBALANCE", + "EXPECTED_TICKS_NUMBER", + "EXPECTED_BUY_IMBALANCE", + "EXPECTED_SELL_IMBALANCE", + "EXPECTED_BUY_TICKS_PROPORTION", + "BUY_TICKS_NUMBER", + "N_TICKS_ON_BAR_FORMATION", + "PREVIOUS_TICK_RULE", + "EXPECTED_IMBALANCE_WINDOW", + "PREVIOUS_BARS_N_TICKS_LIST", + "PREVIOUS_TICK_IMBALANCES_LIST", + "PREVIOUS_TICK_IMBALANCES_BUY_LIST", + "PREVIOUS_TICK_IMBALANCES_SELL_LIST", + "PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST", + "N_PREVIOUS_BARS_FOR_EXPECTED_N_TICKS_ESTIMATION", + # ewma + "ewma", + "compute_exponential_weighted_moving_average", # Alias + # progress + "progress_bar", + # momentum_mean_reverting_strategy_sides + "determine_strategy_side", + # update_figure_layout + "update_figure_layout", + # publication_plots + "setup_publication_style", + "apply_plot_style", + "finalize_plot", +] diff --git a/src/RiskLabAI/utils/constants.py b/src/RiskLabAI/utils/constants.py new file mode 100644 index 0000000..94366c6 --- /dev/null +++ b/src/RiskLabAI/utils/constants.py @@ -0,0 +1,87 @@ +DATE_TIME = "Date Time" +TIMESTAMP = "Timestamp" +TICK_NUMBER = "Tick Number" +OPEN_PRICE = "Open" +HIGH_PRICE = "High" +LOW_PRICE = "Low" +CLOSE_PRICE = "Close" + +CUMULATIVE_TICKS = "Cumulative Ticks" +CUMULATIVE_DOLLAR = "Cumulative Dollar Value" +THRESHOLD = "Threshold" + +CUMULATIVE_VOLUME = "Cumulative Volume" +CUMULATIVE_BUY_VOLUME = "Cumulative Buy Volume" +CUMULATIVE_SELL_VOLUME = "Cumulative Sell Volume" + +CUMULATIVE_THETA = "Cumulative θ" +CUMULATIVE_BUY_THETA = "Cumulative Buy θ" +CUMULATIVE_SELL_THETA = "Cumulative Sell θ" + +# Deprecated non-ASCII aliases (scheduled for removal in 4.0.0). The string *values* are +# unchanged, so stored data and internal dict keys are unaffected; only the +# Python identifier you import differs. These are intentionally excluded from +# ``__all__`` so ``RiskLabAI.utils`` can warn on access via its ``__getattr__``. +CUMULATIVE_θ = CUMULATIVE_THETA +CUMULATIVE_BUY_θ = CUMULATIVE_BUY_THETA +CUMULATIVE_SELL_θ = CUMULATIVE_SELL_THETA + +EXPECTED_IMBALANCE = "expected_imbalance" +EXPECTED_TICKS_NUMBER = "exp_num_ticks" + +EXPECTED_BUY_IMBALANCE = "exp_imbalance_buy" +EXPECTED_SELL_IMBALANCE = "exp_imbalance_sell" +EXPECTED_BUY_TICKS_PROPORTION = "exp_buy_ticks_proportion" +BUY_TICKS_NUMBER = "buy_ticks_num" + +N_TICKS_ON_BAR_FORMATION = "Number of ticks while bar is formed." + +PREVIOUS_TICK_RULE = "Previous tick rule" +EXPECTED_IMBALANCE_WINDOW = "Expected Imbalance Window" + +PREVIOUS_BARS_N_TICKS_LIST = "List of previous bars number of ticks" +PREVIOUS_TICK_IMBALANCES_LIST = "List of previous tick imbalances" + +PREVIOUS_TICK_IMBALANCES_BUY_LIST = "List of previous (buy) tick imbalances" +PREVIOUS_TICK_IMBALANCES_SELL_LIST = "List of previous (sell) tick imbalances" + +PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST = "List of previous bars buy ticks proportion" + +N_PREVIOUS_BARS_FOR_EXPECTED_N_TICKS_ESTIMATION = "Window size for E[T]" + +# Canonical public names. The deprecated non-ASCII θ aliases above are +# deliberately omitted so that `from .constants import *` does not bind them in +# RiskLabAI.utils (whose __getattr__ then warns on access). Scheduled for removal in 4.0.0. +__all__ = [ + "DATE_TIME", + "TIMESTAMP", + "TICK_NUMBER", + "OPEN_PRICE", + "HIGH_PRICE", + "LOW_PRICE", + "CLOSE_PRICE", + "CUMULATIVE_TICKS", + "CUMULATIVE_DOLLAR", + "THRESHOLD", + "CUMULATIVE_VOLUME", + "CUMULATIVE_BUY_VOLUME", + "CUMULATIVE_SELL_VOLUME", + "CUMULATIVE_THETA", + "CUMULATIVE_BUY_THETA", + "CUMULATIVE_SELL_THETA", + "EXPECTED_IMBALANCE", + "EXPECTED_TICKS_NUMBER", + "EXPECTED_BUY_IMBALANCE", + "EXPECTED_SELL_IMBALANCE", + "EXPECTED_BUY_TICKS_PROPORTION", + "BUY_TICKS_NUMBER", + "N_TICKS_ON_BAR_FORMATION", + "PREVIOUS_TICK_RULE", + "EXPECTED_IMBALANCE_WINDOW", + "PREVIOUS_BARS_N_TICKS_LIST", + "PREVIOUS_TICK_IMBALANCES_LIST", + "PREVIOUS_TICK_IMBALANCES_BUY_LIST", + "PREVIOUS_TICK_IMBALANCES_SELL_LIST", + "PREVIOUS_BARS_BUY_TICKS_PROPORTIONS_LIST", + "N_PREVIOUS_BARS_FOR_EXPECTED_N_TICKS_ESTIMATION", +] diff --git a/src/RiskLabAI/utils/ewma.py b/src/RiskLabAI/utils/ewma.py new file mode 100644 index 0000000..ebd002e --- /dev/null +++ b/src/RiskLabAI/utils/ewma.py @@ -0,0 +1,74 @@ +""" +Implements a Numba-jitted Exponentially Weighted Moving Average (EWMA) +that correctly handles the 'adjust=True' logic from pandas. +""" + +import numpy as np + +try: + from numba import jit +except ImportError: + def jit(*decorator_args, **decorator_kwargs): + """Return the original function when optional acceleration is absent.""" + if ( + len(decorator_args) == 1 + and callable(decorator_args[0]) + and not decorator_kwargs + ): + return decorator_args[0] + + def decorator(function): + return function + + return decorator + + +@jit(nopython=True) +def ewma(array: np.ndarray, window: int) -> np.ndarray: + r""" + Calculate the Exponentially Weighted Moving Average (EWMA) of an array. + + This function mimics the pandas `ewm(span=window, adjust=True).mean()` + method, which is crucial for financial calculations as it does not + lose the first `window` observations. + + The EWMA is calculated using the formula: + .. math:: + EWMA_t = \frac{x_t + (1 - \alpha) x_{t-1} + \ldots + (1 - \alpha)^t x_0} + {1 + (1 - \alpha) + \ldots + (1 - \alpha)^t} + + where: + - \(\alpha = 2 / (window + 1)\) + - The denominator \(\omega_t\) is the sum of weights. + + Parameters + ---------- + array : np.ndarray + Input array (e.g., of tick counts or imbalances). + window : int + The span of the EWMA. + + Returns + ------- + np.ndarray + The array of EWMA values. + """ + length = array.shape[0] + result_ewma_array = np.empty(length, dtype=np.float64) + if length == 0: + return result_ewma_array + + alpha = 2.0 / (float(window) + 1.0) + multiplier = 1.0 - alpha + + # Handle the sum of weights (denominator) + weight_sum = 1.0 + current_weighted_sum = array[0] + result_ewma_array[0] = current_weighted_sum + + for i in range(1, length): + weight_sum += multiplier**i + current_weighted_sum = current_weighted_sum * multiplier + array[i] + result_ewma_array[i] = current_weighted_sum / weight_sum + + return result_ewma_array diff --git a/src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py b/src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py new file mode 100644 index 0000000..208f23f --- /dev/null +++ b/src/RiskLabAI/utils/momentum_mean_reverting_strategy_sides.py @@ -0,0 +1,70 @@ +""" +Determines strategy side (long/short) based on moving average crossovers. +""" + +import pandas as pd + + +def determine_strategy_side( + prices: pd.Series, + fast_window: int = 20, + slow_window: int = 50, + exponential: bool = False, + mean_reversion: bool = False, +) -> pd.Series: + r""" + Determines the trading side based on moving average crossovers. + + .. math:: + \text{Momentum:} + \begin{cases} + 1 & \text{if } MA_{fast} \geq MA_{slow} \\ + -1 & \text{otherwise} + \end{cases} + + \text{Mean Reversion:} + \begin{cases} + 1 & \text{if } MA_{fast} < MA_{slow} \\ + -1 & \text{otherwise} + \end{cases} + + Parameters + ---------- + prices : pd.Series + Time series of prices. + fast_window : int, default=20 + Window size for the fast moving average. + slow_window : int, default=50 + Window size for the slow moving average. + exponential : bool, default=False + If True, use Exponential Moving Averages (EWMA). + If False, use Simple Moving Averages (SMA). + mean_reversion : bool, default=False + If True, strategy is mean-reverting (short fast > slow). + If False, strategy is momentum-based (long fast > slow). + + Returns + ------- + pd.Series + A Series of sides (1 for long, -1 for short). + """ + if fast_window >= slow_window: + raise ValueError("fast_window must be smaller than slow_window.") + + if exponential: + fast_ma = prices.ewm(span=fast_window, adjust=False, min_periods=1).mean() + slow_ma = prices.ewm(span=slow_window, adjust=False, min_periods=1).mean() + else: + fast_ma = prices.rolling(window=fast_window, min_periods=1).mean() + slow_ma = prices.rolling(window=slow_window, min_periods=1).mean() + + # Create signal: 1 if fast > slow, -1 if fast < slow + # Explicit int64: plain `int` maps to int32 on Windows, which makes the + # returned dtype platform-dependent. + signal = (fast_ma >= slow_ma).astype("int64") * 2 - 1 + + if mean_reversion: + # Invert the signal for mean reversion + return -signal + + return signal diff --git a/src/RiskLabAI/utils/progress.py b/src/RiskLabAI/utils/progress.py new file mode 100644 index 0000000..ff893fd --- /dev/null +++ b/src/RiskLabAI/utils/progress.py @@ -0,0 +1,68 @@ +""" +Provides a simple terminal-style progress bar. +""" + +import sys +import time + + +def progress_bar( + current_progress: int, + total_progress: int, + start_time: float, + bar_length: int = 20, +) -> None: + r""" + Display a terminal-style progress bar. + + Shows completion percentage and estimated time remaining. + + Parameters + ---------- + current_progress : int + Current iteration number (e.g., `i`). + total_progress : int + Total number of items to process. + start_time : float + The time the task started (from `time.time()`). + bar_length : int, default=20 + Length of the progress bar in characters. + """ + if total_progress == 0: + return + + # Check if task is completed + if current_progress == total_progress: + sys.stdout.write( + f"\rCompleted: [{'-' * bar_length}] 100% - Task completed! \n" + ) + else: + percentage = current_progress / total_progress + + # Handle the very first iteration + if percentage == 0: + arrow = "" + else: + # Safely calculate arrow length, ensuring it doesn't go below -1 + arrow_length = int(round(percentage * bar_length) - 1) + arrow = "-" * max(0, arrow_length) + ">" + + spaces = " " * (bar_length - len(arrow)) + elapsed_time_sec = time.time() - start_time + + # Calculate remaining time + if current_progress == 0: + remaining_time_str = "Calculating..." + else: + remaining_time_min = ( + (elapsed_time_sec / current_progress) + * (total_progress - current_progress) + ) / 60 + remaining_time_str = f"{remaining_time_min:.0f} minutes remaining" + + # Assemble and print the bar + sys.stdout.write( + f"\rCompleted: [{arrow + spaces}] {percentage*100:.0f}% - {remaining_time_str}." + ) + + sys.stdout.flush() diff --git a/src/RiskLabAI/utils/publication_plots.py b/src/RiskLabAI/utils/publication_plots.py new file mode 100644 index 0000000..c6c9a84 --- /dev/null +++ b/src/RiskLabAI/utils/publication_plots.py @@ -0,0 +1,192 @@ +""" +Utilities for creating publication-quality plots with Matplotlib +and Seaborn, using Times New Roman font and high DPI. + +Provides 6 themes and a configuration-based saving function. +""" + +import logging +import os +from typing import Any, Optional + +import matplotlib.figure as fig # For type hinting +import matplotlib.pyplot as plt +import seaborn as sns + +logger = logging.getLogger(__name__) + +# [THEMES dictionary remains the same] +THEMES: dict[str, dict[str, Any]] = { + "light": { + "figure.facecolor": "#FFFFFF", + "axes.facecolor": "#FFFFFF", + "text.color": "#000000", + "axes.labelcolor": "#000000", + "axes.edgecolor": "#000000", + "xtick.color": "#000000", + "ytick.color": "#000000", + "grid.color": "#CCCCCC", + "legend.facecolor": "#FFFFFF", + "legend.edgecolor": "#B0B0B0", + }, + "medium": { + "figure.facecolor": "#B0B0B0", # A more solid, medium grey + "axes.facecolor": "#B0B0B0", + "text.color": "#FFFFFF", # White text (like the dark theme) + "axes.labelcolor": "#FFFFFF", + "axes.edgecolor": "#FFFFFF", + "xtick.color": "#FFFFFF", + "ytick.color": "#FFFFFF", + "grid.color": "#E0E0E0", # Lighter grid lines on medium bg + "legend.facecolor": "#B0B0B0", + "legend.edgecolor": "#FFFFFF", + }, + "dark": { + "figure.facecolor": "#2E2E2E", + "axes.facecolor": "#2E2E2E", + "text.color": "#F0F0F0", + "axes.labelcolor": "#F0F0F0", + "axes.edgecolor": "#F0F0F0", + "xtick.color": "#F0F0F0", + "ytick.color": "#F0F0F0", + "grid.color": "#6A6A6A", + "legend.facecolor": "#2E2E2E", + "legend.edgecolor": "#F0F0F0", + }, +} + +# --- MODULE-LEVEL CONFIGURATION --- +# This dictionary will store the settings from setup_publication_style +_CONFIG = {"save_plots": False, "save_dir": "figs"} + + +# --- UPDATED FUNCTION --- +def setup_publication_style( + theme: str = "light", + quality: int = 300, + save_plots: bool = False, # <-- New parameter + save_dir: str = "figs", # <-- New parameter +) -> None: + """ + Sets the global Matplotlib rcParams and saving configuration. + + Call this function once at the beginning of your notebook. + + Parameters + ---------- + theme : str, optional + The theme to apply. Defaults to 'light'. + quality : int, optional + The DPI for the figures. Defaults to 300. + save_plots : bool, optional + Global switch to enable/disable saving plots. Defaults to False. + save_dir : str, optional + The directory to save figures in. Defaults to 'figs'. + """ + + # [All the theme parsing and styling code remains the same] + # ... (omitted for brevity) ... + is_transparent = False + base_theme_name = theme + if theme.endswith("-transparent"): + is_transparent = True + base_theme_name = theme.replace("-transparent", "") + if base_theme_name not in THEMES: + base_theme_name = "light" + params = THEMES[base_theme_name].copy() + common_params = { + "font.size": 12, + "axes.labelsize": 12, + "axes.titlesize": 14, + "axes.titleweight": "bold", + "xtick.labelsize": 12, + "ytick.labelsize": 12, + "legend.fontsize": 12, + "legend.title_fontsize": 13, + "figure.dpi": quality, + "savefig.dpi": quality, + "axes.grid": True, + "grid.linestyle": "--", + "grid.alpha": 0.7, + "axes.linewidth": 1.2, + } + params.update(common_params) + if is_transparent: + params["figure.facecolor"] = (0, 0, 0, 0) + params["axes.facecolor"] = (0, 0, 0, 0) + params["savefig.transparent"] = True + params["legend.facecolor"] = (0, 0, 0, 0) + else: + params["savefig.transparent"] = False + try: + plt.rc("font", family="Times New Roman") + except Exception: + logger.warning("Times New Roman not found. Defaulting to serif.") + plt.rc("font", family="serif") + plt.rcParams.update(params) + sns_style = "darkgrid" if base_theme_name == "dark" else "whitegrid" + sns.set_style(sns_style, rc=params) + + # --- Store saving configuration --- + _CONFIG["save_plots"] = save_plots + _CONFIG["save_dir"] = save_dir + + logger.info( + "Matplotlib style updated. Theme: '%s', Quality: %s DPI.", theme, quality + ) + if save_plots: + logger.info("Plot saving enabled. Saving to: '%s'", save_dir) + else: + logger.info("Plot saving disabled.") + + +# [apply_plot_style function remains exactly the same] +def apply_plot_style( + ax: plt.Axes, + title: str, + xlabel: str, + ylabel: str, + legend_title: Optional[str] = None, +) -> None: + ax.set_title(title) + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel) + if ax.get_legend() and legend_title is not None: + ax.legend(title=legend_title) + + +# --- UPDATED FUNCTION --- +def finalize_plot(fig: fig.Figure, filename: str) -> None: + """ + Shows the plot and saves it *if* saving was enabled in + setup_publication_style. + + Parameters + ---------- + fig : plt.Figure + The figure object to save. + filename : str + The name of the file (e.g., 'model_performance.png'). + This is required, but only used if saving is enabled. + """ + + # --- 1. Save the figure if global switch is on --- + if _CONFIG["save_plots"]: + save_dir = _CONFIG["save_dir"] + + # Create the directory if it doesn't exist + os.makedirs(save_dir, exist_ok=True) + + # Construct the full path + full_path = os.path.join(save_dir, filename) + + # Save the figure + fig.savefig(full_path, bbox_inches="tight") + + logger.info("Figure saved to: %s", full_path) + + # --- 2. Always show the plot --- + plt.show() + + # --- 3. Close the figure object --- + plt.close(fig) diff --git a/src/RiskLabAI/utils/update_figure_layout.py b/src/RiskLabAI/utils/update_figure_layout.py new file mode 100644 index 0000000..b4357e3 --- /dev/null +++ b/src/RiskLabAI/utils/update_figure_layout.py @@ -0,0 +1,56 @@ +""" +A helper function to apply a consistent, dark-themed layout +to Plotly figures. +""" + +import plotly.graph_objects as go + + +def update_figure_layout( + fig: go.Figure, + title: str, + xaxis_title: str, + yaxis_title: str, + legend_x: float = 1.0, + legend_y: float = 1.0, +) -> go.Figure: + """ + Apply a standardized dark-theme layout to a Plotly figure. + + This function modifies the figure in-place. + + Parameters + ---------- + fig : go.Figure + The Plotly figure object to update. + title : str + The title for the chart. + xaxis_title : str + The title for the x-axis. + yaxis_title : str + The title for the y-axis. + legend_x : float, default=1.0 + The x-position of the legend. + legend_y : float, default=1.0 + The y-position of the legend. + + Returns + ------- + go.Figure + The modified figure. + """ + fig.update_layout( + title=title, + xaxis_title=xaxis_title, + yaxis_title=yaxis_title, + template="plotly_dark", + plot_bgcolor="rgba(0,0,0,0)", # Transparent background + paper_bgcolor="rgba(0,0,0,0)", # Transparent background + legend=dict( + x=legend_x, + y=legend_y, + xanchor="auto", + yanchor="auto", + ), + ) + return fig diff --git a/test/backtest/test_technical_indicators.py b/test/backtest/test_technical_indicators.py new file mode 100644 index 0000000..3b81506 --- /dev/null +++ b/test/backtest/test_technical_indicators.py @@ -0,0 +1,119 @@ +import builtins + +import numpy as np +import pandas as pd +import pytest + +from RiskLabAI.backtest.backtest_overfitting_simulation import ( + _average_directional_index, + _average_true_range, + _commodity_channel_index, + _ichimoku_lines, + _relative_strength_index, + _stochastic_oscillator, + _wilder_change_average, + financial_features_backtest_overfitting_simulation, +) + + +def test_wilder_indicators_match_hand_calculations(): + prices = pd.Series([1.0, 2.0, 3.0, 2.0, 4.0]) + np.testing.assert_allclose( + _relative_strength_index(prices, 2).to_numpy()[2:], + [100.0, 50.0, 100.0 * 1.25 / 1.5], + rtol=0.0, + atol=1e-12, + ) + + true_range_prices = pd.Series([1.0, 3.0, 2.0, 6.0]) + np.testing.assert_allclose( + _average_true_range(true_range_prices, 2).to_numpy()[2:], + [1.5, 2.75], + rtol=0.0, + atol=1e-12, + ) + + directional_prices = pd.Series([1.0, 2.0, 3.0, 2.0, 1.0, 2.0, 3.0]) + np.testing.assert_allclose( + _average_directional_index(directional_prices, 2).to_numpy()[3:], + [50.0, 50.0, 37.5, 50.0], + rtol=0.0, + atol=1e-12, + ) + + +def test_rolling_indicators_match_hand_calculations(): + prices = pd.Series([1.0, 2.0, 3.0, 2.0, 4.0]) + np.testing.assert_allclose( + _commodity_channel_index(prices.iloc[:4], 3).to_numpy()[2:], + [100.0, -50.0], + rtol=0.0, + atol=1e-12, + ) + + oscillator, signal = _stochastic_oscillator(prices, 3, 3) + np.testing.assert_allclose( + oscillator.to_numpy()[2:], [100.0, 0.0, 100.0], rtol=0.0, atol=1e-12 + ) + np.testing.assert_allclose( + signal.to_numpy()[4:], [200.0 / 3.0], rtol=0.0, atol=1e-12 + ) + + conversion, base, span_a, span_b = _ichimoku_lines( + pd.Series(np.arange(1.0, 7.0)), 2, 3, 4 + ) + np.testing.assert_allclose(conversion.to_numpy()[1:], [1.5, 2.5, 3.5, 4.5, 5.5]) + np.testing.assert_allclose(base.to_numpy()[2:], [2.0, 3.0, 4.0, 5.0]) + np.testing.assert_allclose(span_a.to_numpy()[2:], [2.25, 3.25, 4.25, 5.25]) + np.testing.assert_allclose(span_b.to_numpy()[3:], [2.5, 3.5, 4.5]) + + +def test_constant_prices_produce_neutral_mature_indicators(): + prices = pd.Series(np.ones(60)) + assert _relative_strength_index(prices).iloc[-1] == 50.0 + assert _average_true_range(prices).iloc[-1] == 0.0 + assert _average_directional_index(prices).iloc[-1] == 0.0 + assert _commodity_channel_index(prices).iloc[-1] == 0.0 + oscillator, signal = _stochastic_oscillator(prices) + assert oscillator.iloc[-1] == 50.0 + assert signal.iloc[-1] == 50.0 + + +@pytest.mark.parametrize("window", [0, -1, True, 1.5]) +def test_wilder_average_rejects_invalid_windows(window): + with pytest.raises(ValueError, match="positive integer"): + _wilder_change_average(np.array([0.0, 1.0, 2.0]), window) + + +def test_financial_feature_generation_is_self_contained(monkeypatch): + original_import = builtins.__import__ + + def guarded_import(name, *args, **kwargs): + if name == "ta" or name.startswith("ta."): + raise AssertionError("unexpected external technical-analysis import") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + generator = np.random.default_rng(7) + prices = pd.Series( + 100.0 * np.exp(np.cumsum(generator.normal(0.0, 0.01, 300))), + index=pd.date_range("2024-01-01", periods=300), + ) + features = financial_features_backtest_overfitting_simulation(prices) + + assert not features.empty + assert features.index.isin(prices.index).all() + assert np.isfinite(features.dropna().to_numpy()).all() + assert { + "ADX", + "RSI", + "CCI", + "Stochastic", + "ROC", + "ATR", + "Kumo Breakout", + "TK Position", + "Price Kumo Position", + "Cloud Thickness", + "Momentum Confirmation", + }.issubset(features.columns) diff --git a/test/backtest/validation/test_cross_validator_controller.py b/test/backtest/validation/test_cross_validator_controller.py index 306fe2d..72b6a64 100644 --- a/test/backtest/validation/test_cross_validator_controller.py +++ b/test/backtest/validation/test_cross_validator_controller.py @@ -55,6 +55,6 @@ def test_controller_public_attribute(dummy_args): def test_controller_is_deprecated(): """The controller is deprecated in 2.0.0 (delegates to the core registry).""" - with pytest.warns(DeprecationWarning, match="2.1.0"): + with pytest.warns(DeprecationWarning, match="4.0.0"): controller = CrossValidatorController(validator_type="kfold", n_splits=5) assert isinstance(controller.get_validator(), KFold) diff --git a/test/backtest/validation/test_cross_validator_factory.py b/test/backtest/validation/test_cross_validator_factory.py index 7008cd7..68bd331 100644 --- a/test/backtest/validation/test_cross_validator_factory.py +++ b/test/backtest/validation/test_cross_validator_factory.py @@ -91,6 +91,6 @@ def test_factory_invalid_type(): def test_factory_is_deprecated(): """The factory is deprecated in 2.0.0 (delegates to the core registry).""" - with pytest.warns(DeprecationWarning, match="2.1.0"): + with pytest.warns(DeprecationWarning, match="4.0.0"): cv = CrossValidatorFactory.create_cross_validator("kfold", n_splits=5) assert isinstance(cv, KFold) diff --git a/test/causal_factor_analysis/test_cfi_book_graph_examples.py b/test/causal_factor_analysis/test_cfi_book_graph_examples.py new file mode 100644 index 0000000..7b96c9d --- /dev/null +++ b/test/causal_factor_analysis/test_cfi_book_graph_examples.py @@ -0,0 +1,104 @@ +"""Hand-verifiable graph tests for the 2023 book examples.""" + +from __future__ import annotations + +from RiskLabAI.causal_factor_analysis import ( + CausalDAG, + causal_role_evidence, + check_backdoor_adjustment_set, + check_instrument, + d_separation, + minimal_backdoor_adjustment_sets, + minimal_frontdoor_adjustment_sets, +) + + +def _dag(nodes, edges, observed=None): + node_tuple = tuple(nodes) + return CausalDAG( + node_tuple, + tuple(edges), + node_tuple if observed is None else tuple(observed), + ) + + +def test_figures_1_2_and_13_require_the_common_cause_for_backdoor_adjustment(): + graph = _dag(("X", "Y", "Z"), (("Z", "X"), ("Z", "Y"))) + assert d_separation(graph, ("X",), ("Y",)).separated is False + assert d_separation(graph, ("X",), ("Y",), ("Z",)).separated is True + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == (("Z",),) + role = causal_role_evidence(graph, "X", "Y", "Z") + assert role.common_cause_paths == (("X", "Z", "Y"),) + + +def test_figure_3_frontdoor_mediator_identifies_when_confounder_is_latent(): + graph = _dag( + ("M", "U", "X", "Y"), + (("U", "X"), ("U", "Y"), ("X", "M"), ("M", "Y")), + observed=("M", "X", "Y"), + ) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == () + assert minimal_frontdoor_adjustment_sets(graph, "X", "Y") == (("M",),) + + +def test_figure_4_instrument_satisfies_both_named_graph_profiles(): + graph = _dag( + ("U", "W", "X", "Y"), + (("W", "X"), ("X", "Y"), ("U", "X"), ("U", "Y")), + ) + result = check_instrument(graph, "W", "X", "Y") + assert result.pearl_graphical is True + assert result.cfi_simple is True + + +def test_figure_8_hypothetical_factor_mechanism_has_backdoor_and_frontdoor_sets(): + edges = ( + ("MOM", "HML"), + ("MOM", "PC"), + ("HML", "OI"), + ("OI", "PC"), + ) + observed_graph = _dag(("HML", "MOM", "OI", "PC"), edges) + assert minimal_backdoor_adjustment_sets(observed_graph, "HML", "PC") == ( + ("MOM",), + ) + + latent_graph = _dag( + ("HML", "MOM", "OI", "PC"), + edges, + observed=("HML", "OI", "PC"), + ) + assert minimal_backdoor_adjustment_sets(latent_graph, "HML", "PC") == () + assert minimal_frontdoor_adjustment_sets(latent_graph, "HML", "PC") == ( + ("OI",), + ) + + +def test_figures_16_to_18_conditioning_on_collider_opens_the_path(): + graph = _dag(("X", "Y", "Z"), (("X", "Z"), ("Y", "Z"))) + assert d_separation(graph, ("X",), ("Y",)).separated is True + conditioned = d_separation(graph, ("X",), ("Y",), ("Z",)) + assert conditioned.separated is False + assert tuple(path.nodes for path in conditioned.paths if path.is_open) == ( + ("X", "Z", "Y"), + ) + + +def test_figure_19_conditioning_on_chain_mediator_blocks_total_effect_path(): + graph = _dag(("X", "Y", "Z"), (("X", "Z"), ("Z", "Y"))) + assert d_separation(graph, ("X",), ("Y",)).separated is False + assert d_separation(graph, ("X",), ("Y",), ("Z",)).separated is True + role = causal_role_evidence(graph, "X", "Y", "Z") + assert role.mediator_on_directed_paths == (("X", "Z", "Y"),) + + +def test_figure_20_empty_backdoor_set_is_valid_but_conditioning_mediator_is_not(): + graph = _dag( + ("W", "X", "Y", "Z"), + (("X", "Z"), ("W", "Z"), ("Z", "Y"), ("W", "Y")), + ) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == ((),) + assert check_backdoor_adjustment_set(graph, "X", "Y", ()).admissible is True + conditioned = check_backdoor_adjustment_set(graph, "X", "Y", ("Z",)) + assert conditioned.admissible is False + assert conditioned.descendants_in_set == ("Z",) diff --git a/test/causal_factor_analysis/test_factor_mirage.py b/test/causal_factor_analysis/test_factor_mirage.py new file mode 100644 index 0000000..31702d3 --- /dev/null +++ b/test/causal_factor_analysis/test_factor_mirage.py @@ -0,0 +1,408 @@ +"""Tests for the published factor-mirage analytical formulas.""" + +from __future__ import annotations + +import math +import subprocess +import sys +from dataclasses import FrozenInstanceError +from fractions import Fraction +from pathlib import Path + +import numpy as np +import pytest + +from RiskLabAI.causal_factor_analysis import ( + ColliderCoefficients, + ColliderDiagnostics, + StrategyPerformance, + collider_factor_return, + collider_forecast_return, + collider_model_diagnostics, + collider_overcontrolled_coefficients, + confounder_factor_return, + confounder_forecast_return, + confounder_undercontrolled_coefficient, +) + + +@pytest.mark.parametrize( + "beta,gamma,delta", + [ + (1.0, 3.0, 1.0), + (-0.7, 1.2, -0.4), + (0.0, -2.0, 3.0), + (2.5, 0.0, -8.0), + ], +) +def test_confounder_coefficient_matches_equation_2(beta, gamma, delta): + expected = beta + gamma * delta / (1.0 + delta**2) + actual = confounder_undercontrolled_coefficient(beta, gamma, delta) + assert actual == pytest.approx(expected, rel=1e-14, abs=1e-14) + assert actual - beta == pytest.approx( + gamma * delta / (1.0 + delta**2), rel=1e-14, abs=1e-14 + ) + + +def test_confounder_zero_links_remove_omitted_variable_bias(): + assert confounder_undercontrolled_coefficient(2.0, 0.0, 4.0) == 2.0 + assert confounder_undercontrolled_coefficient(2.0, 4.0, 0.0) == 2.0 + + +def test_mirage_confounder_example_matches_equations_5_to_9(): + coefficient = confounder_undercontrolled_coefficient(1.0, 3.0, 1.0) + factor = confounder_factor_return(1.0, 1.0, 1.0, 3.0, 1.0) + forecast = confounder_forecast_return(1.0, 3.0, 1.0) + + assert coefficient == 2.5 + assert factor == StrategyPerformance(correct=16.0, misspecified=6.25) + assert forecast.correct == pytest.approx(17.0, abs=1e-14) + assert forecast.misspecified == pytest.approx(12.5, abs=1e-14) + + +@pytest.mark.parametrize( + "beta,gamma,delta", + [(1.0, 3.0, 1.0), (-0.5, 0.75, 2.0), (2.0, -1.0, -0.25)], +) +def test_static_confounder_forecast_penalty_identity(beta, gamma, delta): + result = confounder_forecast_return(beta, gamma, delta) + expected_gap = gamma**2 / (1.0 + delta**2) + assert result.correct - result.misspecified == pytest.approx( + expected_gap, rel=1e-12, abs=1e-14 + ) + assert result.correct >= 0.0 + assert result.misspecified >= 0.0 + + +def test_confounder_shift_can_create_systematic_losses(): + factor = confounder_factor_return( + 1.0, + 1.0, + 1.0, + 3.0, + 1.0, + delta_realized=-1.0, + ) + forecast = confounder_forecast_return(1.0, 3.0, 1.0, delta_realized=-1.0) + + assert factor == StrategyPerformance(correct=16.0, misspecified=-1.25) + assert forecast.correct == pytest.approx(5.0, abs=1e-14) + assert forecast.misspecified == pytest.approx(-2.5, abs=1e-14) + + +def test_confounder_shift_loss_has_exact_zero_boundaries(): + # beta=1, gamma=3 has roots (-3 +/- sqrt(5))/2. + root = (-3.0 + math.sqrt(5.0)) / 2.0 + factor = confounder_factor_return(2.0, 0.5, 1.0, 3.0, root, delta_realized=1.0) + forecast = confounder_forecast_return(1.0, 3.0, root, delta_realized=1.0) + assert factor.misspecified == pytest.approx(0.0, abs=2e-15) + assert forecast.misspecified == pytest.approx(0.0, abs=2e-15) + assert ( + confounder_factor_return( + 0.0, 0.5, 1.0, 3.0, 1.0, delta_realized=-1.0 + ).misspecified + == 0.0 + ) + + +def test_omitted_realized_delta_is_the_constant_parameter_case(): + static_factor = confounder_factor_return(0.7, -0.2, 0.8, 1.1, -0.6) + shifted_factor = confounder_factor_return( + 0.7, -0.2, 0.8, 1.1, -0.6, delta_realized=-0.6 + ) + static_forecast = confounder_forecast_return(0.8, 1.1, -0.6) + shifted_forecast = confounder_forecast_return(0.8, 1.1, -0.6, delta_realized=-0.6) + assert shifted_factor == static_factor + assert shifted_forecast == static_forecast + + +@pytest.mark.parametrize( + "beta,gamma,delta", + [(0.5, 1.0, 0.8), (1.0, -2.0, 0.25), (-0.7, 0.4, -1.2)], +) +def test_collider_coefficients_match_equations_11_and_12(beta, gamma, delta): + result = collider_overcontrolled_coefficients(beta, gamma, delta) + assert result.beta_hat == pytest.approx( + (beta - delta * gamma) / (1.0 + gamma**2), rel=1e-14, abs=1e-14 + ) + assert result.theta_hat == pytest.approx( + gamma / (1.0 + gamma**2), rel=1e-14, abs=1e-14 + ) + + +def test_collider_unbiased_condition_has_the_correct_sign(): + beta, gamma = 0.7, 1.5 + unbiased = collider_overcontrolled_coefficients(beta, gamma, -beta * gamma) + delta_zero = collider_overcontrolled_coefficients(beta, gamma, 0.0) + gamma_zero = collider_overcontrolled_coefficients(beta, 0.0, 4.0) + + assert unbiased.beta_hat == pytest.approx(beta, rel=1e-14, abs=1e-14) + assert unbiased.theta_hat != 0.0 + assert delta_zero.beta_hat != pytest.approx(beta) + assert gamma_zero == ColliderCoefficients(beta_hat=beta, theta_hat=0.0) + + +def test_conditioning_back_over_x_recovers_the_causal_expectation(): + beta, gamma, delta, x = 0.8, 1.2, -0.4, 2.5 + coefficients = collider_overcontrolled_coefficients(beta, gamma, delta) + expected_proxy = (beta * gamma + delta) * x + conditional_prediction = ( + coefficients.beta_hat * x + coefficients.theta_hat * expected_proxy + ) + assert conditional_prediction == pytest.approx(beta * x, rel=1e-14, abs=1e-14) + + +def test_mirage_collider_factor_example_is_negative(): + result = collider_factor_return(1.0, 1.0, 1.0, 1.0, 3.0) + assert result == StrategyPerformance(correct=1.0, misspecified=-0.5) + + +def test_mirage_collider_forecast_example_is_negative(): + result = collider_forecast_return(1.0, 1.0, 2.0) + assert result == StrategyPerformance(correct=1.0, misspecified=-0.5) + + +def test_collider_loss_boundaries_are_exact(): + assert collider_forecast_return(0.0, 2.0, 3.0).misspecified == 0.0 + assert collider_forecast_return(2.0, 1.0, 2.0).misspecified == 0.0 + assert collider_factor_return(0.0, 4.0, 1.0, 2.0, 3.0).misspecified == 0.0 + assert collider_factor_return(1.0, 2.0, 0.0, 2.0, 3.0).misspecified == 0.0 + + +def test_zero_collider_link_makes_both_strategies_equal(): + factor = collider_factor_return(1.25, -8.0, 0.6, 0.0, 3.0) + forecast = collider_forecast_return(0.6, 0.0, 3.0) + assert factor.misspecified == pytest.approx(factor.correct, abs=1e-15) + assert forecast.misspecified == pytest.approx(forecast.correct, abs=1e-15) + + +def test_collider_diagnostics_match_appendix_d(): + result = collider_model_diagnostics(1.0, 1.0, 2.0, 5) + assert result.residual_variance == 0.5 + assert result.outcome_variance == pytest.approx(2.0, abs=1e-14) + assert result.correct_r_squared == 0.5 + assert result.overcontrolled_r_squared == 0.75 + assert result.correct_adjusted_r_squared == pytest.approx(1.0 / 3.0) + assert result.overcontrolled_adjusted_r_squared == 0.5 + assert result.correct_beta_variance == 0.2 + assert result.overcontrolled_beta_variance == pytest.approx(11.0 / 20.0) + assert result.collider_coefficient_variance == 0.05 + assert result.correct_beta_t_statistic == pytest.approx(math.sqrt(5.0)) + assert result.overcontrolled_beta_t_statistic == pytest.approx( + -math.sqrt(5.0 / 11.0) + ) + assert result.collider_t_statistic == pytest.approx(math.sqrt(5.0)) + assert result.adjusted_r_squared_prefers_overcontrolled is True + assert result.absolute_beta_t_prefers_overcontrolled is False + + +def test_ordinary_r_squared_is_strict_only_for_nonzero_gamma(): + equal = collider_model_diagnostics(0.8, 0.0, 10.0, 50) + larger = collider_model_diagnostics(0.8, -0.2, 10.0, 50) + assert equal.overcontrolled_r_squared == equal.correct_r_squared + assert larger.overcontrolled_r_squared > larger.correct_r_squared + + +def test_small_nonzero_links_do_not_cancel_out_of_r_squared(): + beta_only = collider_model_diagnostics(1e-8, 0.0, 0.0, 20) + gamma_only = collider_model_diagnostics(0.0, 1e-8, 0.0, 20) + assert beta_only.correct_r_squared == pytest.approx(1e-16, rel=1e-15) + assert gamma_only.overcontrolled_r_squared == pytest.approx(1e-16, rel=1e-15) + + +def test_adjusted_r_squared_threshold_is_strict_and_sign_invariant(): + sample_size = 7 + threshold = 1.0 / math.sqrt(sample_size - 3) + at_boundary = collider_model_diagnostics(1.0, threshold, 0.0, sample_size) + below = collider_model_diagnostics(1.0, 0.49, 0.0, sample_size) + above = collider_model_diagnostics(1.0, -0.51, 0.0, sample_size) + + assert at_boundary.correct_adjusted_r_squared == pytest.approx( + at_boundary.overcontrolled_adjusted_r_squared, abs=1e-15 + ) + assert at_boundary.adjusted_r_squared_prefers_overcontrolled is False + assert below.adjusted_r_squared_prefers_overcontrolled is False + assert above.adjusted_r_squared_prefers_overcontrolled is True + + +@pytest.mark.parametrize( + "beta,gamma,delta,n_observations", + [(0.3, 0.8, -0.2, 20), (-1.0, 1.5, 0.4, 100), (0.0, -2.0, 3.0, 8)], +) +def test_expected_t_statistics_match_appendix_d(beta, gamma, delta, n_observations): + result = collider_model_diagnostics(beta, gamma, delta, n_observations) + denominator = math.sqrt((beta * gamma + delta) ** 2 + gamma**2 + 1.0) + expected_overcontrolled = ( + math.sqrt(n_observations) * (beta - delta * gamma) / denominator + ) + assert result.correct_beta_t_statistic == pytest.approx( + math.sqrt(n_observations) * beta, rel=1e-13, abs=1e-14 + ) + assert result.overcontrolled_beta_t_statistic == pytest.approx( + expected_overcontrolled, rel=1e-13, abs=1e-14 + ) + assert result.collider_t_statistic == pytest.approx( + math.sqrt(n_observations) * gamma, rel=1e-13, abs=1e-14 + ) + assert result.absolute_beta_t_prefers_overcontrolled is ( + abs(beta - delta * gamma) / denominator > abs(beta) + ) + + +@pytest.mark.parametrize("sample_size", [True, 4.0, 3, 0, -4]) +def test_invalid_sample_sizes_are_rejected(sample_size): + match = "integer" if isinstance(sample_size, (bool, float)) else "greater than 3" + with pytest.raises(ValueError, match=match): + collider_model_diagnostics(1.0, 1.0, 1.0, sample_size) + + +def test_numpy_integer_sample_size_is_accepted(): + result = collider_model_diagnostics(1.0, 1.0, 1.0, np.int64(5)) + assert isinstance(result, ColliderDiagnostics) + + +@pytest.mark.parametrize( + "invalid", + [True, 1.0 + 2.0j, np.nan, np.inf, -np.inf, [1.0], np.array([1.0])], +) +def test_invalid_real_scalars_are_rejected(invalid): + calls = ( + lambda: confounder_undercontrolled_coefficient(invalid, 1.0, 1.0), + lambda: confounder_factor_return(invalid, 1.0, 1.0, 1.0, 1.0), + lambda: confounder_forecast_return(invalid, 1.0, 1.0), + lambda: collider_overcontrolled_coefficients(invalid, 1.0, 1.0), + lambda: collider_factor_return(invalid, 1.0, 1.0, 1.0, 1.0), + lambda: collider_forecast_return(invalid, 1.0, 1.0), + lambda: collider_model_diagnostics(invalid, 1.0, 1.0, 10), + ) + for call in calls: + with pytest.raises(ValueError, match="real scalar|finite"): + call() + + +def test_extreme_finite_scaling_avoids_spurious_overflow(): + coefficient = confounder_undercontrolled_coefficient(0.0, 1e308, 1e308) + factor = confounder_factor_return(1e200, 0.0, 1e-200, 0.0, 1.0) + collider = collider_overcontrolled_coefficients(1.0, 1e308, 1.0) + + assert coefficient == pytest.approx(1.0, rel=1e-15) + assert factor.correct == pytest.approx(1.0, rel=1e-15) + assert factor.misspecified == pytest.approx(1.0, rel=1e-15) + assert collider.beta_hat == pytest.approx(-1e-308, rel=1e-15, abs=0.0) + assert collider.theta_hat == pytest.approx(1e-308, rel=1e-15, abs=0.0) + + +def test_extreme_collider_scaling_preserves_representable_results(): + coefficients = collider_overcontrolled_coefficients(1e308, 1e163, 0.0) + diagnostics = collider_model_diagnostics(1e154, 1e200, 0.0, 4) + assert coefficients.beta_hat == pytest.approx(1e-18, rel=1e-14, abs=0.0) + assert diagnostics.overcontrolled_beta_variance == pytest.approx( + 2.5e-93, rel=1e-14, abs=0.0 + ) + + +def test_confounder_composites_preserve_latent_scaled_coefficients(): + maximum = sys.float_info.max + finite_large = confounder_factor_return(1e-308, 0.0, maximum, maximum, 1.0) + latent_factor = confounder_factor_return(1e308, 0.0, 0.0, 1e-200, 1e200) + latent_forecast = confounder_forecast_return(0.0, 1e-200, 1.0, delta_realized=1e308) + + assert finite_large.correct == pytest.approx(3.2317006071310996, rel=1e-15) + assert finite_large.misspecified == pytest.approx(7.271326366044974, rel=1e-15) + assert latent_factor.misspecified == pytest.approx(1e-184, rel=1e-14) + assert latent_forecast.misspecified == pytest.approx(5e-93, rel=1e-14) + + +def test_collider_composites_preserve_latent_scaled_coefficients(): + latent_factor = collider_factor_return(1e208, 0.0, 1e-54, 1e200, 0.0) + latent_forecast = collider_forecast_return(1e75, 1e200, 0.0) + tiny_prefactor = collider_factor_return(1e-308, 1e308, 1e-308, 1.0, 0.0) + + assert latent_factor.correct == pytest.approx(1e308, rel=1e-14) + assert latent_factor.misspecified == pytest.approx(1e-92, rel=1e-14) + assert latent_forecast.correct == pytest.approx(1e150, rel=1e-14) + assert latent_forecast.misspecified == pytest.approx(1e-250, rel=1e-14) + assert tiny_prefactor.misspecified == pytest.approx(5e-309, rel=1e-14) + + +def test_expected_t_statistic_preserves_latent_scaled_ratio(): + diagnostics = collider_model_diagnostics(0.0, 1e-200, 1e-200, 10**200) + assert diagnostics.overcontrolled_beta_t_statistic == pytest.approx( + -1e-300, rel=1e-14, abs=0.0 + ) + assert diagnostics.absolute_beta_t_prefers_overcontrolled is True + + +def test_loss_sign_is_preserved_at_ill_conditioned_boundaries(): + confounder = confounder_undercontrolled_coefficient(1.0, 1e10, -1e-10) + collider = collider_forecast_return(1.0, -1e-10, -1e10) + assert confounder == pytest.approx(-3.642219731549774e-17, rel=1e-14) + assert collider.misspecified == pytest.approx(-3.643219731549774e-17, rel=1e-14) + assert confounder < 0.0 + assert collider.misspecified < 0.0 + + +def test_adjusted_r_squared_boolean_uses_the_exact_strict_inequality(): + gamma = 1.0 / math.sqrt(3.0) + diagnostics = collider_model_diagnostics(0.0, gamma, 0.0, 6) + expected = Fraction.from_float(gamma) ** 2 * 3 > 1 + assert diagnostics.adjusted_r_squared_prefers_overcontrolled is expected + + +def test_unrepresentable_result_raises_stable_value_error(): + with pytest.raises(ValueError, match="cannot be represented"): + collider_forecast_return(1e308, 0.0, 0.0) + + +def test_result_records_are_frozen(): + performance = collider_forecast_return(1.0, 1.0, 2.0) + coefficients = collider_overcontrolled_coefficients(1.0, 1.0, 2.0) + diagnostics = collider_model_diagnostics(1.0, 1.0, 2.0, 5) + with pytest.raises(FrozenInstanceError): + performance.correct = 0.0 + with pytest.raises(FrozenInstanceError): + coefficients.beta_hat = 0.0 + with pytest.raises(FrozenInstanceError): + diagnostics.correct_r_squared = 0.0 + + +def test_public_import_is_isolated_and_source_conflicted_api_is_absent(): + repository_root = Path(__file__).resolve().parents[2] + program = """ +import sys +import RiskLabAI + +module = RiskLabAI.causal_factor_analysis +required = { + 'confounder_undercontrolled_coefficient', + 'confounder_factor_return', + 'confounder_forecast_return', + 'collider_overcontrolled_coefficients', + 'collider_factor_return', + 'collider_forecast_return', + 'collider_model_diagnostics', +} +missing = required.difference(module.__all__) +if missing: + raise SystemExit(f'missing public names: {sorted(missing)}') +for forbidden in ('collider_shift_return', 'collider_p_value'): + if hasattr(module, forbidden): + raise SystemExit(f'source-conflicted or unsupported API exposed: {forbidden}') + +blocked = { + 'RiskLabAI.causal', + 'RiskLabAI.optimization', + 'RiskLabAI.backtest', + 'RiskLabAI.risk_control', +} +loaded = blocked.intersection(sys.modules) +if loaded: + raise SystemExit(f'unexpected imports: {sorted(loaded)}') +""" + subprocess.run( + [sys.executable, "-c", program], + cwd=repository_root, + check=True, + capture_output=True, + text=True, + ) diff --git a/test/causal_factor_analysis/test_graph_identification.py b/test/causal_factor_analysis/test_graph_identification.py new file mode 100644 index 0000000..de114cd --- /dev/null +++ b/test/causal_factor_analysis/test_graph_identification.py @@ -0,0 +1,963 @@ +"""Source-locked tests for clean graphical identification evidence.""" + +from __future__ import annotations + +import ast +import itertools +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from RiskLabAI.causal_factor_analysis.graph_identification import ( + BackdoorAdjustmentEvidence, + CausalDAG, + DSeparationEvidence, + FrontdoorAdjustmentEvidence, + InstrumentEvidence, + NodeRoleEvidence, + PathEvidence, + causal_role_evidence, + check_backdoor_adjustment_set, + check_frontdoor_adjustment_set, + check_instrument, + d_separation, + minimal_backdoor_adjustment_sets, + minimal_frontdoor_adjustment_sets, +) + +PUBLIC_NAMES = { + "BackdoorAdjustmentEvidence", + "CausalDAG", + "DSeparationEvidence", + "FrontdoorAdjustmentEvidence", + "InstrumentEvidence", + "NodeRoleEvidence", + "PathEvidence", + "causal_role_evidence", + "check_backdoor_adjustment_set", + "check_frontdoor_adjustment_set", + "check_instrument", + "d_separation", + "minimal_backdoor_adjustment_sets", + "minimal_frontdoor_adjustment_sets", +} + + +def _dag(edges, *, observed=None, extra=()): + nodes = sorted(set(extra) | {node for edge in edges for node in edge}) + if observed is None: + observed = nodes + return CausalDAG(nodes, edges, observed) + + +@pytest.mark.parametrize( + "conditioned,separated", + [((), False), (("U",), True)], +) +def test_fork_d_separation(conditioned, separated): + graph = _dag((("U", "X"), ("U", "Y"))) + evidence = d_separation(graph, ("X",), ("Y",), conditioned) + assert evidence.separated is separated + assert evidence.paths[0].nodes == ("X", "U", "Y") + + +@pytest.mark.parametrize( + "conditioned,separated", + [((), False), (("M",), True)], +) +def test_chain_d_separation(conditioned, separated): + graph = _dag((("X", "M"), ("M", "Y"))) + assert d_separation(graph, ("X",), ("Y",), conditioned).separated is separated + + +def test_collider_and_conditioned_descendant_reverse_path_status(): + graph = _dag((("X", "C"), ("Y", "C"), ("C", "D"))) + closed = d_separation(graph, ("X",), ("Y",)) + opened_at_collider = d_separation(graph, ("X",), ("Y",), ("C",)) + opened_at_descendant = d_separation(graph, ("X",), ("Y",), ("D",)) + + assert closed.separated + assert closed.paths[0].blocking_colliders == ("C",) + assert not opened_at_collider.separated + assert opened_at_collider.paths[0].activated_colliders == ("C",) + assert not opened_at_descendant.separated + assert opened_at_descendant.paths[0].activated_colliders == ("C",) + + +def test_collider_is_path_relative_not_a_global_node_label(): + graph = _dag((("A", "X"), ("A", "C"), ("B", "C"), ("B", "Y"))) + empty = d_separation(graph, ("X",), ("Y",)) + selected = d_separation(graph, ("X",), ("Y",), ("C",)) + roles = causal_role_evidence(graph, "X", "Y", "C") + + assert empty.separated + assert not selected.separated + assert roles.collider_on_paths == (("X", "A", "C", "B", "Y"),) + assert roles.backdoor_noncollider_on_paths == () + + +def test_d_separation_is_symmetric_with_oriented_path_evidence(): + graph = _dag((("U", "X"), ("U", "Y"), ("X", "M"), ("M", "Y"))) + forward = d_separation(graph, ("X",), ("Y",), ("U", "M")) + reverse = d_separation(graph, ("Y",), ("X",), ("U", "M")) + assert forward.separated == reverse.separated + assert {path.nodes for path in forward.paths} == { + tuple(reversed(path.nodes)) for path in reverse.paths + } + + +@pytest.mark.parametrize( + "edges,expected", + [ + (("X", "Y"), ((),)), + (("Y", "X"), ()), + ((), ((),)), + ], +) +def test_empty_or_missing_backdoor_sets_are_distinguished(edges, expected): + graph = _dag( + tuple((edges[index], edges[index + 1]) for index in range(0, len(edges), 2)), + extra=("X", "Y"), + ) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == expected + + +@pytest.mark.parametrize( + "observed,expected", + [ + (("U", "X", "Y", "Z"), (("U",), ("Z",))), + (("X", "Y", "Z"), (("Z",),)), + (("U", "X", "Y"), (("U",),)), + (("X", "Y"), ()), + ], +) +def test_backdoor_search_respects_observability(observed, expected): + graph = _dag( + (("U", "Z"), ("Z", "X"), ("U", "Y"), ("X", "Y")), + observed=observed, + ) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == expected + + +def test_backdoor_returns_all_inclusion_minimal_sets_not_only_smallest(): + graph = _dag( + ( + ("A", "X"), + ("A", "B"), + ("B", "Y"), + ("A", "C"), + ("C", "Y"), + ("X", "Y"), + ) + ) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == ( + ("A",), + ("B", "C"), + ) + + +def test_indegree_two_node_can_be_the_required_backdoor_blocker(): + graph = _dag((("A", "C"), ("B", "C"), ("C", "X"), ("C", "Y"), ("X", "Y"))) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == (("C",),) + + +def test_conditioning_on_a_collider_can_invalidate_an_empty_backdoor_set(): + graph = _dag((("A", "X"), ("A", "C"), ("B", "C"), ("B", "Y"), ("X", "Y"))) + assert check_backdoor_adjustment_set(graph, "X", "Y", ()).admissible + selected = check_backdoor_adjustment_set(graph, "X", "Y", ("C",)) + assert not selected.admissible + assert selected.open_backdoor_paths + + +def test_descendant_control_is_reported_and_rejected(): + graph = _dag((("X", "Y"), ("X", "D"))) + evidence = check_backdoor_adjustment_set(graph, "X", "Y", ("D",)) + assert evidence.descendants_in_set == ("D",) + assert not evidence.admissible + + +def test_classic_frontdoor_set_is_graphically_admissible_but_needs_positivity(): + graph = _dag( + (("U", "X"), ("U", "Y"), ("X", "M"), ("M", "Y")), + observed=("X", "M", "Y"), + ) + evidence = check_frontdoor_adjustment_set(graph, "X", "Y", ("M",)) + assert evidence.graphically_admissible + assert evidence.positivity_required + assert minimal_frontdoor_adjustment_sets(graph, "X", "Y") == (("M",),) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == () + + +def test_frontdoor_checker_accepts_redundant_superset_but_search_removes_it(): + graph = _dag( + (("U", "X"), ("U", "Y"), ("X", "M"), ("M", "Y")), + observed=("X", "M", "N", "Y"), + extra=("N",), + ) + assert check_frontdoor_adjustment_set( + graph, "X", "Y", ("M", "N") + ).graphically_admissible + assert minimal_frontdoor_adjustment_sets(graph, "X", "Y") == (("M",),) + + +@pytest.mark.parametrize( + "extra_edge,failed_field", + [ + (("X", "Y"), "unintercepted_directed_paths"), + (("V", "M"), "open_treatment_mediator_backdoor_paths"), + (("W", "Y"), "open_mediator_outcome_backdoor_paths_given_treatment"), + ], +) +def test_each_frontdoor_criterion_fails_independently(extra_edge, failed_field): + edges = [("U", "X"), ("U", "Y"), ("X", "M"), ("M", "Y")] + if extra_edge == ("V", "M"): + edges.append(("V", "X")) + if extra_edge == ("W", "Y"): + edges.append(("W", "M")) + edges.append(extra_edge) + observed = tuple( + node for node in {item for edge in edges for item in edge} if node != "U" + ) + graph = _dag(tuple(edges), observed=observed) + evidence = check_frontdoor_adjustment_set(graph, "X", "Y", ("M",)) + assert not evidence.graphically_admissible + assert getattr(evidence, failed_field) + + +def test_parallel_and_serial_mediator_sets_have_correct_minimality(): + parallel = _dag( + ( + ("U", "X"), + ("U", "Y"), + ("X", "M1"), + ("M1", "Y"), + ("X", "M2"), + ("M2", "Y"), + ), + observed=("X", "M1", "M2", "Y"), + ) + serial = _dag( + (("U", "X"), ("U", "Y"), ("X", "M1"), ("M1", "M2"), ("M2", "Y")), + observed=("X", "M1", "M2", "Y"), + ) + assert minimal_frontdoor_adjustment_sets(parallel, "X", "Y") == (("M1", "M2"),) + assert minimal_frontdoor_adjustment_sets(serial, "X", "Y") == ( + ("M1",), + ("M2",), + ) + + +def test_latent_or_irrelevant_frontdoor_set_is_not_returned(): + hidden = _dag( + (("U", "X"), ("U", "Y"), ("X", "M"), ("M", "Y")), + observed=("X", "Y"), + ) + unrelated = _dag((), observed=("X", "Y", "M"), extra=("X", "Y", "M")) + assert not check_frontdoor_adjustment_set( + hidden, "X", "Y", ("M",) + ).graphically_admissible + assert minimal_frontdoor_adjustment_sets(hidden, "X", "Y") == () + assert minimal_frontdoor_adjustment_sets(unrelated, "X", "Y") == () + + +def test_pearl_and_cfi_instrument_profiles_are_not_collapsed(): + graph = _dag((("U", "Z"), ("U", "X"), ("X", "Y"))) + evidence = check_instrument(graph, "Z", "X", "Y") + assert evidence.pearl_relevance + assert evidence.pearl_exclusion_exogeneity + assert evidence.pearl_graphical + assert not evidence.cfi_direct_relevance + assert not evidence.cfi_simple + + +def test_simple_direct_instrument_satisfies_both_profiles(): + graph = _dag((("Z", "X"), ("X", "Y"))) + evidence = check_instrument(graph, "Z", "X", "Y") + assert evidence.pearl_graphical + assert evidence.cfi_direct_relevance + assert evidence.cfi_full_mediation + assert evidence.cfi_exogeneity + assert evidence.cfi_simple + + +def test_cfi_simple_profile_is_unconditional_when_pearl_controls_are_supplied(): + graph = _dag((("Z", "X"), ("X", "Y")), extra=("W",)) + evidence = check_instrument(graph, "Z", "X", "Y", ("W",)) + assert evidence.pearl_graphical + assert evidence.cfi_simple + + +def test_conditional_instrument_blocks_observed_common_cause_for_pearl_only(): + graph = _dag((("W", "Z"), ("W", "Y"), ("Z", "X"), ("X", "Y"))) + unconditional = check_instrument(graph, "Z", "X", "Y") + conditional = check_instrument(graph, "Z", "X", "Y", ("W",)) + assert not unconditional.pearl_graphical + assert conditional.pearl_graphical + assert not conditional.cfi_simple + + +@pytest.mark.parametrize( + "edges,conditioned,failed", + [ + ((("Z", "X"), ("X", "Y"), ("Z", "Y")), (), "exclusion"), + ((("U", "Z"), ("U", "Y"), ("Z", "X"), ("X", "Y")), (), "exogeneity"), + ( + (("Z", "C"), ("U", "C"), ("U", "Y"), ("Z", "X"), ("X", "Y")), + ("C",), + "collider", + ), + ], +) +def test_instrument_exclusion_and_exogeneity_failures(edges, conditioned, failed): + graph = _dag(edges) + evidence = check_instrument(graph, "Z", "X", "Y", conditioned) + assert not evidence.pearl_graphical, failed + + +def test_treatment_descendant_conditioning_invalidates_pearl_profile(): + graph = _dag((("Z", "X"), ("X", "Y"), ("X", "D"))) + evidence = check_instrument(graph, "Z", "X", "Y", ("D",)) + assert not evidence.conditioning_unaffected_by_treatment + assert not evidence.pearl_graphical + + +def test_post_treatment_proxy_is_not_a_pearl_instrument(): + graph = _dag( + (("U", "X"), ("U", "Y"), ("X", "Y"), ("X", "Z")), + observed=("X", "Y", "Z"), + ) + evidence = check_instrument(graph, "Z", "X", "Y") + assert evidence.pearl_relevance + assert not evidence.pearl_exclusion_exogeneity + assert not evidence.pearl_graphical + + +def test_node_role_evidence_is_nonexclusive_and_path_witnessed(): + graph = _dag( + ( + ("X", "M"), + ("M", "Y"), + ("A", "M"), + ("A", "X"), + ("B", "M"), + ("B", "Y"), + ) + ) + evidence = causal_role_evidence(graph, "X", "Y", "M") + assert evidence.mediator_on_directed_paths + assert evidence.collider_on_paths + assert evidence.noncollider_on_paths + assert evidence.descendant_of_treatment + assert evidence.ancestor_of_outcome + + +def test_common_cause_and_remote_backdoor_witnesses_are_distinct(): + graph = _dag((("U", "Z"), ("Z", "X"), ("U", "Y"), ("X", "Y"))) + upstream = causal_role_evidence(graph, "X", "Y", "U") + downstream = causal_role_evidence(graph, "X", "Y", "Z") + assert upstream.common_cause_paths + assert upstream.backdoor_noncollider_on_paths + assert downstream.backdoor_noncollider_on_paths + assert downstream.common_cause_paths == () + + +def test_common_cause_witness_requires_a_pathwise_directed_fork(): + graph = _dag((("U", "X"), ("U", "Y"), ("X", "D"), ("U", "A"), ("A", "D"))) + evidence = causal_role_evidence(graph, "X", "Y", "U") + assert evidence.common_cause_paths == (("X", "U", "Y"),) + assert ("X", "D", "A", "U", "Y") in evidence.noncollider_on_paths + + +def test_causal_dag_normalizes_without_mutating_callers_and_is_frozen(): + nodes = ["Y", "X", "U"] + edges = [["U", "Y"], ["U", "X"]] + observed = ["Y", "X"] + graph = CausalDAG(nodes, edges, observed) + nodes.append("LATER") + edges[0][0] = "CHANGED" + observed.clear() + assert graph.nodes == ("U", "X", "Y") + assert graph.directed_edges == (("U", "X"), ("U", "Y")) + assert graph.observed_nodes == ("X", "Y") + assert not hasattr(graph, "__dict__") + with pytest.raises(AttributeError): + graph.nodes = () + + +def test_causal_dag_namedtuple_helpers_cannot_bypass_validation(): + graph = _dag((("X", "Y"),)) + with pytest.raises(ValueError): + graph._replace(nodes=("ONLY",)) + with pytest.raises(ValueError): + CausalDAG._make((("X", "Y"), (("X", "Y"), ("Y", "X")), ("X", "Y"))) + with pytest.raises(TypeError): + CausalDAG._make((graph.nodes, graph.directed_edges)) + + +@pytest.mark.parametrize( + "args,error", + [ + (("XYZ", (), ()), TypeError), + ((("X", 1), (), ("X",)), TypeError), + ((("X", ""), (), ("X",)), ValueError), + ((("X", "X"), (), ("X",)), ValueError), + ((("X", "Y"), (("X",),), ("X", "Y")), ValueError), + ((("X", "Y"), ({"X", "Y"},), ("X", "Y")), TypeError), + ((("X", "Y"), (("X", "Z"),), ("X", "Y")), ValueError), + ((("X", "Y"), (("X", "X"),), ("X", "Y")), ValueError), + ((("X", "Y"), (("X", "Y"), ("Y", "X")), ("X", "Y")), ValueError), + ], +) +def test_malformed_dag_fails_closed(args, error): + with pytest.raises(error): + CausalDAG(*args) + + +def test_query_validation_fails_closed_without_scalar_string_iteration(): + graph = _dag((("X", "Y"),), extra=("Z",)) + with pytest.raises(TypeError): + d_separation(graph, "X", ("Y",)) + with pytest.raises(ValueError): + d_separation(graph, ("X",), ("Y",), ("UNKNOWN",)) + with pytest.raises(ValueError): + d_separation(graph, ("X",), ("Y",), ("X",)) + with pytest.raises(ValueError): + check_backdoor_adjustment_set(graph, "X", "X") + with pytest.raises(ValueError): + check_instrument(graph, "X", "X", "Y") + + +def test_known_latent_and_endpoint_sets_are_substantive_negatives(): + graph = _dag( + (("U", "X"), ("U", "Y"), ("X", "M"), ("M", "Y")), observed=("X", "Y", "M") + ) + latent = check_backdoor_adjustment_set(graph, "X", "Y", ("U",)) + endpoint = check_backdoor_adjustment_set(graph, "X", "Y", ("X",)) + frontdoor = check_frontdoor_adjustment_set(graph, "X", "Y", ("U",)) + assert not latent.all_observed and not latent.admissible + assert endpoint.endpoints_in_set == ("X",) and not endpoint.admissible + assert not frontdoor.all_observed and not frontdoor.graphically_admissible + + +def _oracle_children(nodes, edges): + result = {node: set() for node in nodes} + for source, target in edges: + result[source].add(target) + return result + + +def _oracle_descendants(nodes, edges, start): + children = _oracle_children(nodes, edges) + result = set() + pending = list(children[start]) + while pending: + node = pending.pop() + if node not in result: + result.add(node) + pending.extend(children[node]) + return result + + +def _oracle_simple_paths(nodes, edges, start, finish): + adjacency = {node: set() for node in nodes} + for source, target in edges: + adjacency[source].add(target) + adjacency[target].add(source) + result = [] + + def visit(node, path): + if node == finish: + result.append(tuple(path)) + return + for neighbor in sorted(adjacency[node]): + if neighbor not in path: + visit(neighbor, path + [neighbor]) + + visit(start, [start]) + return result + + +def _oracle_path_open(nodes, edges, path, conditioned): + edge_set = set(edges) + conditioned = set(conditioned) + for index in range(1, len(path) - 1): + previous, node, following = path[index - 1 : index + 2] + collider = (previous, node) in edge_set and (following, node) in edge_set + if collider: + family = {node} | _oracle_descendants(nodes, edges, node) + if not family.intersection(conditioned): + return False + elif node in conditioned: + return False + return True + + +def _oracle_d_separated(nodes, edges, left, right, conditioned): + return not any( + _oracle_path_open(nodes, edges, path, conditioned) + for path in _oracle_simple_paths(nodes, edges, left, right) + ) + + +def _oracle_directed_paths(nodes, edges, start, finish): + edge_set = set(edges) + return tuple( + sorted( + ( + path + for path in _oracle_simple_paths(nodes, edges, start, finish) + if all( + (path[index], path[index + 1]) in edge_set + for index in range(len(path) - 1) + ) + ), + key=lambda path: (len(path), path), + ) + ) + + +def _oracle_frontdoor_admissible(nodes, edges, treatment, outcome, mediators): + mediators = tuple(sorted(mediators)) + directed = _oracle_directed_paths(nodes, edges, treatment, outcome) + if not mediators or not directed: + return False + if any(not set(path[1:-1]).intersection(mediators) for path in directed): + return False + treatment_graph = tuple(edge for edge in edges if edge[0] != treatment) + if any( + not _oracle_d_separated(nodes, treatment_graph, treatment, node, ()) + for node in mediators + ): + return False + mediator_graph = tuple(edge for edge in edges if edge[0] not in mediators) + return all( + _oracle_d_separated(nodes, mediator_graph, node, outcome, (treatment,)) + for node in mediators + ) + + +def _oracle_role(nodes, edges, treatment, outcome, node): + edge_set = set(edges) + paths = tuple( + sorted( + _oracle_simple_paths(nodes, edges, treatment, outcome), + key=lambda path: (len(path), path), + ) + ) + colliders = [] + noncolliders = [] + mediators = [] + backdoors = [] + common_causes = [] + for path in paths: + index = path.index(node) if node in path[1:-1] else None + if index is None: + continue + previous, following = path[index - 1], path[index + 1] + is_collider = (previous, node) in edge_set and (following, node) in edge_set + if is_collider: + colliders.append(path) + else: + noncolliders.append(path) + if all( + (path[position], path[position + 1]) in edge_set + for position in range(len(path) - 1) + ): + mediators.append(path) + if (path[1], treatment) in edge_set and not is_collider: + backdoors.append(path) + directed_to_treatment = all( + (path[position], path[position - 1]) in edge_set + for position in range(index, 0, -1) + ) + directed_to_outcome = all( + (path[position], path[position + 1]) in edge_set + for position in range(index, len(path) - 1) + ) + if not is_collider and directed_to_treatment and directed_to_outcome: + common_causes.append(path) + return { + "collider_on_paths": tuple(colliders), + "noncollider_on_paths": tuple(noncolliders), + "mediator_on_directed_paths": tuple(mediators), + "backdoor_noncollider_on_paths": tuple(backdoors), + "common_cause_paths": tuple(common_causes), + "descendant_of_treatment": node in _oracle_descendants(nodes, edges, treatment), + "ancestor_of_outcome": outcome in _oracle_descendants(nodes, edges, node), + } + + +def _oracle_is_acyclic(nodes, edges): + descendants = _oracle_children(nodes, edges) + indegree = {node: 0 for node in nodes} + for _, target in edges: + indegree[target] += 1 + pending = [node for node in nodes if indegree[node] == 0] + visited = 0 + while pending: + node = pending.pop() + visited += 1 + for child in descendants[node]: + indegree[child] -= 1 + if indegree[child] == 0: + pending.append(child) + return visited == len(nodes) + + +def _all_four_node_dags(): + nodes = ("A", "B", "C", "D") + pairs = tuple(itertools.combinations(nodes, 2)) + for states in itertools.product((0, 1, 2), repeat=len(pairs)): + edges = [] + for state, (first, second) in zip(states, pairs): + if state == 1: + edges.append((first, second)) + elif state == 2: + edges.append((second, first)) + edges = tuple(edges) + if _oracle_is_acyclic(nodes, edges): + yield nodes, edges + + +def test_all_543_four_node_dags_match_independent_d_separation_oracle(): + count = 0 + for nodes, edges in _all_four_node_dags(): + count += 1 + graph = CausalDAG(nodes, tuple(reversed(edges)), tuple(reversed(nodes))) + for left, right in itertools.permutations(nodes, 2): + remaining = tuple(node for node in nodes if node not in {left, right}) + for size in range(len(remaining) + 1): + for conditioned in itertools.combinations(remaining, size): + expected = _oracle_d_separated( + nodes, edges, left, right, conditioned + ) + actual = d_separation( + graph, (left,), (right,), conditioned + ).separated + assert actual is expected + assert count == 543 + + +def test_all_four_node_backdoor_sets_match_independent_powerset_oracle(): + for nodes, edges in _all_four_node_dags(): + graph = CausalDAG(nodes, edges, nodes) + for treatment, outcome in itertools.permutations(nodes, 2): + descendants = _oracle_descendants(nodes, edges, treatment) + candidates = tuple( + node + for node in nodes + if node not in {treatment, outcome} and node not in descendants + ) + mutilated = tuple(edge for edge in edges if edge[0] != treatment) + valid = [] + for size in range(len(candidates) + 1): + for candidate in itertools.combinations(candidates, size): + if _oracle_d_separated( + nodes, mutilated, treatment, outcome, candidate + ): + valid.append(candidate) + expected = tuple( + candidate + for candidate in sorted(valid, key=lambda item: (len(item), item)) + if not any(set(other) < set(candidate) for other in valid) + ) + assert ( + minimal_backdoor_adjustment_sets(graph, treatment, outcome) == expected + ) + + +def test_all_four_node_frontdoor_sets_match_independent_oracle(): + queries = 0 + for nodes, edges in _all_four_node_dags(): + graph = CausalDAG(nodes, tuple(reversed(edges)), tuple(reversed(nodes))) + for treatment, outcome in itertools.permutations(nodes, 2): + queries += 1 + candidates = tuple( + node for node in nodes if node not in {treatment, outcome} + ) + valid = [] + for size in range(1, len(candidates) + 1): + for mediators in itertools.combinations(candidates, size): + expected = _oracle_frontdoor_admissible( + nodes, edges, treatment, outcome, mediators + ) + actual = check_frontdoor_adjustment_set( + graph, treatment, outcome, mediators + ).graphically_admissible + assert actual is expected + if expected: + valid.append(mediators) + expected_minimal = tuple( + candidate + for candidate in sorted(valid, key=lambda item: (len(item), item)) + if not any(set(other) < set(candidate) for other in valid) + ) + assert ( + minimal_frontdoor_adjustment_sets(graph, treatment, outcome) + == expected_minimal + ) + assert queries == 6_516 + + +def test_all_four_node_instrument_profiles_match_independent_oracles(): + assessments = 0 + for nodes, edges in _all_four_node_dags(): + graph = CausalDAG(nodes, tuple(reversed(edges)), tuple(reversed(nodes))) + edge_set = set(edges) + for treatment, outcome in itertools.permutations(nodes, 2): + for instrument in ( + node for node in nodes if node not in {treatment, outcome} + ): + control = next( + node + for node in nodes + if node not in {instrument, treatment, outcome} + ) + for conditioned in ((), (control,)): + assessments += 1 + unaffected = not set(conditioned).intersection( + _oracle_descendants(nodes, edges, treatment) + ) + pearl_relevance = not _oracle_d_separated( + nodes, edges, instrument, treatment, conditioned + ) + intervention_graph = tuple( + edge for edge in edges if edge[1] != treatment + ) + pearl_exclusion = _oracle_d_separated( + nodes, + intervention_graph, + instrument, + outcome, + conditioned, + ) + directed = _oracle_directed_paths(nodes, edges, instrument, outcome) + cfi_direct = (instrument, treatment) in edge_set + cfi_full_mediation = bool(directed) and all( + treatment in path[1:-1] for path in directed + ) + instrument_graph = tuple( + edge for edge in edges if edge[0] != instrument + ) + cfi_exogeneity = _oracle_d_separated( + nodes, instrument_graph, instrument, outcome, () + ) + evidence = check_instrument( + graph, + instrument, + treatment, + outcome, + conditioned, + ) + assert evidence.conditioning_unaffected_by_treatment is unaffected + assert evidence.pearl_relevance is pearl_relevance + assert evidence.pearl_exclusion_exogeneity is pearl_exclusion + assert evidence.pearl_graphical is ( + unaffected and pearl_relevance and pearl_exclusion + ) + assert evidence.cfi_direct_relevance is cfi_direct + assert evidence.cfi_full_mediation is cfi_full_mediation + assert evidence.cfi_exogeneity is cfi_exogeneity + assert evidence.cfi_simple is ( + cfi_direct and cfi_full_mediation and cfi_exogeneity + ) + assert assessments == 26_064 + + +def test_all_four_node_role_evidence_matches_path_oracle(): + assessments = 0 + for nodes, edges in _all_four_node_dags(): + graph = CausalDAG(nodes, tuple(reversed(edges)), tuple(reversed(nodes))) + for treatment, outcome in itertools.permutations(nodes, 2): + for node in nodes: + if node in {treatment, outcome}: + continue + assessments += 1 + expected = _oracle_role(nodes, edges, treatment, outcome, node) + actual = causal_role_evidence(graph, treatment, outcome, node) + for field, value in expected.items(): + assert getattr(actual, field) == value + assert assessments == 13_032 + + +def test_complete_path_evidence_fails_closed_at_explicit_budget(): + graph = _dag((("X", "A"), ("A", "Y"), ("X", "B"), ("B", "Y"))) + assert len(d_separation(graph, ("X",), ("Y",), max_paths=2).paths) == 2 + with pytest.raises(ValueError, match="max_paths"): + d_separation(graph, ("X",), ("Y",), max_paths=1) + with pytest.raises(ValueError, match="max_paths"): + causal_role_evidence(graph, "X", "Y", "A", max_paths=1) + + +def test_path_state_budget_bounds_dense_dead_end_search(): + lobe = tuple(f"A{index:02d}" for index in range(6)) + edges = [("X", "Y")] + edges.extend(("X", node) for node in lobe) + edges.extend( + (left, right) for index, left in enumerate(lobe) for right in lobe[index + 1 :] + ) + graph = _dag(tuple(edges)) + with pytest.raises(ValueError, match="max_path_states"): + d_separation( + graph, + ("X",), + ("Y",), + max_paths=1, + max_path_states=50, + ) + evidence = check_frontdoor_adjustment_set( + graph, + "X", + "Y", + (), + max_paths=1, + max_path_states=2, + ) + assert not evidence.graphically_admissible + + +def test_minimal_search_budget_fails_closed_without_partial_results(): + graph = _dag((("Y", "X"),), extra=("A", "B")) + with pytest.raises(ValueError, match="max_candidates"): + minimal_backdoor_adjustment_sets(graph, "X", "Y", max_candidates=1) + + frontdoor_graph = _dag((("X", "Y"),), extra=("A", "B")) + with pytest.raises(ValueError, match="max_candidates"): + minimal_frontdoor_adjustment_sets(frontdoor_graph, "X", "Y", max_candidates=1) + + +def test_large_candidate_space_hits_budget_without_recursion_failure(): + irrelevant = tuple(f"N{index:04d}" for index in range(1_100)) + graph = _dag((("Y", "X"),), extra=irrelevant) + with pytest.raises(ValueError, match="max_candidates"): + minimal_backdoor_adjustment_sets(graph, "X", "Y", max_candidates=2_000) + + +def test_minimal_search_prunes_supersets_of_many_singleton_solutions(): + chain = tuple(f"A{index:02d}" for index in range(20)) + edges = [(chain[index], chain[index - 1]) for index in range(1, 20)] + edges.extend(((chain[0], "X"), (chain[-1], "Y"))) + graph = _dag(tuple(edges)) + assert minimal_backdoor_adjustment_sets( + graph, "X", "Y", max_candidates=100 + ) == tuple((node,) for node in chain) + + +def test_empty_backdoor_set_short_circuits_candidate_search_exactly(): + irrelevant = tuple(f"N{index:02d}" for index in range(20)) + graph = _dag((("X", "Y"),), extra=irrelevant) + assert minimal_backdoor_adjustment_sets(graph, "X", "Y", max_candidates=1) == ((),) + + +@pytest.mark.parametrize("value,error", [(True, TypeError), (0, ValueError)]) +def test_resource_budgets_require_positive_integers_or_none(value, error): + graph = _dag((("X", "Y"),), extra=("Z",)) + with pytest.raises(error): + d_separation(graph, ("X",), ("Y",), max_paths=value) + with pytest.raises(error): + d_separation(graph, ("X",), ("Y",), max_path_states=value) + with pytest.raises(error): + minimal_backdoor_adjustment_sets(graph, "X", "Y", max_candidates=value) + + +def test_protocol_stage_two_graph_tuples_construct_the_same_dag(): + from RiskLabAI.causal_factor_analysis.protocol import CausalDiscoveryStage + + stage = CausalDiscoveryStage( + method_labels=("PC",), + graph_id="resolved-dag-v1", + graph_nodes=("Y", "X", "U"), + directed_edges=(("U", "Y"), ("X", "Y"), ("U", "X")), + graph_kind="DAG", + ambiguous_edges=(), + assumptions=("Graph accepted after external causal review.",), + ) + graph = CausalDAG( + stage.graph_nodes, + stage.directed_edges, + stage.graph_nodes, + ) + assert graph.nodes == ("U", "X", "Y") + assert minimal_backdoor_adjustment_sets(graph, "X", "Y") == (("U",),) + + +def test_public_records_and_exports_are_exact_and_frozen(): + import RiskLabAI.causal_factor_analysis.graph_identification as module + + assert set(module.__all__) == PUBLIC_NAMES + graph = _dag((("X", "Y"),), extra=("Z",)) + records = ( + d_separation(graph, ("X",), ("Y",)), + check_backdoor_adjustment_set(graph, "X", "Y"), + check_frontdoor_adjustment_set(graph, "X", "Y", ("Z",)), + check_instrument(_dag((("Z", "X"), ("X", "Y"))), "Z", "X", "Y"), + causal_role_evidence(_dag((("X", "Z"), ("Z", "Y"))), "X", "Y", "Z"), + ) + assert isinstance(records[0], DSeparationEvidence) + assert isinstance(records[0].paths[0], PathEvidence) + assert isinstance(records[1], BackdoorAdjustmentEvidence) + assert isinstance(records[2], FrontdoorAdjustmentEvidence) + assert isinstance(records[3], InstrumentEvidence) + assert isinstance(records[4], NodeRoleEvidence) + for record in records: + assert not hasattr(record, "__dict__") + with pytest.raises(AttributeError): + records[0].separated = False + + +def test_module_is_python39_and_standard_library_only(): + module_path = ( + Path(__file__).parents[2] + / "src" + / "RiskLabAI" + / "causal_factor_analysis" + / "graph_identification.py" + ) + source = module_path.read_text(encoding="utf-8") + tree = ast.parse(source, feature_version=(3, 9)) + imported_roots = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported_roots.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported_roots.add(node.module.split(".")[0]) + assert imported_roots <= { + "__future__", + "collections", + "dataclasses", + "itertools", + "typing", + } + + +def test_package_import_does_not_load_legacy_or_private_namespaces(): + project = Path(__file__).parents[2] + public_names = repr(sorted(PUBLIC_NAMES)) + program = f""" +import sys +import RiskLabAI.causal_factor_analysis as package +import RiskLabAI.causal_factor_analysis.graph_identification as module +expected = set({public_names}) +assert expected <= set(package.__all__) +assert expected == set(module.__all__) +blocked = ( + 'RiskLabAI.causal', + 'RiskLabAI.backtest', + 'RiskLabAI.optimization', + 'RiskLabAI.risk_control', +) +assert not any(name == root or name.startswith(root + '.') for name in sys.modules for root in blocked) +""" + environment = os.environ.copy() + environment["PYTHONDONTWRITEBYTECODE"] = "1" + subprocess.run( + [sys.executable, "-B", "-c", program], + cwd=project, + env=environment, + check=True, + capture_output=True, + text=True, + ) diff --git a/test/causal_factor_analysis/test_minimum_variance.py b/test/causal_factor_analysis/test_minimum_variance.py new file mode 100644 index 0000000..f7dc3cf --- /dev/null +++ b/test/causal_factor_analysis/test_minimum_variance.py @@ -0,0 +1,425 @@ +"""Source-locked tests for target-exposure minimum-variance allocation.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest + +from RiskLabAI.causal_factor_analysis import minimum_variance_factor_weights + + +def test_eq4_solution_satisfies_exposure_and_kkt_conditions(): + covariance = np.diag([1.0, 2.0, 4.0]) + exposures = np.array( + [ + [1.0, 0.0], + [0.0, 1.0], + [1.0, 1.0], + ] + ) + targets = np.array([0.0, 1.0]) + + weights = minimum_variance_factor_weights(covariance, exposures, targets) + + expected_weights = np.array([-2.0, 5.0, 2.0]) / 7.0 + expected_lagrange = np.array([4.0, -20.0]) / 7.0 + np.testing.assert_allclose(weights, expected_weights, rtol=0.0, atol=1e-14) + np.testing.assert_allclose(exposures.T @ weights, targets, rtol=0.0, atol=1e-14) + np.testing.assert_allclose( + 2.0 * covariance @ weights + exposures @ expected_lagrange, + np.zeros(3), + rtol=0.0, + atol=1e-14, + ) + assert weights @ covariance @ weights == pytest.approx( + 10.0 / 7.0, rel=1e-14, abs=1e-14 + ) + + +def test_eq4_solution_is_strict_minimum_over_feasible_perturbations(): + covariance = np.diag([1.0, 2.0, 4.0]) + exposures = np.array( + [ + [1.0, 0.0], + [0.0, 1.0], + [1.0, 1.0], + ] + ) + targets = np.array([0.0, 1.0]) + weights = minimum_variance_factor_weights(covariance, exposures, targets) + null_direction = np.array([-1.0, -1.0, 1.0]) + np.testing.assert_allclose( + exposures.T @ null_direction, np.zeros(2), rtol=0.0, atol=0.0 + ) + optimum = weights @ covariance @ weights + + for scale in (-2.0, -0.5, 0.5, 2.0): + candidate = weights + scale * null_direction + np.testing.assert_allclose( + exposures.T @ candidate, targets, rtol=0.0, atol=1e-14 + ) + assert candidate @ covariance @ candidate > optimum + + +def test_eq5_square_solution_is_covariance_independent(): + exposures = np.array([[1.0, 1.0], [1.5, 1.0]]) + targets = np.array([0.0, 1.0]) + expected = np.linalg.solve(exposures.T, targets) + covariance_matrices = ( + np.array([[1.0, 0.2], [0.2, 2.0]]), + np.array([[5.0, -1.0], [-1.0, 0.5]]), + ) + + np.testing.assert_allclose(expected, np.array([3.0, -2.0]), rtol=0.0, atol=1e-14) + for covariance in covariance_matrices: + weights = minimum_variance_factor_weights(covariance, exposures, targets) + np.testing.assert_allclose(weights, expected, rtol=0.0, atol=1e-13) + + +def test_eq6_single_factor_solution(): + covariance = np.array([[2.0, 0.5], [0.5, 1.0]]) + exposures = np.array([[1.0], [1.0]]) + targets = np.array([1.0]) + + weights = minimum_variance_factor_weights(covariance, exposures, targets) + + np.testing.assert_allclose(weights, np.array([0.25, 0.75]), rtol=0.0, atol=1e-14) + np.testing.assert_allclose(exposures.T @ weights, targets, rtol=0.0, atol=1e-14) + assert weights @ covariance @ weights == pytest.approx( + 7.0 / 8.0, rel=1e-14, abs=1e-14 + ) + + +def test_one_asset_one_factor_solution(): + weights = minimum_variance_factor_weights( + np.array([[2.5]]), np.array([[-4.0]]), np.array([3.0]) + ) + np.testing.assert_allclose(weights, np.array([-0.75]), rtol=0.0, atol=0.0) + + +def test_near_collinear_square_constraints_remain_feasible(): + covariance = np.eye(2) + exposures = np.array([[1.0, 1.0], [1.0, 1.0 + 1e-9]]) + targets = np.array([1.0, 0.0]) + + weights = minimum_variance_factor_weights(covariance, exposures, targets) + direct_solution = np.linalg.solve(exposures.T, targets) + + np.testing.assert_allclose(weights, direct_solution, rtol=5e-8, atol=0.0) + np.testing.assert_allclose(exposures.T @ weights, targets, rtol=0.0, atol=1e-12) + + +def test_extreme_finite_exposure_scale_does_not_overflow(): + covariance = np.array([[1.0]]) + exposures = np.array([[1e308]]) + targets = np.array([1.0]) + + weights = minimum_variance_factor_weights(covariance, exposures, targets) + + np.testing.assert_allclose(weights, np.array([1e-308]), rtol=1e-15, atol=0.0) + np.testing.assert_allclose(exposures.T @ weights, targets, rtol=1e-15, atol=0.0) + + +def test_extreme_factor_unit_differences_do_not_create_false_rank_deficiency(): + covariance = np.eye(2) + exposures = np.diag([1e-200, 1e200]) + targets = np.array([1e-200, 1e200]) + + weights = minimum_variance_factor_weights(covariance, exposures, targets) + + np.testing.assert_allclose(weights, np.ones(2), rtol=0.0, atol=0.0) + np.testing.assert_allclose(exposures.T @ weights, targets, rtol=1e-15, atol=0.0) + + +def test_feasible_target_normalization_does_not_overflow(): + covariance = np.eye(2) + exposures = np.array([[1e-308], [1e-308]]) + targets = np.array([2.0]) + + weights = minimum_variance_factor_weights(covariance, exposures, targets) + + np.testing.assert_allclose(weights, np.array([1e308, 1e308]), rtol=1e-15, atol=0.0) + np.testing.assert_allclose(exposures.T @ weights, targets, rtol=1e-15, atol=0.0) + + +def test_unrepresentable_nonzero_scaled_target_is_rejected(): + smallest_positive = np.nextafter(0.0, 1.0) + with pytest.raises(ValueError, match="represented.*numerical precision"): + minimum_variance_factor_weights( + np.array([[1.0]]), + np.array([[1e308]]), + np.array([smallest_positive]), + ) + + +def test_representable_tiny_target_is_not_treated_as_zero(): + weights = minimum_variance_factor_weights( + np.array([[1.0]]), np.array([[1.0]]), np.array([1e-20]) + ) + np.testing.assert_allclose(weights, np.array([1e-20]), rtol=1e-15, atol=0.0) + + +@pytest.mark.parametrize("factor_scale", [1.0, 1e20]) +def test_numerically_unresolved_constraints_are_rejected_independently_of_units( + factor_scale, +): + exposures = factor_scale * np.array([[1.0, 1.0], [1.0, 1.0 + 1e-9]]) + targets = factor_scale * np.array([1e-20, 0.0]) + + with pytest.raises(ValueError, match="resolved.*numerical precision"): + minimum_variance_factor_weights(np.eye(2), exposures, targets) + + +def test_extreme_finite_covariance_scale_does_not_overflow_when_symmetrized(): + weights = minimum_variance_factor_weights( + np.array([[1e308]]), np.array([[1.0]]), np.array([1.0]) + ) + np.testing.assert_allclose(weights, np.array([1.0]), rtol=0.0, atol=0.0) + + +def test_smallest_positive_covariance_is_normalized_before_symmetrization(): + smallest_positive = np.nextafter(0.0, 1.0) + weights = minimum_variance_factor_weights( + np.array([[smallest_positive]]), np.array([[1.0]]), np.array([1.0]) + ) + np.testing.assert_allclose(weights, np.array([1.0]), rtol=0.0, atol=0.0) + + +def test_zero_targets_return_zero_weights(): + covariance = np.diag([1.0, 2.0, 3.0]) + exposures = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]) + weights = minimum_variance_factor_weights(covariance, exposures, np.zeros(2)) + np.testing.assert_array_equal(weights, np.zeros(3)) + + +@pytest.mark.parametrize("covariance_scale", [1e-12, 1.0, 1e12]) +def test_covariance_scale_does_not_change_weights(covariance_scale): + covariance = covariance_scale * np.array( + [[2.0, 0.25, 0.1], [0.25, 1.5, -0.2], [0.1, -0.2, 1.0]] + ) + exposures = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]) + targets = np.array([0.2, -0.4]) + + baseline = minimum_variance_factor_weights( + covariance / covariance_scale, exposures, targets + ) + scaled = minimum_variance_factor_weights(covariance, exposures, targets) + np.testing.assert_allclose(scaled, baseline, rtol=1e-12, atol=1e-12) + + +@pytest.mark.parametrize("target_scale", [-3.0, 0.25, 4.0]) +def test_weights_are_homogeneous_in_targets(target_scale): + covariance = np.diag([1.0, 2.0, 4.0]) + exposures = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]) + targets = np.array([0.5, -0.25]) + baseline = minimum_variance_factor_weights(covariance, exposures, targets) + + scaled = minimum_variance_factor_weights( + covariance, exposures, target_scale * targets + ) + np.testing.assert_allclose(scaled, target_scale * baseline, rtol=1e-12, atol=1e-13) + + +def test_factor_unit_changes_leave_weights_unchanged(): + covariance = np.diag([1.0, 2.0, 4.0]) + exposures = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]) + targets = np.array([0.5, -0.25]) + factor_scales = np.array([100.0, 0.01]) + baseline = minimum_variance_factor_weights(covariance, exposures, targets) + + rescaled = minimum_variance_factor_weights( + covariance, + exposures * factor_scales, + targets * factor_scales, + ) + np.testing.assert_allclose(rescaled, baseline, rtol=1e-12, atol=1e-13) + + +def test_asset_and_factor_permutations_are_equivariant(): + covariance = np.array([[2.0, 0.2, 0.1], [0.2, 1.5, -0.1], [0.1, -0.1, 1.0]]) + exposures = np.array([[1.0, 0.0], [0.2, 1.0], [1.0, -0.5]]) + targets = np.array([0.3, -0.2]) + baseline = minimum_variance_factor_weights(covariance, exposures, targets) + + asset_order = np.array([2, 0, 1]) + factor_order = np.array([1, 0]) + permuted = minimum_variance_factor_weights( + covariance[np.ix_(asset_order, asset_order)], + exposures[asset_order][:, factor_order], + targets[factor_order], + ) + restored = np.empty_like(permuted) + restored[asset_order] = permuted + np.testing.assert_allclose(restored, baseline, rtol=1e-12, atol=1e-13) + + +@pytest.mark.parametrize("covariance_scale", [1e-200, 1.0, 1e200]) +def test_tiny_covariance_asymmetry_is_symmetrized(covariance_scale): + covariance = covariance_scale * np.array([[2.0, 0.3 + 5e-13], [0.3 - 5e-13, 1.0]]) + exposures = np.array([[1.0], [1.0]]) + targets = np.array([1.0]) + symmetric = (covariance + covariance.T) / 2.0 + + weights = minimum_variance_factor_weights(covariance, exposures, targets) + expected = minimum_variance_factor_weights(symmetric, exposures, targets) + np.testing.assert_allclose(weights, expected, rtol=0.0, atol=1e-14) + + +@pytest.mark.parametrize("covariance_scale", [1e-200, 1.0, 1e200]) +def test_material_covariance_asymmetry_is_rejected(covariance_scale): + covariance = covariance_scale * np.array([[2.0, 0.3 + 1e-5], [0.3 - 1e-5, 1.0]]) + with pytest.raises(ValueError, match="symmetric"): + minimum_variance_factor_weights(covariance, np.ones((2, 1)), np.ones(1)) + + +def test_asymmetry_is_measured_relative_to_covariance_scale(): + covariance = np.array([[1e-20, 4e-13], [-4e-13, 1e-20]]) + with pytest.raises(ValueError, match="symmetric"): + minimum_variance_factor_weights(covariance, np.ones((2, 1)), np.ones(1)) + + +@pytest.mark.parametrize( + "covariance", + [ + np.array([[1.0, 1.0], [1.0, 1.0]]), + np.array([[1.0, 2.0], [2.0, 1.0]]), + ], +) +def test_non_positive_definite_covariance_is_rejected(covariance): + with pytest.raises(ValueError, match="positive definite"): + minimum_variance_factor_weights(covariance, np.ones((2, 1)), np.ones(1)) + + +def test_rank_deficient_exposures_are_rejected(): + exposures = np.array([[1.0, 2.0], [2.0, 4.0], [3.0, 6.0]]) + with pytest.raises(ValueError, match="full column rank"): + minimum_variance_factor_weights(np.eye(3), exposures, np.array([1.0, 2.0])) + + +@pytest.mark.parametrize( + "covariance, exposures, targets, match", + [ + (np.ones(2), np.ones((2, 1)), np.ones(1), "2-D square"), + (np.ones((2, 3)), np.ones((2, 1)), np.ones(1), "square"), + (np.empty((0, 0)), np.empty((0, 1)), np.ones(1), "non-empty square"), + (np.eye(2), np.ones(2), np.ones(1), "2-D"), + (np.eye(2), np.ones((3, 1)), np.ones(1), "rows"), + (np.eye(2), np.empty((2, 0)), np.empty(0), "at least one"), + (np.eye(2), np.ones((2, 3)), np.ones(3), "more columns than rows"), + (np.eye(2), np.ones((2, 1)), np.array([[1.0]]), "1-D"), + (np.eye(2), np.ones((2, 1)), np.ones(2), "length"), + ], +) +def test_invalid_shapes_are_rejected(covariance, exposures, targets, match): + with pytest.raises(ValueError, match=match): + minimum_variance_factor_weights(covariance, exposures, targets) + + +@pytest.mark.parametrize("input_name", ["covariance", "exposures", "targets"]) +def test_nonfinite_inputs_are_rejected(input_name): + covariance = np.eye(2) + exposures = np.ones((2, 1)) + targets = np.ones(1) + if input_name == "covariance": + covariance[0, 0] = np.nan + elif input_name == "exposures": + exposures[0, 0] = np.inf + else: + targets[0] = -np.inf + + with pytest.raises(ValueError, match="finite"): + minimum_variance_factor_weights(covariance, exposures, targets) + + +def test_values_outside_float64_range_raise_stable_value_error(): + with pytest.raises(ValueError, match="numeric array"): + minimum_variance_factor_weights([[10**400]], [[1.0]], [1.0]) + + +@pytest.mark.parametrize("input_name", ["covariance", "exposures", "targets"]) +def test_complex_inputs_are_rejected(input_name): + covariance = np.eye(2).astype(complex) + exposures = np.ones((2, 1), dtype=complex) + targets = np.ones(1, dtype=complex) + if input_name != "covariance": + covariance = covariance.real + if input_name != "exposures": + exposures = exposures.real + if input_name != "targets": + targets = targets.real + + with pytest.raises(ValueError, match="real-valued"): + minimum_variance_factor_weights(covariance, exposures, targets) + + +def test_inputs_are_not_mutated_and_output_is_fresh_float64(): + covariance = np.array([[2, 1], [1, 3]], dtype=np.int64) + exposures = np.array([[1], [2]], dtype=np.int64) + targets = np.array([1], dtype=np.int64) + originals = (covariance.copy(), exposures.copy(), targets.copy()) + + first = minimum_variance_factor_weights(covariance, exposures, targets) + second = minimum_variance_factor_weights(covariance, exposures, targets) + + for actual, original in zip((covariance, exposures, targets), originals): + np.testing.assert_array_equal(actual, original) + assert first.dtype == np.float64 + assert first.shape == (2,) + assert np.all(np.isfinite(first)) + assert not np.shares_memory(first, second) + + +def test_explicit_matrix_inverse_is_not_used(monkeypatch): + def fail_if_called(*_args, **_kwargs): + raise AssertionError("explicit matrix inverse must not be used") + + monkeypatch.setattr(np.linalg, "inv", fail_if_called) + weights = minimum_variance_factor_weights(np.eye(2), np.ones((2, 1)), np.ones(1)) + np.testing.assert_allclose(weights, np.array([0.5, 0.5]), rtol=0.0, atol=1e-14) + + +def test_final_factor_exposure_feasibility_is_checked(monkeypatch): + original_qr = np.linalg.qr + + def zero_orthonormal_basis(*args, **kwargs): + basis, triangular_factor = original_qr(*args, **kwargs) + return np.zeros_like(basis), triangular_factor + + monkeypatch.setattr(np.linalg, "qr", zero_orthonormal_basis) + with pytest.raises(ValueError, match="constraints.*numerical precision"): + minimum_variance_factor_weights(np.eye(2), np.ones((2, 1)), np.ones(1)) + + +def test_public_import_is_isolated_from_other_model_namespaces(): + repository_root = Path(__file__).resolve().parents[2] + program = """ +import sys +import RiskLabAI + +if 'causal_factor_analysis' not in RiskLabAI.__all__: + raise SystemExit('root public surface omits causal_factor_analysis') +module = RiskLabAI.causal_factor_analysis +if not hasattr(module, 'minimum_variance_factor_weights'): + raise SystemExit('root lazy import did not resolve the optimizer') + +blocked = { + 'RiskLabAI.causal', + 'RiskLabAI.optimization', + 'RiskLabAI.backtest', + 'RiskLabAI.risk_control', +} +loaded = blocked.intersection(sys.modules) +if loaded: + raise SystemExit(f'unexpected imports: {sorted(loaded)}') +""" + subprocess.run( + [sys.executable, "-c", program], + cwd=repository_root, + check=True, + capture_output=True, + text=True, + ) diff --git a/test/causal_factor_analysis/test_protocol.py b/test/causal_factor_analysis/test_protocol.py new file mode 100644 index 0000000..183420e --- /dev/null +++ b/test/causal_factor_analysis/test_protocol.py @@ -0,0 +1,1173 @@ +"""Source-locked tests for the seven-stage causal-factor protocol.""" + +from __future__ import annotations + +import math +import subprocess +import sys +from dataclasses import FrozenInstanceError, replace +from pathlib import Path + +import pytest + +from RiskLabAI.causal_factor_analysis.protocol import ( + BacktestStage, + CausalAdjustmentSetStage, + CausalDiscoveryStage, + CausalExplanatoryAndPredictivePowerStage, + CausalFactorProtocolReport, + CausalPortfolioConstructionStage, + EventHorizon, + FoldEvidence, + MultipleTestingAdjustmentsStage, + ValidationEvidence, + VariableSelectionStage, + validate_causal_factor_protocol, +) + +PUBLIC_NAMES = { + "BacktestStage", + "CausalAdjustmentSetStage", + "CausalDiscoveryStage", + "CausalExplanatoryAndPredictivePowerStage", + "CausalFactorProtocolReport", + "CausalPortfolioConstructionStage", + "EventHorizon", + "FoldEvidence", + "MultipleTestingAdjustmentsStage", + "ValidationEvidence", + "VariableSelectionStage", + "validate_causal_factor_protocol", +} + + +def _horizons(): + return ( + EventHorizon("o1", 0.0, 2.0), + EventHorizon("o2", 1.0, 3.0), + EventHorizon("o3", 4.0, 5.0), + EventHorizon("o4", 6.0, 7.0), + EventHorizon("o5", 8.0, 9.0), + EventHorizon("o6", 10.0, 11.0), + ) + + +def _fold(fold_id, path_id, train_ids, test_ids, refit_id): + return FoldEvidence( + fold_id=fold_id, + path_id=path_id, + train_ids=train_ids, + test_ids=test_ids, + fit_ids=train_ids, + prediction_ids=test_ids, + refit_id=refit_id, + refit_components=( + "VARIABLE_SELECTION", + "CAUSAL_ESTIMATION", + "PORTFOLIO_CONSTRUCTION", + ), + ) + + +def _validation(method_labels=("PURGED_CROSS_VALIDATION",)): + return ValidationEvidence( + method_labels=method_labels, + event_horizons=_horizons(), + folds=( + _fold("f1", "p1", ("o3", "o4", "o5", "o6"), ("o1", "o2"), "r1"), + _fold("f2", "p1", ("o1", "o2", "o5", "o6"), ("o3", "o4"), "r2"), + _fold("f3", "p1", ("o1", "o2", "o3", "o4"), ("o5", "o6"), "r3"), + ), + embargo=1.0, + ) + + +def _cpcv_validation(): + return ValidationEvidence( + method_labels=("COMBINATORIAL_PURGED_CROSS_VALIDATION",), + event_horizons=_horizons(), + folds=( + _fold("p1-f1", "p1", ("o3", "o4", "o5", "o6"), ("o1", "o2"), "p1-r1"), + _fold("p1-f2", "p1", ("o1", "o2", "o5", "o6"), ("o3", "o4"), "p1-r2"), + _fold("p1-f3", "p1", ("o1", "o2", "o3", "o4"), ("o5", "o6"), "p1-r3"), + _fold("p2-f1", "p2", ("o4", "o5", "o6"), ("o1", "o3"), "p2-r1"), + _fold("p2-f2", "p2", ("o3", "o4", "o6"), ("o2", "o5"), "p2-r2"), + _fold("p2-f3", "p2", ("o1", "o2", "o3"), ("o4", "o6"), "p2-r3"), + ), + embargo=1.0, + ) + + +def _valid_report(): + trial_ids = ("strategy-a", "strategy-b", "strategy-c") + family_id = "preregistered-family" + selection_validation = _validation() + evaluation_validation = _validation() + backtest_validation = _cpcv_validation() + + return CausalFactorProtocolReport( + trial_family_id=family_id, + declared_trial_ids=trial_ids, + variable_selection=VariableSelectionStage( + purpose="RISK_PREMIA_HARVESTING", + selected_variables=("X", "Z", "I", "M", "C", "D"), + method_labels=( + "MUTUAL_INFORMATION", + "SHAPLEY_VALUES", + "MEAN_DECREASE_IMPURITY", + "PERMUTATION_FEATURE_IMPORTANCE", + ), + overlapping_returns=True, + strong_time_dependence=True, + validation=selection_validation, + ), + causal_discovery=CausalDiscoveryStage( + method_labels=( + "PC", + "ECONOMIC_REASONING", + "DOMAIN_EXPERTISE", + "OBSERVED_OUTCOMES", + ), + graph_id="resolved-dag", + graph_nodes=("X", "Y", "Z", "I", "M", "C", "D"), + directed_edges=( + ("Z", "X"), + ("Z", "Y"), + ("I", "X"), + ("X", "M"), + ("M", "Y"), + ("X", "Y"), + ("X", "C"), + ("Y", "C"), + ("X", "D"), + ), + graph_kind="DAG", + ambiguous_edges=(), + assumptions=( + "The recorded variables are causally sufficient for the target effect.", + "Edge directions use temporal and economic restrictions.", + ), + ), + causal_adjustment_set=CausalAdjustmentSetStage( + treatment="X", + outcome="Y", + method_label="BACKDOOR_ADJUSTMENT", + identified=True, + admissible_adjustment_sets=(("Z",),), + selected_adjustment_set=("Z",), + confounders=("Z",), + descendants=("M", "C", "D"), + mediators=("M",), + colliders=("C",), + instruments=("I",), + control_justifications=(("Z", "Common cause of factor and return."),), + open_backdoor_paths=(), + ), + causal_explanatory_and_predictive_power=( + CausalExplanatoryAndPredictivePowerStage( + task_types=("RETURN_SIZE",), + estimator_label="fold-refit causal regression", + explanatory_metric_labels=("R_SQUARED",), + predictive_metric_labels=( + "MEAN_SQUARED_ERROR", + "SPEARMAN_CORRELATION", + ), + explanatory_evidence_id="explanatory-evidence", + predictive_evidence_id="predictive-evidence", + naive_benchmark_id="training-mean-benchmark", + validation=evaluation_validation, + ) + ), + causal_portfolio_construction=CausalPortfolioConstructionStage( + method_labels=( + "POSITION_SIZING", + "EXPOSURE_CONTROL", + "ECONOMIC_RATIONALE", + "FRAGILITY_STRESS_TEST", + "TRANSACTION_COST_OPTIMIZATION", + "TRANSFER_COEFFICIENT", + ), + causal_exposures=("X",), + controlled_unintended_exposures=("C", "D"), + cost_model_id="cost-model-v1", + constraint_set_id="constraints-v1", + economic_rationale="Positions target the identified X-to-Y effect.", + fragility_scenarios=("weaken X-to-Y", "perturb Z-to-X"), + transfer_coefficient=0.82, + ), + backtest=BacktestStage( + method_labels=("COMBINATORIAL_PURGED_CROSS_VALIDATION",), + trial_family_id=family_id, + declared_trial_ids=trial_ids, + validation=backtest_validation, + ), + multiple_testing_adjustments=MultipleTestingAdjustmentsStage( + method_labels=("HOLM", "DEFLATED_SHARPE_RATIO"), + trial_family_id=family_id, + declared_trial_ids=trial_ids, + family_partitions=( + ("primary", ("strategy-a", "strategy-b")), + ("robustness", ("strategy-c",)), + ), + alpha=0.05, + backtests_independent=False, + p_value_estimator="dependence-adjusted-estimator", + time_dependence_model="purged temporal folds", + p_value_inputs_assume_independence=False, + sharpe_variance=0.04, + effective_trials=2.0, + sample_length=252, + skewness=0.1, + kurtosis=3.2, + selection_bias_evidence_id="selection-bias-evidence", + ), + ) + + +def _validate_with(report, **changes): + return validate_causal_factor_protocol(replace(report, **changes)) + + +def test_protocol_01_success_has_exact_stage_order_and_provenance(): + report = validate_causal_factor_protocol(_valid_report()) + + assert report.stage_names == ( + "Variable Selection", + "Causal Discovery", + "Causal Adjustment Set", + "Causal Explanatory and Predictive Power", + "Causal Portfolio Construction", + "Backtest", + "Multiple Testing Adjustments", + ) + assert len(report.stages) == 7 + assert report.source_oracle == "PROTOCOL-01" + assert report.source_locator == ( + "PDF pages 18-22 (journal pages 27-31), Exhibit 10" + ) + + +def test_valid_records_are_frozen_and_sequence_inputs_are_tuple_backed(): + stage = VariableSelectionStage( + purpose="CAUSAL_ATTRIBUTION", + selected_variables=["X"], + method_labels=["MUTUAL_INFORMATION"], + overlapping_returns=False, + strong_time_dependence=False, + validation=_validation(), + ) + assert stage.selected_variables == ("X",) + assert stage.method_labels == ("MUTUAL_INFORMATION",) + with pytest.raises(FrozenInstanceError): + stage.purpose = "RISK_PREMIA_HARVESTING" + + +def test_wrong_stage_record_type_is_rejected(): + report = _valid_report() + with pytest.raises(ValueError, match="Stage 2"): + _validate_with(report, causal_discovery=report.variable_selection) + + +def test_boundary_touching_horizons_are_not_overlapping(): + report = _valid_report() + horizons = tuple( + EventHorizon(f"b{index}", float(index), float(index + 1)) + for index in range(6) + ) + validation = replace( + report.variable_selection.validation, + event_horizons=horizons, + folds=( + _fold( + "b1", "bp", ("b2", "b3", "b4", "b5"), ("b0", "b1"), "br1" + ), + _fold("b2", "bp", ("b0", "b1", "b4", "b5"), ("b2", "b3"), "br2"), + _fold("b3", "bp", ("b0", "b1", "b2", "b3"), ("b4", "b5"), "br3"), + ), + embargo=0.0, + ) + selection = replace( + report.variable_selection, + overlapping_returns=False, + strong_time_dependence=False, + validation=validation, + ) + evaluation = replace( + report.causal_explanatory_and_predictive_power, + validation=validation, + ) + validate_causal_factor_protocol( + replace( + report, + variable_selection=selection, + causal_explanatory_and_predictive_power=evaluation, + ) + ) + + +def test_overlap_declaration_and_purging_are_required(): + report = _valid_report() + selection = replace(report.variable_selection, overlapping_returns=False) + with pytest.raises(ValueError, match="Stage 1"): + _validate_with(report, variable_selection=selection) + + +def test_cpcv_is_a_source_supported_purged_validation_method(): + report = _valid_report() + cpcv = _cpcv_validation() + selection = replace(report.variable_selection, validation=cpcv) + evaluation = replace( + report.causal_explanatory_and_predictive_power, + validation=cpcv, + ) + validate_causal_factor_protocol( + replace( + report, + variable_selection=selection, + causal_explanatory_and_predictive_power=evaluation, + ) + ) + + validation = replace( + report.variable_selection.validation, + method_labels=("WALK_FORWARD",), + ) + selection = replace(report.variable_selection, validation=validation) + with pytest.raises(ValueError, match="Stage 1"): + _validate_with(report, variable_selection=selection) + + +def test_strong_time_dependence_requires_positive_embargo(): + report = _valid_report() + validation = replace(report.variable_selection.validation, embargo=0.0) + selection = replace(report.variable_selection, validation=validation) + with pytest.raises(ValueError, match="Stage 1"): + _validate_with(report, variable_selection=selection) + + +def test_in_sample_stage1_methods_do_not_require_temporal_folds_without_guards(): + report = _valid_report() + selection = replace( + report.variable_selection, + method_labels=("MUTUAL_INFORMATION", "SHAPLEY_VALUES"), + overlapping_returns=False, + strong_time_dependence=False, + validation=None, + ) + validate_causal_factor_protocol( + replace(report, variable_selection=selection) + ) + + with pytest.raises(ValueError, match="Stage 1"): + validate_causal_factor_protocol( + replace( + report, + variable_selection=replace( + selection, + method_labels=("PERMUTATION_FEATURE_IMPORTANCE",), + ), + ) + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("train_ids", ("o3", "o4", "missing")), + ("test_ids", ("o1", "o3")), + ("fit_ids", ("o1",)), + ("fit_ids", ("o3",)), + ("prediction_ids", ("o1",)), + ("refit_id", ""), + ("refit_components", ("CAUSAL_ESTIMATION",)), + ], +) +def test_fold_membership_and_training_only_refit_evidence(field, value): + report = _valid_report() + validation = report.causal_explanatory_and_predictive_power.validation + bad_fold = replace(validation.folds[0], **{field: value}) + bad_validation = replace(validation, folds=(bad_fold,) + validation.folds[1:]) + stage = replace( + report.causal_explanatory_and_predictive_power, + validation=bad_validation, + ) + with pytest.raises(ValueError, match="Stage 4"): + _validate_with(report, causal_explanatory_and_predictive_power=stage) + + +def test_refit_id_cannot_represent_different_training_sets(): + report = _valid_report() + validation = report.causal_explanatory_and_predictive_power.validation + second = replace(validation.folds[1], refit_id=validation.folds[0].refit_id) + stage = replace( + report.causal_explanatory_and_predictive_power, + validation=replace(validation, folds=(validation.folds[0], second, validation.folds[2])), + ) + with pytest.raises(ValueError, match="Stage 4"): + _validate_with(report, causal_explanatory_and_predictive_power=stage) + + +def test_temporal_overlap_and_embargo_leakage_are_rejected(): + report = _valid_report() + validation = report.causal_explanatory_and_predictive_power.validation + overlapping_fold = _fold("bad", "p2", ("o2", "o3"), ("o1",), "bad-refit") + stage = replace( + report.causal_explanatory_and_predictive_power, + validation=replace(validation, folds=(overlapping_fold,)), + ) + with pytest.raises(ValueError, match="Stage 4"): + _validate_with(report, causal_explanatory_and_predictive_power=stage) + + horizons = tuple( + EventHorizon(item.observation_id, item.start, item.end) + for item in validation.event_horizons + ) + (EventHorizon("embargoed", 3.5, 3.8),) + embargo_fold = _fold("bad2", "p3", ("embargoed", "o3"), ("o1", "o2"), "bad2-refit") + bad_validation = replace(validation, event_horizons=horizons, folds=(embargo_fold,)) + stage = replace( + report.causal_explanatory_and_predictive_power, + validation=bad_validation, + ) + with pytest.raises(ValueError, match="Stage 4"): + _validate_with(report, causal_explanatory_and_predictive_power=stage) + + +@pytest.mark.parametrize( + "changes", + [ + {"graph_kind": "CPDAG"}, + {"ambiguous_edges": (("X", "Y"),)}, + {"directed_edges": (("X", "X"),)}, + {"directed_edges": (("X", "M"), ("M", "X"))}, + {"assumptions": ()}, + ], +) +def test_discovery_requires_explicit_resolved_acyclic_dag(changes): + report = _valid_report() + discovery = replace(report.causal_discovery, **changes) + with pytest.raises(ValueError, match="Stage 2"): + _validate_with(report, causal_discovery=discovery) + + +def test_discovery_graph_contains_every_selected_variable(): + report = _valid_report() + discovery = replace( + report.causal_discovery, + graph_nodes=tuple( + node for node in report.causal_discovery.graph_nodes if node != "D" + ), + directed_edges=tuple( + edge for edge in report.causal_discovery.directed_edges if "D" not in edge + ), + ) + with pytest.raises(ValueError, match="Stage 2"): + _validate_with(report, causal_discovery=discovery) + + +@pytest.mark.parametrize("forbidden", ["C", "M", "D"]) +def test_ordinary_controls_exclude_colliders_mediators_and_descendants(forbidden): + report = _valid_report() + adjustment = replace( + report.causal_adjustment_set, + admissible_adjustment_sets=((forbidden,),), + selected_adjustment_set=(forbidden,), + control_justifications=((forbidden, "Not admissible as an ordinary control."),), + ) + with pytest.raises(ValueError, match="Stage 3"): + _validate_with(report, causal_adjustment_set=adjustment) + + +@pytest.mark.parametrize("inadmissible", ["C", "M", "D", "I"]) +def test_every_declared_admissible_adjustment_set_is_graph_valid(inadmissible): + report = _valid_report() + adjustment = replace( + report.causal_adjustment_set, + admissible_adjustment_sets=(("Z",), (inadmissible,)), + ) + with pytest.raises(ValueError, match="Stage 3"): + _validate_with(report, causal_adjustment_set=adjustment) + + +def test_declared_descendants_must_exactly_match_the_graph(): + report = _valid_report() + adjustment = replace( + report.causal_adjustment_set, + descendants=("M", "C"), + ) + + with pytest.raises(ValueError, match="exactly match the graph"): + _validate_with(report, causal_adjustment_set=adjustment) + + +@pytest.mark.parametrize( + "changes", + [ + {"identified": False}, + {"open_backdoor_paths": ("X<-Z->Y",)}, + {"control_justifications": ()}, + {"selected_adjustment_set": ("I",)}, + ], +) +def test_backdoor_adjustment_fails_closed(changes): + report = _valid_report() + adjustment = replace(report.causal_adjustment_set, **changes) + with pytest.raises(ValueError, match="Stage 3"): + _validate_with(report, causal_adjustment_set=adjustment) + + +def test_frontdoor_mediator_is_a_mechanism_not_an_ordinary_control(): + report = _valid_report() + adjustment = replace( + report.causal_adjustment_set, + method_label="FRONT_DOOR_ADJUSTMENT", + admissible_adjustment_sets=((),), + selected_adjustment_set=(), + control_justifications=(), + frontdoor_criteria_satisfied=True, + ) + with pytest.raises(ValueError, match="Stage 3"): + validate_causal_factor_protocol( + replace(report, causal_adjustment_set=adjustment) + ) + + discovery = replace( + report.causal_discovery, + directed_edges=tuple( + edge + for edge in report.causal_discovery.directed_edges + if edge != ("X", "Y") + ), + ) + validate_causal_factor_protocol( + replace( + report, + causal_discovery=discovery, + causal_adjustment_set=adjustment, + ) + ) + with pytest.raises(ValueError, match="Stage 3"): + validate_causal_factor_protocol( + replace( + report, + causal_adjustment_set=replace( + adjustment, + admissible_adjustment_sets=(("M",),), + selected_adjustment_set=("M",), + control_justifications=(("M", "ordinary control"),), + ), + ) + ) + + +def test_instrumental_variables_requires_all_identification_premises(): + report = _valid_report() + adjustment = replace( + report.causal_adjustment_set, + method_label="INSTRUMENTAL_VARIABLES", + admissible_adjustment_sets=((),), + selected_adjustment_set=(), + control_justifications=(), + instrument_relevance_satisfied=True, + instrument_exclusion_satisfied=True, + instrument_exogeneity_satisfied=True, + ) + validate_causal_factor_protocol( + replace(report, causal_adjustment_set=adjustment) + ) + for field in ( + "instrument_relevance_satisfied", + "instrument_exclusion_satisfied", + "instrument_exogeneity_satisfied", + ): + with pytest.raises(ValueError, match="Stage 3"): + validate_causal_factor_protocol( + replace( + report, + causal_adjustment_set=replace(adjustment, **{field: False}), + ) + ) + + discovery = replace( + report.causal_discovery, + directed_edges=tuple( + edge + for edge in report.causal_discovery.directed_edges + if edge != ("I", "X") + ), + ) + with pytest.raises(ValueError, match="Stage 3"): + validate_causal_factor_protocol( + replace( + report, + causal_discovery=discovery, + causal_adjustment_set=adjustment, + ) + ) + + +def test_instrumental_variable_can_be_conditionally_exogenous(): + report = _valid_report() + discovery = replace( + report.causal_discovery, + directed_edges=( + ("Z", "I"), + ("I", "X"), + ("Z", "Y"), + ("X", "M"), + ("M", "Y"), + ("X", "Y"), + ("X", "C"), + ("Y", "C"), + ("X", "D"), + ), + ) + adjustment = replace( + report.causal_adjustment_set, + method_label="INSTRUMENTAL_VARIABLES", + instrument_relevance_satisfied=True, + instrument_exclusion_satisfied=True, + instrument_exogeneity_satisfied=True, + ) + validate_causal_factor_protocol( + replace( + report, + causal_discovery=discovery, + causal_adjustment_set=adjustment, + ) + ) + + +def test_instrument_cannot_also_be_an_ordinary_control(): + report = _valid_report() + adjustment = replace( + report.causal_adjustment_set, + method_label="INSTRUMENTAL_VARIABLES", + admissible_adjustment_sets=(("I",),), + selected_adjustment_set=("I",), + control_justifications=(("I", "Not an ordinary control."),), + instrument_relevance_satisfied=True, + instrument_exclusion_satisfied=True, + instrument_exogeneity_satisfied=True, + ) + with pytest.raises(ValueError, match="Stage 3"): + _validate_with(report, causal_adjustment_set=adjustment) + + +def test_selected_controls_cannot_block_instrument_relevance(): + report = _valid_report() + discovery = replace( + report.causal_discovery, + directed_edges=( + ("I", "Z"), + ("Z", "X"), + ("D", "X"), + ("D", "Y"), + ("X", "M"), + ("M", "Y"), + ("X", "Y"), + ("X", "C"), + ("Y", "C"), + ), + ) + adjustment = replace( + report.causal_adjustment_set, + method_label="INSTRUMENTAL_VARIABLES", + admissible_adjustment_sets=(("D", "Z"),), + selected_adjustment_set=("D", "Z"), + confounders=("D",), + descendants=("M", "C"), + control_justifications=( + ("D", "Blocks the graph-implied backdoor path."), + ("Z", "Declared conditional control."), + ), + instrument_relevance_satisfied=True, + instrument_exclusion_satisfied=True, + instrument_exogeneity_satisfied=True, + ) + with pytest.raises(ValueError, match="Stage 3"): + validate_causal_factor_protocol( + replace( + report, + causal_discovery=discovery, + causal_adjustment_set=adjustment, + ) + ) + + +def test_declared_instruments_are_graph_valid_for_every_adjustment_method(): + report = _valid_report() + adjustment = replace(report.causal_adjustment_set, instruments=("D",)) + with pytest.raises(ValueError, match="Stage 3"): + _validate_with(report, causal_adjustment_set=adjustment) + + +def test_graph_implied_confounders_cannot_be_omitted_or_used_as_instruments(): + report = _valid_report() + omitted = replace( + report.causal_adjustment_set, + admissible_adjustment_sets=((),), + selected_adjustment_set=(), + confounders=(), + control_justifications=(), + ) + with pytest.raises(ValueError, match="Stage 3"): + validate_causal_factor_protocol( + replace(report, causal_adjustment_set=omitted) + ) + + invalid_instrument = replace( + report.causal_adjustment_set, + method_label="INSTRUMENTAL_VARIABLES", + admissible_adjustment_sets=((),), + selected_adjustment_set=(), + instruments=("Z",), + control_justifications=(), + instrument_relevance_satisfied=True, + instrument_exclusion_satisfied=True, + instrument_exogeneity_satisfied=True, + ) + with pytest.raises(ValueError, match="Stage 3"): + validate_causal_factor_protocol( + replace(report, causal_adjustment_set=invalid_instrument) + ) + + +@pytest.mark.parametrize( + "field", + [ + "frontdoor_criteria_satisfied", + "instrument_relevance_satisfied", + "instrument_exclusion_satisfied", + "instrument_exogeneity_satisfied", + ], +) +def test_adjustment_method_specific_evidence_is_inapplicable_by_default(field): + report = _valid_report() + adjustment = replace(report.causal_adjustment_set, **{field: []}) + with pytest.raises(ValueError, match="Stage 3"): + _validate_with(report, causal_adjustment_set=adjustment) + + +def test_graph_valid_backdoor_blocker_need_not_be_the_remote_common_cause(): + report = _valid_report() + discovery = replace( + report.causal_discovery, + directed_edges=( + ("Z", "I"), + ("I", "X"), + ("Z", "Y"), + ("X", "M"), + ("M", "Y"), + ("X", "Y"), + ("X", "C"), + ("Y", "C"), + ("X", "D"), + ), + ) + adjustment = replace( + report.causal_adjustment_set, + admissible_adjustment_sets=(("I",),), + selected_adjustment_set=("I",), + instruments=(), + control_justifications=(("I", "Blocks the graph-implied backdoor path."),), + ) + validate_causal_factor_protocol( + replace( + report, + causal_discovery=discovery, + causal_adjustment_set=adjustment, + ) + ) + + +@pytest.mark.parametrize( + "changes", + [ + {"predictive_metric_labels": ("LOG_LOSS",)}, + {"naive_benchmark_id": ""}, + {"predictive_evidence_id": "explanatory-evidence"}, + {"validation": replace(_validation(), method_labels=("WALK_FORWARD",))}, + ], +) +def test_stage4_metrics_baseline_separation_and_purging(changes): + report = _valid_report() + stage = replace(report.causal_explanatory_and_predictive_power, **changes) + with pytest.raises(ValueError, match="Stage 4"): + _validate_with(report, causal_explanatory_and_predictive_power=stage) + + +def test_stage4_refits_variable_selection_inside_each_training_fold(): + report = _valid_report() + validation = report.causal_explanatory_and_predictive_power.validation + first_fold = replace( + validation.folds[0], + refit_components=("CAUSAL_ESTIMATION",), + ) + stage = replace( + report.causal_explanatory_and_predictive_power, + validation=replace( + validation, + folds=(first_fold,) + validation.folds[1:], + ), + ) + with pytest.raises(ValueError, match="Stage 4"): + _validate_with(report, causal_explanatory_and_predictive_power=stage) + + +def test_purged_cross_validation_covers_every_out_of_sample_observation(): + report = _valid_report() + validation = report.causal_explanatory_and_predictive_power.validation + stage = replace( + report.causal_explanatory_and_predictive_power, + validation=replace(validation, folds=validation.folds[:2]), + ) + with pytest.raises(ValueError, match="Stage 4"): + _validate_with(report, causal_explanatory_and_predictive_power=stage) + + +def test_one_source_supported_stage4_aspect_is_sufficient(): + report = _valid_report() + stage = replace( + report.causal_explanatory_and_predictive_power, + explanatory_metric_labels=(), + explanatory_evidence_id=None, + ) + validate_causal_factor_protocol( + replace(report, causal_explanatory_and_predictive_power=stage) + ) + + +def test_complementary_stage4_tasks_and_multiclass_options_are_supported(): + report = _valid_report() + stage = replace( + report.causal_explanatory_and_predictive_power, + task_types=("PROBABILITY", "RANKING", "RETURN_SIZE"), + explanatory_metric_labels=("BRIER_SCORE", "R_SQUARED"), + predictive_metric_labels=( + "LOG_LOSS", + "ROC_CURVE", + "CLASSIFICATION_ACCURACY", + "MEAN_SQUARED_ERROR", + ), + multiclass_encoding="ONE_VS_REST", + averaging_method="MACRO", + ) + validate_causal_factor_protocol( + replace(report, causal_explanatory_and_predictive_power=stage) + ) + + +@pytest.mark.parametrize( + "changes", + [ + {"cost_model_id": ""}, + {"constraint_set_id": ""}, + {"economic_rationale": ""}, + {"fragility_scenarios": ()}, + {"transfer_coefficient": math.inf}, + {"transfer_coefficient": -1.01}, + {"transfer_coefficient": 1.01}, + {"causal_exposures": ()}, + { + "causal_exposures": ("X", "D"), + "controlled_unintended_exposures": ("C",), + }, + {"controlled_unintended_exposures": ("C", "D", "NOT_IN_GRAPH")}, + ], +) +def test_portfolio_construction_requires_complete_evidence(changes): + report = _valid_report() + stage = replace(report.causal_portfolio_construction, **changes) + with pytest.raises(ValueError, match="Stage 5"): + _validate_with(report, causal_portfolio_construction=stage) + + +def test_monte_carlo_requires_an_explicit_dgp(): + report = _valid_report() + stage = replace( + report.backtest, + method_labels=("MONTE_CARLO",), + validation=None, + monte_carlo_dgp="structural market simulator v1", + ) + validate_causal_factor_protocol(replace(report, backtest=stage)) + with pytest.raises(ValueError, match="Stage 6"): + validate_causal_factor_protocol( + replace(report, backtest=replace(stage, monte_carlo_dgp="")) + ) + with pytest.raises(ValueError, match="Stage 6"): + validate_causal_factor_protocol( + replace(report, backtest=replace(stage, validation="invalid")) + ) + with pytest.raises(ValueError, match="Stage 6"): + _validate_with( + report, + backtest=replace(report.backtest, monte_carlo_dgp=[]), + ) + + +def test_backtest_labels_and_cpcv_path_contributions_fail_closed(): + report = _valid_report() + with pytest.raises(ValueError, match="Stage 6"): + _validate_with( + report, + backtest=replace(report.backtest, method_labels=("PBO",)), + ) + + validation = report.backtest.validation + duplicate = replace( + validation.folds[1], + path_id=validation.folds[0].path_id, + test_ids=("o1",), + prediction_ids=("o1",), + ) + stage = replace( + report.backtest, + validation=replace(validation, folds=(validation.folds[0], duplicate)), + ) + with pytest.raises(ValueError, match="Stage 6"): + _validate_with(report, backtest=stage) + + incomplete = replace( + report.backtest, + validation=replace( + report.backtest.validation, + folds=(report.backtest.validation.folds[0],), + ), + ) + with pytest.raises(ValueError, match="Stage 6"): + _validate_with(report, backtest=incomplete) + + +def test_cpcv_paths_must_encode_distinct_test_partitions(): + report = _valid_report() + first_path = report.backtest.validation.folds[:3] + copied_path = tuple( + replace( + fold, + fold_id=f"copy-{fold.fold_id}", + path_id="copied-path", + refit_id=f"copy-{fold.refit_id}", + ) + for fold in first_path + ) + backtest = replace( + report.backtest, + validation=replace( + report.backtest.validation, + folds=first_path + copied_path, + ), + ) + with pytest.raises(ValueError, match="Stage 6"): + _validate_with(report, backtest=backtest) + + +def test_walk_forward_cannot_train_on_future_observations(): + report = _valid_report() + stage = replace( + report.backtest, + method_labels=("WALK_FORWARD",), + validation=_validation(("WALK_FORWARD",)), + ) + with pytest.raises(ValueError, match="Stage 6"): + _validate_with(report, backtest=stage) + + +def test_walk_forward_cannot_duplicate_out_of_sample_predictions(): + report = _valid_report() + first = _fold("wf1", "wf-path-1", ("o1", "o2"), ("o3", "o4"), "wf-r1") + second = replace( + first, + fold_id="wf2", + path_id="wf-path-2", + refit_id="wf-r2", + ) + validation = ValidationEvidence( + method_labels=("WALK_FORWARD",), + event_horizons=_horizons(), + folds=(first, second), + embargo=1.0, + ) + backtest = replace( + report.backtest, + method_labels=("WALK_FORWARD",), + validation=validation, + ) + with pytest.raises(ValueError, match="Stage 6"): + _validate_with(report, backtest=backtest) + + +def test_strong_time_dependence_requires_a_backtest_embargo(): + report = _valid_report() + backtest = replace( + report.backtest, + validation=replace(report.backtest.validation, embargo=0.0), + ) + with pytest.raises(ValueError, match="Stage 6"): + _validate_with(report, backtest=backtest) + + +@pytest.mark.parametrize( + "stage_name,changes", + [ + ("backtest", {"trial_family_id": "other-family"}), + ("backtest", {"declared_trial_ids": ("strategy-a", "fabricated")}), + ("multiple_testing_adjustments", {"trial_family_id": "other-family"}), + ("multiple_testing_adjustments", {"declared_trial_ids": ("strategy-a",)}), + ], +) +def test_trial_family_identity_is_exact_across_stages(stage_name, changes): + report = _valid_report() + stage = replace(getattr(report, stage_name), **changes) + with pytest.raises(ValueError, match="Stage [67]"): + _validate_with(report, **{stage_name: stage}) + + +@pytest.mark.parametrize( + "changes", + [ + {"method_labels": ("BHY",)}, + {"alpha": True}, + {"alpha": 1.0}, + {"backtests_independent": True}, + {"family_partitions": (("primary", ("strategy-a",)),)}, + {"p_value_estimator": ""}, + {"time_dependence_model": ""}, + {"p_value_inputs_assume_independence": True}, + {"p_value_inputs_assume_independence": None}, + {"sharpe_variance": 0.0}, + {"effective_trials": 0.1}, + {"effective_trials": 3.0}, + {"sample_length": 1}, + {"skewness": math.nan}, + {"skewness": 3.0, "kurtosis": 2.0}, + {"kurtosis": -100.0}, + {"selection_bias_evidence_id": ""}, + ], +) +def test_multiple_testing_inputs_and_methods_fail_closed(changes): + report = _valid_report() + stage = replace(report.multiple_testing_adjustments, **changes) + with pytest.raises(ValueError, match="Stage 7"): + _validate_with(report, multiple_testing_adjustments=stage) + + +def test_non_iid_p_values_and_dsr_only_evidence_are_supported(): + report = _valid_report() + dependence_adjusted = replace( + report.multiple_testing_adjustments, + p_value_estimator="non-IID block bootstrap", + time_dependence_model="serial-dependence block bootstrap", + p_value_inputs_assume_independence=False, + ) + _validate_with(report, multiple_testing_adjustments=dependence_adjusted) + + dsr_only = replace( + report.multiple_testing_adjustments, + method_labels=("DEFLATED_SHARPE_RATIO",), + p_value_estimator=None, + time_dependence_model=None, + p_value_inputs_assume_independence=None, + ) + _validate_with(report, multiple_testing_adjustments=dsr_only) + + p_value_only = replace( + report.multiple_testing_adjustments, + method_labels=("HOLM",), + sharpe_variance=None, + effective_trials=None, + sample_length=None, + skewness=None, + kurtosis=None, + selection_bias_evidence_id=None, + ) + _validate_with(report, multiple_testing_adjustments=p_value_only) + + +@pytest.mark.parametrize( + "stage_name,stage", + [ + ( + "variable_selection", + replace(_valid_report().variable_selection, purpose=[]), + ), + ( + "causal_discovery", + replace(_valid_report().causal_discovery, method_labels=(["PC"],)), + ), + ( + "causal_adjustment_set", + replace(_valid_report().causal_adjustment_set, method_label=[]), + ), + ( + "causal_adjustment_set", + replace(_valid_report().causal_adjustment_set, treatment=[]), + ), + ( + "causal_adjustment_set", + replace(_valid_report().causal_adjustment_set, outcome=[]), + ), + ( + "causal_adjustment_set", + replace(_valid_report().causal_adjustment_set, colliders=None), + ), + ( + "causal_explanatory_and_predictive_power", + replace( + _valid_report().causal_explanatory_and_predictive_power, + task_types=(["RETURN_SIZE"],), + ), + ), + ( + "causal_explanatory_and_predictive_power", + replace( + _valid_report().causal_explanatory_and_predictive_power, + averaging_method=[], + ), + ), + ( + "variable_selection", + replace( + _valid_report().variable_selection, + validation=replace( + _valid_report().variable_selection.validation, + method_labels=(["PURGED_CROSS_VALIDATION"],), + ), + ), + ), + ], +) +def test_malformed_labels_fail_with_value_error(stage_name, stage): + with pytest.raises(ValueError): + _validate_with(_valid_report(), **{stage_name: stage}) + + +def test_module_and_package_exports_are_exact_and_isolated(): + repository_root = Path(__file__).resolve().parents[2] + program = f""" +import sys +import RiskLabAI.causal_factor_analysis as package +import RiskLabAI.causal_factor_analysis.protocol as module + +expected = {sorted(PUBLIC_NAMES)!r} +if sorted(module.__all__) != expected: + raise SystemExit('protocol public surface mismatch') +if not set(expected).issubset(package.__all__): + raise SystemExit('package public surface omits protocol names') + +blocked = {{ + 'RiskLabAI.causal', + 'RiskLabAI.optimization', + 'RiskLabAI.backtest', + 'RiskLabAI.risk_control', +}} +if blocked.intersection(sys.modules): + raise SystemExit('private-dependent namespace loaded') +""" + subprocess.run( + [sys.executable, "-c", program], + cwd=repository_root, + check=True, + capture_output=True, + text=True, + ) diff --git a/test/causal_factor_analysis/test_specification_diagnostics.py b/test/causal_factor_analysis/test_specification_diagnostics.py new file mode 100644 index 0000000..9e1161a --- /dev/null +++ b/test/causal_factor_analysis/test_specification_diagnostics.py @@ -0,0 +1,260 @@ +"""Analytical and source-example tests for the book's specification experiments.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import numpy as np +import pytest + +from RiskLabAI.causal_factor_analysis import ( + PopulationRegressionDiagnostics, + RegressionDiagnostics, + SpecificationExperimentResult, + SpecificationPopulationResult, + collider_population_diagnostics, + collider_specification_experiment, + confounder_undercontrolled_coefficient, + confounded_mediator_population_diagnostics, + confounded_mediator_specification_experiment, + fork_population_diagnostics, + fork_specification_experiment, +) + + +def _independent_population_regression( + predictor_covariance, + predictor_outcome_covariance, + outcome_variance, +): + covariance = np.asarray(predictor_covariance, dtype=float) + cross_covariance = np.asarray(predictor_outcome_covariance, dtype=float) + slopes = np.linalg.solve(covariance, cross_covariance) + r_squared = float(cross_covariance @ slopes / outcome_variance) + return tuple(float(value) for value in slopes), r_squared + + +def test_equation_16_and_appendix_equations_43_to_53_close_analytically(): + beta, gamma, delta = 1.25, -0.75, 2.0 + covariance_v_x = 1.0 + variance_x = 1.0 + delta**2 + conditional_v_coefficient = covariance_v_x / variance_x + conditional_v_variance = 1.0 - covariance_v_x**2 / variance_x + assert conditional_v_coefficient == 1.0 / (1.0 + delta**2) + assert conditional_v_variance == delta**2 / (1.0 + delta**2) + expected = beta + gamma * delta / (1.0 + delta**2) + assert confounder_undercontrolled_coefficient(beta, gamma, delta) == pytest.approx( + expected, rel=1.0e-15, abs=1.0e-15 + ) + + +def test_appendix_equations_54_to_61_give_exact_conditional_gaussian_moments(): + covariance_xy = np.eye(2) + covariance_xy_z = np.array([[1.0], [1.0]]) + variance_z = np.array([[3.0]]) + conditional_mean_coefficient = covariance_xy_z @ np.linalg.inv(variance_z) + conditional_covariance = ( + covariance_xy + - conditional_mean_coefficient @ covariance_xy_z.T + ) + np.testing.assert_allclose( + conditional_mean_coefficient, + np.array([[1.0 / 3.0], [1.0 / 3.0]]), + rtol=0.0, + atol=1.0e-15, + ) + np.testing.assert_allclose( + conditional_covariance, + np.array([[2.0 / 3.0, -1.0 / 3.0], [-1.0 / 3.0, 2.0 / 3.0]]), + rtol=0.0, + atol=1.0e-15, + ) + + +def test_fork_population_targets_follow_from_covariance_oracle(): + unconditioned_slopes, unconditioned_r2 = _independent_population_regression( + ((2.0,),), + (1.0,), + 2.0, + ) + conditioned_slopes, conditioned_r2 = _independent_population_regression( + ((2.0, 1.0), (1.0, 1.0)), + (1.0, 1.0), + 2.0, + ) + result = fork_population_diagnostics() + assert result == SpecificationPopulationResult( + structure="fork", + conditioning_variable_role="confounder", + unconditioned=PopulationRegressionDiagnostics( + ("intercept", "X"), + (0.0, *unconditioned_slopes), + unconditioned_r2, + ), + conditioned=PopulationRegressionDiagnostics( + ("intercept", "X", "Z"), + (0.0, *conditioned_slopes), + conditioned_r2, + ), + ) + + +def test_collider_population_targets_and_equation_31_are_exact(): + conditional_covariance = 0.0 - 1.0 * (1.0 / 3.0) * 1.0 + assert conditional_covariance == pytest.approx(-1.0 / 3.0) + unconditioned_slopes, unconditioned_r2 = _independent_population_regression( + ((1.0,),), + (0.0,), + 1.0, + ) + conditioned_slopes, conditioned_r2 = _independent_population_regression( + ((1.0, 1.0), (1.0, 3.0)), + (0.0, 1.0), + 1.0, + ) + result = collider_population_diagnostics() + assert result.unconditioned.coefficients == (0.0, *unconditioned_slopes) + assert result.unconditioned.r_squared == unconditioned_r2 + assert result.conditioned.coefficients == (0.0, *conditioned_slopes) + assert result.conditioned.r_squared == conditioned_r2 + assert result.conditioned.coefficients[1:] == (-0.5, 0.5) + + +def test_confounded_mediator_population_targets_follow_from_covariance_oracle(): + unconditioned_slopes, unconditioned_r2 = _independent_population_regression( + ((1.0,),), + (1.0,), + 7.0, + ) + conditioned_slopes, conditioned_r2 = _independent_population_regression( + ((1.0, 1.0), (1.0, 3.0)), + (1.0, 4.0), + 7.0, + ) + result = confounded_mediator_population_diagnostics() + assert result.unconditioned.coefficients == (0.0, *unconditioned_slopes) + assert result.unconditioned.r_squared == unconditioned_r2 + assert result.conditioned.coefficients == (0.0, *conditioned_slopes) + assert result.conditioned.r_squared == conditioned_r2 + assert result.unconditioned.r_squared == 1.0 / 7.0 + assert result.conditioned.r_squared == 11.0 / 14.0 + + +@pytest.mark.parametrize( + "function,expected_unconditioned,expected_conditioned,expected_r_squared", + [ + ( + fork_specification_experiment, + (0.008987566774752393, 0.4963512366313372), + (0.0053778294681722984, 0.0007202300104839902, 0.9957125101775881), + (0.2470257605084264, 0.49480398566467254), + ), + ( + collider_specification_experiment, + (-0.022119251900918794, 0.00150355284008128), + (-0.013760210010524129, -0.4962958394242535, 0.4988282874959081), + (2.2405222220855947e-06, 0.49918868068004596), + ), + ( + confounded_mediator_specification_experiment, + (-0.022225744034636837, 1.0055025493825809), + (0.002741745742420054, -0.4813665679581204, 1.4899423082054921), + (0.14399915797065876, 0.7840010726224431), + ), + ], +) +def test_seed_zero_source_experiments_reproduce_reported_specification_patterns( + function, + expected_unconditioned, + expected_conditioned, + expected_r_squared, +): + result = function() + assert result.unconditioned.coefficients == pytest.approx( + expected_unconditioned, rel=1.0e-12, abs=1.0e-12 + ) + assert result.conditioned.coefficients == pytest.approx( + expected_conditioned, rel=1.0e-12, abs=1.0e-12 + ) + assert result.unconditioned.r_squared == pytest.approx( + expected_r_squared[0], rel=1.0e-12, abs=1.0e-12 + ) + assert result.conditioned.r_squared == pytest.approx( + expected_r_squared[1], rel=1.0e-12, abs=1.0e-12 + ) + assert result.unconditioned.n_observations == 5_000 + assert result.conditioned.n_observations == 5_000 + + +def test_seeded_experiments_are_reproducible_and_seed_sensitive(): + first = fork_specification_experiment(sample_size=200, seed=17) + second = fork_specification_experiment(sample_size=200, seed=17) + different = fork_specification_experiment(sample_size=200, seed=18) + assert first == second + assert first != different + + +@pytest.mark.parametrize( + "function", + [ + fork_specification_experiment, + collider_specification_experiment, + confounded_mediator_specification_experiment, + ], +) +@pytest.mark.parametrize( + "sample_size,seed", + [ + (3, 0), + (True, 0), + (4.5, 0), + (1_000_001, 0), + (10, True), + (10, -1), + (10, 2**32), + ], +) +def test_experiments_reject_invalid_resource_and_seed_inputs( + function, + sample_size, + seed, +): + with pytest.raises(ValueError): + function(sample_size=sample_size, seed=seed) + + +def test_public_diagnostic_records_are_immutable_tuple_records(): + records = ( + fork_population_diagnostics(), + fork_population_diagnostics().conditioned, + fork_specification_experiment(sample_size=20, seed=3), + fork_specification_experiment(sample_size=20, seed=3).conditioned, + ) + assert isinstance(records[0], SpecificationPopulationResult) + assert isinstance(records[1], PopulationRegressionDiagnostics) + assert isinstance(records[2], SpecificationExperimentResult) + assert isinstance(records[3], RegressionDiagnostics) + for record in records: + assert not hasattr(record, "__dict__") + with pytest.raises(AttributeError): + record.structure = "changed" + + +def test_module_is_python_39_grammar_and_uses_only_numpy_plus_standard_library(): + source_path = ( + Path(__file__).resolve().parents[2] + / "src" + / "RiskLabAI" + / "causal_factor_analysis" + / "specification_diagnostics.py" + ) + source = source_path.read_text(encoding="utf-8") + tree = ast.parse(source, feature_version=(3, 9)) + roots = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + roots.update(alias.name.split(".", 1)[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + roots.add(node.module.split(".", 1)[0]) + assert roots == {"__future__", "math", "numbers", "typing", "numpy"} diff --git a/test/causal_factor_analysis/test_treatment_effects.py b/test/causal_factor_analysis/test_treatment_effects.py new file mode 100644 index 0000000..1651c15 --- /dev/null +++ b/test/causal_factor_analysis/test_treatment_effects.py @@ -0,0 +1,180 @@ +"""Exact tests for the 2023 book's treatment-effect estimands.""" + +from __future__ import annotations + +import math + +import numpy as np +import pytest + +from RiskLabAI.causal_factor_analysis import ( + DifferenceInDifferencesEstimate, + TreatmentEffectDecomposition, + average_treatment_effect, + backdoor_adjusted_average_treatment_effect, + backdoor_adjusted_expectation, + difference_in_differences, + frontdoor_adjusted_probability, + linear_instrumental_variable_effect, + randomized_mean_difference, + treatment_effect_decomposition, +) + + +def test_equation_2_average_treatment_effect_is_exact_mean_contrast(): + assert average_treatment_effect(7.25, 2.0) == 5.25 + assert average_treatment_effect(-2, -5) == 3.0 + + +def test_equation_3_decomposition_closes_exactly(): + result = treatment_effect_decomposition( + observed_treated_mean=10.0, + observed_control_mean=4.0, + counterfactual_control_mean_for_treated=6.0, + ) + assert result == TreatmentEffectDecomposition( + observed_difference=6.0, + average_treatment_effect_on_treated=4.0, + sample_selection_bias=2.0, + ) + assert result.observed_difference == ( + result.average_treatment_effect_on_treated + result.sample_selection_bias + ) + + +def test_equations_4_to_6_randomized_difference_uses_both_groups(): + treated = np.array([4.0, 6.0, 8.0]) + control = np.array([1.0, 3.0, 5.0]) + treated_before = treated.copy() + control_before = control.copy() + assert randomized_mean_difference(treated, control) == 3.0 + np.testing.assert_array_equal(treated, treated_before) + np.testing.assert_array_equal(control, control_before) + + +def test_two_group_two_period_difference_in_differences_is_exact(): + result = difference_in_differences( + treated_before=10.0, + treated_after=15.0, + control_before=20.0, + control_after=22.0, + ) + assert result == DifferenceInDifferencesEstimate( + treated_change=5.0, + control_change=2.0, + estimate=3.0, + ) + + +def test_equation_8_backdoor_standardization_matches_hand_sum(): + result = backdoor_adjusted_expectation( + conditional_outcome_means=(0.1, 0.8), + adjustment_probabilities=(0.75, 0.25), + ) + assert result == pytest.approx(0.1 * 0.75 + 0.8 * 0.25, abs=1.0e-15) + + +def test_equation_7_backdoor_ate_matches_stratum_contrasts(): + result = backdoor_adjusted_average_treatment_effect( + treated_conditional_means=(0.4, 0.9), + control_conditional_means=(0.1, 0.2), + adjustment_probabilities=(0.75, 0.25), + ) + assert result == pytest.approx( + (0.4 - 0.1) * 0.75 + (0.9 - 0.2) * 0.25, + abs=1.0e-15, + ) + + +def test_equation_9_frontdoor_probability_matches_nested_hand_sum(): + result = frontdoor_adjusted_probability( + mediator_probabilities_given_treatment=(0.25, 0.75), + outcome_probabilities_given_mediator_and_treatment=( + (0.1, 0.5), + (0.4, 0.8), + ), + treatment_probabilities=(0.6, 0.4), + ) + expected = 0.25 * (0.1 * 0.6 + 0.5 * 0.4) + 0.75 * ( + 0.4 * 0.6 + 0.8 * 0.4 + ) + assert result == pytest.approx(expected, abs=1.0e-15) + + +def test_linear_instrumental_variable_ratio_is_exact(): + assert linear_instrumental_variable_effect(0.9, 0.3) == pytest.approx(3.0) + assert linear_instrumental_variable_effect(-2.0, 0.5) == -4.0 + + +@pytest.mark.parametrize( + "function,args", + [ + (average_treatment_effect, (True, 0.0)), + (average_treatment_effect, (math.inf, 0.0)), + (treatment_effect_decomposition, (1.0, 0.0, math.nan)), + (randomized_mean_difference, ((), (1.0,))), + (randomized_mean_difference, (((1.0, 2.0),), (1.0,))), + (randomized_mean_difference, (("1",), (1.0,))), + (difference_in_differences, (0.0, 1.0, 0.0, math.inf)), + (backdoor_adjusted_expectation, ((1.0,), (0.4, 0.6))), + (backdoor_adjusted_expectation, ((1.0, 2.0), (0.4, 0.4))), + ( + backdoor_adjusted_average_treatment_effect, + ((1.0,), (0.0, 1.0), (1.0,)), + ), + ( + frontdoor_adjusted_probability, + ((1.0,), ((0.2, 0.3),), (1.0,)), + ), + ( + frontdoor_adjusted_probability, + ((1.0,), ((1.2,),), (1.0,)), + ), + (linear_instrumental_variable_effect, (1.0, 0.0)), + ], +) +def test_estimands_reject_malformed_or_undefined_inputs(function, args): + with pytest.raises(ValueError): + function(*args) + + +def test_probability_tolerance_accepts_only_roundoff_scale_error(): + assert backdoor_adjusted_expectation( + (2.0, 4.0), + (0.5, 0.5 + 5.0e-13), + ) == pytest.approx(3.0 + 2.0e-12, abs=1.0e-14) + with pytest.raises(ValueError, match="sum to one"): + backdoor_adjusted_expectation((2.0, 4.0), (0.5, 0.50001)) + + +@pytest.mark.parametrize( + "function,args", + [ + (average_treatment_effect, (np.finfo(float).max, -np.finfo(float).max)), + ( + treatment_effect_decomposition, + (np.finfo(float).max, -np.finfo(float).max, 0.0), + ), + ( + difference_in_differences, + (-np.finfo(float).max, np.finfo(float).max, 0.0, 0.0), + ), + ( + backdoor_adjusted_average_treatment_effect, + ((np.finfo(float).max,), (-np.finfo(float).max,), (1.0,)), + ), + ], +) +def test_finite_inputs_that_overflow_the_estimand_fail_closed(function, args): + with np.errstate(over="ignore", invalid="ignore"): + with pytest.raises(ValueError): + function(*args) + + +def test_public_result_records_are_tuple_immutable_without_instance_dicts(): + treatment = treatment_effect_decomposition(3.0, 1.0, 2.0) + did = difference_in_differences(1.0, 3.0, 4.0, 5.0) + for result in (treatment, did): + assert not hasattr(result, "__dict__") + with pytest.raises(AttributeError): + result.estimate = 0.0 diff --git a/test/data/synthetic_data/test_synthetic_controlled_environment.py b/test/data/synthetic_data/test_synthetic_controlled_environment.py index 3329cd6..154df59 100644 --- a/test/data/synthetic_data/test_synthetic_controlled_environment.py +++ b/test/data/synthetic_data/test_synthetic_controlled_environment.py @@ -58,6 +58,7 @@ def test_align_params_length(): def test_generate_prices_from_regimes(sample_regimes): """Test the single-path price generation.""" + pytest.importorskip("quantecon", reason="requires RiskLabAI[synth]") regimes, tm = sample_regimes n_steps = 100 @@ -76,6 +77,7 @@ def test_generate_prices_from_regimes(sample_regimes): def test_parallel_generate_prices(sample_regimes): """Test the parallel price generation.""" + pytest.importorskip("quantecon", reason="requires RiskLabAI[synth]") regimes, tm = sample_regimes n_steps = 50 n_paths = 4 diff --git a/test/features/feature_importance/test_feature_importance.py b/test/features/feature_importance/test_feature_importance.py index 9d1c10e..b2cc3f3 100644 --- a/test/features/feature_importance/test_feature_importance.py +++ b/test/features/feature_importance/test_feature_importance.py @@ -45,7 +45,7 @@ def mock_data(): def test_controller_is_deprecated(mock_data): """The controller is deprecated in 2.0.0 (delegates to the core registry).""" _, _, classifier, _ = mock_data - with pytest.warns(DeprecationWarning, match="2.1.0"): + with pytest.warns(DeprecationWarning, match="4.0.0"): FeatureImportanceController("MDI", classifier=classifier) diff --git a/test/optimization/test_hyper_parameter_tuning.py b/test/optimization/test_hyper_parameter_tuning.py index a92955c..89bf583 100644 --- a/test/optimization/test_hyper_parameter_tuning.py +++ b/test/optimization/test_hyper_parameter_tuning.py @@ -53,7 +53,7 @@ def fit(self, X, y, **kwargs): def test_my_pipeline_deprecated_alias_warns(): """MyPipeline is a deprecated subclass of SampleWeightedPipeline.""" assert issubclass(MyPipeline, SampleWeightedPipeline) - with pytest.warns(DeprecationWarning, match="MyPipeline.*2.1.0"): + with pytest.warns(DeprecationWarning, match="MyPipeline.*4.0.0"): pipe = MyPipeline([("clf", LogisticRegression())]) assert isinstance(pipe, SampleWeightedPipeline) diff --git a/test/test_deprecation.py b/test/test_deprecation.py index 5e8ab67..fa6c5da 100644 --- a/test/test_deprecation.py +++ b/test/test_deprecation.py @@ -20,9 +20,9 @@ def new_func(a, b=2): """Add two numbers.""" return a + b - old_func = deprecated_alias(new_func, "old_func", removed_in="2.1.0") + old_func = deprecated_alias(new_func, "old_func", removed_in="4.0.0") - with pytest.warns(DeprecationWarning, match="old_func.*2.1.0.*new_func"): + with pytest.warns(DeprecationWarning, match="old_func.*4.0.0.*new_func"): result = old_func(3, b=4) assert result == 7 @@ -34,12 +34,12 @@ class New: def __init__(self, x): self.x = x - Old = deprecated_class(New, "Old", removed_in="2.1.0") + Old = deprecated_class(New, "Old", removed_in="4.0.0") assert issubclass(Old, New) assert Old is not New - with pytest.warns(DeprecationWarning, match="Old.*2.1.0.*New"): + with pytest.warns(DeprecationWarning, match="Old.*4.0.0.*New"): obj = Old(5) assert isinstance(obj, New) @@ -47,15 +47,15 @@ def __init__(self, x): def test_warn_deprecated_name_emits_warning(): - with pytest.warns(DeprecationWarning, match="OLD_NAME.*2.1.0.*NEW_NAME"): - warn_deprecated_name("OLD_NAME", "NEW_NAME", removed_in="2.1.0") + with pytest.warns(DeprecationWarning, match="OLD_NAME.*4.0.0.*NEW_NAME"): + warn_deprecated_name("OLD_NAME", "NEW_NAME", removed_in="4.0.0") def test_deprecated_theta_constant_alias_warns(): """The non-ASCII θ constants warn (and return the ASCII value) via utils.""" import RiskLabAI.utils as utils - with pytest.warns(DeprecationWarning, match="2.1.0"): + with pytest.warns(DeprecationWarning, match="4.0.0"): value = utils.CUMULATIVE_θ # Identifier changed; the string value is intentionally unchanged. From 5bbcc2cf3d302e32d749fde84567b2d53a360026 Mon Sep 17 00:00:00 2001 From: hamid-arian Date: Sun, 23 Aug 2026 21:24:32 -0400 Subject: [PATCH 2/6] Fix Python 3.0.0 release CI and metadata --- .github/workflows/ci.yml | 115 +++++++---- MANIFEST.in | 1 + PACKAGE_FILES.json | 6 +- PACKAGE_INSPECTION.md | 5 +- PUBLIC_API.json | 12 +- RELEASE_NOTES_NEXT_MAJOR.md | 15 +- TEST_INVENTORY.json | 21 +- docs/compatibility.md | 27 ++- pyproject.toml | 1 - .../causal_factor_analysis/protocol.py | 189 ++++++++++++------ .../specification_diagnostics.py | 12 +- .../treatment_effects.py | 16 +- .../test_cfi_book_graph_examples.py | 8 +- test/causal_factor_analysis/test_protocol.py | 27 +-- .../test_specification_diagnostics.py | 3 +- .../test_treatment_effects.py | 4 +- 16 files changed, 285 insertions(+), 177 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5baf55..9de1af4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,71 +5,108 @@ on: branches: [main] pull_request: +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + PYTHONDONTWRITEBYTECODE: "1" + PYTHONHASHSEED: "0" + jobs: - test: - name: pytest (py${{ matrix.python-version }}) + base: + name: base py${{ matrix.python-version }} / ${{ matrix.dependency-set }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + include: + - python-version: "3.12" + dependency-set: minimum + numpy-spec: "numpy==2.2.0" + - python-version: "3.12" + dependency-set: latest + numpy-spec: "numpy>=2.2,<3" + - python-version: "3.13" + dependency-set: minimum + numpy-spec: "numpy==2.2.0" + - python-version: "3.13" + dependency-set: latest + numpy-spec: "numpy>=2.2,<3" + - python-version: "3.14" + dependency-set: minimum + numpy-spec: "numpy==2.3.2" + - python-version: "3.14" + dependency-set: latest + numpy-spec: "numpy>=2.3.2,<3" steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: pip - - name: Install package with all extras + - name: Install the base distribution run: | python -m pip install --upgrade pip - pip install -e ".[all]" pytest - - name: Run tests - # test/pde requires a working torch runtime; revisit when pde is - # re-validated (see IMPROVEMENT_PLAN.md phase 3). - run: pytest -q --ignore=test/pde + python -m pip install "${{ matrix.numpy-spec }}" + python -m pip install . "pytest>=8.4,<10" + - name: Verify installed-package origin + working-directory: ${{ runner.temp }} + env: + RISKLABAI_REPOSITORY_ROOT: ${{ github.workspace }} + run: python -B -c "import os; from pathlib import Path; import RiskLabAI; origin = Path(RiskLabAI.__file__).resolve(); repository = Path(os.environ['RISKLABAI_REPOSITORY_ROOT']).resolve(); assert not origin.is_relative_to(repository), (origin, repository); assert RiskLabAI.__version__ == '3.0.0'; print(origin)" + - name: Run the installed base package tests + working-directory: ${{ runner.temp }} + run: python -B -m pytest -q -p no:cacheprovider -c "$GITHUB_WORKSPACE/pyproject.toml" "$GITHUB_WORKSPACE/test" - minimal-install: - name: base install is importable + optional: + name: optional features py${{ matrix.python-version }} runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ matrix.python-version }} cache: pip - - name: Install base package only (no extras) + - name: Install feature-specific optional groups run: | python -m pip install --upgrade pip - pip install . - - name: Import every public sub-package + python -m pip install ".[speed,pde,synth,hpo,plot,symbolic,profile,simulation,changepoints,test]" + - name: Verify changepoint availability policy run: | - python -c " - import RiskLabAI - print('version:', RiskLabAI.__version__) - for sub in ['backtest', 'cluster', 'controller', 'data', - 'ensemble', 'features', 'hpc', 'optimization', 'utils']: - getattr(RiskLabAI, sub) - print('ok:', sub) - " - - format: - name: black - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - run: pip install "black==26.5.1" - - run: black --check RiskLabAI test + if [[ "${{ matrix.python-version }}" == "3.14" ]]; then + python -B -c "import importlib.util; assert importlib.util.find_spec('ruptures') is None" + else + python -B -c "import ruptures" + fi + - name: Verify installed-package origin + working-directory: ${{ runner.temp }} + env: + RISKLABAI_REPOSITORY_ROOT: ${{ github.workspace }} + run: python -B -c "import os; from pathlib import Path; import RiskLabAI; origin = Path(RiskLabAI.__file__).resolve(); repository = Path(os.environ['RISKLABAI_REPOSITORY_ROOT']).resolve(); assert not origin.is_relative_to(repository), (origin, repository); assert RiskLabAI.__version__ == '3.0.0'; print(origin)" + - name: Run the installed optional-feature tests + working-directory: ${{ runner.temp }} + run: python -B -m pytest -q -p no:cacheprovider -c "$GITHUB_WORKSPACE/pyproject.toml" "$GITHUB_WORKSPACE/test" - lint: - name: ruff + quality: + name: static quality runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.11" - - run: pip install "ruff==0.15.17" - - run: ruff check RiskLabAI test + python-version: "3.14" + - name: Install pinned quality tools + run: python -m pip install "black==26.5.1" "ruff==0.15.17" + - name: Black + run: black --check src/RiskLabAI/causal_factor_analysis test/causal_factor_analysis + - name: Ruff + run: ruff check src/RiskLabAI/causal_factor_analysis test/causal_factor_analysis diff --git a/MANIFEST.in b/MANIFEST.in index fbb55a8..f572cee 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,7 @@ include PUBLIC_API.json include TEST_INVENTORY.json include PACKAGE_FILES.json include PACKAGE_INSPECTION.md +include .github/workflows/ci.yml recursive-include src *.py recursive-include test *.py recursive-include docs *.md diff --git a/PACKAGE_FILES.json b/PACKAGE_FILES.json index 98bfac2..9f4dd1c 100644 --- a/PACKAGE_FILES.json +++ b/PACKAGE_FILES.json @@ -8,8 +8,9 @@ "wheel", "source_distribution" ], - "candidate_file_count": 215, + "candidate_file_count": 216, "candidate_files": [ + ".github/workflows/ci.yml", "LICENSE", "MANIFEST.in", "PACKAGE_FILES.json", @@ -229,8 +230,9 @@ "local_control_files_excluded_from_artifacts": [ "PACKAGE_IDENTITY.json" ], - "release_source_file_count": 214, + "release_source_file_count": 215, "release_source_files": [ + ".github/workflows/ci.yml", "LICENSE", "MANIFEST.in", "PACKAGE_FILES.json", diff --git a/PACKAGE_INSPECTION.md b/PACKAGE_INSPECTION.md index 08fc6c4..bce7a14 100644 --- a/PACKAGE_INSPECTION.md +++ b/PACKAGE_INSPECTION.md @@ -22,7 +22,8 @@ The wheel may contain only the 124 declared Python runtime modules plus the standard metadata directory created for RiskLabAI 3.0.0. Tests, examples, local readiness controls, caches, bytecode, credentials, and undeclared data must not be present in the wheel. The source distribution must contain exactly -the release-source members declared in `PACKAGE_FILES.json` plus only the +the 215 release-source members declared in `PACKAGE_FILES.json`, including the +governed `.github/workflows/ci.yml`, plus only the backend-generated metadata members that the inspection record enumerates and hashes explicitly. @@ -46,6 +47,8 @@ lane, verify the exact module origins, `RiskLabAI.__version__ == "3.0.0"`, the complete 57-feature causal API. Run the exact package-scoped test inventory and the valid optional-feature lanes. Confirm that optional groups do not become base dependencies and that changepoints alone is unavailable on Python 3.14. +Run installed-package tests outside the source checkout and reject any import +whose resolved path remains inside the repository. ## Final review diff --git a/PUBLIC_API.json b/PUBLIC_API.json index fbdf042..5159a32 100644 --- a/PUBLIC_API.json +++ b/PUBLIC_API.json @@ -1116,8 +1116,8 @@ "path": "src/RiskLabAI/causal_factor_analysis/protocol.py", "module": "RiskLabAI.causal_factor_analysis.protocol", "module_kind": "module", - "sha256": "75CC2191038FAEBBDE7AD5AB38B8842FF9B9FE3772E732FB4F0764B83C432F4D", - "size": 63829, + "sha256": "97A78E100A01192A9C7511DC8C18CF54300F46BF36187A87E3FF607A5C91011B", + "size": 64712, "defined_public_names": [ "BacktestStage", "CausalAdjustmentSetStage", @@ -1199,8 +1199,8 @@ "path": "src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py", "module": "RiskLabAI.causal_factor_analysis.specification_diagnostics", "module_kind": "module", - "sha256": "EB8066295C2C6374E817AE2A1EA4459443CF76B1F84692CB11EEC6B66A687429", - "size": 9661, + "sha256": "5AF8F882F798FF9D0AC8F3955A9A2C9533589AA328B929030CF16930FA28FA65", + "size": 9595, "defined_public_names": [ "PopulationRegressionDiagnostics", "RegressionDiagnostics", @@ -1270,8 +1270,8 @@ "path": "src/RiskLabAI/causal_factor_analysis/treatment_effects.py", "module": "RiskLabAI.causal_factor_analysis.treatment_effects", "module_kind": "module", - "sha256": "692B945C0B44FA3B87FE23957B4400E717F4D27903ACBC12003C44528A6F6919", - "size": 11888, + "sha256": "B3D980B969803C12F0F3C5A6C6C55C37D844827131F45924F53E629F02A1E980", + "size": 11832, "defined_public_names": [ "DifferenceInDifferencesEstimate", "TreatmentEffectDecomposition", diff --git a/RELEASE_NOTES_NEXT_MAJOR.md b/RELEASE_NOTES_NEXT_MAJOR.md index 3bdece2..5728990 100644 --- a/RELEASE_NOTES_NEXT_MAJOR.md +++ b/RELEASE_NOTES_NEXT_MAJOR.md @@ -18,12 +18,19 @@ Compatibility aliases that remain present in 3.0.0 now consistently warn of removal in 4.0.0. No alias is silently deleted, and the warning contract is locked by package-scoped tests. +The release metadata uses the PEP 639 `BSD-3-Clause` expression and license +file without the superseded license classifier. The governed CI workflow now +installs the distribution, rejects imports shadowed by the preserved repository +root, tests the six approved base runtime/NumPy combinations, exercises +feature-specific optional lanes, and isolates the Python 3.14 changepoint +limitation. Pinned static checks cover the clean causal source and tests. + No source-conflicted causal result is admitted. The metadata contract is complete; artifacts, publication, upload, and release remain blocked. The static distributable metadata, exact 124-module runtime inventory, 78-file package-scoped test inventory, public API inventory, documentation, -examples, dependency groups, and intended package file list are now frozen. -No package artifact has been created. The remaining gates are future artifact -inspection and separate human authorization for version-control, publication, -and release actions. +examples, dependency groups, governed CI workflow, and 215-file intended +source-distribution list are now frozen. No retained package artifact has been +created. The remaining gates are future artifact inspection and separate human +authorization for version-control, publication, and release actions. diff --git a/TEST_INVENTORY.json b/TEST_INVENTORY.json index 720cf31..6f63042 100644 --- a/TEST_INVENTORY.json +++ b/TEST_INVENTORY.json @@ -4,7 +4,7 @@ "product": "RiskLabAI", "language": "Python", "version": "3.0.0", - "runner": "pytest", + "runner": "pytest / six base lanes / three optional-feature lanes", "file_count": 78, "file_groups": { "CAUSAL_FACTOR_ANALYSIS": 7, @@ -23,6 +23,9 @@ "technical_indicator_oracles": 8, "total_applicable": 680 }, + "base_lane_count": 6, + "optional_feature_lane_count": 3, + "ci_execution_count": 9, "required_lane_count": 6, "required_lane_case_total": 4080, "files": [ @@ -203,8 +206,8 @@ }, { "path": "test/causal_factor_analysis/test_cfi_book_graph_examples.py", - "size": 3672, - "sha256": "6A840BBD03FC9C098455C0DAE641ACA279D6C8E8A4F70B040DB2A555D7E50078", + "size": 3644, + "sha256": "195A3291A6CB6EAEE929F02951BAA45BD6E933AB751CCC30469402BBDA12DE9A", "group": "CAUSAL_FACTOR_ANALYSIS", "declared_test_functions": 7 }, @@ -231,22 +234,22 @@ }, { "path": "test/causal_factor_analysis/test_protocol.py", - "size": 39100, - "sha256": "8781036062F8734967F115FA63A52F66A14552B1EA56CB9B422A71B1E25070B2", + "size": 39048, + "sha256": "1CAACE494F327A7B135E3A0D8F38BC15EBEA57BBC2D2CB77CB9D6BA6B8E17930", "group": "CAUSAL_FACTOR_ANALYSIS", "declared_test_functions": 43 }, { "path": "test/causal_factor_analysis/test_specification_diagnostics.py", - "size": 9096, - "sha256": "DDED9E16AA02CBA5284BACE5B61E4CA53CBB1FBC2A72E665F289CD252E81930E", + "size": 9088, + "sha256": "6FD3A8C9F1DFDC665AE9DCD80E41798D5D1EAE48BB9B2F4EE5AAB61B1B3393D4", "group": "CAUSAL_FACTOR_ANALYSIS", "declared_test_functions": 10 }, { "path": "test/causal_factor_analysis/test_treatment_effects.py", - "size": 5969, - "sha256": "0B67456180473825489748AD99098C8BFCE6D15293EFE136546785AC19FE836E", + "size": 5955, + "sha256": "434750E2D2889FA64E49BAE08372D0E08C9DC145BE88AB5CD58B942B4D1F5B6B", "group": "CAUSAL_FACTOR_ANALYSIS", "declared_test_functions": 12 }, diff --git a/docs/compatibility.md b/docs/compatibility.md index 41f73bf..59ae18a 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -66,6 +66,24 @@ The financial-feature simulation no longer relies on the stale `ta` package. ADX, RSI, CCI, stochastic momentum, ROC, ATR, and Ichimoku lines are computed internally from standard definitions and have hand-calculated oracle tests. +## Continuous integration contract + +The governed CI workflow installs the distribution non-editably before every +runtime test. Import-origin probes run outside the repository checkout and +reject any `RiskLabAI` module resolved from the preserved repository root. + +The base matrix contains six valid interpreter/NumPy combinations: the lowest +supported NumPy binary and the latest supported NumPy 2.x release on each of +CPython 3.12, 3.13, and 3.14. Three additional optional-feature lanes install +the feature-specific groups on those interpreter lines. The `speed` group +keeps its narrower NumPy `<2.6` constraint, and the Python 3.14 lane verifies +that only the unavailable changepoint backend is skipped. + +Pinned Black and Ruff checks cover the independently maintained clean causal +source and tests. Preserved source and regression tests remain governed by the +complete installed-package runtime matrix and are not rewritten solely to +satisfy a new style tool. + ## Deprecation policy Public compatibility aliases retained in RiskLabAI 3.0.0 remain callable and @@ -104,7 +122,8 @@ did not import or reintroduce the excluded legacy causal implementation. ## Remaining release gates -The metadata, public surface, tests, and source allowlists are complete. No -package has been built or installed. Temporary artifact inspection and separate -human authorization for version-control, publication, and release remain -outstanding. +The metadata, public surface, tests, source allowlists, installed-origin probes, +and CI contract are complete. Source installs used isolated verification +environments and retained no wheel or source-distribution artifact. Formal +artifact inspection and separate human authorization for version-control, +publication, and release remain outstanding. diff --git a/pyproject.toml b/pyproject.toml index ce9a2ac..154c63b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,6 @@ classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Financial and Insurance Industry", "Intended Audience :: Science/Research", - "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.12", diff --git a/src/RiskLabAI/causal_factor_analysis/protocol.py b/src/RiskLabAI/causal_factor_analysis/protocol.py index 281afe2..35edfcf 100644 --- a/src/RiskLabAI/causal_factor_analysis/protocol.py +++ b/src/RiskLabAI/causal_factor_analysis/protocol.py @@ -188,8 +188,12 @@ class ValidationEvidence: embargo: Real = 0.0 def __post_init__(self): - _store_normalized_field(self, "method_labels", _freeze_tuple(self.method_labels)) - _store_normalized_field(self, "event_horizons", _freeze_tuple(self.event_horizons)) + _store_normalized_field( + self, "method_labels", _freeze_tuple(self.method_labels) + ) + _store_normalized_field( + self, "event_horizons", _freeze_tuple(self.event_horizons) + ) _store_normalized_field(self, "folds", _freeze_tuple(self.folds)) @@ -208,7 +212,9 @@ def __post_init__(self): _store_normalized_field( self, "selected_variables", _freeze_tuple(self.selected_variables) ) - _store_normalized_field(self, "method_labels", _freeze_tuple(self.method_labels)) + _store_normalized_field( + self, "method_labels", _freeze_tuple(self.method_labels) + ) @dataclass(frozen=True) @@ -224,7 +230,9 @@ class CausalDiscoveryStage: assumptions: tuple[str, ...] def __post_init__(self): - _store_normalized_field(self, "method_labels", _freeze_tuple(self.method_labels)) + _store_normalized_field( + self, "method_labels", _freeze_tuple(self.method_labels) + ) _store_normalized_field(self, "graph_nodes", _freeze_tuple(self.graph_nodes)) _store_normalized_field( self, "directed_edges", _freeze_nested_tuple(self.directed_edges) @@ -343,7 +351,9 @@ class BacktestStage: monte_carlo_dgp: Optional[str] = None def __post_init__(self): - _store_normalized_field(self, "method_labels", _freeze_tuple(self.method_labels)) + _store_normalized_field( + self, "method_labels", _freeze_tuple(self.method_labels) + ) _store_normalized_field( self, "declared_trial_ids", _freeze_tuple(self.declared_trial_ids) ) @@ -370,7 +380,9 @@ class MultipleTestingAdjustmentsStage: selection_bias_evidence_id: Optional[str] = None def __post_init__(self): - _store_normalized_field(self, "method_labels", _freeze_tuple(self.method_labels)) + _store_normalized_field( + self, "method_labels", _freeze_tuple(self.method_labels) + ) _store_normalized_field( self, "declared_trial_ids", _freeze_tuple(self.declared_trial_ids) ) @@ -434,9 +446,7 @@ def _nonblank(value) -> bool: def _safe_label_set(value) -> set[str]: - if not isinstance(value, tuple) or not all( - isinstance(item, str) for item in value - ): + if not isinstance(value, tuple) or not all(isinstance(item, str) for item in value): return set() return set(value) @@ -527,15 +537,15 @@ def _validate_event_horizons( start = float(horizon.start) end = float(horizon.end) if not start < end: - _add_error(errors, f"{stage}: every event horizon must have positive length.") + _add_error( + errors, f"{stage}: every event horizon must have positive length." + ) continue result[horizon.observation_id] = (start, end) return result -def _intervals_overlap( - left: tuple[float, float], right: tuple[float, float] -) -> bool: +def _intervals_overlap(left: tuple[float, float], right: tuple[float, float]) -> bool: return left[0] < right[1] and right[0] < left[1] @@ -630,7 +640,9 @@ def _validate_fold_evidence( fit_set = set(fold.fit_ids) prediction_set = set(fold.prediction_ids) if train_set.intersection(test_set): - _add_error(errors, f"{stage}: training and test membership must be disjoint.") + _add_error( + errors, f"{stage}: training and test membership must be disjoint." + ) if fit_set != train_set or fit_set.intersection(test_set): _add_error( errors, @@ -639,10 +651,10 @@ def _validate_fold_evidence( if prediction_set != test_set: _add_error(errors, f"{stage}: predictions must match the test membership.") if not (train_set | test_set | fit_set | prediction_set).issubset(known_ids): - _add_error(errors, f"{stage}: every fold identifier must have an event horizon.") - if validation_methods.intersection( - {"PURGED_CROSS_VALIDATION", "WALK_FORWARD"} - ): + _add_error( + errors, f"{stage}: every fold identifier must have an event horizon." + ) + if validation_methods.intersection({"PURGED_CROSS_VALIDATION", "WALK_FORWARD"}): if nonresampled_test_ids.intersection(test_set): _add_error( errors, @@ -715,7 +727,9 @@ def _validate_fold_evidence( tuple(sorted(partition)) for partition in path_partitions.values() ] if len(set(partition_signatures)) != len(partition_signatures): - _add_error(errors, f"{stage}: CPCV paths must have distinct test partitions.") + _add_error( + errors, f"{stage}: CPCV paths must have distinct test partitions." + ) if ( "PURGED_CROSS_VALIDATION" in validation_methods and known_ids @@ -896,7 +910,9 @@ def _validate_discovery( if edges_valid: for source, target in stage.directed_edges: if source not in nodes or target not in nodes: - _add_error(errors, f"{label}: every directed edge must use graph nodes.") + _add_error( + errors, f"{label}: every directed edge must use graph nodes." + ) if source == target: _add_error(errors, f"{label}: directed self-loops are not admissible.") if nodes and not _is_acyclic(nodes, stage.directed_edges): @@ -907,7 +923,9 @@ def _validate_discovery( _add_error(errors, f"{label}: ambiguous edges must be a tuple.") elif stage.ambiguous_edges: _valid_edge_tuple(stage.ambiguous_edges, label, "ambiguous edges", errors) - _add_error(errors, f"{label}: identification-relevant ambiguity must be resolved.") + _add_error( + errors, f"{label}: identification-relevant ambiguity must be resolved." + ) _validate_string_tuple(stage.assumptions, label, "graph assumptions", errors) safe_edges = stage.directed_edges if edges_valid else () return nodes, _adjacency(nodes, safe_edges), safe_edges @@ -975,7 +993,10 @@ def _validate_adjustment( _add_error(errors, f"{label}: treatment and outcome must be distinct.") if treatment not in graph_nodes or outcome not in graph_nodes: _add_error(errors, f"{label}: treatment and outcome must be graph nodes.") - if not _nonblank(stage.method_label) or stage.method_label not in _ADJUSTMENT_METHODS: + if ( + not _nonblank(stage.method_label) + or stage.method_label not in _ADJUSTMENT_METHODS + ): _add_error(errors, f"{label}: adjustment method is not source-listed.") if not isinstance(stage.identified, bool) or not stage.identified: _add_error(errors, f"{label}: the causal effect must be identified.") @@ -1006,14 +1027,16 @@ def _validate_adjustment( ("instruments", "instruments"), ): value = getattr(stage, field_name) - valid = _validate_string_tuple( - value, label, subject, errors, allow_empty=True - ) + valid = _validate_string_tuple(value, label, subject, errors, allow_empty=True) role_names[field_name] = set(value) if valid else set() if valid and not set(value).issubset(graph_nodes): - _add_error(errors, f"{label}: every declared causal role must be a graph node.") + _add_error( + errors, f"{label}: every declared causal role must be a graph node." + ) if valid and {treatment, outcome}.intersection(value): - _add_error(errors, f"{label}: treatment and outcome cannot be role variables.") + _add_error( + errors, f"{label}: treatment and outcome cannot be role variables." + ) computed_descendants = _reachable(adjacency, treatment) computed_descendants.discard(outcome) @@ -1026,7 +1049,9 @@ def _validate_adjustment( if mediator not in computed_descendants or outcome not in _reachable( adjacency, mediator ): - _add_error(errors, f"{label}: declared mediators must lie on a causal path.") + _add_error( + errors, f"{label}: declared mediators must lie on a causal path." + ) incoming = {node: 0 for node in graph_nodes} for targets in adjacency.values(): for target in targets: @@ -1034,7 +1059,9 @@ def _validate_adjustment( if any(incoming.get(collider, 0) < 2 for collider in role_names["colliders"]): _add_error(errors, f"{label}: declared colliders must have converging arrows.") if outcome not in _reachable(adjacency, treatment): - _add_error(errors, f"{label}: the graph must retain a treatment-to-outcome path.") + _add_error( + errors, f"{label}: the graph must retain a treatment-to-outcome path." + ) graph_confounders = { node @@ -1073,9 +1100,7 @@ def _validate_adjustment( errors, f"{label}: a variable cannot be both a confounder and an instrument.", ) - instrument_edges = tuple( - edge for edge in directed_edges if edge[0] != treatment - ) + instrument_edges = tuple(edge for edge in directed_edges if edge[0] != treatment) for instrument in role_names["instruments"]: if treatment not in _reachable_avoiding(adjacency, instrument, selected): _add_error( @@ -1179,8 +1204,13 @@ def _validate_adjustment( f"{label}: selected controls must block every graph-implied backdoor path.", ) elif stage.method_label == "FRONT_DOOR_ADJUSTMENT": - if not role_names["mediators"] or stage.frontdoor_criteria_satisfied is not True: - _add_error(errors, f"{label}: front-door criteria must be explicitly satisfied.") + if ( + not role_names["mediators"] + or stage.frontdoor_criteria_satisfied is not True + ): + _add_error( + errors, f"{label}: front-door criteria must be explicitly satisfied." + ) elif outcome in _reachable_avoiding( adjacency, treatment, role_names["mediators"] ): @@ -1202,7 +1232,9 @@ def _validate_adjustment( if not role_names["instruments"] or any( value is not True for value in instrument_premises ): - _add_error(errors, f"{label}: instrumental-variable premises must all hold.") + _add_error( + errors, f"{label}: instrumental-variable premises must all hold." + ) def _same_horizons( @@ -1274,9 +1306,7 @@ def validate_causal_factor_protocol( selected_valid = _validate_string_tuple( selection.selected_variables, label, "selected variables", errors ) - selected_variables = ( - set(selection.selected_variables) if selected_valid else set() - ) + selected_variables = set(selection.selected_variables) if selected_valid else set() selection_methods_valid = _validate_labels( selection.method_labels, _VARIABLE_SELECTION_METHODS, @@ -1303,7 +1333,9 @@ def validate_causal_factor_protocol( f"{label}: the declared selection design requires temporal validation.", ) elif not isinstance(selection.validation, ValidationEvidence): - _add_error(errors, f"{label}: temporal validation evidence has an invalid type.") + _add_error( + errors, f"{label}: temporal validation evidence has an invalid type." + ) selection_horizons = {} else: selection_horizons = _validate_fold_evidence( @@ -1316,12 +1348,16 @@ def validate_causal_factor_protocol( if isinstance(selection.overlapping_returns, bool) and ( selection.overlapping_returns != actual_overlap ): - _add_error(errors, f"{label}: overlap declaration must match event horizons.") + _add_error( + errors, f"{label}: overlap declaration must match event horizons." + ) validation_methods = _safe_label_set(selection.validation.method_labels) if actual_overlap and not validation_methods.intersection( _PURGED_VALIDATION_METHODS ): - _add_error(errors, f"{label}: overlapping returns require purged validation.") + _add_error( + errors, f"{label}: overlapping returns require purged validation." + ) if selection.strong_time_dependence and ( not _finite_real(selection.validation.embargo) or float(selection.validation.embargo) <= 0.0 @@ -1394,7 +1430,9 @@ def validate_causal_factor_protocol( if has_explanatory and not _nonblank(stage4.explanatory_evidence_id): _add_error(errors, f"{label}: explanatory evidence identifier is required.") if not has_explanatory and stage4.explanatory_evidence_id is not None: - _add_error(errors, f"{label}: explanatory evidence must match reported metrics.") + _add_error( + errors, f"{label}: explanatory evidence must match reported metrics." + ) if has_predictive and not _nonblank(stage4.predictive_evidence_id): _add_error(errors, f"{label}: predictive evidence identifier is required.") if not has_predictive and stage4.predictive_evidence_id is not None: @@ -1416,11 +1454,17 @@ def validate_causal_factor_protocol( and has_predictive and stage4.explanatory_evidence_id == stage4.predictive_evidence_id ): - _add_error(errors, f"{label}: explanatory and predictive evidence must be separate.") + _add_error( + errors, f"{label}: explanatory and predictive evidence must be separate." + ) if selection.purpose == "CAUSAL_ATTRIBUTION" and not has_explanatory: - _add_error(errors, f"{label}: causal attribution requires explanatory evidence.") + _add_error( + errors, f"{label}: causal attribution requires explanatory evidence." + ) if selection.purpose == "RISK_PREMIA_HARVESTING" and not has_predictive: - _add_error(errors, f"{label}: risk-premia harvesting requires predictive evidence.") + _add_error( + errors, f"{label}: risk-premia harvesting requires predictive evidence." + ) if not _nonblank(stage4.naive_benchmark_id): _add_error(errors, f"{label}: a naive benchmark identifier is required.") multiclass_fields = (stage4.multiclass_encoding, stage4.averaging_method) @@ -1430,14 +1474,15 @@ def validate_causal_factor_protocol( ): _add_error(errors, f"{label}: multiclass encoding is not source-listed.") if not isinstance(stage4.averaging_method, str) or ( - stage4.averaging_method - not in {"MICRO", "MACRO", "WEIGHTED", "SAMPLES"} + stage4.averaging_method not in {"MICRO", "MACRO", "WEIGHTED", "SAMPLES"} ): _add_error(errors, f"{label}: multiclass averaging is not source-listed.") if not _safe_label_set(stage4.task_types).intersection( {"PROBABILITY", "RANKING"} ): - _add_error(errors, f"{label}: multiclass settings require a classification task.") + _add_error( + errors, f"{label}: multiclass settings require a classification task." + ) if not isinstance(stage4.validation, ValidationEvidence): _add_error(errors, f"{label}: purged validation evidence is required.") else: @@ -1471,7 +1516,9 @@ def validate_causal_factor_protocol( errors, ) if portfolio_labels_valid and set(portfolio.method_labels) != _PORTFOLIO_METHODS: - _add_error(errors, f"{label}: all source portfolio considerations are required.") + _add_error( + errors, f"{label}: all source portfolio considerations are required." + ) causal_valid = _validate_string_tuple( portfolio.causal_exposures, label, "causal exposures", errors ) @@ -1503,9 +1550,13 @@ def validate_causal_factor_protocol( ) safe_treatment = adjustment.treatment if _nonblank(adjustment.treatment) else "" if safe_treatment not in causal_exposures: - _add_error(errors, f"{label}: the target causal factor must drive position sizing.") + _add_error( + errors, f"{label}: the target causal factor must drive position sizing." + ) if causal_exposures.intersection(neutral_exposures): - _add_error(errors, f"{label}: causal and neutralized exposures must be disjoint.") + _add_error( + errors, f"{label}: causal and neutralized exposures must be disjoint." + ) declared_colliders = _safe_label_set(adjustment.colliders) if not declared_colliders.issubset(neutral_exposures): _add_error(errors, f"{label}: declared colliders must be neutralized.") @@ -1550,7 +1601,9 @@ def validate_causal_factor_protocol( ) if non_monte_carlo: if not isinstance(backtest.validation, ValidationEvidence): - _add_error(errors, f"{label}: temporal backtests require validation evidence.") + _add_error( + errors, f"{label}: temporal backtests require validation evidence." + ) else: _validate_fold_evidence( backtest.validation, @@ -1582,7 +1635,9 @@ def validate_causal_factor_protocol( if not _nonblank(backtest.monte_carlo_dgp): _add_error(errors, f"{label}: Monte Carlo requires an explicit DGP.") elif backtest.monte_carlo_dgp is not None: - _add_error(errors, f"{label}: a Monte Carlo DGP requires the Monte Carlo method.") + _add_error( + errors, f"{label}: a Monte Carlo DGP requires the Monte Carlo method." + ) multiple_testing = report.multiple_testing_adjustments label = "Stage 7" @@ -1613,7 +1668,9 @@ def validate_causal_factor_protocol( _add_error(errors, f"{label}: a family partition is invalid.") continue if partition[0] in partition_ids: - _add_error(errors, f"{label}: family partition identifiers must be unique.") + _add_error( + errors, f"{label}: family partition identifiers must be unique." + ) partition_ids.add(partition[0]) if not _validate_string_tuple( partition[1], label, "partition trial identifiers", errors @@ -1624,12 +1681,17 @@ def validate_causal_factor_protocol( _add_error(errors, f"{label}: family partitions must be disjoint.") covered_trials.add(trial_id) if trials_valid and covered_trials != set(report.declared_trial_ids): - _add_error(errors, f"{label}: family partitions must cover declared trials exactly.") + _add_error( + errors, + f"{label}: family partitions must cover declared trials exactly.", + ) if not _finite_real(multiple_testing.alpha) or not ( 0.0 < float(multiple_testing.alpha) < 1.0 ): - _add_error(errors, f"{label}: alpha must be a finite number between zero and one.") + _add_error( + errors, f"{label}: alpha must be a finite number between zero and one." + ) if not isinstance(multiple_testing.backtests_independent, bool): _add_error(errors, f"{label}: backtest-dependence flag must be boolean.") elif multiple_testing.backtests_independent: @@ -1670,9 +1732,7 @@ def validate_causal_factor_protocol( ): _add_error(errors, f"{label}: positive Sharpe-ratio variance is required.") if not _finite_real(multiple_testing.effective_trials) or not ( - 1.0 - <= float(multiple_testing.effective_trials) - < trial_count + 1.0 <= float(multiple_testing.effective_trials) < trial_count ): _add_error(errors, f"{label}: effective trials must be below total trials.") if ( @@ -1683,10 +1743,13 @@ def validate_causal_factor_protocol( _add_error(errors, f"{label}: sample length must be an integer above one.") if not _finite_real(multiple_testing.skewness): _add_error(errors, f"{label}: skewness must be finite.") - if not _finite_real(multiple_testing.kurtosis) or float( - multiple_testing.kurtosis - ) < 1.0: - _add_error(errors, f"{label}: Pearson kurtosis must be finite and at least one.") + if ( + not _finite_real(multiple_testing.kurtosis) + or float(multiple_testing.kurtosis) < 1.0 + ): + _add_error( + errors, f"{label}: Pearson kurtosis must be finite and at least one." + ) elif _finite_real(multiple_testing.skewness) and abs( float(multiple_testing.skewness) ) > math.sqrt(float(multiple_testing.kurtosis) - 1.0): diff --git a/src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py b/src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py index 3ad6407..8f929ae 100644 --- a/src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py +++ b/src/RiskLabAI/causal_factor_analysis/specification_diagnostics.py @@ -168,9 +168,7 @@ def fork_population_diagnostics() -> SpecificationPopulationResult: return SpecificationPopulationResult( structure="fork", conditioning_variable_role="confounder", - unconditioned=_population_regression( - ("intercept", "X"), (0.0, 0.5), 0.25 - ), + unconditioned=_population_regression(("intercept", "X"), (0.0, 0.5), 0.25), conditioned=_population_regression( ("intercept", "X", "Z"), (0.0, 0.0, 1.0), 0.5 ), @@ -183,9 +181,7 @@ def collider_population_diagnostics() -> SpecificationPopulationResult: return SpecificationPopulationResult( structure="collider", conditioning_variable_role="collider", - unconditioned=_population_regression( - ("intercept", "X"), (0.0, 0.0), 0.0 - ), + unconditioned=_population_regression(("intercept", "X"), (0.0, 0.0), 0.0), conditioned=_population_regression( ("intercept", "X", "Z"), (0.0, -0.5, 0.5), 0.5 ), @@ -198,9 +194,7 @@ def confounded_mediator_population_diagnostics() -> SpecificationPopulationResul return SpecificationPopulationResult( structure="confounded_mediator", conditioning_variable_role="confounded_mediator", - unconditioned=_population_regression( - ("intercept", "X"), (0.0, 1.0), 1.0 / 7.0 - ), + unconditioned=_population_regression(("intercept", "X"), (0.0, 1.0), 1.0 / 7.0), conditioned=_population_regression( ("intercept", "X", "Z"), (0.0, -0.5, 1.5), diff --git a/src/RiskLabAI/causal_factor_analysis/treatment_effects.py b/src/RiskLabAI/causal_factor_analysis/treatment_effects.py index 8d5bab9..3c6d64a 100644 --- a/src/RiskLabAI/causal_factor_analysis/treatment_effects.py +++ b/src/RiskLabAI/causal_factor_analysis/treatment_effects.py @@ -124,15 +124,11 @@ def treatment_effect_decomposition( "counterfactual_control_mean_for_treated", counterfactual_control_mean_for_treated, ) - observed_difference = _as_finite_real( - "observed_difference", treated - control - ) + observed_difference = _as_finite_real("observed_difference", treated - control) effect_on_treated = _as_finite_real( "average_treatment_effect_on_treated", treated - counterfactual ) - selection_bias = _as_finite_real( - "sample_selection_bias", counterfactual - control - ) + selection_bias = _as_finite_real("sample_selection_bias", counterfactual - control) return TreatmentEffectDecomposition( observed_difference=observed_difference, average_treatment_effect_on_treated=effect_on_treated, @@ -218,12 +214,8 @@ def backdoor_adjusted_average_treatment_effect( ) -> float: """Return the standardized average treatment effect in Equation 7.""" - treated = _as_finite_vector( - "treated_conditional_means", treated_conditional_means - ) - control = _as_finite_vector( - "control_conditional_means", control_conditional_means - ) + treated = _as_finite_vector("treated_conditional_means", treated_conditional_means) + control = _as_finite_vector("control_conditional_means", control_conditional_means) probabilities = _as_probability_vector( "adjustment_probabilities", adjustment_probabilities ) diff --git a/test/causal_factor_analysis/test_cfi_book_graph_examples.py b/test/causal_factor_analysis/test_cfi_book_graph_examples.py index 7b96c9d..4536415 100644 --- a/test/causal_factor_analysis/test_cfi_book_graph_examples.py +++ b/test/causal_factor_analysis/test_cfi_book_graph_examples.py @@ -59,9 +59,7 @@ def test_figure_8_hypothetical_factor_mechanism_has_backdoor_and_frontdoor_sets( ("OI", "PC"), ) observed_graph = _dag(("HML", "MOM", "OI", "PC"), edges) - assert minimal_backdoor_adjustment_sets(observed_graph, "HML", "PC") == ( - ("MOM",), - ) + assert minimal_backdoor_adjustment_sets(observed_graph, "HML", "PC") == (("MOM",),) latent_graph = _dag( ("HML", "MOM", "OI", "PC"), @@ -69,9 +67,7 @@ def test_figure_8_hypothetical_factor_mechanism_has_backdoor_and_frontdoor_sets( observed=("HML", "OI", "PC"), ) assert minimal_backdoor_adjustment_sets(latent_graph, "HML", "PC") == () - assert minimal_frontdoor_adjustment_sets(latent_graph, "HML", "PC") == ( - ("OI",), - ) + assert minimal_frontdoor_adjustment_sets(latent_graph, "HML", "PC") == (("OI",),) def test_figures_16_to_18_conditioning_on_collider_opens_the_path(): diff --git a/test/causal_factor_analysis/test_protocol.py b/test/causal_factor_analysis/test_protocol.py index 183420e..d93e525 100644 --- a/test/causal_factor_analysis/test_protocol.py +++ b/test/causal_factor_analysis/test_protocol.py @@ -271,16 +271,13 @@ def test_wrong_stage_record_type_is_rejected(): def test_boundary_touching_horizons_are_not_overlapping(): report = _valid_report() horizons = tuple( - EventHorizon(f"b{index}", float(index), float(index + 1)) - for index in range(6) + EventHorizon(f"b{index}", float(index), float(index + 1)) for index in range(6) ) validation = replace( report.variable_selection.validation, event_horizons=horizons, folds=( - _fold( - "b1", "bp", ("b2", "b3", "b4", "b5"), ("b0", "b1"), "br1" - ), + _fold("b1", "bp", ("b2", "b3", "b4", "b5"), ("b0", "b1"), "br1"), _fold("b2", "bp", ("b0", "b1", "b4", "b5"), ("b2", "b3"), "br2"), _fold("b3", "bp", ("b0", "b1", "b2", "b3"), ("b4", "b5"), "br3"), ), @@ -354,9 +351,7 @@ def test_in_sample_stage1_methods_do_not_require_temporal_folds_without_guards() strong_time_dependence=False, validation=None, ) - validate_causal_factor_protocol( - replace(report, variable_selection=selection) - ) + validate_causal_factor_protocol(replace(report, variable_selection=selection)) with pytest.raises(ValueError, match="Stage 1"): validate_causal_factor_protocol( @@ -401,7 +396,9 @@ def test_refit_id_cannot_represent_different_training_sets(): second = replace(validation.folds[1], refit_id=validation.folds[0].refit_id) stage = replace( report.causal_explanatory_and_predictive_power, - validation=replace(validation, folds=(validation.folds[0], second, validation.folds[2])), + validation=replace( + validation, folds=(validation.folds[0], second, validation.folds[2]) + ), ) with pytest.raises(ValueError, match="Stage 4"): _validate_with(report, causal_explanatory_and_predictive_power=stage) @@ -421,7 +418,9 @@ def test_temporal_overlap_and_embargo_leakage_are_rejected(): horizons = tuple( EventHorizon(item.observation_id, item.start, item.end) for item in validation.event_horizons - ) + (EventHorizon("embargoed", 3.5, 3.8),) + ) + ( + EventHorizon("embargoed", 3.5, 3.8), + ) embargo_fold = _fold("bad2", "p3", ("embargoed", "o3"), ("o1", "o2"), "bad2-refit") bad_validation = replace(validation, event_horizons=horizons, folds=(embargo_fold,)) stage = replace( @@ -571,9 +570,7 @@ def test_instrumental_variables_requires_all_identification_premises(): instrument_exclusion_satisfied=True, instrument_exogeneity_satisfied=True, ) - validate_causal_factor_protocol( - replace(report, causal_adjustment_set=adjustment) - ) + validate_causal_factor_protocol(replace(report, causal_adjustment_set=adjustment)) for field in ( "instrument_relevance_satisfied", "instrument_exclusion_satisfied", @@ -711,9 +708,7 @@ def test_graph_implied_confounders_cannot_be_omitted_or_used_as_instruments(): control_justifications=(), ) with pytest.raises(ValueError, match="Stage 3"): - validate_causal_factor_protocol( - replace(report, causal_adjustment_set=omitted) - ) + validate_causal_factor_protocol(replace(report, causal_adjustment_set=omitted)) invalid_instrument = replace( report.causal_adjustment_set, diff --git a/test/causal_factor_analysis/test_specification_diagnostics.py b/test/causal_factor_analysis/test_specification_diagnostics.py index 9e1161a..35c0249 100644 --- a/test/causal_factor_analysis/test_specification_diagnostics.py +++ b/test/causal_factor_analysis/test_specification_diagnostics.py @@ -55,8 +55,7 @@ def test_appendix_equations_54_to_61_give_exact_conditional_gaussian_moments(): variance_z = np.array([[3.0]]) conditional_mean_coefficient = covariance_xy_z @ np.linalg.inv(variance_z) conditional_covariance = ( - covariance_xy - - conditional_mean_coefficient @ covariance_xy_z.T + covariance_xy - conditional_mean_coefficient @ covariance_xy_z.T ) np.testing.assert_allclose( conditional_mean_coefficient, diff --git a/test/causal_factor_analysis/test_treatment_effects.py b/test/causal_factor_analysis/test_treatment_effects.py index 1651c15..f971fe1 100644 --- a/test/causal_factor_analysis/test_treatment_effects.py +++ b/test/causal_factor_analysis/test_treatment_effects.py @@ -95,9 +95,7 @@ def test_equation_9_frontdoor_probability_matches_nested_hand_sum(): ), treatment_probabilities=(0.6, 0.4), ) - expected = 0.25 * (0.1 * 0.6 + 0.5 * 0.4) + 0.75 * ( - 0.4 * 0.6 + 0.8 * 0.4 - ) + expected = 0.25 * (0.1 * 0.6 + 0.5 * 0.4) + 0.75 * (0.4 * 0.6 + 0.8 * 0.4) assert result == pytest.approx(expected, abs=1.0e-15) From fa76ac7921e7c068d35b003e31e96981404f936e Mon Sep 17 00:00:00 2001 From: hamid-arian Date: Mon, 24 Aug 2026 10:19:27 -0400 Subject: [PATCH 3/6] Fix Python release CI subprocess isolation --- PACKAGE_INSPECTION.md | 4 +++- RELEASE_NOTES_NEXT_MAJOR.md | 7 ++++--- TEST_INVENTORY.json | 16 ++++++++-------- docs/compatibility.md | 3 +++ .../causal_factor_analysis/test_factor_mirage.py | 7 ++----- .../test_graph_identification.py | 5 ++--- .../test_minimum_variance.py | 7 ++----- test/causal_factor_analysis/test_protocol.py | 7 ++----- 8 files changed, 26 insertions(+), 30 deletions(-) diff --git a/PACKAGE_INSPECTION.md b/PACKAGE_INSPECTION.md index bce7a14..9c093be 100644 --- a/PACKAGE_INSPECTION.md +++ b/PACKAGE_INSPECTION.md @@ -48,7 +48,9 @@ complete 57-feature causal API. Run the exact package-scoped test inventory and the valid optional-feature lanes. Confirm that optional groups do not become base dependencies and that changepoints alone is unavailable on Python 3.14. Run installed-package tests outside the source checkout and reject any import -whose resolved path remains inside the repository. +whose resolved path remains inside the repository. Tests that launch child +interpreters must give those processes clean temporary working directories and +apply the same origin rejection. ## Final review diff --git a/RELEASE_NOTES_NEXT_MAJOR.md b/RELEASE_NOTES_NEXT_MAJOR.md index 5728990..e0af397 100644 --- a/RELEASE_NOTES_NEXT_MAJOR.md +++ b/RELEASE_NOTES_NEXT_MAJOR.md @@ -21,9 +21,10 @@ locked by package-scoped tests. The release metadata uses the PEP 639 `BSD-3-Clause` expression and license file without the superseded license classifier. The governed CI workflow now installs the distribution, rejects imports shadowed by the preserved repository -root, tests the six approved base runtime/NumPy combinations, exercises -feature-specific optional lanes, and isolates the Python 3.14 changepoint -limitation. Pinned static checks cover the clean causal source and tests. +root in both parent and child interpreters, tests the six approved base +runtime/NumPy combinations, exercises feature-specific optional lanes, and +isolates the Python 3.14 changepoint limitation. Pinned static checks cover the +clean causal source and tests. No source-conflicted causal result is admitted. The metadata contract is complete; artifacts, publication, upload, and release remain blocked. diff --git a/TEST_INVENTORY.json b/TEST_INVENTORY.json index 6f63042..5a8d041 100644 --- a/TEST_INVENTORY.json +++ b/TEST_INVENTORY.json @@ -213,29 +213,29 @@ }, { "path": "test/causal_factor_analysis/test_factor_mirage.py", - "size": 16394, - "sha256": "268DB559600AF1D56EDADEEB0FC20DCF642D4FF5AF4DBBBD4517CC5EE4F94DFF", + "size": 16311, + "sha256": "B64153FE7B39D103022268E503A132F76E967A05DB103D3339A662A9DA0A890C", "group": "CAUSAL_FACTOR_ANALYSIS", "declared_test_functions": 32 }, { "path": "test/causal_factor_analysis/test_graph_identification.py", - "size": 35186, - "sha256": "64D386BDD80E76CEAA4762D429B5E3ADE083AB7E509BE348A7A632B9C11E4C35", + "size": 35155, + "sha256": "16CA24092CF9D05D2A7718FF1BACBF050592EFDDE18C7B7CCF2B20121250F789", "group": "CAUSAL_FACTOR_ANALYSIS", "declared_test_functions": 47 }, { "path": "test/causal_factor_analysis/test_minimum_variance.py", - "size": 15983, - "sha256": "954E52279A50DADEAE6519B2CF6679B7AB4CE0C3F01CE70AC81AD829E9423F16", + "size": 15900, + "sha256": "F0C7A037F7A2AE1652E2E69362898572095EA6342EAB5699367C8FC84E929D9D", "group": "CAUSAL_FACTOR_ANALYSIS", "declared_test_functions": 32 }, { "path": "test/causal_factor_analysis/test_protocol.py", - "size": 39048, - "sha256": "1CAACE494F327A7B135E3A0D8F38BC15EBEA57BBC2D2CB77CB9D6BA6B8E17930", + "size": 38965, + "sha256": "E1E456BE476A5D990F781E240868B6C344FA0345FC78DC3C027BE63271BCA240", "group": "CAUSAL_FACTOR_ANALYSIS", "declared_test_functions": 43 }, diff --git a/docs/compatibility.md b/docs/compatibility.md index 59ae18a..a7f09a6 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -71,6 +71,9 @@ internally from standard definitions and have hand-calculated oracle tests. The governed CI workflow installs the distribution non-editably before every runtime test. Import-origin probes run outside the repository checkout and reject any `RiskLabAI` module resolved from the preserved repository root. +Tests that launch child interpreters also use clean temporary working +directories, so a subprocess cannot silently resolve the preserved root +package in place of the installed `src` distribution. The base matrix contains six valid interpreter/NumPy combinations: the lowest supported NumPy binary and the latest supported NumPy 2.x release on each of diff --git a/test/causal_factor_analysis/test_factor_mirage.py b/test/causal_factor_analysis/test_factor_mirage.py index 31702d3..ef32f86 100644 --- a/test/causal_factor_analysis/test_factor_mirage.py +++ b/test/causal_factor_analysis/test_factor_mirage.py @@ -7,8 +7,6 @@ import sys from dataclasses import FrozenInstanceError from fractions import Fraction -from pathlib import Path - import numpy as np import pytest @@ -366,8 +364,7 @@ def test_result_records_are_frozen(): diagnostics.correct_r_squared = 0.0 -def test_public_import_is_isolated_and_source_conflicted_api_is_absent(): - repository_root = Path(__file__).resolve().parents[2] +def test_public_import_is_isolated_and_source_conflicted_api_is_absent(tmp_path): program = """ import sys import RiskLabAI @@ -401,7 +398,7 @@ def test_public_import_is_isolated_and_source_conflicted_api_is_absent(): """ subprocess.run( [sys.executable, "-c", program], - cwd=repository_root, + cwd=tmp_path, check=True, capture_output=True, text=True, diff --git a/test/causal_factor_analysis/test_graph_identification.py b/test/causal_factor_analysis/test_graph_identification.py index de114cd..b8fafd5 100644 --- a/test/causal_factor_analysis/test_graph_identification.py +++ b/test/causal_factor_analysis/test_graph_identification.py @@ -933,8 +933,7 @@ def test_module_is_python39_and_standard_library_only(): } -def test_package_import_does_not_load_legacy_or_private_namespaces(): - project = Path(__file__).parents[2] +def test_package_import_does_not_load_legacy_or_private_namespaces(tmp_path): public_names = repr(sorted(PUBLIC_NAMES)) program = f""" import sys @@ -955,7 +954,7 @@ def test_package_import_does_not_load_legacy_or_private_namespaces(): environment["PYTHONDONTWRITEBYTECODE"] = "1" subprocess.run( [sys.executable, "-B", "-c", program], - cwd=project, + cwd=tmp_path, env=environment, check=True, capture_output=True, diff --git a/test/causal_factor_analysis/test_minimum_variance.py b/test/causal_factor_analysis/test_minimum_variance.py index f7dc3cf..817e0a4 100644 --- a/test/causal_factor_analysis/test_minimum_variance.py +++ b/test/causal_factor_analysis/test_minimum_variance.py @@ -4,8 +4,6 @@ import subprocess import sys -from pathlib import Path - import numpy as np import pytest @@ -394,8 +392,7 @@ def zero_orthonormal_basis(*args, **kwargs): minimum_variance_factor_weights(np.eye(2), np.ones((2, 1)), np.ones(1)) -def test_public_import_is_isolated_from_other_model_namespaces(): - repository_root = Path(__file__).resolve().parents[2] +def test_public_import_is_isolated_from_other_model_namespaces(tmp_path): program = """ import sys import RiskLabAI @@ -418,7 +415,7 @@ def test_public_import_is_isolated_from_other_model_namespaces(): """ subprocess.run( [sys.executable, "-c", program], - cwd=repository_root, + cwd=tmp_path, check=True, capture_output=True, text=True, diff --git a/test/causal_factor_analysis/test_protocol.py b/test/causal_factor_analysis/test_protocol.py index d93e525..5064f0f 100644 --- a/test/causal_factor_analysis/test_protocol.py +++ b/test/causal_factor_analysis/test_protocol.py @@ -6,8 +6,6 @@ import subprocess import sys from dataclasses import FrozenInstanceError, replace -from pathlib import Path - import pytest from RiskLabAI.causal_factor_analysis.protocol import ( @@ -1137,8 +1135,7 @@ def test_malformed_labels_fail_with_value_error(stage_name, stage): _validate_with(_valid_report(), **{stage_name: stage}) -def test_module_and_package_exports_are_exact_and_isolated(): - repository_root = Path(__file__).resolve().parents[2] +def test_module_and_package_exports_are_exact_and_isolated(tmp_path): program = f""" import sys import RiskLabAI.causal_factor_analysis as package @@ -1161,7 +1158,7 @@ def test_module_and_package_exports_are_exact_and_isolated(): """ subprocess.run( [sys.executable, "-c", program], - cwd=repository_root, + cwd=tmp_path, check=True, capture_output=True, text=True, From d8bf7be61eb972f12c64f3308637779303f8c7eb Mon Sep 17 00:00:00 2001 From: hamid-arian Date: Mon, 24 Aug 2026 13:47:33 -0400 Subject: [PATCH 4/6] Fix square-system causal optimizer stability --- PUBLIC_API.json | 168 +++++++++--------- TEST_INVENTORY.json | 18 +- docs/compatibility.md | 12 +- .../causal_factor_analysis/optimizer.py | 33 ++-- .../test_minimum_variance.py | 16 ++ 5 files changed, 134 insertions(+), 113 deletions(-) diff --git a/PUBLIC_API.json b/PUBLIC_API.json index 5159a32..1852cd1 100644 --- a/PUBLIC_API.json +++ b/PUBLIC_API.json @@ -67,8 +67,8 @@ "path": "src/RiskLabAI/backtest/__init__.py", "module": "RiskLabAI.backtest", "module_kind": "package", - "sha256": "6AE9F8763745BD3331A53D77441E4AB76BA51727916F8516C3181CC7F88590FB", - "size": 6381, + "sha256": "BC1D1DB5707902C0C61EFBB1DB1CDC88101388B065BCA30FE7A8CA621667058C", + "size": 6172, "defined_public_names": [], "definitions": [] }, @@ -289,8 +289,8 @@ "path": "src/RiskLabAI/backtest/backtest_synthetic_data.py", "module": "RiskLabAI.backtest.backtest_synthetic_data", "module_kind": "module", - "sha256": "9ACDA6D9D81702F611CBC4EBE11899C31463DA899709D9975931F592CD59AF39", - "size": 3523, + "sha256": "F67B8239A8E5F9EF5BA73F84938614F211C71197CD87AAFCFE10A8EDEBABC627", + "size": 3424, "defined_public_names": [ "synthetic_back_testing" ], @@ -693,8 +693,8 @@ "path": "src/RiskLabAI/backtest/validation/__init__.py", "module": "RiskLabAI.backtest.validation", "module_kind": "package", - "sha256": "503E9653E49B6606D1534CD677A731F1D5EB07563ED39AE62B64CF39F4BED5B1", - "size": 1951, + "sha256": "88C99E37BE96E2BD23FD54ED0491C962F2BDEDD696E97F9B2627A410D1F8660F", + "size": 1896, "defined_public_names": [], "definitions": [] }, @@ -787,8 +787,8 @@ "path": "src/RiskLabAI/backtest/validation/cross_validator_interface.py", "module": "RiskLabAI.backtest.validation.cross_validator_interface", "module_kind": "module", - "sha256": "9BB6CFBFF867C8F863CF0CA0B1FF6384463A4385D88C2FD1FEFFA0775B85FD0D", - "size": 8202, + "sha256": "A6FB69106047B94AF0E7ECCDE88FBDCCB5178BE08ED8EE8E03B04564F8E50A7E", + "size": 7967, "defined_public_names": [ "CrossValidator" ], @@ -890,8 +890,8 @@ "path": "src/RiskLabAI/backtest/validation/purged_kfold.py", "module": "RiskLabAI.backtest.validation.purged_kfold", "module_kind": "module", - "sha256": "86802CB8E32ECDD092A522FBAC196C2BA5F05FAAAF4227091C705EDBD2DBFCD8", - "size": 19810, + "sha256": "8E2385391624CFB31E3B9C97B0D8E27B774E1985DF6ED5997F59A1D49ACFF902", + "size": 19267, "defined_public_names": [ "PurgedKFold" ], @@ -1099,8 +1099,8 @@ "path": "src/RiskLabAI/causal_factor_analysis/optimizer.py", "module": "RiskLabAI.causal_factor_analysis.optimizer", "module_kind": "module", - "sha256": "9C15ABD41FE232686553E37911A4F123C9FD745F877BC52C03E35CC0D36101A8", - "size": 8555, + "sha256": "C83DFD2630E82E3D31D45E47E3895FD8926CB2FC67A66842C4C25BDEEF2379A5", + "size": 8835, "defined_public_names": [ "minimum_variance_factor_weights" ], @@ -1341,8 +1341,8 @@ "path": "src/RiskLabAI/cluster/__init__.py", "module": "RiskLabAI.cluster", "module_kind": "package", - "sha256": "CC1993BE2778895842431BD777BFEF3B620341E6DB13B4BEDB5A58891CAF14DC", - "size": 669, + "sha256": "1313FCD1DFC97508FB0ECAE567B46BAF5CE9997748C302ABACD0AA4606D931B3", + "size": 642, "defined_public_names": [], "definitions": [] }, @@ -1403,8 +1403,8 @@ "path": "src/RiskLabAI/controller/__init__.py", "module": "RiskLabAI.controller", "module_kind": "package", - "sha256": "875DCD9EC172311DF8E9DF665C60102BDC0F56218A5CDDE0760E2A081E369230", - "size": 75, + "sha256": "0DED3F4FAB2B94562CC9337E7D1A170B6ACA95D4F4477C2A6730DF29ABE0A8FF", + "size": 73, "defined_public_names": [], "definitions": [] }, @@ -1412,8 +1412,8 @@ "path": "src/RiskLabAI/controller/bars_initializer.py", "module": "RiskLabAI.controller.bars_initializer", "module_kind": "module", - "sha256": "28284D4877900DE9D3582C2B821FC66173E474E1F62C95F27385190C4A88426F", - "size": 13999, + "sha256": "6C45A2CACEB4BD9AAB6B08EA0EDD240E4144CF823FCE361C5297F2DC9FFF9022", + "size": 13700, "defined_public_names": [ "BarsInitializerController" ], @@ -1429,8 +1429,8 @@ "path": "src/RiskLabAI/controller/data_structure_controller.py", "module": "RiskLabAI.controller.data_structure_controller", "module_kind": "module", - "sha256": "237843F3AD293C366607FA555C218F4F3CD8ACCD881F784030B3AA6B12BC8E23", - "size": 6102, + "sha256": "4D4CAFBA0B0059A89398A1CD453F57A803E80466C93DC3AC4A0F11BB894DDF8C", + "size": 5914, "defined_public_names": [ "BAR_COLUMNS", "Controller" @@ -1538,8 +1538,8 @@ "path": "src/RiskLabAI/data/__init__.py", "module": "RiskLabAI.data", "module_kind": "package", - "sha256": "F238157F6B54DB5E12E2F910821431BED049AB6E99665B64AD81F95EB7B1488B", - "size": 664, + "sha256": "BF1701C8581DC602A5D37E10BAE6CDB837D489BD8791781B5F50BA4D067876FD", + "size": 630, "defined_public_names": [], "definitions": [] }, @@ -1556,8 +1556,8 @@ "path": "src/RiskLabAI/data/denoise/denoising.py", "module": "RiskLabAI.data.denoise.denoising", "module_kind": "module", - "sha256": "411FFBD20F902E19426506D6C5472570960F9091E67E3D0F8626DE09E63B8B92", - "size": 11538, + "sha256": "AE33547DD525D4905A7322995E491D8D361D434E05F56CDD1155F7AF04CD7371", + "size": 11161, "defined_public_names": [ "corr_to_cov", "cov_to_corr", @@ -1692,8 +1692,8 @@ "path": "src/RiskLabAI/data/differentiation/differentiation.py", "module": "RiskLabAI.data.differentiation.differentiation", "module_kind": "module", - "sha256": "FD6A1B4247CC3F7B6C794866CD1809770460BD2AFC08375F2F96B02AF0D79E02", - "size": 12258, + "sha256": "4E9E45C90AEB0B8949E5F6F774D1AF20AA5973188F5FD23FFB4344E6EF5C5E9F", + "size": 11859, "defined_public_names": [ "calculate_weights_ffd", "calculate_weights_std", @@ -1783,8 +1783,8 @@ "path": "src/RiskLabAI/data/distance/distance_metric.py", "module": "RiskLabAI.data.distance.distance_metric", "module_kind": "module", - "sha256": "6399EFDB1E12A82ED3BB6831FF6F985E0EAB18D2420D82670CF1599189D51E73", - "size": 8377, + "sha256": "89073FAA39E9FF195C7028BEA970410562745534AE52AAFD310C5499E4DC914D", + "size": 8073, "defined_public_names": [ "calculate_cross_entropy", "calculate_distance", @@ -1836,8 +1836,8 @@ "path": "src/RiskLabAI/data/labeling/__init__.py", "module": "RiskLabAI.data.labeling", "module_kind": "package", - "sha256": "E84DB73F915081EF60F6C8441D10BFE5E759037EE7D42945B1AD2FBB77A10CC3", - "size": 1193, + "sha256": "CBDD997697143DFFD4287A26E568C257FDE2634BDBA325955438353D65DC86F7", + "size": 1144, "defined_public_names": [], "definitions": [] }, @@ -1947,8 +1947,8 @@ "path": "src/RiskLabAI/data/structures/abstract_imbalance_bars.py", "module": "RiskLabAI.data.structures.abstract_imbalance_bars", "module_kind": "module", - "sha256": "350535B7AAF4E997E19242E07991EA7AB22BB41C041490D2243F25B2D4D90111", - "size": 6388, + "sha256": "E7A54B451B412FD1F2463E59E8706131A1D7AD8C84D3FE6CC1E2B5A3DC8B1B99", + "size": 6210, "defined_public_names": [ "AbstractImbalanceBars" ], @@ -1964,8 +1964,8 @@ "path": "src/RiskLabAI/data/structures/abstract_information_driven_bars.py", "module": "RiskLabAI.data.structures.abstract_information_driven_bars", "module_kind": "module", - "sha256": "2035EA576F53B0CF6B7A4A44CEB2DDC64E531BC91E8F6C98ADACF7578A11306A", - "size": 4150, + "sha256": "8601CD7E405A10F66BF6A82192E8EA261F9D71CA2A8C85E48FFA334EBC00D8DF", + "size": 4021, "defined_public_names": [ "AbstractInformationDrivenBars" ], @@ -1981,8 +1981,8 @@ "path": "src/RiskLabAI/data/structures/abstract_run_bars.py", "module": "RiskLabAI.data.structures.abstract_run_bars", "module_kind": "module", - "sha256": "51DDD1B7C662E2FB2D749C5C8483782C76EF00584ACC36FD8314787239DAFFED", - "size": 10204, + "sha256": "1B7E12ED9C1B2505F014FE5487CF1AFC938463198C8B40F7360ED1739E035EFD", + "size": 9943, "defined_public_names": [ "AbstractRunBars" ], @@ -2212,8 +2212,8 @@ "path": "src/RiskLabAI/data/weights/sample_weights.py", "module": "RiskLabAI.data.weights.sample_weights", "module_kind": "module", - "sha256": "A1AEBA7541663238C20B3B7A76B8706AB62D6860CD171DB6000E2EF92C492724", - "size": 7225, + "sha256": "4DA73877840EF49BDFF35B087CE1FC52354D14E9136217D16B1948C5577A9BB4", + "size": 6991, "defined_public_names": [ "calculate_average_uniqueness", "calculate_time_decay", @@ -2256,8 +2256,8 @@ "path": "src/RiskLabAI/ensemble/bagging_classifier_accuracy.py", "module": "RiskLabAI.ensemble.bagging_classifier_accuracy", "module_kind": "module", - "sha256": "DF1570D491B4452412103BE1ECDC6B0C4CFDA3D9AAF55D9DDD7620B611134F92", - "size": 1300, + "sha256": "9B2150DA7DB358161927137E5E2ED9C3FA0249FF71E86401F29C9F96258B839C", + "size": 1255, "defined_public_names": [ "bagging_classifier_accuracy" ], @@ -2302,8 +2302,8 @@ "path": "src/RiskLabAI/features/__init__.py", "module": "RiskLabAI.features", "module_kind": "package", - "sha256": "8E6EF92F8D1E0E9BCA37E3436806A37308B8B3CB08C221481F4A1261327C5A18", - "size": 123, + "sha256": "CE30ECC88CBFF140893C4FADFBE38DC90EE954024126EF66F1A5200AE0146C2B", + "size": 117, "defined_public_names": [], "definitions": [] }, @@ -2349,8 +2349,8 @@ "path": "src/RiskLabAI/features/entropy_features/kontoyiannis.py", "module": "RiskLabAI.features.entropy_features.kontoyiannis", "module_kind": "module", - "sha256": "08FCF8EEEF1D9C4FFFC4F812A39B77F406050091ED287C9AE1A577E849998A4B", - "size": 2902, + "sha256": "A69F1FEB3DAC762724F5ED9F17BCAD0E9D9EC88B5946348FA80EA4059AD04119", + "size": 2793, "defined_public_names": [ "kontoyiannis_entropy", "longest_match_length" @@ -2372,8 +2372,8 @@ "path": "src/RiskLabAI/features/entropy_features/lempel_ziv.py", "module": "RiskLabAI.features.entropy_features.lempel_ziv", "module_kind": "module", - "sha256": "295BFA3C0E3970C75C924C6D19E2D0B384E29A2B50AED7C8C08E1723A975BF4F", - "size": 1017, + "sha256": "1F015F187028C1957CC91BC91D944B10202D7D3EF37F3CAAF29D4CFB238C8FF8", + "size": 976, "defined_public_names": [ "lempel_ziv_entropy" ], @@ -2389,8 +2389,8 @@ "path": "src/RiskLabAI/features/entropy_features/plug_in.py", "module": "RiskLabAI.features.entropy_features.plug_in", "module_kind": "module", - "sha256": "90DB453E95DD416771EAC11232E33584582C7FF1EA9B79DD441588185BADC483", - "size": 981, + "sha256": "799C9ABB1AD8A864BF15E5F1E3267C5FCB482B83C38A4F0FCF9E4C5F2C81B763", + "size": 942, "defined_public_names": [ "plug_in_entropy_estimator" ], @@ -2406,8 +2406,8 @@ "path": "src/RiskLabAI/features/entropy_features/pmf.py", "module": "RiskLabAI.features.entropy_features.pmf", "module_kind": "module", - "sha256": "649221CC7406A3CF9607DE10CCA44C0F7D661E2B877209626704EBFF93F0BBC9", - "size": 1862, + "sha256": "55A1BA2FAC7BDC384C15E36B7874E01C0DD0B40D5D8FF6E6113972B81FED8D24", + "size": 1798, "defined_public_names": [ "ngram_counts", "probability_mass_function" @@ -2429,8 +2429,8 @@ "path": "src/RiskLabAI/features/entropy_features/shannon.py", "module": "RiskLabAI.features.entropy_features.shannon", "module_kind": "module", - "sha256": "B594B49278CB1904E605CC44083B0A545F71E14B2BDDA6EF0A2BE03FAE2A2DA8", - "size": 723, + "sha256": "FBA55BE6A0B6E529ABD77BE038F5F885945B034A04DA042CFFD99A278735B37E", + "size": 689, "defined_public_names": [ "shannon_entropy" ], @@ -2446,8 +2446,8 @@ "path": "src/RiskLabAI/features/feature_importance/__init__.py", "module": "RiskLabAI.features.feature_importance", "module_kind": "package", - "sha256": "AA2D447A39D488D12DBDAA7E54F158AC9F4E94F7839D4F0BE56BBE6F9E0EC27D", - "size": 2142, + "sha256": "24C80AA3C46B3237D6FBC513C4E4A7EED88A54304C94792E5F24F295A00AFB73", + "size": 2076, "defined_public_names": [], "definitions": [] }, @@ -2455,8 +2455,8 @@ "path": "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mda.py", "module": "RiskLabAI.features.feature_importance.clustered_feature_importance_mda", "module_kind": "module", - "sha256": "656BBC12FE44777CE8557AF6A0F3292DA11A69B23C0F812E10501E9AFF4E2BF1", - "size": 5518, + "sha256": "7BF977CB4D65189BB478CAA8378D3094E7D0A8FD672C498AC3BD881FF0B818A6", + "size": 5361, "defined_public_names": [ "ClusteredFeatureImportanceMDA" ], @@ -2472,8 +2472,8 @@ "path": "src/RiskLabAI/features/feature_importance/clustered_feature_importance_mdi.py", "module": "RiskLabAI.features.feature_importance.clustered_feature_importance_mdi", "module_kind": "module", - "sha256": "965BFFF4A3EF79239B54C7DD5A09B7590C3EA1DF8F2B2F10695ADFA095CA45C7", - "size": 3511, + "sha256": "4B2B7A7F98BA4030A8B0B885C2D9077F22445F3D5B07FF470D1E247BF4346516", + "size": 3403, "defined_public_names": [ "ClusteredFeatureImportanceMDI" ], @@ -2546,8 +2546,8 @@ "path": "src/RiskLabAI/features/feature_importance/feature_importance_mda.py", "module": "RiskLabAI.features.feature_importance.feature_importance_mda", "module_kind": "module", - "sha256": "5E22AE26672B9B3A608CC237791625386C5B500FFBF768AEEC5478154F727CF0", - "size": 4650, + "sha256": "872D141D6D96B4CED9660710ACE3EFBFE4FCCB74065BEEA6FEF5344DF1A61861", + "size": 4515, "defined_public_names": [ "FeatureImportanceMDA" ], @@ -2563,8 +2563,8 @@ "path": "src/RiskLabAI/features/feature_importance/feature_importance_mdi.py", "module": "RiskLabAI.features.feature_importance.feature_importance_mdi", "module_kind": "module", - "sha256": "82F93E0DA408EB9D4984856C3C432704DC8093ADC90FE9DC898D2109E7381A92", - "size": 2739, + "sha256": "C201C6FA3986268AE49D7D2AFE177DBB468AA4E5D32CD9B9CA805DAEAA9EE5AE", + "size": 2651, "defined_public_names": [ "FeatureImportanceMDI" ], @@ -2580,8 +2580,8 @@ "path": "src/RiskLabAI/features/feature_importance/feature_importance_sfi.py", "module": "RiskLabAI.features.feature_importance.feature_importance_sfi", "module_kind": "module", - "sha256": "67CB65A25ACA73457000C23AC4853A1D281CA3C2DB3CE6614C1262AF51B808BF", - "size": 3984, + "sha256": "69ED5D5AEA7F977E72459492D278ED5A1A3B0B5A7099CED6C96D1B5EE26C4833", + "size": 3865, "defined_public_names": [ "FeatureImportanceSFI" ], @@ -2597,8 +2597,8 @@ "path": "src/RiskLabAI/features/feature_importance/feature_importance_strategy.py", "module": "RiskLabAI.features.feature_importance.feature_importance_strategy", "module_kind": "module", - "sha256": "E007852540710974A4C56089DB6A5DE46F11B19760690E400BAFCE154156BD1D", - "size": 894, + "sha256": "0B5BF2F9EA40AEF208FAF010308BE4E0AF32B9378B395A8B555CF2CB1D97CCAA", + "size": 857, "defined_public_names": [ "FeatureImportanceStrategy" ], @@ -2665,8 +2665,8 @@ "path": "src/RiskLabAI/features/microstructural_features/__init__.py", "module": "RiskLabAI.features.microstructural_features", "module_kind": "package", - "sha256": "69C079797912E65103C77DD079B00F420C30479FEFEB7A3C0C3E70AF7BCC855A", - "size": 737, + "sha256": "5A47422A191048EB40286C2BA1169BA3FF04396BFA08F9582439A26521A06353", + "size": 708, "defined_public_names": [], "definitions": [] }, @@ -2775,8 +2775,8 @@ "path": "src/RiskLabAI/features/structural_breaks/structural_breaks.py", "module": "RiskLabAI.features.structural_breaks.structural_breaks", "module_kind": "module", - "sha256": "9F5E690FDAC43FFE12728DBF696DF2B250235B192604F8CA9DDE9BC922E14656", - "size": 26104, + "sha256": "0C562C044C063DFA0606EB88D74A33B54B08A69FA19A126D1B8DB7FCD3276242", + "size": 25396, "defined_public_names": [ "compute_beta", "get_bsadf_sequence", @@ -2878,8 +2878,8 @@ "path": "src/RiskLabAI/hpc/hpc.py", "module": "RiskLabAI.hpc.hpc", "module_kind": "module", - "sha256": "B5AEFF775459A0AD8A5AD0E10B0806E7FC08137F1CCD0CD85966FFF60D03D1C5", - "size": 11685, + "sha256": "CC6377F6E22F2BB25D87723DE0D6C36DEB555F4D08C19BDD4B55C64D5FABEED1", + "size": 11306, "defined_public_names": [ "expand_call", "linear_partitions", @@ -2937,8 +2937,8 @@ "path": "src/RiskLabAI/optimization/__init__.py", "module": "RiskLabAI.optimization", "module_kind": "package", - "sha256": "E67CA20C00441769F46EA11BFDC7113646556FF45EFD63C1E748CC2BA93805A1", - "size": 1257, + "sha256": "8B178E855B01B7AAA4A6988060A26B794A9E4A867CF5E5F3125D1CBC0AE6900C", + "size": 1211, "defined_public_names": [], "definitions": [] }, @@ -2963,8 +2963,8 @@ "path": "src/RiskLabAI/optimization/hrp.py", "module": "RiskLabAI.optimization.hrp", "module_kind": "module", - "sha256": "5A5F6F3D3145C9F7870CD1FEE15BD88D11A7C7E707D4A6EDCF83F9F43F92AED1", - "size": 6323, + "sha256": "5DD1818F65CDCE5C07870D96353627600C325ADE1C4297AAB933AB5954538047", + "size": 6108, "defined_public_names": [ "cluster_variance", "distance_corr", @@ -3010,8 +3010,8 @@ "path": "src/RiskLabAI/optimization/hyper_parameter_tuning.py", "module": "RiskLabAI.optimization.hyper_parameter_tuning", "module_kind": "module", - "sha256": "93619B3E22168D701F2DF2E17ABA0134B3E3BF1345C2B6CF234E14A9A5083A33", - "size": 6347, + "sha256": "DCAFAFB9A5541BA40FE48FC8DBDD57E804BBD9DC5C22430F7979B0080AD81D29", + "size": 6152, "defined_public_names": [ "MyPipeline", "SampleWeightedPipeline", @@ -3039,8 +3039,8 @@ "path": "src/RiskLabAI/optimization/nco.py", "module": "RiskLabAI.optimization.nco", "module_kind": "module", - "sha256": "96D9A482B49D9E784B6790B46E28EFE706E3C63A262521D721BD75C46A59085A", - "size": 4122, + "sha256": "4FB1EE548E62CEF2BDD71E289F8F3DAD403F92AEE95820DA2BF1DA9409CAAA53", + "size": 3992, "defined_public_names": [ "get_optimal_portfolio_weights", "get_optimal_portfolio_weights_nco" @@ -3233,8 +3233,8 @@ "path": "src/RiskLabAI/utils/constants.py", "module": "RiskLabAI.utils.constants", "module_kind": "module", - "sha256": "A151D28E3AC9C3860C823F60097360532230864FF049ADDB5524ABBD99A88ABA", - "size": 3088, + "sha256": "751B87152BF9EABFE4AE5BB9CF35F38FBD56E4FE2691BF52B749CC3DAFAACE5A", + "size": 3003, "defined_public_names": [ "BUY_TICKS_NUMBER", "CLOSE_PRICE", @@ -3274,8 +3274,8 @@ "path": "src/RiskLabAI/utils/ewma.py", "module": "RiskLabAI.utils.ewma", "module_kind": "module", - "sha256": "31C124A43EA1CD2BACE221F76C3EB3518E71C19D6BF547AF20D555DC8ACD97FC", - "size": 2173, + "sha256": "48306B7F06272DFBABFB0CFE2B147F232038884BFCE77FEE072583A8F8F6355B", + "size": 2118, "defined_public_names": [ "ewma" ], diff --git a/TEST_INVENTORY.json b/TEST_INVENTORY.json index 5a8d041..5d2f7c9 100644 --- a/TEST_INVENTORY.json +++ b/TEST_INVENTORY.json @@ -13,21 +13,21 @@ }, "collected_cases_in_reference_full_feature_environment": { "preserved_library": 343, - "causal_factor_analysis": 341, + "causal_factor_analysis": 342, "technical_indicator_oracles": 8, - "total": 692 + "total": 693 }, "verified_required_matrix_cases_per_lane": { "preserved_library_applicable": 331, - "causal_factor_analysis": 341, + "causal_factor_analysis": 342, "technical_indicator_oracles": 8, - "total_applicable": 680 + "total_applicable": 681 }, + "required_lane_count": 6, + "required_lane_case_total": 4086, "base_lane_count": 6, "optional_feature_lane_count": 3, "ci_execution_count": 9, - "required_lane_count": 6, - "required_lane_case_total": 4080, "files": [ { "path": "test/backtest/test_advanced_bet_sizing.py", @@ -227,10 +227,10 @@ }, { "path": "test/causal_factor_analysis/test_minimum_variance.py", - "size": 15900, - "sha256": "F0C7A037F7A2AE1652E2E69362898572095EA6342EAB5699367C8FC84E929D9D", + "size": 16521, + "sha256": "4DC42E571B7B2485B362366F8257F27CCC8C69B019BA50D1D3D74DC95F5571C7", "group": "CAUSAL_FACTOR_ANALYSIS", - "declared_test_functions": 32 + "declared_test_functions": 33 }, { "path": "test/causal_factor_analysis/test_protocol.py", diff --git a/docs/compatibility.md b/docs/compatibility.md index a7f09a6..61e80e7 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -96,14 +96,14 @@ deleted in this release. ## Matrix evidence -Every lane passed all 341 frozen causal-factor tests. Across six lanes this is -2,046 causal assertions with no failures. +Every required base lane passed all 342 frozen causal-factor tests. Across six +lanes this is 2,052 causal assertions with no failures. -The final self-contained package test tree collects exactly 692 tests: 343 -preserved-library tests, 341 causal-factor tests, and 8 independent technical- +The final self-contained package test tree collects exactly 693 tests: 343 +preserved-library tests, 342 causal-factor tests, and 8 independent technical- indicator oracles. Its complete Python 3.12 current-dependency run succeeded -with 691 passes and one declared Windows platform skip. The package inventory -records both the collected suite and the 680-test applicable base matrix used +with 692 passes and one declared Windows platform skip. The package inventory +records both the collected suite and the 681-test applicable base matrix used on every supported interpreter/dependency lane. The complete preserved base suite passed 331 applicable tests per lane. Five diff --git a/src/RiskLabAI/causal_factor_analysis/optimizer.py b/src/RiskLabAI/causal_factor_analysis/optimizer.py index c39a124..c5ade99 100644 --- a/src/RiskLabAI/causal_factor_analysis/optimizer.py +++ b/src/RiskLabAI/causal_factor_analysis/optimizer.py @@ -85,14 +85,16 @@ def minimum_variance_factor_weights( Notes ----- - The published inverse formula is evaluated by Cholesky-whitening the - exposures and solving the resulting minimum-norm problem with reduced QR. - This avoids both explicit inverses and the condition-number squaring of a - normal-equation Gram matrix. Covariance and constraint units are normalized - by positive scalars without changing the optimizer, and the requested - exposures are verified before return. No diagonal jitter or pseudoinverse - is applied: singular covariance matrices and redundant factor constraints - are outside the stated assumptions of the model. + For square full-rank exposure systems, Equation (5) is evaluated directly + by solving :math:`B^\mathsf{T}\omega=c`. Otherwise, the published inverse + formula is evaluated by Cholesky-whitening the exposures and solving the + resulting minimum-norm problem with reduced QR. This avoids both explicit + inverses and the condition-number squaring of a normal-equation Gram matrix. + Covariance and constraint units are normalized by positive scalars without + changing the optimizer, and the requested exposures are verified before + return. No diagonal jitter or pseudoinverse is applied: singular covariance + matrices and redundant factor constraints are outside the stated assumptions + of the model. Examples -------- @@ -181,12 +183,15 @@ def minimum_variance_factor_weights( if np.linalg.matrix_rank(whitened_exposures) != n_factors: raise ValueError("factor_exposures must have full column rank") - orthonormal_basis, triangular_factor = np.linalg.qr( - whitened_exposures, mode="reduced" - ) - transformed_targets = np.linalg.solve(triangular_factor.T, scaled_targets) - whitened_weights = orthonormal_basis @ transformed_targets - weights = np.linalg.solve(covariance_cholesky.T, whitened_weights) + if n_factors == n_assets: + weights = np.linalg.solve(scaled_exposures.T, scaled_targets) + else: + orthonormal_basis, triangular_factor = np.linalg.qr( + whitened_exposures, mode="reduced" + ) + transformed_targets = np.linalg.solve(triangular_factor.T, scaled_targets) + whitened_weights = orthonormal_basis @ transformed_targets + weights = np.linalg.solve(covariance_cholesky.T, whitened_weights) except np.linalg.LinAlgError as exc: raise ValueError("factor_exposures must have full column rank") from exc diff --git a/test/causal_factor_analysis/test_minimum_variance.py b/test/causal_factor_analysis/test_minimum_variance.py index 817e0a4..7d04f08 100644 --- a/test/causal_factor_analysis/test_minimum_variance.py +++ b/test/causal_factor_analysis/test_minimum_variance.py @@ -78,6 +78,22 @@ def test_eq5_square_solution_is_covariance_independent(): np.testing.assert_allclose(weights, expected, rtol=0.0, atol=1e-13) +def test_eq5_square_solution_does_not_use_qr(monkeypatch): + def fail_if_called(*_args, **_kwargs): + raise AssertionError("square exposure systems must use the Equation (5) solve") + + monkeypatch.setattr(np.linalg, "qr", fail_if_called) + exposures = np.array([[1.0, 1.0], [1.0, 1.0 + 1e-9]]) + targets = np.array([1.0, 0.0]) + + weights = minimum_variance_factor_weights(np.eye(2), exposures, targets) + + np.testing.assert_allclose( + weights, np.linalg.solve(exposures.T, targets), rtol=5e-8, atol=0.0 + ) + np.testing.assert_allclose(exposures.T @ weights, targets, rtol=0.0, atol=1e-12) + + def test_eq6_single_factor_solution(): covariance = np.array([[2.0, 0.5], [0.5, 1.0]]) exposures = np.array([[1.0], [1.0]]) From 31a13f82755adb7c63166c309355ecd0700dfba3 Mon Sep 17 00:00:00 2001 From: hamid-arian Date: Mon, 24 Aug 2026 22:27:40 -0400 Subject: [PATCH 5/6] Fix cross-platform optimizer residual test --- TEST_INVENTORY.json | 4 ++-- .../test_minimum_variance.py | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/TEST_INVENTORY.json b/TEST_INVENTORY.json index 5d2f7c9..f2b3476 100644 --- a/TEST_INVENTORY.json +++ b/TEST_INVENTORY.json @@ -227,8 +227,8 @@ }, { "path": "test/causal_factor_analysis/test_minimum_variance.py", - "size": 16521, - "sha256": "4DC42E571B7B2485B362366F8257F27CCC8C69B019BA50D1D3D74DC95F5571C7", + "size": 17057, + "sha256": "9B2D772C018F99D358168AF9462DC48E0B7C53C924370A8B564AC1C4C66CE1FB", "group": "CAUSAL_FACTOR_ANALYSIS", "declared_test_functions": 33 }, diff --git a/test/causal_factor_analysis/test_minimum_variance.py b/test/causal_factor_analysis/test_minimum_variance.py index 7d04f08..454d775 100644 --- a/test/causal_factor_analysis/test_minimum_variance.py +++ b/test/causal_factor_analysis/test_minimum_variance.py @@ -10,6 +10,18 @@ from RiskLabAI.causal_factor_analysis import minimum_variance_factor_weights +def _assert_factor_constraints_within_roundoff(exposures, weights, targets): + """Check componentwise residuals against a float64 backward-error bound.""" + constraint_matrix = exposures.T + achieved = constraint_matrix @ weights + residuals = np.abs(achieved - targets) + roundoff_scales = np.abs(constraint_matrix) @ np.abs(weights) + np.abs(targets) + tolerances = ( + 8.0 * constraint_matrix.shape[1] * np.finfo(np.float64).eps * roundoff_scales + ) + np.testing.assert_array_less(residuals, np.nextafter(tolerances, np.inf)) + + def test_eq4_solution_satisfies_exposure_and_kkt_conditions(): covariance = np.diag([1.0, 2.0, 4.0]) exposures = np.array( @@ -91,7 +103,7 @@ def fail_if_called(*_args, **_kwargs): np.testing.assert_allclose( weights, np.linalg.solve(exposures.T, targets), rtol=5e-8, atol=0.0 ) - np.testing.assert_allclose(exposures.T @ weights, targets, rtol=0.0, atol=1e-12) + _assert_factor_constraints_within_roundoff(exposures, weights, targets) def test_eq6_single_factor_solution(): @@ -124,7 +136,7 @@ def test_near_collinear_square_constraints_remain_feasible(): direct_solution = np.linalg.solve(exposures.T, targets) np.testing.assert_allclose(weights, direct_solution, rtol=5e-8, atol=0.0) - np.testing.assert_allclose(exposures.T @ weights, targets, rtol=0.0, atol=1e-12) + _assert_factor_constraints_within_roundoff(exposures, weights, targets) def test_extreme_finite_exposure_scale_does_not_overflow(): From 10ec9ad5ef0331dbd14319bdf9c7e29765400771 Mon Sep 17 00:00:00 2001 From: hamid-arian Date: Tue, 25 Aug 2026 15:38:41 -0400 Subject: [PATCH 6/6] Restore targeted-shrinkage denoising API --- PACKAGE_INSPECTION.md | 17 +++-- PUBLIC_API.json | 24 +++--- README.md | 9 ++- RELEASE_NOTES_NEXT_MAJOR.md | 11 ++- TEST_INVENTORY.json | 16 ++-- docs/compatibility.md | 8 +- src/RiskLabAI/data/denoise/__init__.py | 1 + src/RiskLabAI/data/denoise/denoising.py | 97 +++++++++++++++++++++++++ test/data/denoise/test_denoising.py | 78 ++++++++++++++++++++ 9 files changed, 227 insertions(+), 34 deletions(-) diff --git a/PACKAGE_INSPECTION.md b/PACKAGE_INSPECTION.md index 9c093be..fceac72 100644 --- a/PACKAGE_INSPECTION.md +++ b/PACKAGE_INSPECTION.md @@ -44,13 +44,16 @@ the reviewed source bytes. In fresh, isolated environments for every supported Python and valid NumPy lane, verify the exact module origins, `RiskLabAI.__version__ == "3.0.0"`, the 13 root exports, all module and package exports in `PUBLIC_API.json`, and the -complete 57-feature causal API. Run the exact package-scoped test inventory and -the valid optional-feature lanes. Confirm that optional groups do not become -base dependencies and that changepoints alone is unavailable on Python 3.14. -Run installed-package tests outside the source checkout and reject any import -whose resolved path remains inside the repository. Tests that launch child -interpreters must give those processes clean temporary working directories and -apply the same origin rejection. +complete 57-feature causal API. For every module that declares `__all__`, +require every listed name to be bound on that installed module and resolvable by +an explicit import; an advertised but unbound name rejects the artifact. Run +the exact package-scoped test inventory and the valid optional-feature lanes. +Confirm that optional groups do not become base dependencies and that +changepoints alone is unavailable on Python 3.14. Run installed-package tests +outside the source checkout and reject any import whose resolved path remains +inside the repository. Tests that launch child interpreters must give those +processes clean temporary working directories and apply the same origin +rejection. ## Final review diff --git a/PUBLIC_API.json b/PUBLIC_API.json index 1852cd1..026794f 100644 --- a/PUBLIC_API.json +++ b/PUBLIC_API.json @@ -1547,8 +1547,8 @@ "path": "src/RiskLabAI/data/denoise/__init__.py", "module": "RiskLabAI.data.denoise", "module_kind": "package", - "sha256": "90FC750BE16266C6625CCA5423D69A3FA734A66B5E81CC294CD7B8450DF36A87", - "size": 824, + "sha256": "72E4069E071BF4ED278796A6FF14751B99ED9B64D71D0175F57150C1F00427F0", + "size": 844, "defined_public_names": [], "definitions": [] }, @@ -1556,13 +1556,14 @@ "path": "src/RiskLabAI/data/denoise/denoising.py", "module": "RiskLabAI.data.denoise.denoising", "module_kind": "module", - "sha256": "AE33547DD525D4905A7322995E491D8D361D434E05F56CDD1155F7AF04CD7371", - "size": 11161, + "sha256": "4F78D615A8B429309E2840AFBE1429D6340231F5DE73BEAA93C4741545AAA06E", + "size": 15138, "defined_public_names": [ "corr_to_cov", "cov_to_corr", "denoise_cov", "denoised_corr", + "denoised_corr2", "find_max_eval", "fit_kde", "marcenko_pastur_pdf", @@ -1586,11 +1587,16 @@ "kind": "function", "signature": "(eigenvalues: np.ndarray, q: float, bandwidth: float)" }, - { - "name": "denoised_corr", - "kind": "function", - "signature": "(eigenvalues: np.ndarray, eigenvectors: np.ndarray, num_facts: int)" - }, + { + "name": "denoised_corr", + "kind": "function", + "signature": "(eigenvalues: np.ndarray, eigenvectors: np.ndarray, num_facts: int)" + }, + { + "name": "denoised_corr2", + "kind": "function", + "signature": "(eigenvalues: np.ndarray, eigenvectors: np.ndarray, num_factors: int, alpha: float=0.0)" + }, { "name": "pca", "kind": "function", diff --git a/README.md b/README.md index 0ba418b..4d74882 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,11 @@ Python distribution and import namespace remain `RiskLabAI`; the Julia package remains `RiskLabAI` with UUID `a72881da-fdaa-49c1-8962-99caf4ccfee8`. The distributable metadata contract is complete, but no wheel or source -distribution has been created, installed, published, registered, or released. -The approved target version is `3.0.0`. The source remains blocked until a -separately authorized temporary artifact is inspected and the owner separately -authorizes version-control and release actions. +distribution exists for this corrected source state, and nothing has been +published, registered, or released. The approved target version is `3.0.0`. +The source remains blocked until a separately authorized temporary artifact is +inspected and the owner separately authorizes version-control and release +actions. ## Final Python source scope diff --git a/RELEASE_NOTES_NEXT_MAJOR.md b/RELEASE_NOTES_NEXT_MAJOR.md index e0af397..cfcdc1d 100644 --- a/RELEASE_NOTES_NEXT_MAJOR.md +++ b/RELEASE_NOTES_NEXT_MAJOR.md @@ -14,6 +14,12 @@ dependencies, a Python-3.14-specific Joblib floor, and self-contained standard technical indicators. Python 3.14 remains supported when the optional changepoint backend is unavailable. +The preserved targeted-shrinkage API `RiskLabAI.data.denoise.denoised_corr2` +is restored after a refactor left its public export without the corresponding +implementation. Its historical parameter contract and published analytical +formula are covered by exact matrix, boundary, validation, and export-binding +tests. + Compatibility aliases that remain present in 3.0.0 now consistently warn of removal in 4.0.0. No alias is silently deleted, and the warning contract is locked by package-scoped tests. @@ -33,5 +39,6 @@ The static distributable metadata, exact 124-module runtime inventory, 78-file package-scoped test inventory, public API inventory, documentation, examples, dependency groups, governed CI workflow, and 215-file intended source-distribution list are now frozen. No retained package artifact has been -created. The remaining gates are future artifact inspection and separate human -authorization for version-control, publication, and release actions. +created for this corrected source state. The remaining gates are future +artifact inspection and separate human authorization for version-control, +publication, and release actions. diff --git a/TEST_INVENTORY.json b/TEST_INVENTORY.json index f2b3476..5f68554 100644 --- a/TEST_INVENTORY.json +++ b/TEST_INVENTORY.json @@ -12,19 +12,19 @@ "TECHNICAL_INDICATOR_ORACLES": 1 }, "collected_cases_in_reference_full_feature_environment": { - "preserved_library": 343, + "preserved_library": 354, "causal_factor_analysis": 342, "technical_indicator_oracles": 8, - "total": 693 + "total": 704 }, "verified_required_matrix_cases_per_lane": { - "preserved_library_applicable": 331, + "preserved_library_applicable": 342, "causal_factor_analysis": 342, "technical_indicator_oracles": 8, - "total_applicable": 681 + "total_applicable": 692 }, "required_lane_count": 6, - "required_lane_case_total": 4086, + "required_lane_case_total": 4152, "base_lane_count": 6, "optional_feature_lane_count": 3, "ci_execution_count": 9, @@ -297,10 +297,10 @@ }, { "path": "test/data/denoise/test_denoising.py", - "size": 2818, - "sha256": "DAC2EA0D49203308445314A87BBBD73EDA875C197996D75F88C7F5074C4E9D33", + "size": 6035, + "sha256": "4CAE78A888AEEE68C65C7A4625208E5EFC0CAD101D7DE05C35D2C0B4F0272FFE", "group": "PRESERVED_LIBRARY", - "declared_test_functions": 4 + "declared_test_functions": 11 }, { "path": "test/data/denoise/test_nercome.py", diff --git a/docs/compatibility.md b/docs/compatibility.md index 61e80e7..245f817 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -99,14 +99,14 @@ deleted in this release. Every required base lane passed all 342 frozen causal-factor tests. Across six lanes this is 2,052 causal assertions with no failures. -The final self-contained package test tree collects exactly 693 tests: 343 +The final self-contained package test tree collects exactly 704 tests: 354 preserved-library tests, 342 causal-factor tests, and 8 independent technical- indicator oracles. Its complete Python 3.12 current-dependency run succeeded -with 692 passes and one declared Windows platform skip. The package inventory -records both the collected suite and the 681-test applicable base matrix used +with 703 passes and one declared Windows platform skip. The package inventory +records both the collected suite and the 692-test applicable base matrix used on every supported interpreter/dependency lane. -The complete preserved base suite passed 331 applicable tests per lane. Five +The complete preserved base suite passed 342 applicable tests per lane. Five platform or optional-capability cases were skipped by their declared guards; the two QuantEcon-dependent cases were exercised separately in the `synth` lanes. The new indicator-oracle suite passed 8 tests in every lane. Numba's diff --git a/src/RiskLabAI/data/denoise/__init__.py b/src/RiskLabAI/data/denoise/__init__.py index 64ba8f1..3506d53 100644 --- a/src/RiskLabAI/data/denoise/__init__.py +++ b/src/RiskLabAI/data/denoise/__init__.py @@ -15,6 +15,7 @@ cov_to_corr, denoise_cov, denoised_corr, + denoised_corr2, find_max_eval, fit_kde, marcenko_pastur_pdf, diff --git a/src/RiskLabAI/data/denoise/denoising.py b/src/RiskLabAI/data/denoise/denoising.py index 27d56d6..61646e8 100644 --- a/src/RiskLabAI/data/denoise/denoising.py +++ b/src/RiskLabAI/data/denoise/denoising.py @@ -12,6 +12,7 @@ John Wiley & Sons, Chapter 2. """ +from numbers import Integral, Real from typing import Optional import numpy as np @@ -239,6 +240,102 @@ def denoised_corr( return corr1 +def denoised_corr2( + eigenvalues: np.ndarray, + eigenvectors: np.ndarray, + num_factors: int, + alpha: float = 0.0, +) -> np.ndarray: + r"""Denoise a correlation matrix by targeted eigenvector shrinkage. + + The leading ``num_factors`` eigenpairs are retained as signal. The + remaining eigenpairs are treated as noise, and their off-diagonal + contribution is multiplied by ``alpha`` while their diagonal contribution + is preserved. Thus ``alpha=0`` applies full targeted shrinkage and + ``alpha=1`` reconstructs the unshrunk matrix. + + This is the targeted-shrinkage construction in López de Prado (2020), + *Machine Learning for Asset Managers*, Section 2.5.2, Code Snippet 2.6. + + Parameters + ---------- + eigenvalues : np.ndarray + Square diagonal matrix of eigenvalues, ordered consistently with + ``eigenvectors`` and conventionally from largest to smallest. + eigenvectors : np.ndarray + Square matrix whose columns are the corresponding eigenvectors. + num_factors : int + Number of leading eigenpairs classified as signal. Values from zero + through the matrix dimension are allowed. + alpha : float, default=0.0 + Noise shrinkage intensity in the closed interval ``[0, 1]``. + + Returns + ------- + np.ndarray + The targeted-shrinkage matrix. + + Raises + ------ + TypeError + If the arrays are not real numeric arrays, ``num_factors`` is not an + integer, or ``alpha`` is not real. + ValueError + If the arrays are empty or incompatible, the eigenvalue matrix is not + diagonal, an input is non-finite, or a parameter lies outside its + permitted range. + """ + eigenvalues_array = np.asarray(eigenvalues) + eigenvectors_array = np.asarray(eigenvectors) + + if ( + eigenvalues_array.ndim != 2 + or eigenvalues_array.shape[0] != eigenvalues_array.shape[1] + ): + raise ValueError("eigenvalues must be a non-empty square diagonal matrix") + if eigenvalues_array.shape[0] == 0: + raise ValueError("eigenvalues must be a non-empty square diagonal matrix") + if eigenvectors_array.shape != eigenvalues_array.shape: + raise ValueError("eigenvectors must be square and match eigenvalues") + + for name, array in ( + ("eigenvalues", eigenvalues_array), + ("eigenvectors", eigenvectors_array), + ): + if not np.issubdtype(array.dtype, np.number) or np.iscomplexobj(array): + raise TypeError(f"{name} must be a real numeric array") + if not np.all(np.isfinite(array)): + raise ValueError(f"{name} must contain only finite values") + + diagonal = np.diag(np.diag(eigenvalues_array)) + if not np.allclose(eigenvalues_array, diagonal, rtol=0.0, atol=1e-12): + raise ValueError("eigenvalues must be a diagonal matrix") + + if isinstance(num_factors, (bool, np.bool_)) or not isinstance( + num_factors, Integral + ): + raise TypeError("num_factors must be an integer") + num_factors = int(num_factors) + dimension = eigenvalues_array.shape[0] + if not 0 <= num_factors <= dimension: + raise ValueError("num_factors must be between zero and the matrix dimension") + + if isinstance(alpha, (bool, np.bool_)) or not isinstance(alpha, Real): + raise TypeError("alpha must be a real number") + alpha = float(alpha) + if not np.isfinite(alpha) or not 0.0 <= alpha <= 1.0: + raise ValueError("alpha must be finite and between zero and one") + + signal_values = eigenvalues_array[:num_factors, :num_factors] + signal_vectors = eigenvectors_array[:, :num_factors] + noise_values = eigenvalues_array[num_factors:, num_factors:] + noise_vectors = eigenvectors_array[:, num_factors:] + + signal = signal_vectors @ signal_values @ signal_vectors.T + noise = noise_vectors @ noise_values @ noise_vectors.T + return signal + alpha * noise + (1.0 - alpha) * np.diag(np.diag(noise)) + + # --- Utility Functions --- diff --git a/test/data/denoise/test_denoising.py b/test/data/denoise/test_denoising.py index cbcf59e..378d190 100644 --- a/test/data/denoise/test_denoising.py +++ b/test/data/denoise/test_denoising.py @@ -6,10 +6,12 @@ import pandas as pd import pytest +import RiskLabAI.data.denoise as denoise from RiskLabAI.data.denoise import ( corr_to_cov, cov_to_corr, denoise_cov, + denoised_corr2, marcenko_pastur_pdf, optimal_portfolio, ) @@ -71,6 +73,82 @@ def test_cov_corr_conversion(): assert np.allclose(cov, cov_new) +def test_denoise_public_exports_are_bound(): + """Every advertised denoising name must be bound on the package.""" + assert all(hasattr(denoise, name) for name in denoise.__all__) + + +def test_denoised_corr2_hand_verifiable_rotation(): + """Targeted shrinkage matches an exact two-factor calculation.""" + eigenvalues = np.diag([3.0, 1.0]) + eigenvectors = np.array([[1.0, 1.0], [1.0, -1.0]]) / np.sqrt(2.0) + + fully_shrunk = denoised_corr2(eigenvalues, eigenvectors, 1, alpha=0.0) + half_shrunk = denoised_corr2(eigenvalues, eigenvectors, 1, alpha=0.5) + unshrunk = denoised_corr2(eigenvalues, eigenvectors, 1, alpha=1.0) + + assert np.allclose(fully_shrunk, [[2.0, 1.5], [1.5, 2.0]]) + assert np.allclose(half_shrunk, [[2.0, 1.25], [1.25, 2.0]]) + assert np.allclose(unshrunk, [[2.0, 1.0], [1.0, 2.0]]) + + +def test_denoised_corr2_preserves_diagonal_trace_and_interpolates(): + """Shrinkage changes only the noise off-diagonal contribution.""" + eigenvalues = np.diag([4.0, 2.0, 0.5]) + eigenvectors, _ = np.linalg.qr( + np.array([[1.0, 2.0, 3.0], [2.0, -1.0, 1.0], [1.0, 1.0, -1.0]]) + ) + original = eigenvectors @ eigenvalues @ eigenvectors.T + + fully_shrunk = denoised_corr2(eigenvalues, eigenvectors, 1, alpha=0.0) + partly_shrunk = denoised_corr2(eigenvalues, eigenvectors, 1, alpha=0.25) + unshrunk = denoised_corr2(eigenvalues, eigenvectors, 1, alpha=1.0) + + assert np.allclose(unshrunk, original) + assert np.allclose(np.diag(fully_shrunk), np.diag(original)) + assert np.isclose(np.trace(fully_shrunk), np.trace(original)) + assert np.allclose(partly_shrunk, 0.75 * fully_shrunk + 0.25 * unshrunk) + + +def test_denoised_corr2_factor_count_boundaries(): + """The all-noise and all-signal partitions have exact meanings.""" + eigenvalues = np.diag([3.0, 1.0]) + eigenvectors = np.array([[1.0, 1.0], [1.0, -1.0]]) / np.sqrt(2.0) + original = eigenvectors @ eigenvalues @ eigenvectors.T + + assert np.allclose( + denoised_corr2(eigenvalues, eigenvectors, 0, alpha=0.0), + np.diag(np.diag(original)), + ) + assert np.allclose( + denoised_corr2(eigenvalues, eigenvectors, 2, alpha=0.0), original + ) + + +@pytest.mark.parametrize("alpha", [-0.01, 1.01, np.inf, np.nan]) +def test_denoised_corr2_rejects_invalid_alpha(alpha): + eigenvalues = np.diag([2.0, 1.0]) + eigenvectors = np.eye(2) + + with pytest.raises(ValueError, match="alpha"): + denoised_corr2(eigenvalues, eigenvectors, 1, alpha=alpha) + + +@pytest.mark.parametrize("num_factors", [-1, 3]) +def test_denoised_corr2_rejects_invalid_factor_count(num_factors): + with pytest.raises(ValueError, match="num_factors"): + denoised_corr2(np.diag([2.0, 1.0]), np.eye(2), num_factors) + + +def test_denoised_corr2_rejects_incompatible_inputs(): + with pytest.raises(ValueError, match="diagonal"): + denoised_corr2(np.array([[2.0, 0.1], [0.0, 1.0]]), np.eye(2), 1) + with pytest.raises(ValueError, match="match eigenvalues"): + denoised_corr2(np.diag([2.0, 1.0]), np.eye(3), 1) + with pytest.raises(TypeError, match="num_factors"): + denoised_corr2(np.diag([2.0, 1.0]), np.eye(2), 1.0) + + def test_denoise_cov(noisy_cov_matrix): """Test the end-to-end denoising function.""" cov, q = noisy_cov_matrix